diff --git a/libpolyml/exporter.cpp b/libpolyml/exporter.cpp index 4bd3032c..45bffacf 100644 --- a/libpolyml/exporter.cpp +++ b/libpolyml/exporter.cpp @@ -1,984 +1,984 @@ /* Title: exporter.cpp - Export a function as an object or C file Copyright (c) 2006-7, 2015, 2016-21 David C.J. Matthews This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 2.1 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifdef HAVE_CONFIG_H #include "config.h" #elif defined(_WIN32) #include "winconfig.h" #else #error "No configuration file" #endif #ifdef HAVE_ASSERT_H #include #define ASSERT(x) assert(x) #else #define ASSERT(x) #endif #ifdef HAVE_STRING_H #include #endif #ifdef HAVE_ERRNO_H #include #endif #ifdef HAVE_SYS_PARAM_H #include #endif #ifdef HAVE_STDLIB_H #include #endif #if (defined(_WIN32)) #include #else #define _T(x) x #define _tcslen strlen #define _tcscmp strcmp #define _tcscat strcat #endif #include "exporter.h" #include "save_vec.h" #include "polystring.h" #include "run_time.h" #include "osmem.h" #include "scanaddrs.h" #include "gc.h" #include "machine_dep.h" #include "diagnostics.h" #include "memmgr.h" #include "processes.h" // For IO_SPACING #include "sys.h" // For EXC_Fail #include "rtsentry.h" #include "pexport.h" #ifdef HAVE_PECOFF #include "pecoffexport.h" #elif defined(HAVE_ELF_H) || defined(HAVE_ELF_ABI_H) #include "elfexport.h" #elif defined(HAVE_MACH_O_RELOC_H) #include "machoexport.h" #endif #if (defined(_WIN32)) #define NOMEMORY ERROR_NOT_ENOUGH_MEMORY #define ERRORNUMBER _doserrno #else #define NOMEMORY ENOMEM #define ERRORNUMBER errno #endif extern "C" { POLYEXTERNALSYMBOL POLYUNSIGNED PolyExport(FirstArgument threadId, PolyWord fileName, PolyWord root); POLYEXTERNALSYMBOL POLYUNSIGNED PolyExportPortable(FirstArgument threadId, PolyWord fileName, PolyWord root); } /* To export the function and everything reachable from it we need to copy all the objects into a new area. We leave tombstones in the original objects by overwriting the length word. That prevents us from copying an object twice and breaks loops. Once we've copied the objects we then have to go back over the memory and turn the tombstones back into length words. */ GraveYard::~GraveYard() { free(graves); } // Used to calculate the space required for the ordinary mutables // and the no-overwrite mutables. They are interspersed in local space. class MutSizes : public ScanAddress { public: MutSizes() : mutSize(0), noOverSize(0) {} virtual PolyObject *ScanObjectAddress(PolyObject *base) { return base; }// No Actually used virtual void ScanAddressesInObject(PolyObject *base, POLYUNSIGNED lengthWord) { const POLYUNSIGNED words = OBJ_OBJECT_LENGTH(lengthWord) + 1; // Include length word if (OBJ_IS_NO_OVERWRITE(lengthWord)) noOverSize += words; else mutSize += words; } POLYUNSIGNED mutSize, noOverSize; }; CopyScan::CopyScan(unsigned h/*=0*/): hierarchy(h) { defaultImmSize = defaultMutSize = defaultCodeSize = defaultNoOverSize = 0; tombs = 0; graveYard = 0; } void CopyScan::initialise(bool isExport/*=true*/) { ASSERT(gMem.eSpaces.size() == 0); // Set the space sizes to a proportion of the space currently in use. // Computing these sizes is not obvious because CopyScan is used both // for export and for saved states. For saved states in particular we // want to use a smaller size because they are retained after we save // the state and if we have many child saved states it's important not // to waste memory. if (hierarchy == 0) { graveYard = new GraveYard[gMem.pSpaces.size()]; if (graveYard == 0) { if (debugOptions & DEBUG_SAVING) Log("SAVE: Unable to allocate graveyard, size: %lu.\n", gMem.pSpaces.size()); throw MemoryException(); } } for (std::vector::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++) { PermanentMemSpace *space = *i; if (space->hierarchy >= hierarchy) { // Include this if we're exporting (hierarchy=0) or if we're saving a state // and will include this in the new state. size_t size = (space->top-space->bottom)/4; if (space->noOverwrite) defaultNoOverSize += size; else if (space->isMutable) defaultMutSize += size; else if (space->isCode) defaultCodeSize += size; else defaultImmSize += size; if (space->hierarchy == 0 && ! space->isMutable) { // We need a separate area for the tombstones because this is read-only graveYard[tombs].graves = (PolyWord*)calloc(space->spaceSize(), sizeof(PolyWord)); if (graveYard[tombs].graves == 0) { if (debugOptions & DEBUG_SAVING) Log("SAVE: Unable to allocate graveyard for permanent space, size: %lu.\n", space->spaceSize() * sizeof(PolyWord)); throw MemoryException(); } if (debugOptions & DEBUG_SAVING) Log("SAVE: Allocated graveyard for permanent space, %p size: %lu.\n", graveYard[tombs].graves, space->spaceSize() * sizeof(PolyWord)); graveYard[tombs].startAddr = space->bottom; graveYard[tombs].endAddr = space->top; tombs++; } } } for (std::vector::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++) { LocalMemSpace *space = *i; uintptr_t size = space->allocatedSpace(); // It looks as though the mutable size generally gets // overestimated while the immutable size is correct. if (space->isMutable) { MutSizes sizeMut; sizeMut.ScanAddressesInRegion(space->bottom, space->lowerAllocPtr); sizeMut.ScanAddressesInRegion(space->upperAllocPtr, space->top); defaultNoOverSize += sizeMut.noOverSize / 4; defaultMutSize += sizeMut.mutSize / 4; } else defaultImmSize += size/2; } for (std::vector::iterator i = gMem.cSpaces.begin(); i < gMem.cSpaces.end(); i++) { CodeSpace *space = *i; uintptr_t size = space->spaceSize(); defaultCodeSize += size/2; } if (isExport) { // Minimum 1M words. if (defaultMutSize < 1024*1024) defaultMutSize = 1024*1024; if (defaultImmSize < 1024*1024) defaultImmSize = 1024*1024; if (defaultCodeSize < 1024*1024) defaultCodeSize = 1024*1024; #ifdef MACOSX // Limit the segment size for Mac OS X. The linker has a limit of 2^24 relocations // in a segment so this is a crude way of ensuring the limit isn't exceeded. // It's unlikely to be exceeded by the code itself. // Actually, from trial-and-error, the limit seems to be around 6M. if (defaultMutSize > 6 * 1024 * 1024) defaultMutSize = 6 * 1024 * 1024; if (defaultImmSize > 6 * 1024 * 1024) defaultImmSize = 6 * 1024 * 1024; #endif if (defaultNoOverSize < 4096) defaultNoOverSize = 4096; // Except for the no-overwrite area } else { // Much smaller minimum sizes for saved states. if (defaultMutSize < 1024) defaultMutSize = 1024; if (defaultImmSize < 4096) defaultImmSize = 4096; if (defaultCodeSize < 4096) defaultCodeSize = 4096; if (defaultNoOverSize < 4096) defaultNoOverSize = 4096; // Set maximum sizes as well. We may have insufficient contiguous space for // very large areas. if (defaultMutSize > 1024 * 1024) defaultMutSize = 1024 * 1024; if (defaultImmSize > 1024 * 1024) defaultImmSize = 1024 * 1024; if (defaultCodeSize > 1024 * 1024) defaultCodeSize = 1024 * 1024; if (defaultNoOverSize > 1024 * 1024) defaultNoOverSize = 1024 * 1024; } if (debugOptions & DEBUG_SAVING) Log("SAVE: Copyscan default sizes: Immutable: %" POLYUFMT ", Mutable: %" POLYUFMT ", Code: %" POLYUFMT ", No-overwrite %" POLYUFMT ".\n", defaultImmSize, defaultMutSize, defaultCodeSize, defaultNoOverSize); } CopyScan::~CopyScan() { gMem.DeleteExportSpaces(); if (graveYard) delete[](graveYard); } // This function is called for each address in an object // once it has been copied to its new location. We copy first // then scan to update the addresses. POLYUNSIGNED CopyScan::ScanAddressAt(PolyWord *pt) { PolyWord val = *pt; // Ignore integers. if (IS_INT(val) || val == PolyWord::FromUnsigned(0)) return 0; PolyObject *obj = val.AsObjPtr(); POLYUNSIGNED l = ScanAddress(&obj); *pt = obj; return l; } // This function is called for each address in an object // once it has been copied to its new location. We copy first // then scan to update the addresses. POLYUNSIGNED CopyScan::ScanAddress(PolyObject **pt) { PolyObject *obj = *pt; MemSpace *space = gMem.SpaceForObjectAddress(obj); ASSERT(space != 0); // We may sometimes get addresses that have already been updated // to point to the new area. e.g. (only?) in the case of constants // that have been updated in ScanConstantsWithinCode. if (space->spaceType == ST_EXPORT) return 0; // If this is at a lower level than the hierarchy we are saving // then leave it untouched. if (space->spaceType == ST_PERMANENT) { PermanentMemSpace *pmSpace = (PermanentMemSpace*)space; if (pmSpace->hierarchy < hierarchy) return 0; } // Have we already scanned this? if (obj->ContainsForwardingPtr()) { // Update the address to the new value. #ifdef POLYML32IN64 PolyObject *newAddr; if (space->isCode) newAddr = (PolyObject*)(globalCodeBase + ((obj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); else newAddr = obj->GetForwardingPtr(); #else PolyObject *newAddr = obj->GetForwardingPtr(); #endif *pt = newAddr; return 0; // No need to scan it again. } else if (space->spaceType == ST_PERMANENT) { // See if we have this in the grave-yard. for (unsigned i = 0; i < tombs; i++) { GraveYard *g = &graveYard[i]; if ((PolyWord*)obj >= g->startAddr && (PolyWord*)obj < g->endAddr) { PolyWord *tombAddr = g->graves + ((PolyWord*)obj - g->startAddr); PolyObject *tombObject = (PolyObject*)tombAddr; if (tombObject->ContainsForwardingPtr()) { #ifdef POLYML32IN64 PolyObject *newAddr; if (space->isCode) newAddr = (PolyObject*)(globalCodeBase + ((tombObject->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); else newAddr = tombObject->GetForwardingPtr(); #else PolyObject *newAddr = tombObject->GetForwardingPtr(); #endif *pt = newAddr; return 0; } break; // No need to look further } } } // No, we need to copy it. ASSERT(space->spaceType == ST_LOCAL || space->spaceType == ST_PERMANENT || space->spaceType == ST_CODE); POLYUNSIGNED lengthWord = obj->LengthWord(); POLYUNSIGNED originalLengthWord = lengthWord; POLYUNSIGNED words = OBJ_OBJECT_LENGTH(lengthWord); enum _newAddrType naType; if (obj->IsMutable()) { if (obj->IsNoOverwriteObject()) naType = NANoOverwriteMutable; else naType = NAMutable; } else if (obj->IsCodeObject()) naType = NACode; else if (obj->IsByteObject()) naType = NAByte; else naType = NAWord; PolyObject* newObj; #if((defined(HOSTARCHITECTURE_X86_64) || defined(HOSTARCHITECTURE_AARCH64)) && ! defined(POLYML32IN64)) // Split the constant area off into a separate object. This allows us to create a // position-independent executable. if (obj->IsCodeObject() && hierarchy == 0) { PolyWord* constPtr; POLYUNSIGNED numConsts; machineDependent->GetConstSegmentForCode(obj, constPtr, numConsts); // Newly generated code will have the constants included with the code // but if this is in the executable the constants will have been extracted before. bool constsWereIncluded = constPtr > (PolyWord*)obj && constPtr < ((PolyWord*)obj) + words; POLYUNSIGNED codeAreaSize = words; if (constsWereIncluded) codeAreaSize -= numConsts + 1; newObj = newAddressForObject(codeAreaSize, NACode); PolyObject* writable = gMem.SpaceForObjectAddress(newObj)->writeAble(newObj); writable->SetLengthWord(codeAreaSize, F_CODE_OBJ); // set length word lengthWord = newObj->LengthWord(); // Get the actual length word used memcpy(writable, obj, codeAreaSize * sizeof(PolyWord)); PolyObject* newConsts = newAddressForObject(numConsts, NACodeConst); newConsts->SetLengthWord(numConsts); memcpy(newConsts, constPtr, numConsts * sizeof(PolyWord)); machineDependent->SetAddressOfConstants(newObj, writable, codeAreaSize, (PolyWord*)newConsts); } else #endif { newObj = newAddressForObject(words, naType); PolyObject* writAble = gMem.SpaceForObjectAddress(newObj)->writeAble(newObj); writAble->SetLengthWord(lengthWord); // copy length word if (hierarchy == 0 /* Exporting object module */ && obj->IsNoOverwriteObject() && ! obj->IsByteObject()) { // These are not exported. They are used for special values e.g. mutexes // that should be set to 0/nil/NONE at start-up. // Weak+No-overwrite byte objects are used for entry points and volatiles // in the foreign-function interface and have to be treated specially. // Note: this must not be done when exporting a saved state because the // copied version is used as the local data for the rest of the session. for (POLYUNSIGNED i = 0; i < words; i++) writAble->Set(i, TAGGED(0)); } else memcpy(writAble, obj, words * sizeof(PolyWord)); } if (space->spaceType == ST_PERMANENT && !space->isMutable && ((PermanentMemSpace*)space)->hierarchy == 0) { // The immutable permanent areas are read-only. unsigned m; for (m = 0; m < tombs; m++) { GraveYard *g = &graveYard[m]; if ((PolyWord*)obj >= g->startAddr && (PolyWord*)obj < g->endAddr) { PolyWord *tombAddr = g->graves + ((PolyWord*)obj - g->startAddr); PolyObject *tombObject = (PolyObject*)tombAddr; #ifdef POLYML32IN64 - if (isCodeObj) + if (naType == NACode) { POLYUNSIGNED ll = (POLYUNSIGNED)(((PolyWord*)newObj - globalCodeBase) >> 1 | _OBJ_TOMBSTONE_BIT); tombObject->SetLengthWord(ll); } else tombObject->SetForwardingPtr(newObj); #else tombObject->SetForwardingPtr(newObj); #endif break; // No need to look further } } ASSERT(m < tombs); // Should be there. } else if (naType == NACode) #ifdef POLYML32IN64 // If this is a code address we can't use the usual forwarding pointer format. // Instead we have to compute the offset relative to the base of the code. { POLYUNSIGNED ll = (POLYUNSIGNED)(((PolyWord*)newObj-globalCodeBase) >> 1 | _OBJ_TOMBSTONE_BIT); gMem.SpaceForObjectAddress(obj)->writeAble(obj)->SetLengthWord(ll); } #else gMem.SpaceForObjectAddress(obj)->writeAble(obj)->SetForwardingPtr(newObj); #endif else obj->SetForwardingPtr(newObj); // Put forwarding pointer in old object. if (naType == NACode) { // We should flush the instruction cache here since we will execute the code // at this location if this is a saved state. machineDependent->FlushInstructionCache(newObj, newObj->Length()); // We have to update any relative addresses within the code // to take account of its new position. We have to do that now // even though ScanAddressesInObject will do it again because this // is the only point where we have both the old and the new addresses. PolyWord *oldConstAddr; POLYUNSIGNED count; machineDependent->GetConstSegmentForCode(obj, OBJ_OBJECT_LENGTH(originalLengthWord), oldConstAddr, count); PolyWord *newConstAddr = machineDependent->ConstPtrForCode(newObj); machineDependent->ScanConstantsWithinCode(newObj, obj, words, newConstAddr, oldConstAddr, count, this); } *pt = newObj; // Update it to the newly copied object. return lengthWord; // This new object needs to be scanned. } PolyObject* CopyScan::newAddressForObject(POLYUNSIGNED words, enum _newAddrType naType) { PolyObject* newObj = 0; // Allocate a new address for the object. for (std::vector::iterator i = gMem.eSpaces.begin(); i < gMem.eSpaces.end(); i++) { PermanentMemSpace* space = *i; bool match = false; switch (naType) { case NAWord: match = !space->isMutable && !space->byteOnly && !space->isCode; case NAMutable: match = space->isMutable && !space->noOverwrite; break; case NANoOverwriteMutable: match = space->isMutable && space->noOverwrite; break; case NAByte: match = !space->isMutable && space->byteOnly; break; case NACode: match = !space->isMutable && space->isCode && !space->constArea; break; case NACodeConst: match = !space->isMutable && space->isCode && space->constArea; break; } if (match) { ASSERT(space->topPointer <= space->top && space->topPointer >= space->bottom); size_t spaceLeft = space->top - space->topPointer; if (spaceLeft > words) { newObj = (PolyObject*)(space->topPointer + 1); space->topPointer += words + 1; #ifdef POLYML32IN64 // Maintain the odd-word alignment of topPointer if ((words & 1) == 0 && space->topPointer < space->top) { *space->writeAble(space->topPointer) = PolyWord::FromUnsigned(0); space->topPointer++; } #endif break; } } } if (newObj == 0) { // Didn't find room in the existing spaces. Create a new space. uintptr_t spaceWords; switch (naType) { case NAMutable: spaceWords = defaultMutSize; break; case NANoOverwriteMutable: spaceWords = defaultNoOverSize; break; case NACode: spaceWords = defaultCodeSize; break; case NACodeConst: spaceWords = defaultCodeSize; break; default: spaceWords = defaultImmSize; } if (spaceWords <= words) spaceWords = words + 1; // Make sure there's space for this object. PermanentMemSpace* space = gMem.NewExportSpace(spaceWords, naType == NAMutable || naType == NANoOverwriteMutable, naType == NANoOverwriteMutable, naType == NACode || naType == NACodeConst); if (naType == NAByte) space->byteOnly = true; if (naType == NACodeConst) space->constArea = true; if (space == 0) { if (debugOptions & DEBUG_SAVING) Log("SAVE: Unable to allocate export space, size: %lu.\n", spaceWords); // Unable to allocate this. throw MemoryException(); } newObj = (PolyObject*)(space->topPointer + 1); space->topPointer += words + 1; #ifdef POLYML32IN64 // Maintain the odd-word alignment of topPointer if ((words & 1) == 0 && space->topPointer < space->top) { *space->writeAble(space->topPointer) = PolyWord::FromUnsigned(0); space->topPointer++; } #endif ASSERT(space->topPointer <= space->top && space->topPointer >= space->bottom); } return newObj; } // The address of code in the code area. We treat this as a normal heap cell. // We will probably need to copy this and to process addresses within it. POLYUNSIGNED CopyScan::ScanCodeAddressAt(PolyObject **pt) { POLYUNSIGNED lengthWord = ScanAddress(pt); if (lengthWord) ScanAddressesInObject(*pt, lengthWord); return 0; } PolyObject *CopyScan::ScanObjectAddress(PolyObject *base) { PolyWord val = base; // Scan this as an address. POLYUNSIGNED lengthWord = CopyScan::ScanAddressAt(&val); if (lengthWord) ScanAddressesInObject(val.AsObjPtr(), lengthWord); return val.AsObjPtr(); } #define MAX_EXTENSION 4 // The longest extension we may need to add is ".obj" // Convert the forwarding pointers in a region back into length words. // Generally if this object has a forwarding pointer that's // because we've moved it into the export region. We can, // though, get multiple levels of forwarding if there is an object // that has been shifted up by a garbage collection, leaving a forwarding // pointer and then that object has been moved to the export region. // We mustn't turn locally forwarded values back into ordinary objects // because they could contain addresses that are no longer valid. static POLYUNSIGNED GetObjLength(PolyObject *obj) { if (obj->ContainsForwardingPtr()) { PolyObject *forwardedTo; #ifdef POLYML32IN64 { MemSpace *space = gMem.SpaceForObjectAddress(obj); if (space->isCode) forwardedTo = (PolyObject*)(globalCodeBase + ((obj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); else forwardedTo = obj->GetForwardingPtr(); } #else forwardedTo = obj->GetForwardingPtr(); #endif POLYUNSIGNED length = GetObjLength(forwardedTo); MemSpace *space = gMem.SpaceForObjectAddress(forwardedTo); if (space->spaceType == ST_EXPORT) { // If this is a code object whose constant area has been split off we // need to add the length of the constant area. if (forwardedTo->IsCodeObject()) { PolyWord* constPtr; POLYUNSIGNED numConsts; machineDependent->GetConstSegmentForCode(forwardedTo, constPtr, numConsts); if (!(constPtr > (PolyWord*)forwardedTo && constPtr < ((PolyWord*)forwardedTo) + OBJ_OBJECT_LENGTH(length))) length += numConsts + 1; } gMem.SpaceForObjectAddress(obj)->writeAble(obj)->SetLengthWord(length); } return length; } else { ASSERT(obj->ContainsNormalLengthWord()); return obj->LengthWord(); } } static void FixForwarding(PolyWord *pt, size_t space) { while (space) { pt++; PolyObject *obj = (PolyObject*)pt; #ifdef POLYML32IN64 if ((uintptr_t)obj & 4) { // Skip filler words needed to align to an even word space--; continue; // We've added 1 to pt so just loop. } #endif size_t length = OBJ_OBJECT_LENGTH(GetObjLength(obj)); pt += length; ASSERT(space > length); space -= length+1; } } class ExportRequest: public MainThreadRequest { public: ExportRequest(Handle root, Exporter *exp): MainThreadRequest(MTP_EXPORTING), exportRoot(root), exporter(exp) {} virtual void Perform() { exporter->RunExport(exportRoot->WordP()); } Handle exportRoot; Exporter *exporter; }; static void exporter(TaskData *taskData, Handle fileName, Handle root, const TCHAR *extension, Exporter *exports) { size_t extLen = _tcslen(extension); TempString fileNameBuff(Poly_string_to_T_alloc(fileName->Word(), extLen)); if (fileNameBuff == NULL) raise_syscall(taskData, "Insufficient memory", NOMEMORY); size_t length = _tcslen(fileNameBuff); // Does it already have the extension? If not add it on. if (length < extLen || _tcscmp(fileNameBuff + length - extLen, extension) != 0) _tcscat(fileNameBuff, extension); #if (defined(_WIN32) && defined(UNICODE)) exports->exportFile = _wfopen(fileNameBuff, L"wb"); #else exports->exportFile = fopen(fileNameBuff, "wb"); #endif if (exports->exportFile == NULL) raise_syscall(taskData, "Cannot open export file", ERRORNUMBER); // Request a full GC to reduce the size of fix-ups. FullGC(taskData); // Request the main thread to do the export. ExportRequest request(root, exports); processes->MakeRootRequest(taskData, &request); if (exports->errorMessage) raise_fail(taskData, exports->errorMessage); } // This is called by the initial thread to actually do the export. void Exporter::RunExport(PolyObject *rootFunction) { Exporter *exports = this; PolyObject *copiedRoot = 0; CopyScan copyScan(hierarchy); try { copyScan.initialise(); // Copy the root and everything reachable from it into the temporary area. copiedRoot = copyScan.ScanObjectAddress(rootFunction); } catch (MemoryException &) { // If we ran out of memory. copiedRoot = 0; } // Fix the forwarding pointers. for (std::vector::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++) { LocalMemSpace *space = *i; // Local areas only have objects from the allocation pointer to the top. FixForwarding(space->bottom, space->lowerAllocPtr - space->bottom); FixForwarding(space->upperAllocPtr, space->top - space->upperAllocPtr); } for (std::vector::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++) { MemSpace *space = *i; // Permanent areas are filled with objects from the bottom. FixForwarding(space->bottom, space->top - space->bottom); } for (std::vector::iterator i = gMem.cSpaces.begin(); i < gMem.cSpaces.end(); i++) { MemSpace *space = *i; // Code areas are filled with objects from the bottom. FixForwarding(space->bottom, space->top - space->bottom); } // Reraise the exception after cleaning up the forwarding pointers. if (copiedRoot == 0) { exports->errorMessage = "Insufficient Memory"; return; } // Copy the areas into the export object. size_t tableEntries = gMem.eSpaces.size(); unsigned memEntry = 0; if (hierarchy != 0) tableEntries += gMem.pSpaces.size(); exports->memTable = new memoryTableEntry[tableEntries]; // If we're constructing a module we need to include the global spaces. if (hierarchy != 0) { // Permanent spaces from the executable. for (std::vector::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++) { PermanentMemSpace *space = *i; if (space->hierarchy < hierarchy) { memoryTableEntry *entry = &exports->memTable[memEntry++]; entry->mtOriginalAddr = entry->mtCurrentAddr = space->bottom; entry->mtLength = (space->topPointer-space->bottom)*sizeof(PolyWord); entry->mtIndex = space->index; entry->mtFlags = 0; if (space->isMutable) entry->mtFlags |= MTF_WRITEABLE; if (space->isCode) entry->mtFlags |= MTF_EXECUTABLE; } } newAreas = memEntry; } for (std::vector::iterator i = gMem.eSpaces.begin(); i < gMem.eSpaces.end(); i++) { memoryTableEntry *entry = &exports->memTable[memEntry++]; PermanentMemSpace *space = *i; entry->mtOriginalAddr = entry->mtCurrentAddr = space->bottom; entry->mtLength = (space->topPointer-space->bottom)*sizeof(PolyWord); entry->mtIndex = hierarchy == 0 ? memEntry-1 : space->index; entry->mtFlags = 0; if (space->isMutable) { entry->mtFlags = MTF_WRITEABLE; if (space->noOverwrite) entry->mtFlags |= MTF_NO_OVERWRITE; } if (space->isCode && !space->constArea) entry->mtFlags |= MTF_EXECUTABLE; if (space->byteOnly) entry->mtFlags |= MTF_BYTES; } ASSERT(memEntry == tableEntries); exports->memTableEntries = memEntry; exports->rootFunction = copiedRoot; try { // This can raise MemoryException at least in PExport::exportStore. exports->exportStore(); } catch (MemoryException &) { exports->errorMessage = "Insufficient Memory"; } } // Functions called via the RTS call. Handle exportNative(TaskData *taskData, Handle args) { #ifdef HAVE_PECOFF // Windows including Cygwin #if (defined(_WIN32)) const TCHAR *extension = _T(".obj"); // Windows #else const char *extension = ".o"; // Cygwin #endif PECOFFExport exports; exporter(taskData, taskData->saveVec.push(args->WordP()->Get(0)), taskData->saveVec.push(args->WordP()->Get(1)), extension, &exports); #elif defined(HAVE_ELF_H) || defined(HAVE_ELF_ABI_H) // Most Unix including Linux, FreeBSD and Solaris. const char *extension = ".o"; ELFExport exports; exporter(taskData, taskData->saveVec.push(args->WordP()->Get(0)), taskData->saveVec.push(args->WordP()->Get(1)), extension, &exports); #elif defined(HAVE_MACH_O_RELOC_H) // Mac OS-X const char *extension = ".o"; MachoExport exports; exporter(taskData, taskData->saveVec.push(args->WordP()->Get(0)), taskData->saveVec.push(args->WordP()->Get(1)), extension, &exports); #else raise_exception_string (taskData, EXC_Fail, "Native export not available for this platform"); #endif return taskData->saveVec.push(TAGGED(0)); } Handle exportPortable(TaskData *taskData, Handle args) { PExport exports; exporter(taskData, taskData->saveVec.push(args->WordP()->Get(0)), taskData->saveVec.push(args->WordP()->Get(1)), _T(".txt"), &exports); return taskData->saveVec.push(TAGGED(0)); } POLYUNSIGNED PolyExport(FirstArgument threadId, PolyWord fileName, PolyWord root) { TaskData *taskData = TaskData::FindTaskForId(threadId); ASSERT(taskData != 0); taskData->PreRTSCall(); Handle reset = taskData->saveVec.mark(); Handle pushedName = taskData->saveVec.push(fileName); Handle pushedRoot = taskData->saveVec.push(root); try { #ifdef HAVE_PECOFF // Windows including Cygwin #if (defined(_WIN32)) const TCHAR *extension = _T(".obj"); // Windows #else const char *extension = ".o"; // Cygwin #endif PECOFFExport exports; exporter(taskData, pushedName, pushedRoot, extension, &exports); #elif defined(HAVE_ELF_H) || defined(HAVE_ELF_ABI_H) // Most Unix including Linux, FreeBSD and Solaris. const char *extension = ".o"; ELFExport exports; exporter(taskData, pushedName, pushedRoot, extension, &exports); #elif defined(HAVE_MACH_O_RELOC_H) // Mac OS-X const char *extension = ".o"; MachoExport exports; exporter(taskData, pushedName, pushedRoot, extension, &exports); #else raise_exception_string (taskData, EXC_Fail, "Native export not available for this platform"); #endif } catch (...) { } // If an ML exception is raised taskData->saveVec.reset(reset); taskData->PostRTSCall(); return TAGGED(0).AsUnsigned(); // Returns unit } POLYUNSIGNED PolyExportPortable(FirstArgument threadId, PolyWord fileName, PolyWord root) { TaskData *taskData = TaskData::FindTaskForId(threadId); ASSERT(taskData != 0); taskData->PreRTSCall(); Handle reset = taskData->saveVec.mark(); Handle pushedName = taskData->saveVec.push(fileName); Handle pushedRoot = taskData->saveVec.push(root); try { PExport exports; exporter(taskData, pushedName, pushedRoot, _T(".txt"), &exports); } catch (...) { } // If an ML exception is raised taskData->saveVec.reset(reset); taskData->PostRTSCall(); return TAGGED(0).AsUnsigned(); // Returns unit } // Helper functions for exporting. We need to produce relocation information // and this code is common to every method. Exporter::Exporter(unsigned int h): exportFile(NULL), errorMessage(0), hierarchy(h), memTable(0), newAreas(0) { } Exporter::~Exporter() { delete[](memTable); if (exportFile) fclose(exportFile); } void Exporter::relocateValue(PolyWord *pt) { #ifndef POLYML32IN64 PolyWord q = *pt; if (IS_INT(q) || q == PolyWord::FromUnsigned(0)) {} else createRelocation(pt); #endif } void Exporter::createRelocation(PolyWord* pt) { *gMem.SpaceForAddress(pt)->writeAble(pt) = createRelocation(*pt, pt); } // Check through the areas to see where the address is. It must be // in one of them. unsigned Exporter::findArea(void *p) { for (unsigned i = 0; i < memTableEntries; i++) { if (p > memTable[i].mtOriginalAddr && p <= (char*)memTable[i].mtOriginalAddr + memTable[i].mtLength) return i; } { ASSERT(0); } return 0; } void Exporter::relocateObject(PolyObject *p) { if (p->IsByteObject()) { if (p->IsMutable() && p->IsWeakRefObject()) { // Weak mutable byte refs are used for external references and // also in the FFI for non-persistent values. bool isFuncPtr = true; const char *entryName = getEntryPointName(p, &isFuncPtr); if (entryName != 0) addExternalReference(p, entryName, isFuncPtr); // Clear the first word of the data. ASSERT(p->Length() >= sizeof(uintptr_t)/sizeof(PolyWord)); *(uintptr_t*)p = 0; } } else if (p->IsCodeObject()) { POLYUNSIGNED constCount; PolyWord *cp; ASSERT(! p->IsMutable() ); machineDependent->GetConstSegmentForCode(p, cp, constCount); /* Now the constants. */ for (POLYUNSIGNED i = 0; i < constCount; i++) relocateValue(&(cp[i])); } else // Closure and ordinary objects { POLYUNSIGNED length = p->Length(); for (POLYUNSIGNED i = 0; i < length; i++) relocateValue(p->Offset(i)); } } ExportStringTable::ExportStringTable(): strings(0), stringSize(0), stringAvailable(0) { } ExportStringTable::~ExportStringTable() { free(strings); } // Add a string to the string table, growing it if necessary. unsigned long ExportStringTable::makeEntry(const char *str) { unsigned len = (unsigned)strlen(str); unsigned long entry = stringSize; if (stringSize + len + 1 > stringAvailable) { stringAvailable = stringAvailable+stringAvailable/2; if (stringAvailable < stringSize + len + 1) stringAvailable = stringSize + len + 1 + 500; char* newStrings = (char*)realloc(strings, stringAvailable); if (newStrings == 0) { if (debugOptions & DEBUG_SAVING) Log("SAVE: Unable to realloc string table, size: %lu.\n", stringAvailable); throw MemoryException(); } else strings = newStrings; } strcpy(strings + stringSize, str); stringSize += len + 1; return entry; } struct _entrypts exporterEPT[] = { { "PolyExport", (polyRTSFunction)&PolyExport}, { "PolyExportPortable", (polyRTSFunction)&PolyExportPortable}, { NULL, NULL} // End of list. }; diff --git a/libpolyml/machoexport.cpp b/libpolyml/machoexport.cpp index bb7e1ac1..23477a09 100644 --- a/libpolyml/machoexport.cpp +++ b/libpolyml/machoexport.cpp @@ -1,564 +1,607 @@ /* Title: Write out a database as a Mach object file Author: David Matthews. Copyright (c) 2006-7, 2011-2, 2016-18, 2020-21 David C. J. Matthews This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 2.1 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR H PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include "config.h" #ifdef HAVE_STDLIB_H #include #endif #ifdef HAVE_STDIO_H #include #endif #ifdef HAVE_STDDEF_H #include #endif #ifdef HAVE_UNISTD_H #include #endif #ifdef HAVE_ERRNO_H #include #endif #ifdef HAVE_TIME_H #include #endif #ifdef HAVE_ASSERT_H #include #define ASSERT(x) assert(x) #else #define ASSERT(x) #endif // If we haven't got the Mach header files we shouldn't be building this. #include #include #include #include #include #ifdef HAVE_STRING_H #include #endif #ifdef HAVE_SYS_UTSNAME_H #include #endif #include "globals.h" #include "diagnostics.h" #include "sys.h" #include "machine_dep.h" #include "gc.h" #include "mpoly.h" #include "scanaddrs.h" #include "machoexport.h" #include "run_time.h" #include "version.h" #include "polystring.h" #include "timing.h" // Mach-O seems to require each section to have a discrete virtual address range // so we have to adjust various offsets to fit. void MachoExport::adjustOffset(unsigned area, size_t &offset) { // Add in the offset. If sect is memTableEntries it's actually the // descriptors so doesn't have any additional offset. if (area != memTableEntries) { offset += sizeof(exportDescription)+sizeof(memoryTableEntry)*memTableEntries; for (unsigned i = 0; i < area; i++) offset += memTable[i].mtLength; } } void MachoExport::addExternalReference(void *relocAddr, const char *name, bool /*isFuncPtr*/) { externTable.makeEntry(name); writeRelocation(0, relocAddr, symbolNum++, true); } // Generate the address relative to the start of the segment. void MachoExport::setRelocationAddress(void *p, int32_t *reloc) { unsigned area = findArea(p); size_t offset = (char*)p - (char*)memTable[area].mtOriginalAddr; *reloc = offset; } /* Get the index corresponding to an address. */ PolyWord MachoExport::createRelocation(PolyWord p, void *relocAddr) { void *addr = p.AsAddress(); unsigned addrArea = findArea(addr); size_t offset = (char*)addr - (char*)memTable[addrArea].mtOriginalAddr; adjustOffset(addrArea, offset); return writeRelocation(offset, relocAddr, addrArea+1 /* Sections count from 1 */, false); } PolyWord MachoExport::writeRelocation(POLYUNSIGNED offset, void *relocAddr, unsigned symbolNumber, bool isExtern) { // It looks as though struct relocation_info entries are only used // with GENERIC_RELOC_VANILLA types. struct relocation_info relInfo; setRelocationAddress(relocAddr, &relInfo.r_address); relInfo.r_symbolnum = symbolNumber; relInfo.r_pcrel = 0; #if (SIZEOF_VOIDP == 8) relInfo.r_length = 3; // 8 bytes #else relInfo.r_length = 2; // 4 bytes #endif relInfo.r_type = GENERIC_RELOC_VANILLA; relInfo.r_extern = isExtern ? 1 : 0; fwrite(&relInfo, sizeof(relInfo), 1, exportFile); relocationCount++; return PolyWord::FromUnsigned(offset); } /* This is called for each constant within the code. Print a relocation entry for the word and return a value that means that the offset is saved in original word. */ void MachoExport::ScanConstant(PolyObject *base, byte *addr, ScanRelocationKind code, intptr_t displacement) { #ifndef POLYML32IN64 PolyObject *p = GetConstantValue(addr, code, displacement); if (p == 0) return; void *a = p; unsigned aArea = findArea(a); // Set the value at the address to the offset relative to the symbol. size_t offset = (char*)a - (char*)memTable[aArea].mtOriginalAddr; - adjustOffset(aArea, offset); switch (code) { case PROCESS_RELOC_DIRECT: // 32/64 bit address of target { struct relocation_info reloc; setRelocationAddress(addr, &reloc.r_address); reloc.r_symbolnum = aArea+1; // Section numbers start at 1 reloc.r_pcrel = 0; #if (SIZEOF_VOIDP == 8) reloc.r_length = 3; // 8 bytes #else reloc.r_length = 2; // 4 bytes #endif reloc.r_type = GENERIC_RELOC_VANILLA; reloc.r_extern = 0; // r_symbolnum is a section number. It should be 1 if we make the IO area a common. + // Adjust offset since this is section-relative + adjustOffset(aArea, offset); for (unsigned i = 0; i < sizeof(PolyWord); i++) { addr[i] = (byte)(offset & 0xff); offset >>= 8; } fwrite(&reloc, sizeof(reloc), 1, exportFile); relocationCount++; } break; #if (defined(HOSTARCHITECTURE_X86) || defined(HOSTARCHITECTURE_X86_64)) case PROCESS_RELOC_I386RELATIVE: // 32 bit relative address { unsigned addrArea = findArea(addr); // If it's in the same area we don't need a relocation because the // relative offset will be unchanged. if (addrArea != aArea) { struct relocation_info reloc; setRelocationAddress(addr, &reloc.r_address); reloc.r_symbolnum = aArea+1; // Section numbers start at 1 reloc.r_pcrel = 1; reloc.r_length = 2; // 4 bytes #if (defined(HOSTARCHITECTURE_X86_64)) reloc.r_type = X86_64_RELOC_SIGNED; // Or X86_64_RELOC_BRANCH ? #else reloc.r_type = GENERIC_RELOC_VANILLA; #endif reloc.r_extern = 0; // r_symbolnum is a section number. fwrite(&reloc, sizeof(reloc), 1, exportFile); relocationCount++; + // Adjust offset since this is section-relative + adjustOffset(aArea, offset); size_t addrOffset = (char*)addr - (char*)memTable[addrArea].mtOriginalAddr; adjustOffset(addrArea, addrOffset); offset -= addrOffset + 4; for (unsigned i = 0; i < 4; i++) { addr[i] = (byte)(offset & 0xff); offset >>= 8; } } } break; +#endif +#if (defined(HOSTARCHITECTURE_AARCH64)) case PROCESS_RELOC_ARM64ADRPLDR: { + // This seems to be completely undocumented and has been worked out + // by some reverse-engineering and trial-and-error. Have to use + // symbol-relative addressing with the offset provided by a + // ARM64_RELOC_ADDEND "relocation" that affects the next entry. unsigned addrArea = findArea(addr); struct relocation_info reloc; + // We need four relocations here. // The first instruction is ADRP setRelocationAddress(addr, &reloc.r_address); - reloc.r_symbolnum = aArea + 1; // Section numbers start at 1 reloc.r_pcrel = 1; reloc.r_length = 2; // 4 bytes - reloc.r_rtype = ARM64_RELOC_PAGE21; - reloc.r_extern = 0; // r_symbolnum is a section number. + reloc.r_type = ARM64_RELOC_ADDEND; + reloc.r_symbolnum = offset; // Used for offset + reloc.r_extern = 1; // Needed for ARM64_RELOC_PAGE21 and ARM64_RELOC_PAGEOFF12 + fwrite(&reloc, sizeof(reloc), 1, exportFile); + relocationCount++; + reloc.r_type = ARM64_RELOC_PAGE21; + reloc.r_symbolnum = aArea+1; // Symbol number fwrite(&reloc, sizeof(reloc), 1, exportFile); relocationCount++; // The second instruction is LDR setRelocationAddress(addr+4, &reloc.r_address); reloc.r_pcrel = 0; // This is an absolute 12-bit value - reloc.r_rtype = ARM64_RELOC_PAGEOFF12; + reloc.r_type = ARM64_RELOC_ADDEND; + reloc.r_symbolnum = offset; // Used for offset + fwrite(&reloc, sizeof(reloc), 1, exportFile); + relocationCount++; + reloc.r_type = ARM64_RELOC_PAGEOFF12; + reloc.r_symbolnum = aArea+1; // Symbol number fwrite(&reloc, sizeof(reloc), 1, exportFile); relocationCount++; // Now we've found the target we can zero the data in the instructions. uint32_t* instrAddr = (uint32_t*)addr; - addr[0] &= 0x9f00001f; - addr[1] &= 0xffc003ff; + instrAddr[0] &= 0x9f00001f; + instrAddr[1] &= 0xffc003ff; } break; #endif default: ASSERT(0); // Wrong type of relocation for this architecture. } #endif } // Set the file alignment. void MachoExport::alignFile(int align) { char pad[32] = {0}; // Maximum alignment int offset = ftell(exportFile); if ((offset % align) == 0) return; fwrite(&pad, align - (offset % align), 1, exportFile); } void MachoExport::createStructsRelocation(unsigned sect, size_t offset) { struct relocation_info reloc; reloc.r_address = offset; reloc.r_symbolnum = sect+1; // Section numbers start at 1 reloc.r_pcrel = 0; #if (SIZEOF_VOIDP == 8) reloc.r_length = 3; // 8 bytes #else reloc.r_length = 2; // 4 bytes #endif reloc.r_type = GENERIC_RELOC_VANILLA; reloc.r_extern = 0; // r_symbolnum is a section number. fwrite(&reloc, sizeof(reloc), 1, exportFile); relocationCount++; } void MachoExport::exportStore(void) { PolyWord *p; #if (SIZEOF_VOIDP == 8) struct mach_header_64 fhdr; struct segment_command_64 sHdr; struct section_64 *sections = new section_64[memTableEntries+1]; size_t sectionSize = sizeof(section_64); #else struct mach_header fhdr; struct segment_command sHdr; struct section *sections = new section[memTableEntries+1]; size_t sectionSize = sizeof(section); #endif struct symtab_command symTab; unsigned i; // Write out initial values for the headers. These are overwritten at the end. // File header memset(&fhdr, 0, sizeof(fhdr)); fhdr.filetype = MH_OBJECT; fhdr.ncmds = 2; // One for the segment and one for the symbol table. fhdr.sizeofcmds = sizeof(sHdr) + sectionSize * (memTableEntries+1) + sizeof(symTab); fhdr.flags = 0; // The machine needs to match the machine we're compiling for // even if this is actually portable code. #if (SIZEOF_VOIDP == 8) fhdr.magic = MH_MAGIC_64; // (0xfeedfacf) 64-bit magic number #else fhdr.magic = MH_MAGIC; // Feed Face (0xfeedface) #endif #if defined(HOSTARCHITECTURE_X86) fhdr.cputype = CPU_TYPE_I386; fhdr.cpusubtype = CPU_SUBTYPE_I386_ALL; #elif defined(HOSTARCHITECTURE_PPC) fhdr.cputype = CPU_TYPE_POWERPC; fhdr.cpusubtype = CPU_SUBTYPE_POWERPC_ALL; #elif defined(HOSTARCHITECTURE_X86_64) fhdr.cputype = CPU_TYPE_X86_64; fhdr.cpusubtype = CPU_SUBTYPE_X86_64_ALL; #elif defined(HOSTARCHITECTURE_AARCH64) fhdr.cputype = CPU_TYPE_ARM64; fhdr.cpusubtype = CPU_SUBTYPE_ARM64_ALL; #else #error "No support for exporting on this architecture" #endif fwrite(&fhdr, sizeof(fhdr), 1, exportFile); // Write it for the moment. - symbolNum = 1; // The first symbol is poly_exports + symbolNum = memTableEntries+1; // Symbols start after table entries and poly_exports // Segment header. memset(&sHdr, 0, sizeof(sHdr)); #if (SIZEOF_VOIDP == 8) sHdr.cmd = LC_SEGMENT_64; #else sHdr.cmd = LC_SEGMENT; #endif sHdr.nsects = memTableEntries+1; // One for each entry plus one for the tables. sHdr.cmdsize = sizeof(sHdr) + sectionSize * sHdr.nsects; // Add up the sections to give the file size sHdr.filesize = 0; for (i = 0; i < memTableEntries; i++) sHdr.filesize += memTable[i].mtLength; // Do we need any alignment? sHdr.filesize += sizeof(exportDescription) + memTableEntries * sizeof(memoryTableEntry); sHdr.vmsize = sHdr.filesize; // Set them the same since we don't have any "common" area. // sHdr.fileOff is set later. sHdr.maxprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; sHdr.initprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE; sHdr.flags = 0; // Write it initially. fwrite(&sHdr, sizeof(sHdr), 1, exportFile); // Section header for each entry in the table POLYUNSIGNED sectAddr = sizeof(exportDescription)+sizeof(memoryTableEntry)*memTableEntries; for (i = 0; i < memTableEntries; i++) { memset(&(sections[i]), 0, sectionSize); if (memTable[i].mtFlags & MTF_WRITEABLE) { // Mutable areas ASSERT(!(memTable[i].mtFlags & MTF_EXECUTABLE)); // Executable areas can't be writable. sprintf(sections[i].sectname, "__data"); sprintf(sections[i].segname, "__DATA"); sections[i].flags = S_ATTR_LOC_RELOC | S_REGULAR; } #ifndef CODEISNOTEXECUTABLE // Not if we're building the interpreted version. else if (memTable[i].mtFlags & MTF_EXECUTABLE) { sprintf(sections[i].sectname, "__text"); sprintf(sections[i].segname, "__TEXT"); sections[i].flags = S_ATTR_LOC_RELOC | S_ATTR_SOME_INSTRUCTIONS | S_REGULAR; } #endif else { sprintf(sections[i].sectname, "__const"); sprintf(sections[i].segname, "__DATA"); sections[i].flags = S_ATTR_LOC_RELOC | S_REGULAR; } sections[i].addr = sectAddr; sections[i].size = memTable[i].mtLength; sectAddr += memTable[i].mtLength; //sections[i].offset is set later //sections[i].reloff is set later //sections[i].nreloc is set later sections[i].align = 3; // 8 byte alignment // theSection.size is set later } // For the tables. memset(&(sections[memTableEntries]), 0, sectionSize); sprintf(sections[memTableEntries].sectname, "__const"); sprintf(sections[memTableEntries].segname, "__DATA"); sections[memTableEntries].addr = 0; sections[memTableEntries].size = sizeof(exportDescription)+sizeof(memoryTableEntry)*memTableEntries; sections[memTableEntries].align = 3; // 8 byte alignment // theSection.size is set later sections[memTableEntries].flags = S_ATTR_LOC_RELOC | S_ATTR_SOME_INSTRUCTIONS | S_REGULAR; // Write them out for the moment. fwrite(sections, sectionSize * (memTableEntries+1), 1, exportFile); // Symbol table header. memset(&symTab, 0, sizeof(symTab)); symTab.cmd = LC_SYMTAB; symTab.cmdsize = sizeof(symTab); //symTab.symoff is set later //symTab.nsyms is set later //symTab.stroff is set later //symTab.strsize is set later fwrite(&symTab, sizeof(symTab), 1, exportFile); // Create and write out the relocations. for (i = 0; i < memTableEntries; i++) { sections[i].reloff = ftell(exportFile); relocationCount = 0; // Create the relocation table and turn all addresses into offsets. char *start = (char*)memTable[i].mtOriginalAddr; char *end = start + memTable[i].mtLength; for (p = (PolyWord*)start; p < (PolyWord*)end; ) { p++; PolyObject *obj = (PolyObject*)p; POLYUNSIGNED length = obj->Length(); if (length != 0 && obj->IsCodeObject()) { POLYUNSIGNED constCount; PolyWord* cp; // Get the constant area pointer first because ScanConstantsWithinCode // may alter it. machineDependent->GetConstSegmentForCode(obj, cp, constCount); // Update any constants before processing the object // We need that for relative jumps/calls in X86/64. machineDependent->ScanConstantsWithinCode(obj, this); if (cp > (PolyWord*)obj && cp < ((PolyWord*)obj) + length) { // Process the constants if they're in the area but not if they've been moved. for (POLYUNSIGNED i = 0; i < constCount; i++) relocateValue(&(cp[i])); } } else relocateObject(obj); p += length; } sections[i].nreloc = relocationCount; } // Additional relocations for the descriptors. sections[memTableEntries].reloff = ftell(exportFile); relocationCount = 0; // Address of "memTable" within "exports". We can't use createRelocation because // the position of the relocation is not in either the mutable or the immutable area. createStructsRelocation(memTableEntries, offsetof(exportDescription, memTable)); // Address of "rootFunction" within "exports" unsigned rootAddrArea = findArea(rootFunction); size_t rootOffset = (char*)rootFunction - (char*)memTable[rootAddrArea].mtOriginalAddr; adjustOffset(rootAddrArea, rootOffset); createStructsRelocation(rootAddrArea, offsetof(exportDescription, rootFunction)); // Addresses of the areas within memtable. for (i = 0; i < memTableEntries; i++) { createStructsRelocation(i, sizeof(exportDescription) + i * sizeof(memoryTableEntry) + offsetof(memoryTableEntry, mtCurrentAddr)); } sections[memTableEntries].nreloc = relocationCount; // The symbol table. symTab.symoff = ftell(exportFile); + // Global symbols: Just one. { #if (SIZEOF_VOIDP == 8) struct nlist_64 symbol; #else struct nlist symbol; #endif memset(&symbol, 0, sizeof(symbol)); // Zero unused fields symbol.n_un.n_strx = stringTable.makeEntry("_poly_exports"); symbol.n_type = N_EXT | N_SECT; symbol.n_sect = memTableEntries+1; // Sections count from 1. symbol.n_desc = REFERENCE_FLAG_DEFINED; fwrite(&symbol, sizeof(symbol), 1, exportFile); } + // Create a symbol entry for each memTable entry. + // ARM relocations need to be relative to an external. + for (i = 0; i < memTableEntries; i++) + { +#if (SIZEOF_VOIDP == 8) + struct nlist_64 symbol; +#else + struct nlist symbol; +#endif + memset(&symbol, 0, sizeof(symbol)); // Zero unused fields + char symName[100]; + sprintf(symName, "area%d", i); + symbol.n_un.n_strx = stringTable.makeEntry(symName); + symbol.n_type = N_SECT; + symbol.n_sect = i+1; // Sections count from 1. + symbol.n_desc = REFERENCE_FLAG_PRIVATE_DEFINED; + symbol.n_value = 0; + size_t offset = 0; + adjustOffset(i, offset); + symbol.n_value = offset; + fwrite(&symbol, sizeof(symbol), 1, exportFile); + } + // External references. for (unsigned i = 0; i < externTable.stringSize; i += (unsigned)strlen(externTable.strings+i) + 1) { const char *symbolName = externTable.strings+i; #if (SIZEOF_VOIDP == 8) struct nlist_64 symbol; #else struct nlist symbol; #endif memset(&symbol, 0, sizeof(symbol)); // Zero unused fields // Have to add an underscore to the symbols. TempCString fullSymbol; fullSymbol = (char*)malloc(strlen(symbolName) + 2); if (fullSymbol == 0) throw MemoryException(); sprintf(fullSymbol, "_%s", symbolName); symbol.n_un.n_strx = stringTable.makeEntry(fullSymbol); symbol.n_type = N_EXT | N_UNDF; symbol.n_sect = NO_SECT; symbol.n_desc = REFERENCE_FLAG_UNDEFINED_NON_LAZY; fwrite(&symbol, sizeof(symbol), 1, exportFile); } symTab.nsyms = symbolNum; // The symbol name table symTab.stroff = ftell(exportFile); fwrite(stringTable.strings, stringTable.stringSize, 1, exportFile); symTab.strsize = stringTable.stringSize; alignFile(4); exportDescription exports; memset(&exports, 0, sizeof(exports)); exports.structLength = sizeof(exportDescription); exports.memTableSize = sizeof(memoryTableEntry); exports.memTableEntries = memTableEntries; exports.memTable = (memoryTableEntry *)sizeof(exportDescription); // It follows immediately after this. // Set the value to be the offset relative to the base of the area. We have set a relocation // already which will add the base of the area. exports.rootFunction = (void*)rootOffset; exports.timeStamp = getBuildTime(); exports.architecture = machineDependent->MachineArchitecture(); exports.rtsVersion = POLY_version_number; #ifdef POLYML32IN64 exports.originalBaseAddr = globalHeapBase; #else exports.originalBaseAddr = 0; #endif sections[memTableEntries].offset = ftell(exportFile); fwrite(&exports, sizeof(exports), 1, exportFile); size_t addrOffset = sizeof(exports)+sizeof(memoryTableEntry)*memTableEntries; for (i = 0; i < memTableEntries; i++) { void *save = memTable[i].mtCurrentAddr; memTable[i].mtCurrentAddr = (void*)addrOffset; // Set this to the relative address. addrOffset += memTable[i].mtLength; fwrite(&memTable[i], sizeof(memoryTableEntry), 1, exportFile); memTable[i].mtCurrentAddr = save; } // Now the binary data. for (i = 0; i < memTableEntries; i++) { alignFile(4); sections[i].offset = ftell(exportFile); fwrite(memTable[i].mtOriginalAddr, 1, memTable[i].mtLength, exportFile); } // Rewind to rewrite the headers with the actual offsets. rewind(exportFile); fwrite(&fhdr, sizeof(fhdr), 1, exportFile); // File header fwrite(&sHdr, sizeof(sHdr), 1, exportFile); // Segment header fwrite(sections, sectionSize * (memTableEntries+1), 1, exportFile); // Section headers fwrite(&symTab, sizeof(symTab), 1, exportFile); // Symbol table header fclose(exportFile); exportFile = NULL; delete[](sections); } diff --git a/libpolyml/memmgr.cpp b/libpolyml/memmgr.cpp index f13e4068..bf7dcc49 100644 --- a/libpolyml/memmgr.cpp +++ b/libpolyml/memmgr.cpp @@ -1,1392 +1,1392 @@ /* Title: memmgr.cpp Memory segment manager Copyright (c) 2006-7, 2011-12, 2016-18 David C. J. Matthews This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 2.1 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifdef HAVE_CONFIG_H #include "config.h" #elif defined(_WIN32) #include "winconfig.h" #else #error "No configuration file" #endif #ifdef HAVE_ASSERT_H #include #define ASSERT(x) assert(x) #else #define ASSERT(x) #endif #include #include #include "globals.h" #include "memmgr.h" #include "osmem.h" #include "scanaddrs.h" #include "bitmap.h" #include "mpoly.h" #include "diagnostics.h" #include "statistics.h" #include "processes.h" #include "machine_dep.h" #ifdef POLYML32IN64 // This contains the address of the base of the heap. PolyWord *globalHeapBase, *globalCodeBase; #endif // heap resizing policy option requested on command line unsigned heapsizingOption = 0; // If we are building for the interpreted version we don't need or want the // code to be executable. static const enum OSMem::_MemUsage executableCodeWhereNecessary = #ifdef CODEISNOTEXECUTABLE OSMem::UsageData; #else OSMem::UsageExecutableCode; #endif MemSpace::MemSpace(OSMem *alloc): SpaceTree(true) { spaceType = ST_PERMANENT; isMutable = false; bottom = 0; top = 0; isCode = false; allocator = alloc; shadowSpace = 0; } MemSpace::~MemSpace() { if (allocator != 0 && bottom != 0) { if (isCode) allocator->FreeCodeArea(bottom, shadowSpace, (char*)top - (char*)bottom); else allocator->FreeDataArea(bottom, (char*)top - (char*)bottom); } } MarkableSpace::MarkableSpace(OSMem *alloc): MemSpace(alloc), spaceLock("Local space") { } LocalMemSpace::LocalMemSpace(OSMem *alloc): MarkableSpace(alloc) { spaceType = ST_LOCAL; upperAllocPtr = lowerAllocPtr = 0; for (unsigned i = 0; i < NSTARTS; i++) start[i] = 0; start_index = 0; i_marked = m_marked = updated = 0; allocationSpace = false; } bool LocalMemSpace::InitSpace(PolyWord *heapSpace, uintptr_t size, bool mut) { isMutable = mut; bottom = heapSpace; top = bottom + size; // Initialise all the fields. The partial GC in particular relies on this. upperAllocPtr = partialGCTop = fullGCRescanStart = fullGCLowerLimit = lowestWeak = top; lowerAllocPtr = partialGCScan = partialGCRootBase = partialGCRootTop = fullGCRescanEnd = highestWeak = bottom; #ifdef POLYML32IN64 // The address must be on an odd-word boundary so that after the length // word is put in the actual cell address is on an even-word boundary. lowerAllocPtr[0] = PolyWord::FromUnsigned(0); lowerAllocPtr = bottom + 1; #endif spaceOwner = 0; allocationSpace = false; // Bitmap for the space. return bitmap.Create(size); } MemMgr::MemMgr(): allocLock("Memmgr alloc"), codeBitmapLock("Code bitmap") { nextIndex = 0; reservedSpace = 0; nextAllocator = 0; defaultSpaceSize = 0; spaceBeforeMinorGC = 0; spaceForHeap = 0; currentAllocSpace = currentHeapSize = 0; defaultSpaceSize = 1024 * 1024 / sizeof(PolyWord); // 1Mbyte segments. spaceTree = new SpaceTreeTree; } MemMgr::~MemMgr() { delete(spaceTree); // Have to do this before we delete the spaces. for (std::vector::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) delete(*i); for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) delete(*i); for (std::vector::iterator i = eSpaces.begin(); i < eSpaces.end(); i++) delete(*i); for (std::vector::iterator i = sSpaces.begin(); i < sSpaces.end(); i++) delete(*i); for (std::vector::iterator i = cSpaces.begin(); i < cSpaces.end(); i++) delete(*i); } bool MemMgr::Initialise() { #ifdef POLYML32IN64 // Reserve a single 16G area but with no access. void *heapBase; if (!osHeapAlloc.Initialise(OSMem::UsageData, (size_t)16 * 1024 * 1024 * 1024, &heapBase)) return false; globalHeapBase = (PolyWord*)heapBase; // Allocate a 4 gbyte area for the stacks. // It's important that the stack and code areas have addresses with // non-zero top 32-bits. - if (!osStackAlloc.Initialise(OSMem::UsageStack, (size_t)4 * 1024 * 1024 * 1024)) + if (!osStackAlloc.Initialise(OSMem::UsageStack, (size_t)4 * 1024 * 1024 * 1024, 0)) return false; #else if (!osHeapAlloc.Initialise(OSMem::UsageData) || !osStackAlloc.Initialise(OSMem::UsageStack)) return false; #endif #if (defined(POLYML32IN64) || defined(HOSTARCHITECTURE_X86_64) || defined(HOSTARCHITECTURE_AARCH64)) // Reserve a 2G area for the code. void* codeBase; if (!osCodeAlloc.Initialise(executableCodeWhereNecessary, (size_t)2 * 1024 * 1024 * 1024, &codeBase)) return false; #ifdef POLYML32IN64 globalCodeBase = (PolyWord*)codeBase; #endif return true; #else return osCodeAlloc.Initialise(executableCodeWhereNecessary); #endif } // Create and initialise a new local space and add it to the table. LocalMemSpace* MemMgr::NewLocalSpace(uintptr_t size, bool mut) { try { LocalMemSpace *space = new LocalMemSpace(&osHeapAlloc); // Before trying to allocate the heap temporarily allocate the // reserved space. This ensures that this much space will always // be available for C stacks and the C++ heap. void *reservation = 0; size_t rSpace = reservedSpace*sizeof(PolyWord); if (reservedSpace != 0) { reservation = osHeapAlloc.AllocateDataArea(rSpace); if (reservation == NULL) { // Insufficient space for the reservation. Can't allocate this local space. if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New local %smutable space: insufficient reservation space\n", mut ? "": "im"); delete space; return 0; } } // Allocate the heap itself. size_t iSpace = size * sizeof(PolyWord); PolyWord* heapSpace = (PolyWord*)osHeapAlloc.AllocateDataArea(iSpace); // The size may have been rounded up to a block boundary. size = iSpace / sizeof(PolyWord); bool success = heapSpace != 0 && space->InitSpace(heapSpace, size, mut) && AddLocalSpace(space); if (reservation != 0) osHeapAlloc.FreeDataArea(reservation, rSpace); if (success) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New local %smutable space %p, size=%luk words, bottom=%p, top=%p\n", mut ? "": "im", space, space->spaceSize()/1024, space->bottom, space->top); currentHeapSize += space->spaceSize(); globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); return space; } // If something went wrong. delete space; if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New local %smutable space: insufficient space\n", mut ? "": "im"); return 0; } catch (std::bad_alloc&) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New local %smutable space: \"new\" failed\n", mut ? "": "im"); return 0; } } // Create a local space for initial allocation. LocalMemSpace *MemMgr::CreateAllocationSpace(uintptr_t size) { LocalMemSpace *result = NewLocalSpace(size, true); if (result) { result->allocationSpace = true; currentAllocSpace += result->spaceSize(); globalStats.incSize(PSS_ALLOCATION, result->spaceSize()*sizeof(PolyWord)); globalStats.incSize(PSS_ALLOCATION_FREE, result->freeSpace()*sizeof(PolyWord)); } return result; } // If an allocation space has a lot of data left in it after a GC, particularly // a single large object we should turn it into a local area. void MemMgr::ConvertAllocationSpaceToLocal(LocalMemSpace *space) { ASSERT(space->allocationSpace); space->allocationSpace = false; // Currently it is left as a mutable area but if the contents are all // immutable e.g. a large vector it could be better to turn it into an // immutable area. currentAllocSpace -= space->spaceSize(); } // Add a local memory space to the table. bool MemMgr::AddLocalSpace(LocalMemSpace *space) { // Add to the table. // Update the B-tree. try { AddTree(space); // The entries in the local table are ordered so that the copy phase of the full // GC simply has to copy to an entry earlier in the table. Immutable spaces come // first, followed by mutable spaces and finally allocation spaces. if (space->allocationSpace) lSpaces.push_back(space); // Just add at the end else if (space->isMutable) { // Add before the allocation spaces std::vector::iterator i = lSpaces.begin(); while (i != lSpaces.end() && ! (*i)->allocationSpace) i++; lSpaces.insert(i, space); } else { // Immutable space: Add before the mutable spaces std::vector::iterator i = lSpaces.begin(); while (i != lSpaces.end() && ! (*i)->isMutable) i++; lSpaces.insert(i, space); } } catch (std::bad_alloc&) { RemoveTree(space); return false; } return true; } // Create an entry for a permanent space. PermanentMemSpace* MemMgr::NewPermanentSpace(PolyWord *base, uintptr_t words, unsigned flags, unsigned index, unsigned hierarchy /*= 0*/) { try { PermanentMemSpace *space = new PermanentMemSpace(0/* Not freed */); space->bottom = base; space->topPointer = space->top = space->bottom + words; space->spaceType = ST_PERMANENT; space->isMutable = flags & MTF_WRITEABLE ? true : false; space->noOverwrite = flags & MTF_NO_OVERWRITE ? true : false; space->byteOnly = flags & MTF_BYTES ? true : false; space->isCode = flags & MTF_EXECUTABLE ? true : false; space->index = index; space->hierarchy = hierarchy; if (index >= nextIndex) nextIndex = index+1; // Extend the permanent memory table and add this space to it. try { AddTree(space); pSpaces.push_back(space); } catch (std::exception&) { RemoveTree(space); delete space; return 0; } return space; } catch (std::bad_alloc&) { return 0; } } PermanentMemSpace *MemMgr::AllocateNewPermanentSpace(uintptr_t byteSize, unsigned flags, unsigned index, unsigned hierarchy) { try { OSMem *alloc = flags & MTF_EXECUTABLE ? (OSMem*)&osCodeAlloc : (OSMem*)&osHeapAlloc; PermanentMemSpace *space = new PermanentMemSpace(alloc); size_t actualSize = byteSize; PolyWord* base; void* newShadow=0; if (flags & MTF_EXECUTABLE) base = (PolyWord*)alloc->AllocateCodeArea(actualSize, newShadow); else base = (PolyWord*)alloc->AllocateDataArea(actualSize); if (base == 0) { delete(space); return 0; } space->bottom = base; space->shadowSpace = (PolyWord*)newShadow; space->topPointer = space->top = space->bottom + actualSize/sizeof(PolyWord); space->spaceType = ST_PERMANENT; space->isMutable = flags & MTF_WRITEABLE ? true : false; space->noOverwrite = flags & MTF_NO_OVERWRITE ? true : false; space->byteOnly = flags & MTF_BYTES ? true : false; space->isCode = flags & MTF_EXECUTABLE ? true : false; space->index = index; space->hierarchy = hierarchy; if (index >= nextIndex) nextIndex = index + 1; // Extend the permanent memory table and add this space to it. try { AddTree(space); pSpaces.push_back(space); } catch (std::exception&) { RemoveTree(space); delete space; return 0; } return space; } catch (std::bad_alloc&) { return 0; } } bool MemMgr::CompletePermanentSpaceAllocation(PermanentMemSpace *space) { // Remove write access unless it is mutable. // Don't remove write access unless this is top-level. Share-data assumes only hierarchy 0 is write-protected. if (!space->isMutable && space->hierarchy == 0) { if (space->isCode) osCodeAlloc.DisableWriteForCode(space->bottom, space->shadowSpace, (char*)space->top - (char*)space->bottom); else osHeapAlloc.EnableWrite(false, space->bottom, (char*)space->top - (char*)space->bottom); } return true; } // Delete a local space and remove it from the table. void MemMgr::DeleteLocalSpace(std::vector::iterator &iter) { LocalMemSpace *sp = *iter; if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Deleted local %s space %p at %p size %zu\n", sp->spaceTypeString(), sp, sp->bottom, sp->spaceSize()); currentHeapSize -= sp->spaceSize(); globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); if (sp->allocationSpace) currentAllocSpace -= sp->spaceSize(); RemoveTree(sp); delete(sp); iter = lSpaces.erase(iter); } // Remove local areas that are now empty after a GC. // It isn't clear if we always want to do this. void MemMgr::RemoveEmptyLocals() { for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); ) { LocalMemSpace *space = *i; if (space->isEmpty()) DeleteLocalSpace(i); else i++; } } // Create and initialise a new export space and add it to the table. PermanentMemSpace* MemMgr::NewExportSpace(uintptr_t size, bool mut, bool noOv, bool code) { try { OSMem *alloc = code ? (OSMem*)&osCodeAlloc : (OSMem*)&osHeapAlloc; PermanentMemSpace *space = new PermanentMemSpace(alloc); space->spaceType = ST_EXPORT; space->isMutable = mut; space->noOverwrite = noOv; space->isCode = code; space->index = nextIndex++; // Allocate the memory itself. size_t iSpace = size*sizeof(PolyWord); if (code) { void* shadow; space->bottom = (PolyWord*)alloc->AllocateCodeArea(iSpace, shadow); if (space->bottom != 0) space->shadowSpace = (PolyWord*)shadow; } else space->bottom = (PolyWord*)alloc->AllocateDataArea(iSpace); if (space->bottom == 0) { delete space; if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New export %smutable space: insufficient space\n", mut ? "" : "im"); return 0; } // The size may have been rounded up to a block boundary. size = iSpace/sizeof(PolyWord); space->top = space->bottom + size; space->topPointer = space->bottom; #ifdef POLYML32IN64 // The address must be on an odd-word boundary so that after the length // word is put in the actual cell address is on an even-word boundary. space->writeAble(space->topPointer)[0] = PolyWord::FromUnsigned(0); space->topPointer = space->bottom + 1; #endif if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New export %smutable %s%sspace %p, size=%luk words, bottom=%p, top=%p\n", mut ? "" : "im", noOv ? "no-overwrite " : "", code ? "code " : "", space, space->spaceSize() / 1024, space->bottom, space->top); // Add to the table. try { AddTree(space); eSpaces.push_back(space); } catch (std::exception&) { RemoveTree(space); delete space; if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New export %smutable space: Adding to tree failed\n", mut ? "" : "im"); return 0; } return space; } catch (std::bad_alloc&) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New export %smutable space: \"new\" failed\n", mut ? "" : "im"); return 0; } } void MemMgr::DeleteExportSpaces(void) { for (std::vector::iterator i = eSpaces.begin(); i < eSpaces.end(); i++) { PermanentMemSpace *space = *i; RemoveTree(space); delete(space); } eSpaces.clear(); } // If we have saved the state rather than exported a function we turn the exported // spaces into permanent ones, removing existing permanent spaces at the same or // lower level. bool MemMgr::PromoteExportSpaces(unsigned hierarchy) { // Save permanent spaces at a lower hierarchy. Others are converted into // local spaces. Most or all items will have been copied from these spaces // into an export space but there could be items reachable only from the stack. std::vector::iterator i = pSpaces.begin(); while (i != pSpaces.end()) { PermanentMemSpace *pSpace = *i; if (pSpace->hierarchy < hierarchy) i++; else { try { // Turn this into a local space or a code space // Remove this from the tree - AddLocalSpace will make an entry for the local version. RemoveTree(pSpace); if (pSpace->isCode) { // Enable write access. Permanent spaces are read-only. // osCodeAlloc.SetPermissions(pSpace->bottom, (char*)pSpace->top - (char*)pSpace->bottom, // PERMISSION_READ | PERMISSION_WRITE | PERMISSION_EXEC); CodeSpace *space = new CodeSpace(pSpace->bottom, pSpace->shadowSpace, pSpace->spaceSize(), &osCodeAlloc); if (! space->headerMap.Create(space->spaceSize())) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Unable to create header map for state space %p\n", pSpace); return false; } if (!AddCodeSpace(space)) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Unable to convert saved state space %p into code space\n", pSpace); return false; } if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Converted saved state space %p into code space %p\n", pSpace, space); // Set the bits in the header map. for (PolyWord *ptr = space->bottom; ptr < space->top; ) { PolyObject *obj = (PolyObject*)(ptr+1); // We may have forwarded this if this has been // copied to the exported area. Restore the original length word. if (obj->ContainsForwardingPtr()) { #ifdef POLYML32IN64 PolyObject *forwardedTo = obj; // This is relative to globalCodeBase not globalHeapBase while (forwardedTo->ContainsForwardingPtr()) forwardedTo = (PolyObject*)(globalCodeBase + ((forwardedTo->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); #else PolyObject *forwardedTo = obj->FollowForwardingChain(); #endif obj->SetLengthWord(forwardedTo->LengthWord()); } // Set the "start" bit if this is allocated. It will be a byte seg if not. if (obj->IsCodeObject()) space->headerMap.SetBit(ptr-space->bottom); ASSERT(!obj->IsClosureObject()); ptr += obj->Length() + 1; } } else { // Enable write access. Permanent spaces are read-only. // osHeapAlloc.SetPermissions(pSpace->bottom, (char*)pSpace->top - (char*)pSpace->bottom, // PERMISSION_READ | PERMISSION_WRITE); LocalMemSpace *space = new LocalMemSpace(&osHeapAlloc); space->top = pSpace->top; // Space is allocated in local areas from the top down. This area is full and // all data is in the old generation. The area can be recovered by a full GC. space->bottom = space->upperAllocPtr = space->lowerAllocPtr = space->fullGCLowerLimit = pSpace->bottom; space->isMutable = pSpace->isMutable; space->isCode = false; if (! space->bitmap.Create(space->top-space->bottom) || ! AddLocalSpace(space)) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Unable to convert saved state space %p into local space\n", pSpace); return false; } if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Converted saved state space %p into local %smutable space %p\n", pSpace, pSpace->isMutable ? "im": "", space); currentHeapSize += space->spaceSize(); globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord)); } i = pSpaces.erase(i); } catch (std::bad_alloc&) { return false; } } } // Save newly exported spaces. for(std::vector::iterator j = eSpaces.begin(); j < eSpaces.end(); j++) { PermanentMemSpace *space = *j; space->hierarchy = hierarchy; // Set the hierarchy of the new spaces. space->spaceType = ST_PERMANENT; // Put a dummy object to fill up the unused space. if (space->topPointer != space->top) FillUnusedSpace(space->writeAble(space->topPointer), space->top - space->topPointer); // Put in a dummy object to fill the rest of the space. pSpaces.push_back(space); } eSpaces.clear(); return true; } // Before we import a hierarchical saved state we need to turn any previously imported // spaces into local spaces. bool MemMgr::DemoteImportSpaces() { return PromoteExportSpaces(1); // Only truly permanent spaces are retained. } // Return the space for a given index PermanentMemSpace *MemMgr::SpaceForIndex(unsigned index) { for (std::vector::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) { PermanentMemSpace *space = *i; if (space->index == index) return space; } return NULL; } // In several places we assume that segments are filled with valid // objects. This fills unused memory with one or more "byte" objects. void MemMgr::FillUnusedSpace(PolyWord *base, uintptr_t words) { PolyWord *pDummy = base+1; while (words > 0) { #ifdef POLYML32IN64 // Make sure that any dummy object we insert is properly aligned. if (((uintptr_t)pDummy) & 4) { *pDummy++ = PolyWord::FromUnsigned(0); words--; continue; } #endif POLYUNSIGNED oSize; // If the space is larger than the maximum object size // we will need several objects. if (words > MAX_OBJECT_SIZE) oSize = MAX_OBJECT_SIZE; else oSize = (POLYUNSIGNED)(words-1); // Make this a byte object so it's always skipped. ((PolyObject*)pDummy)->SetLengthWord(oSize, F_BYTE_OBJ); words -= oSize+1; pDummy += oSize+1; } } // Allocate an area of the heap of at least minWords and at most maxWords. // This is used both when allocating single objects (when minWords and maxWords // are the same) and when allocating heap segments. If there is insufficient // space to satisfy the minimum it will return 0. PolyWord *MemMgr::AllocHeapSpace(uintptr_t minWords, uintptr_t &maxWords, bool doAllocation) { PLocker locker(&allocLock); // We try to distribute the allocations between the memory spaces // so that at the next GC we don't have all the most recent cells in // one space. The most recent cells will be more likely to survive a // GC so distibuting them improves the load balance for a multi-thread GC. nextAllocator++; if (nextAllocator > gMem.lSpaces.size()) nextAllocator = 0; unsigned j = nextAllocator; for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) { if (j >= gMem.lSpaces.size()) j = 0; LocalMemSpace *space = gMem.lSpaces[j++]; if (space->allocationSpace) { uintptr_t available = space->freeSpace(); if (available > 0 && available >= minWords) { // Reduce the maximum value if we had less than that. if (available < maxWords) maxWords = available; #ifdef POLYML32IN64 // If necessary round down to an even boundary if (maxWords & 1) { maxWords--; space->lowerAllocPtr[maxWords] = PolyWord::FromUnsigned(0); } #endif PolyWord *result = space->lowerAllocPtr; // Return the address. if (doAllocation) space->lowerAllocPtr += maxWords; // Allocate it. #ifdef POLYML32IN64 ASSERT((uintptr_t)result & 4); // Must be odd-word aligned #endif return result; } } } // There isn't space in the existing areas - can we create a new area? // The reason we don't have enough space could simply be that we want to // allocate an object larger than the default space size. Try deleting // some other spaces to bring currentAllocSpace below spaceBeforeMinorGC - minWords. if (minWords > defaultSpaceSize && minWords < spaceBeforeMinorGC) RemoveExcessAllocation(spaceBeforeMinorGC - minWords); if (currentAllocSpace/* + minWords */ < spaceBeforeMinorGC) { // i.e. the current allocation space is less than the space allowed for the minor GC // but it may be that allocating this object will take us over the limit. We allow // that to happen so that we can successfully allocate very large objects even if // we have a new GC very shortly. uintptr_t spaceSize = defaultSpaceSize; #ifdef POLYML32IN64 // When we create the allocation space we take one word so that the first // length word is on an odd-word boundary. We need to allow for that otherwise // we may have available < minWords. if (minWords >= spaceSize) spaceSize = minWords+1; // If we really want a large space. #else if (minWords > spaceSize) spaceSize = minWords; // If we really want a large space. #endif LocalMemSpace *space = CreateAllocationSpace(spaceSize); if (space == 0) return 0; // Can't allocate it // Allocate our space in this new area. uintptr_t available = space->freeSpace(); ASSERT(available >= minWords); if (available < maxWords) { maxWords = available; #ifdef POLYML32IN64 // If necessary round down to an even boundary if (maxWords & 1) { maxWords--; space->lowerAllocPtr[maxWords] = PolyWord::FromUnsigned(0); } #endif } PolyWord *result = space->lowerAllocPtr; // Return the address. if (doAllocation) space->lowerAllocPtr += maxWords; // Allocate it. #ifdef POLYML32IN64 ASSERT((uintptr_t)result & 4); // Must be odd-word aligned #endif return result; } return 0; // There isn't space even for the minimum. } CodeSpace::CodeSpace(PolyWord *start, PolyWord *shadow, uintptr_t spaceSize, OSMem *alloc): MarkableSpace(alloc) { bottom = start; shadowSpace = shadow; top = start+spaceSize; isMutable = true; // Make it mutable just in case. This will cause it to be scanned. isCode = true; spaceType = ST_CODE; #ifdef POLYML32IN64 // Dummy word so that the cell itself, after the length word, is on an 8-byte boundary. writeAble(start)[0] = PolyWord::FromUnsigned(0); largestFree = spaceSize - 2; firstFree = start+1; #else largestFree = spaceSize - 1; firstFree = start; #endif } CodeSpace *MemMgr::NewCodeSpace(uintptr_t size) { // Allocate a new area and add it at the end of the table. CodeSpace *allocSpace = 0; // Allocate a new mutable, code space. N.B. This may round up "actualSize". size_t actualSize = size * sizeof(PolyWord); void* shadow; PolyWord *mem = (PolyWord*)osCodeAlloc.AllocateCodeArea(actualSize, shadow); if (mem != 0) { try { allocSpace = new CodeSpace(mem, (PolyWord*)shadow, actualSize / sizeof(PolyWord), &osCodeAlloc); allocSpace->shadowSpace = (PolyWord*)shadow; if (!allocSpace->headerMap.Create(allocSpace->spaceSize())) { delete allocSpace; allocSpace = 0; } else if (!AddCodeSpace(allocSpace)) { delete allocSpace; allocSpace = 0; } else if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New code space %p allocated at %p size %lu\n", allocSpace, allocSpace->bottom, allocSpace->spaceSize()); // Put in a byte cell to mark the area as unallocated. FillUnusedSpace(allocSpace->writeAble(allocSpace->firstFree), allocSpace->top- allocSpace->firstFree); } catch (std::bad_alloc&) { } if (allocSpace == 0) { osCodeAlloc.FreeCodeArea(mem, shadow, actualSize); mem = 0; } } return allocSpace; } // Allocate memory for a piece of code. This needs to be both mutable and executable, // at least for native code. The interpreted version need not (should not?) make the // area executable. It will not be executed until the mutable bit has been cleared. // Once code is allocated it is not GCed or moved. // initCell is a byte cell that is copied into the new code area. PolyObject* MemMgr::AllocCodeSpace(POLYUNSIGNED requiredSize) { PLocker locker(&codeSpaceLock); // Search the code spaces until we find a free area big enough. size_t i = 0; while (true) { if (i != cSpaces.size()) { CodeSpace *space = cSpaces[i]; if (space->largestFree >= requiredSize) { POLYUNSIGNED actualLargest = 0; while (space->firstFree < space->top) { PolyObject *obj = (PolyObject*)(space->firstFree+1); // Skip over allocated areas or free areas that are too small. if (obj->IsCodeObject() || obj->Length() < 8) space->firstFree += obj->Length()+1; else break; } PolyWord *pt = space->firstFree; while (pt < space->top) { PolyObject *obj = (PolyObject*)(pt+1); POLYUNSIGNED length = obj->Length(); if (obj->IsByteObject()) { if (length >= requiredSize) { // Free and large enough PolyWord *next = pt+requiredSize+1; POLYUNSIGNED spare = length - requiredSize; #ifdef POLYML32IN64 // Maintain alignment. if (((requiredSize + 1) & 1) && spare != 0) { space->writeAble(next++)[0] = PolyWord::FromUnsigned(0); spare--; } #endif if (spare != 0) FillUnusedSpace(space->writeAble(next), spare); space->isMutable = true; // Set this - it ensures the area is scanned on GC. space->headerMap.SetBit(pt-space->bottom); // Set the "header" bit // Set the length word of the code area and copy the byte cell in. // The code bit must be set before the lock is released to ensure // another thread doesn't reuse this. space->writeAble(obj)->SetLengthWord(requiredSize, F_CODE_OBJ|F_MUTABLE_BIT); return obj; } else if (length >= actualLargest) actualLargest = length+1; } pt += length+1; } // Reached the end without finding what we wanted. Update the largest size. space->largestFree = actualLargest; } i++; // Next area } else { // Allocate a new area and add it at the end of the table. uintptr_t spaceSize = requiredSize + 1; #ifdef POLYML32IN64 // We need to allow for the extra alignment word otherwise we // may allocate less than we need. spaceSize += 1; #endif CodeSpace *allocSpace = NewCodeSpace(spaceSize); if (allocSpace == 0) return 0; // Try a GC. globalStats.incSize(PSS_CODE_SPACE, allocSpace->spaceSize() * sizeof(PolyWord)); } } } // Remove code areas that are completely empty. This is probably better than waiting to reuse them. // It's particularly important if we reload a saved state because the code areas for old saved states // are made into local code areas just in case they are currently in use or reachable. void MemMgr::RemoveEmptyCodeAreas() { for (std::vector::iterator i = cSpaces.begin(); i != cSpaces.end(); ) { CodeSpace *space = *i; PolyObject *start = (PolyObject *)(space->bottom+1); if (start->IsByteObject() && start->Length() == space->spaceSize()-1) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Deleted code space %p at %p size %zu\n", space, space->bottom, space->spaceSize()); globalStats.decSize(PSS_CODE_SPACE, space->spaceSize() * sizeof(PolyWord)); // We have an empty cell that fills the whole space. RemoveTree(space); delete(space); i = cSpaces.erase(i); } else i++; } } // Add a code space to the tables. Used both for newly compiled code and also demoted saved spaces. bool MemMgr::AddCodeSpace(CodeSpace *space) { try { AddTree(space); cSpaces.push_back(space); } catch (std::exception&) { RemoveTree(space); return false; } return true; } // Check that we have sufficient space for an allocation to succeed. // Called from the GC to ensure that we will not get into an infinite // loop trying to allocate, failing and garbage-collecting again. bool MemMgr::CheckForAllocation(uintptr_t words) { uintptr_t allocated = 0; return AllocHeapSpace(words, allocated, false) != 0; } // Adjust the allocation area by removing free areas so that the total // size of the allocation area is less than the required value. This // is used after the quick GC and also if we need to allocate a large // object. void MemMgr::RemoveExcessAllocation(uintptr_t words) { // First remove any non-standard allocation areas. for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end();) { LocalMemSpace *space = *i; if (space->allocationSpace && space->isEmpty() && space->spaceSize() != defaultSpaceSize) DeleteLocalSpace(i); else i++; } for (std::vector::iterator i = lSpaces.begin(); currentAllocSpace > words && i < lSpaces.end(); ) { LocalMemSpace *space = *i; if (space->allocationSpace && space->isEmpty()) DeleteLocalSpace(i); else i++; } } // Return number of words free in all allocation spaces. uintptr_t MemMgr::GetFreeAllocSpace() { uintptr_t freeSpace = 0; PLocker lock(&allocLock); for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) { LocalMemSpace *space = *i; if (space->allocationSpace) freeSpace += space->freeSpace(); } return freeSpace; } StackSpace *MemMgr::NewStackSpace(uintptr_t size) { PLocker lock(&stackSpaceLock); try { StackSpace *space = new StackSpace(&osStackAlloc); size_t iSpace = size*sizeof(PolyWord); space->bottom = (PolyWord*)osStackAlloc.AllocateDataArea(iSpace); if (space->bottom == 0) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New stack space: insufficient space\n"); delete space; return 0; } // The size may have been rounded up to a block boundary. size = iSpace/sizeof(PolyWord); space->top = space->bottom + size; space->spaceType = ST_STACK; space->isMutable = true; // Add the stack space to the tree. This ensures that operations such as // LocalSpaceForAddress will work for addresses within the stack. We can // get them in the RTS with functions such as quot_rem and exception stack. // It's not clear whether they really appear in the GC. try { AddTree(space); sSpaces.push_back(space); } catch (std::exception&) { RemoveTree(space); delete space; return 0; } if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New stack space %p allocated at %p size %lu\n", space, space->bottom, space->spaceSize()); globalStats.incSize(PSS_STACK_SPACE, space->spaceSize() * sizeof(PolyWord)); return space; } catch (std::bad_alloc&) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: New stack space: \"new\" failed\n"); return 0; } } // If checkmem is given write protect the immutable areas except during a GC. void MemMgr::ProtectImmutable(bool on) { if (debugOptions & DEBUG_CHECK_OBJECTS) { for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) { LocalMemSpace *space = *i; if (!space->isMutable) { if (!space->isCode) osHeapAlloc.EnableWrite(!on, space->bottom, (char*)space->top - (char*)space->bottom); } } } } bool MemMgr::GrowOrShrinkStack(TaskData *taskData, uintptr_t newSize) { StackSpace *space = taskData->stack; size_t iSpace = newSize*sizeof(PolyWord); PolyWord *newSpace = (PolyWord*)osStackAlloc.AllocateDataArea(iSpace); if (newSpace == 0) { if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Unable to change size of stack %p from %lu to %lu: insufficient space\n", space, space->spaceSize(), newSize); return false; } // The size may have been rounded up to a block boundary. newSize = iSpace/sizeof(PolyWord); try { AddTree(space, newSpace, newSpace+newSize); } catch (std::bad_alloc&) { RemoveTree(space, newSpace, newSpace+newSize); delete space; return 0; } taskData->CopyStackFrame(space->stack(), space->spaceSize(), (StackObject*)newSpace, newSize); if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Size of stack %p changed from %lu to %lu at %p\n", space, space->spaceSize(), newSize, newSpace); globalStats.incSize(PSS_STACK_SPACE, (newSize - space->spaceSize()) * sizeof(PolyWord)); RemoveTree(space); // Remove it BEFORE freeing the space - another thread may allocate it PolyWord *oldBottom = space->bottom; size_t oldSize = (char*)space->top - (char*)space->bottom; space->bottom = newSpace; // Switch this before freeing - We could get a profile trap during the free space->top = newSpace+newSize; osStackAlloc.FreeDataArea(oldBottom, oldSize); return true; } // Delete a stack when a thread has finished. // This can be called by an ML thread so needs an interlock. bool MemMgr::DeleteStackSpace(StackSpace *space) { PLocker lock(&stackSpaceLock); for (std::vector::iterator i = sSpaces.begin(); i < sSpaces.end(); i++) { if (*i == space) { globalStats.decSize(PSS_STACK_SPACE, space->spaceSize() * sizeof(PolyWord)); RemoveTree(space); delete space; sSpaces.erase(i); if (debugOptions & DEBUG_MEMMGR) Log("MMGR: Deleted stack space %p at %p size %zu\n", space, space->bottom, space->spaceSize()); return true; } } ASSERT(false); // It should always be in the table. return false; } SpaceTreeTree::SpaceTreeTree(): SpaceTree(false) { for (unsigned i = 0; i < 256; i++) tree[i] = 0; } SpaceTreeTree::~SpaceTreeTree() { for (unsigned i = 0; i < 256; i++) { if (tree[i] && ! tree[i]->isSpace) delete(tree[i]); } } // Add and remove entries in the space tree. void MemMgr::AddTree(MemSpace *space, PolyWord *startS, PolyWord *endS) { // It isn't clear we need to lock here but it's probably sensible. PLocker lock(&spaceTreeLock); AddTreeRange(&spaceTree, space, (uintptr_t)startS, (uintptr_t)endS); } void MemMgr::RemoveTree(MemSpace *space, PolyWord *startS, PolyWord *endS) { PLocker lock(&spaceTreeLock); RemoveTreeRange(&spaceTree, space, (uintptr_t)startS, (uintptr_t)endS); } void MemMgr::AddTreeRange(SpaceTree **tt, MemSpace *space, uintptr_t startS, uintptr_t endS) { if (*tt == 0) *tt = new SpaceTreeTree; ASSERT(! (*tt)->isSpace); SpaceTreeTree *t = (SpaceTreeTree*)*tt; const unsigned shift = (sizeof(void*)-1) * 8; // Takes the high-order byte uintptr_t r = startS >> shift; ASSERT(r < 256); const uintptr_t s = endS == 0 ? 256 : endS >> shift; ASSERT(s >= r && s <= 256); if (r == s) // Wholly within this entry AddTreeRange(&(t->tree[r]), space, startS << 8, endS << 8); else { // Deal with any remainder at the start. if ((r << shift) != startS) { AddTreeRange(&(t->tree[r]), space, startS << 8, 0 /*End of range*/); r++; } // Whole entries. while (r < s) { ASSERT(t->tree[r] == 0); t->tree[r] = space; r++; } // Remainder at the end. if ((s << shift) != endS) AddTreeRange(&(t->tree[r]), space, 0, endS << 8); } } // Remove an entry from the tree for a range. Strictly speaking we don't need the // space argument here but it's useful as a check. // This may be called to remove a partially installed structure if we have // run out of space in AddTreeRange. void MemMgr::RemoveTreeRange(SpaceTree **tt, MemSpace *space, uintptr_t startS, uintptr_t endS) { SpaceTreeTree *t = (SpaceTreeTree*)*tt; if (t == 0) return; // This can only occur if we're recovering. ASSERT(! t->isSpace); const unsigned shift = (sizeof(void*)-1) * 8; uintptr_t r = startS >> shift; const uintptr_t s = endS == 0 ? 256 : endS >> shift; if (r == s) RemoveTreeRange(&(t->tree[r]), space, startS << 8, endS << 8); else { // Deal with any remainder at the start. if ((r << shift) != startS) { RemoveTreeRange(&(t->tree[r]), space, startS << 8, 0); r++; } // Whole entries. while (r < s) { ASSERT(t->tree[r] == space || t->tree[r] == 0 /* Recovery only */); t->tree[r] = 0; r++; } // Remainder at the end. if ((s << shift) != endS) RemoveTreeRange(&(t->tree[r]), space, 0, endS << 8); } // See if the whole vector is now empty. for (unsigned j = 0; j < 256; j++) { if (t->tree[j]) return; // It's not empty - we're done. } delete(t); *tt = 0; } uintptr_t MemMgr::AllocatedInAlloc() { uintptr_t inAlloc = 0; for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) { LocalMemSpace *sp = *i; if (sp->allocationSpace) inAlloc += sp->allocatedSpace(); } return inAlloc; } // Report heap sizes and occupancy before and after GC void MemMgr::ReportHeapSizes(const char *phase) { uintptr_t alloc = 0, nonAlloc = 0, inAlloc = 0, inNonAlloc = 0; for (std::vector::iterator i = lSpaces.begin(); i < lSpaces.end(); i++) { LocalMemSpace *sp = *i; if (sp->allocationSpace) { alloc += sp->spaceSize(); inAlloc += sp->allocatedSpace(); } else { nonAlloc += sp->spaceSize(); inNonAlloc += sp->allocatedSpace(); } } Log("Heap: %s Major heap used ", phase); LogSize(inNonAlloc); Log(" of "); LogSize(nonAlloc); Log(" (%1.0f%%). Alloc space used ", (float)inNonAlloc / (float)nonAlloc * 100.0F); LogSize(inAlloc); Log(" of "); LogSize(alloc); Log(" (%1.0f%%). Total space ", (float)inAlloc / (float)alloc * 100.0F); LogSize(spaceForHeap); Log(" %1.0f%% full.\n", (float)(inAlloc + inNonAlloc) / (float)spaceForHeap * 100.0F); Log("Heap: Local spaces %" PRI_SIZET ", permanent spaces %" PRI_SIZET ", code spaces %" PRI_SIZET ", stack spaces %" PRI_SIZET "\n", lSpaces.size(), pSpaces.size(), cSpaces.size(), sSpaces.size()); uintptr_t cTotal = 0, cOccupied = 0; for (std::vector::iterator c = cSpaces.begin(); c != cSpaces.end(); c++) { cTotal += (*c)->spaceSize(); PolyWord *pt = (*c)->bottom; while (pt < (*c)->top) { pt++; PolyObject *obj = (PolyObject*)pt; if (obj->ContainsForwardingPtr()) { #ifdef POLYML32IN64 // This is relative to globalCodeBase not globalHeapBase while (obj->ContainsForwardingPtr()) obj = (PolyObject*)(globalCodeBase + ((obj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); #else obj = obj->FollowForwardingChain(); #endif pt += obj->Length(); } else { if (obj->IsCodeObject()) cOccupied += obj->Length() + 1; pt += obj->Length(); } } } Log("Heap: Code area: total "); LogSize(cTotal); Log(" occupied: "); LogSize(cOccupied); Log("\n"); uintptr_t stackSpace = 0; for (std::vector::iterator s = sSpaces.begin(); s != sSpaces.end(); s++) { stackSpace += (*s)->spaceSize(); } Log("Heap: Stack area: total "); LogSize(stackSpace); Log("\n"); } // Profiling - Find a code object or return zero if not found. // This can be called on a "user" thread. PolyObject *MemMgr::FindCodeObject(const byte *addr) { MemSpace *space = SpaceForAddress(addr); if (space == 0) return 0; Bitmap *profMap = 0; if (! space->isCode) return 0; if (space->spaceType == ST_CODE) { CodeSpace *cSpace = (CodeSpace*)space; profMap = &cSpace->headerMap; } else if (space->spaceType == ST_PERMANENT) { PermanentMemSpace *pSpace = (PermanentMemSpace*)space; profMap = &pSpace->profileCode; } else return 0; // Must be in code or permanent code. // For the permanent areas the header maps are created and initialised on demand. if (! profMap->Created()) { PLocker lock(&codeBitmapLock); if (! profMap->Created()) // Second check now we've got the lock. { // Create the bitmap. If it failed just say "not in this area" if (! profMap->Create(space->spaceSize())) return 0; // Set the first bit before releasing the lock. profMap->SetBit(0); } } // A bit is set if it is a length word. while ((uintptr_t)addr & (sizeof(POLYUNSIGNED)-1)) addr--; // Make it word aligned PolyWord *wordAddr = (PolyWord*)addr; // Work back to find the first set bit before this. // Normally we will find one but if we're looking up a value that // is actually an integer it might be in a piece of code that is now free. uintptr_t bitOffset = profMap->FindLastSet(wordAddr - space->bottom); if (space->spaceType == ST_CODE) { PolyWord *ptr = space->bottom+bitOffset; if (ptr >= space->top) return 0; // This will find the last non-free code cell or the first cell. // Return zero if the value was not actually in the cell or it wasn't code. PolyObject *obj = (PolyObject*)(ptr+1); #ifdef POLYML32IN64 PolyObject *lastObj = obj; // This is relative to globalCodeBase not globalHeapBase. while (lastObj->ContainsForwardingPtr()) lastObj = (PolyObject*)(globalCodeBase + ((lastObj->LengthWord() & ~_OBJ_TOMBSTONE_BIT) << 1)); #else PolyObject *lastObj = obj->FollowForwardingChain(); #endif // We normally replace forwarding pointers but when scanning to update // addresses after a saved state we may not have yet done that. if (wordAddr > ptr && wordAddr < ptr + 1 + lastObj->Length() && lastObj->IsCodeObject()) return obj; else return 0; } // Permanent area - the bits are set on demand. // Now work forward, setting any bits if necessary. We don't need a lock // because this is monotonic. for (;;) { PolyWord *ptr = space->bottom+bitOffset; if (ptr >= space->top) return 0; PolyObject *obj = (PolyObject*)(ptr+1); ASSERT(obj->ContainsNormalLengthWord()); if (wordAddr > ptr && wordAddr < ptr + obj->Length()) return obj; bitOffset += obj->Length()+1; profMap->SetBit(bitOffset); } return 0; } // Remove profiling bitmaps from permanent areas to free up memory. void MemMgr::RemoveProfilingBitmaps() { for (std::vector::iterator i = pSpaces.begin(); i < pSpaces.end(); i++) (*i)->profileCode.Destroy(); } #ifdef POLYML32IN64DEBUG POLYOBJECTPTR PolyWord::AddressToObjectPtr(void *address) { ASSERT(address >= globalHeapBase); uintptr_t offset = (PolyWord*)address - globalHeapBase; ASSERT(offset <= 0x7fffffff); // Currently limited to 8Gbytes ASSERT((offset & 1) == 0); return (POLYOBJECTPTR)offset; } #endif MemMgr gMem; // The one and only memory manager object