diff options
Diffstat (limited to 'contrib/llvm/lib/ExecutionEngine')
47 files changed, 4712 insertions, 757 deletions
diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index c59885753a8f..ae96c7f5955f 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -49,14 +49,13 @@ STATISTIC(NumGlobals , "Number of global vars initialized"); ExecutionEngine *(*ExecutionEngine::MCJITCtor)( std::unique_ptr<Module> M, std::string *ErrorStr, std::shared_ptr<MCJITMemoryManager> MemMgr, - - std::shared_ptr<JITSymbolResolver> Resolver, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, std::unique_ptr<TargetMachine> TM) = nullptr; ExecutionEngine *(*ExecutionEngine::OrcMCJITReplacementCtor)( - std::string *ErrorStr, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver, - std::unique_ptr<TargetMachine> TM) = nullptr; + std::string *ErrorStr, std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, + std::unique_ptr<TargetMachine> TM) = nullptr; ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr<Module> M, std::string *ErrorStr) =nullptr; @@ -97,14 +96,14 @@ ExecutionEngine::~ExecutionEngine() { } namespace { -/// \brief Helper class which uses a value handler to automatically deletes the +/// Helper class which uses a value handler to automatically deletes the /// memory block when the GlobalVariable is destroyed. class GVMemoryBlock final : public CallbackVH { GVMemoryBlock(const GlobalVariable *GV) : CallbackVH(const_cast<GlobalVariable*>(GV)) {} public: - /// \brief Returns the address the GlobalVariable should be written into. The + /// Returns the address the GlobalVariable should be written into. The /// GVMemoryBlock object prefixes that. static char *Create(const GlobalVariable *GV, const DataLayout& TD) { Type *ElTy = GV->getValueType(); @@ -215,7 +214,7 @@ void ExecutionEngine::addGlobalMapping(StringRef Name, uint64_t Addr) { assert(!Name.empty() && "Empty GlobalMapping symbol name!"); - DEBUG(dbgs() << "JIT: Map \'" << Name << "\' to [" << Addr << "]\n";); + LLVM_DEBUG(dbgs() << "JIT: Map \'" << Name << "\' to [" << Addr << "]\n";); uint64_t &CurVal = EEState.getGlobalAddressMap()[Name]; assert((!CurVal || !Addr) && "GlobalMapping already established!"); CurVal = Addr; @@ -344,13 +343,14 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, unsigned PtrSize = EE->getDataLayout().getPointerSize(); Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize); - DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array.get() << "\n"); + LLVM_DEBUG(dbgs() << "JIT: ARGV = " << (void *)Array.get() << "\n"); Type *SBytePtr = Type::getInt8PtrTy(C); for (unsigned i = 0; i != InputArgv.size(); ++i) { unsigned Size = InputArgv[i].size()+1; auto Dest = make_unique<char[]>(Size); - DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void*)Dest.get() << "\n"); + LLVM_DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void *)Dest.get() + << "\n"); std::copy(InputArgv[i].begin(), InputArgv[i].end(), Dest.get()); Dest[Size-1] = 0; @@ -502,9 +502,9 @@ EngineBuilder::setMemoryManager(std::unique_ptr<MCJITMemoryManager> MM) { return *this; } -EngineBuilder& -EngineBuilder::setSymbolResolver(std::unique_ptr<JITSymbolResolver> SR) { - Resolver = std::shared_ptr<JITSymbolResolver>(std::move(SR)); +EngineBuilder & +EngineBuilder::setSymbolResolver(std::unique_ptr<LegacyJITSymbolResolver> SR) { + Resolver = std::shared_ptr<LegacyJITSymbolResolver>(std::move(SR)); return *this; } @@ -532,7 +532,6 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // Unless the interpreter was explicitly selected or the JIT is not linked, // try making a JIT. if ((WhichEngine & EngineKind::JIT) && TheTM) { - Triple TT(M->getTargetTriple()); if (!TM->getTarget().hasJIT()) { errs() << "WARNING: This target JIT is not designed for the host" << " you are running. If bad things happen, please choose" @@ -591,7 +590,7 @@ void *ExecutionEngine::getPointerToGlobal(const GlobalValue *GV) { return getPointerToGlobalIfAvailable(GV); } -/// \brief Converts a Constant* into a GenericValue, including handling of +/// Converts a Constant* into a GenericValue, including handling of /// ConstantExpr values. GenericValue ExecutionEngine::getConstantValue(const Constant *C) { // If its undefined, return the garbage. @@ -904,6 +903,9 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { Result.IntVal = cast<ConstantInt>(C)->getValue(); break; case Type::PointerTyID: + while (auto *A = dyn_cast<GlobalAlias>(C)) { + C = A->getAliasee(); + } if (isa<ConstantPointerNull>(C)) Result.PointerVal = nullptr; else if (const Function *F = dyn_cast<Function>(C)) @@ -1182,8 +1184,8 @@ void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, } void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) { - DEBUG(dbgs() << "JIT: Initializing " << Addr << " "); - DEBUG(Init->dump()); + LLVM_DEBUG(dbgs() << "JIT: Initializing " << Addr << " "); + LLVM_DEBUG(Init->dump()); if (isa<UndefValue>(Init)) return; @@ -1230,7 +1232,7 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) { return; } - DEBUG(dbgs() << "Bad Type: " << *Init->getType() << "\n"); + LLVM_DEBUG(dbgs() << "Bad Type: " << *Init->getType() << "\n"); llvm_unreachable("Unknown constant type to initialize memory with!"); } diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 20251c23b17c..abcdaeba8eb0 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -14,11 +14,12 @@ #include "llvm-c/ExecutionEngine.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/GenericValue.h" +#include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Module.h" -#include "llvm/Support/CodeGenCWrappers.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Target/CodeGenCWrappers.h" #include "llvm/Target/TargetOptions.h" #include <cstring> @@ -411,3 +412,26 @@ void LLVMDisposeMCJITMemoryManager(LLVMMCJITMemoryManagerRef MM) { delete unwrap(MM); } +/*===-- JIT Event Listener functions -------------------------------------===*/ + + +#if !LLVM_USE_INTEL_JITEVENTS +LLVMJITEventListenerRef LLVMCreateIntelJITEventListener(void) +{ + return nullptr; +} +#endif + +#if !LLVM_USE_OPROFILE +LLVMJITEventListenerRef LLVMCreateOProfileJITEventListener(void) +{ + return nullptr; +} +#endif + +#if !LLVM_USE_PERF +LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void) +{ + return nullptr; +} +#endif diff --git a/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp b/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp index dad099d73c96..fd4f0746f7f9 100644 --- a/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm-c/ExecutionEngine.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/Object/ObjectFile.h" @@ -235,3 +236,8 @@ JITEventListener* JITEventListener::createGDBRegistrationListener() { } } // namespace llvm + +LLVMJITEventListenerRef LLVMCreateGDBRegistrationListener(void) +{ + return wrap(JITEventListener::createGDBRegistrationListener()); +} diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index cb6dd5e57283..211f5216811f 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "IntelJITEventsWrapper.h" +#include "llvm-c/ExecutionEngine.h" #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/Config/config.h" @@ -100,15 +101,17 @@ void IntelJITEventListener::NotifyObjectEmitted( const RuntimeDyld::LoadedObjectInfo &L) { OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); - const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); + const ObjectFile *DebugObj = DebugObjOwner.getBinary(); + if (!DebugObj) + return; // Get the address of the object image for use as a unique identifier - const void* ObjData = DebugObj.getData().data(); - std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj); + const void* ObjData = DebugObj->getData().data(); + std::unique_ptr<DIContext> Context = DWARFContext::create(*DebugObj); MethodAddressVector Functions; // Use symbol info to iterate functions in the object. - for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) { + for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(*DebugObj)) { SymbolRef Sym = P.first; std::vector<LineNumberInfo> LineInfo; std::string SourceFileName; @@ -238,3 +241,7 @@ JITEventListener *JITEventListener::createIntelJITEventListener( } // namespace llvm +LLVMJITEventListenerRef LLVMCreateIntelJITEventListener(void) +{ + return wrap(JITEventListener::createIntelJITEventListener()); +} diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c index f2d36a76a315..bc8fea148749 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c @@ -26,7 +26,6 @@ #include <pthread.h> #include <stdint.h> #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ -#include <malloc.h> #include <stdlib.h> #include "jitprofiling.h" diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp index 96844439e721..9e77d160c30b 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -974,11 +974,11 @@ void Interpreter::visitAllocaInst(AllocaInst &I) { unsigned MemToAlloc = std::max(1U, NumElements * TypeSize); // Allocate enough memory to hold the type... - void *Memory = malloc(MemToAlloc); + void *Memory = safe_malloc(MemToAlloc); - DEBUG(dbgs() << "Allocated Type: " << *Ty << " (" << TypeSize << " bytes) x " - << NumElements << " (Total: " << MemToAlloc << ") at " - << uintptr_t(Memory) << '\n'); + LLVM_DEBUG(dbgs() << "Allocated Type: " << *Ty << " (" << TypeSize + << " bytes) x " << NumElements << " (Total: " << MemToAlloc + << ") at " << uintptr_t(Memory) << '\n'); GenericValue Result = PTOGV(Memory); assert(Result.PointerVal && "Null pointer returned by malloc!"); @@ -1025,7 +1025,7 @@ GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I, GenericValue Result; Result.PointerVal = ((char*)getOperandValue(Ptr, SF).PointerVal) + Total; - DEBUG(dbgs() << "GEP Index " << Total << " bytes.\n"); + LLVM_DEBUG(dbgs() << "GEP Index " << Total << " bytes.\n"); return Result; } @@ -2118,7 +2118,7 @@ void Interpreter::run() { // Track the number of dynamic instructions executed. ++NumDynamicInsts; - DEBUG(dbgs() << "About to interpret: " << I); + LLVM_DEBUG(dbgs() << "About to interpret: " << I); visit(I); // Dispatch to one of the visit* methods... } } diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp index f7b8a3b657ee..2c663c2e1edf 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -39,11 +39,10 @@ static struct RegisterJIT { extern "C" void LLVMLinkInMCJIT() { } -ExecutionEngine* -MCJIT::createJIT(std::unique_ptr<Module> M, - std::string *ErrorStr, +ExecutionEngine * +MCJIT::createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, std::unique_ptr<TargetMachine> TM) { // Try to register the program as a source of symbols to resolve against. // @@ -64,7 +63,7 @@ MCJIT::createJIT(std::unique_ptr<Module> M, MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> TM, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver) + std::shared_ptr<LegacyJITSymbolResolver> Resolver) : ExecutionEngine(TM->createDataLayout(), std::move(M)), TM(std::move(TM)), Ctx(nullptr), MemMgr(std::move(MemMgr)), Resolver(*this, std::move(Resolver)), Dyld(*this->MemMgr, this->Resolver), @@ -143,8 +142,14 @@ void MCJIT::setObjectCache(ObjectCache* NewCache) { } std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { + assert(M && "Can not emit a null module"); + MutexGuard locked(lock); + // Materialize all globals in the module if they have not been + // materialized already. + cantFail(M->materializeAll()); + // This must be a module which has already been added but not loaded to this // MCJIT instance, since these conditions are tested by our caller, // generateCodeForModule. @@ -165,7 +170,7 @@ std::unique_ptr<MemoryBuffer> MCJIT::emitObject(Module *M) { // Flush the output buffer to get the generated code into memory std::unique_ptr<MemoryBuffer> CompiledObjBuffer( - new ObjectMemoryBuffer(std::move(ObjBufferSV))); + new SmallVectorMemoryBuffer(std::move(ObjBufferSV))); // If we have an object cache, tell it about the new object. // Note that we're using the compiled image, not the loaded image (as below). @@ -666,3 +671,5 @@ LinkingSymbolResolver::findSymbol(const std::string &Name) { return nullptr; return ClientResolver->findSymbol(Name); } + +void LinkingSymbolResolver::anchor() {} diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h index daf578f5daae..943b14942a0f 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -14,10 +14,10 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/ObjectCache.h" -#include "llvm/ExecutionEngine/ObjectMemoryBuffer.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Module.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" namespace llvm { class MCJIT; @@ -26,11 +26,11 @@ class MCJIT; // functions across modules that it owns. It aggregates the memory manager // that is passed in to the MCJIT constructor and defers most functionality // to that object. -class LinkingSymbolResolver : public JITSymbolResolver { +class LinkingSymbolResolver : public LegacyJITSymbolResolver { public: LinkingSymbolResolver(MCJIT &Parent, - std::shared_ptr<JITSymbolResolver> Resolver) - : ParentEngine(Parent), ClientResolver(std::move(Resolver)) {} + std::shared_ptr<LegacyJITSymbolResolver> Resolver) + : ParentEngine(Parent), ClientResolver(std::move(Resolver)) {} JITSymbol findSymbol(const std::string &Name) override; @@ -41,7 +41,8 @@ public: private: MCJIT &ParentEngine; - std::shared_ptr<JITSymbolResolver> ClientResolver; + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; + void anchor() override; }; // About Module states: added->loaded->finalized. @@ -67,7 +68,7 @@ private: class MCJIT : public ExecutionEngine { MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver); + std::shared_ptr<LegacyJITSymbolResolver> Resolver); typedef llvm::SmallPtrSet<Module *, 4> ModulePtrSet; @@ -300,11 +301,10 @@ public: MCJITCtor = createJIT; } - static ExecutionEngine* - createJIT(std::unique_ptr<Module> M, - std::string *ErrorStr, + static ExecutionEngine * + createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, std::unique_ptr<TargetMachine> TM); // @} diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/ObjectBuffer.h b/contrib/llvm/lib/ExecutionEngine/MCJIT/ObjectBuffer.h deleted file mode 100644 index 92310f3eb54a..000000000000 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/ObjectBuffer.h +++ /dev/null @@ -1,48 +0,0 @@ -//===--- ObjectBuffer.h - Utility class to wrap object memory ---*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares a wrapper class to hold the memory into which an -// object will be generated. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_EXECUTIONENGINE_OBJECTBUFFER_H -#define LLVM_EXECUTIONENGINE_OBJECTBUFFER_H - -#include "llvm/ADT/SmallVector.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { - -class ObjectMemoryBuffer : public MemoryBuffer { -public: - template <unsigned N> - ObjectMemoryBuffer(SmallVector<char, N> SV) - : SV(SV), BufferName("<in-memory object>") { - init(this->SV.begin(), this->SV.end(), false); - } - - template <unsigned N> - ObjectMemoryBuffer(SmallVector<char, N> SV, StringRef Name) - : SV(SV), BufferName(Name) { - init(this->SV.begin(), this->SV.end(), false); - } - const char* getBufferIdentifier() const override { return BufferName.c_str(); } - - BufferKind getBufferKind() const override { return MemoryBuffer_Malloc; } - -private: - SmallVector<char, 4096> SV; - std::string BufferName; -}; - -} // namespace llvm - -#endif diff --git a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index 3581d6458395..6f0825fb38da 100644 --- a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -12,8 +12,10 @@ // //===----------------------------------------------------------------------===// +#include "llvm-c/ExecutionEngine.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/Config/config.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/OProfileWrapper.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" @@ -57,9 +59,10 @@ public: void OProfileJITEventListener::initialize() { if (!Wrapper->op_open_agent()) { const std::string err_str = sys::StrError(); - DEBUG(dbgs() << "Failed to connect to OProfile agent: " << err_str << "\n"); + LLVM_DEBUG(dbgs() << "Failed to connect to OProfile agent: " << err_str + << "\n"); } else { - DEBUG(dbgs() << "Connected to OProfile agent.\n"); + LLVM_DEBUG(dbgs() << "Connected to OProfile agent.\n"); } } @@ -67,10 +70,10 @@ OProfileJITEventListener::~OProfileJITEventListener() { if (Wrapper->isAgentAvailable()) { if (Wrapper->op_close_agent() == -1) { const std::string err_str = sys::StrError(); - DEBUG(dbgs() << "Failed to disconnect from OProfile agent: " - << err_str << "\n"); + LLVM_DEBUG(dbgs() << "Failed to disconnect from OProfile agent: " + << err_str << "\n"); } else { - DEBUG(dbgs() << "Disconnected from OProfile agent.\n"); + LLVM_DEBUG(dbgs() << "Disconnected from OProfile agent.\n"); } } } @@ -84,6 +87,7 @@ void OProfileJITEventListener::NotifyObjectEmitted( OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); + std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj); // Use symbol info to iterate functions in the object. for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) { @@ -103,12 +107,34 @@ void OProfileJITEventListener::NotifyObjectEmitted( if (Wrapper->op_write_native_code(Name.data(), Addr, (void *)Addr, Size) == -1) { - DEBUG(dbgs() << "Failed to tell OProfile about native function " << Name - << " at [" << (void *)Addr << "-" << ((char *)Addr + Size) - << "]\n"); + LLVM_DEBUG(dbgs() << "Failed to tell OProfile about native function " + << Name << " at [" << (void *)Addr << "-" + << ((char *)Addr + Size) << "]\n"); + continue; + } + + DILineInfoTable Lines = Context->getLineInfoForAddressRange(Addr, Size); + size_t i = 0; + size_t num_entries = Lines.size(); + struct debug_line_info *debug_line; + debug_line = (struct debug_line_info *)calloc( + num_entries, sizeof(struct debug_line_info)); + + for (auto& It : Lines) { + debug_line[i].vma = (unsigned long)It.first; + debug_line[i].lineno = It.second.Line; + debug_line[i].filename = + const_cast<char *>(Lines.front().second.FileName.c_str()); + ++i; + } + + if (Wrapper->op_write_debug_line_info((void *)Addr, num_entries, + debug_line) == -1) { + LLVM_DEBUG(dbgs() << "Failed to tell OProfiler about debug object at [" + << (void *)Addr << "-" << ((char *)Addr + Size) + << "]\n"); continue; } - // TODO: support line number info (similar to IntelJITEventListener.cpp) } DebugObjects[Obj.getData().data()] = std::move(DebugObjOwner); @@ -135,9 +161,10 @@ void OProfileJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) { uint64_t Addr = *AddrOrErr; if (Wrapper->op_unload_native_code(Addr) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about unload of native function at " - << (void*)Addr << "\n"); + LLVM_DEBUG( + dbgs() + << "Failed to tell OProfile about unload of native function at " + << (void *)Addr << "\n"); continue; } } @@ -156,3 +183,7 @@ JITEventListener *JITEventListener::createOProfileJITEventListener() { } // namespace llvm +LLVMJITEventListenerRef LLVMCreateOProfileJITEventListener(void) +{ + return wrap(JITEventListener::createOProfileJITEventListener()); +} diff --git a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp index d96278a8137b..b473ac3faf4c 100644 --- a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp +++ b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp @@ -64,15 +64,16 @@ bool OProfileWrapper::initialize() { // If the oprofile daemon is not running, don't load the opagent library if (!isOProfileRunning()) { - DEBUG(dbgs() << "OProfile daemon is not detected.\n"); + LLVM_DEBUG(dbgs() << "OProfile daemon is not detected.\n"); return false; } std::string error; if(!DynamicLibrary::LoadLibraryPermanently("libopagent.so", &error)) { - DEBUG(dbgs() - << "OProfile connector library libopagent.so could not be loaded: " - << error << "\n"); + LLVM_DEBUG( + dbgs() + << "OProfile connector library libopagent.so could not be loaded: " + << error << "\n"); } // Get the addresses of the opagent functions diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp new file mode 100644 index 000000000000..d42e7b05ba67 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -0,0 +1,343 @@ +//===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/Cloning.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +template <typename MaterializerFtor> +class LambdaValueMaterializer final : public ValueMaterializer { +public: + LambdaValueMaterializer(MaterializerFtor M) : M(std::move(M)) {} + + Value *materialize(Value *V) final { return M(V); } + +private: + MaterializerFtor M; +}; + +template <typename MaterializerFtor> +LambdaValueMaterializer<MaterializerFtor> +createLambdaValueMaterializer(MaterializerFtor M) { + return LambdaValueMaterializer<MaterializerFtor>(std::move(M)); +} +} // namespace + +static void extractAliases(MaterializationResponsibility &R, Module &M, + MangleAndInterner &Mangle) { + SymbolAliasMap Aliases; + + std::vector<GlobalAlias *> ModAliases; + for (auto &A : M.aliases()) + ModAliases.push_back(&A); + + for (auto *A : ModAliases) { + Constant *Aliasee = A->getAliasee(); + assert(A->hasName() && "Anonymous alias?"); + assert(Aliasee->hasName() && "Anonymous aliasee"); + std::string AliasName = A->getName(); + + Aliases[Mangle(AliasName)] = SymbolAliasMapEntry( + {Mangle(Aliasee->getName()), JITSymbolFlags::fromGlobalValue(*A)}); + + if (isa<Function>(Aliasee)) { + auto *F = cloneFunctionDecl(M, *cast<Function>(Aliasee)); + A->replaceAllUsesWith(F); + A->eraseFromParent(); + F->setName(AliasName); + } else if (isa<GlobalValue>(Aliasee)) { + auto *G = cloneGlobalVariableDecl(M, *cast<GlobalVariable>(Aliasee)); + A->replaceAllUsesWith(G); + A->eraseFromParent(); + G->setName(AliasName); + } + } + + R.replace(symbolAliases(std::move(Aliases))); +} + +static std::unique_ptr<Module> +extractAndClone(Module &M, LLVMContext &NewContext, StringRef Suffix, + function_ref<bool(const GlobalValue *)> ShouldCloneDefinition) { + SmallVector<char, 1> ClonedModuleBuffer; + + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDefinition(GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); + + for (auto *GV : ClonedDefsInSrc) { + // Delete the definition and bump the linkage in the source module. + if (isa<Function>(GV)) { + auto &F = *cast<Function>(GV); + F.deleteBody(); + F.setPersonalityFn(nullptr); + } else if (isa<GlobalVariable>(GV)) { + cast<GlobalVariable>(GV)->setInitializer(nullptr); + } else + llvm_unreachable("Unsupported global type"); + + GV->setLinkage(GlobalValue::ExternalLinkage); + } + + BitcodeWriter BCWriter(ClonedModuleBuffer); + + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } + + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + + auto ClonedModule = + cantFail(parseBitcodeFile(ClonedModuleBufferRef, NewContext)); + ClonedModule->setModuleIdentifier((M.getName() + Suffix).str()); + return ClonedModule; +} + +static std::unique_ptr<Module> extractGlobals(Module &M, + LLVMContext &NewContext) { + return extractAndClone(M, NewContext, ".globals", [](const GlobalValue *GV) { + return isa<GlobalVariable>(GV); + }); +} + +namespace llvm { +namespace orc { + +class ExtractingIRMaterializationUnit : public IRMaterializationUnit { +public: + ExtractingIRMaterializationUnit(ExecutionSession &ES, + CompileOnDemandLayer2 &Parent, + std::unique_ptr<Module> M) + : IRMaterializationUnit(ES, std::move(M)), Parent(Parent) {} + + ExtractingIRMaterializationUnit(std::unique_ptr<Module> M, + SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition, + CompileOnDemandLayer2 &Parent) + : IRMaterializationUnit(std::move(M), std::move(SymbolFlags), + std::move(SymbolToDefinition)), + Parent(Parent) {} + +private: + void materialize(MaterializationResponsibility R) override { + // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the + // extracted module key, extracted module, and source module key + // together. This could be used, for example, to provide a specific + // memory manager instance to the linking layer. + + auto RequestedSymbols = R.getRequestedSymbols(); + + // Extract the requested functions into a new module. + std::unique_ptr<Module> ExtractedFunctionsModule; + if (!RequestedSymbols.empty()) { + std::string Suffix; + std::set<const GlobalValue *> FunctionsToClone; + for (auto &Name : RequestedSymbols) { + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && I->second != nullptr && + "Should have a non-null definition"); + FunctionsToClone.insert(I->second); + Suffix += "."; + Suffix += *Name; + } + + std::lock_guard<std::mutex> Lock(SourceModuleMutex); + ExtractedFunctionsModule = + extractAndClone(*M, Parent.GetAvailableContext(), Suffix, + [&](const GlobalValue *GV) -> bool { + return FunctionsToClone.count(GV); + }); + } + + // Build a new ExtractingIRMaterializationUnit to delegate the unrequested + // symbols to. + SymbolFlagsMap DelegatedSymbolFlags; + IRMaterializationUnit::SymbolNameToDefinitionMap + DelegatedSymbolToDefinition; + for (auto &KV : SymbolToDefinition) { + if (RequestedSymbols.count(KV.first)) + continue; + DelegatedSymbolFlags[KV.first] = + JITSymbolFlags::fromGlobalValue(*KV.second); + DelegatedSymbolToDefinition[KV.first] = KV.second; + } + + if (!DelegatedSymbolFlags.empty()) { + assert(DelegatedSymbolFlags.size() == + DelegatedSymbolToDefinition.size() && + "SymbolFlags and SymbolToDefinition should have the same number " + "of entries"); + R.replace(llvm::make_unique<ExtractingIRMaterializationUnit>( + std::move(M), std::move(DelegatedSymbolFlags), + std::move(DelegatedSymbolToDefinition), Parent)); + } + + if (ExtractedFunctionsModule) + Parent.emitExtractedFunctionsModule(std::move(R), + std::move(ExtractedFunctionsModule)); + } + + void discard(const VSO &V, SymbolStringPtr Name) override { + // All original symbols were materialized by the CODLayer and should be + // final. The function bodies provided by M should never be overridden. + llvm_unreachable("Discard should never be called on an " + "ExtractingIRMaterializationUnit"); + } + + mutable std::mutex SourceModuleMutex; + CompileOnDemandLayer2 &Parent; +}; + +CompileOnDemandLayer2::CompileOnDemandLayer2( + ExecutionSession &ES, IRLayer &BaseLayer, JITCompileCallbackManager &CCMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager, + GetAvailableContextFunction GetAvailableContext) + : IRLayer(ES), BaseLayer(BaseLayer), CCMgr(CCMgr), + BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)), + GetAvailableContext(std::move(GetAvailableContext)) {} + +Error CompileOnDemandLayer2::add(VSO &V, VModuleKey K, + std::unique_ptr<Module> M) { + return IRLayer::add(V, K, std::move(M)); +} + +void CompileOnDemandLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + auto &ES = getExecutionSession(); + assert(M && "M should not be null"); + + for (auto &GV : M->global_values()) + if (GV.hasWeakLinkage()) + GV.setLinkage(GlobalValue::ExternalLinkage); + + MangleAndInterner Mangle(ES, M->getDataLayout()); + + extractAliases(R, *M, Mangle); + + auto GlobalsModule = extractGlobals(*M, GetAvailableContext()); + + // Delete the bodies of any available externally functions, rename the + // rest, and build the compile callbacks. + std::map<SymbolStringPtr, std::pair<JITTargetAddress, JITSymbolFlags>> + StubCallbacksAndLinkages; + auto &TargetVSO = R.getTargetVSO(); + + for (auto &F : M->functions()) { + if (F.isDeclaration()) + continue; + + if (F.hasAvailableExternallyLinkage()) { + F.deleteBody(); + F.setPersonalityFn(nullptr); + continue; + } + + assert(F.hasName() && "Function should have a name"); + std::string StubUnmangledName = F.getName(); + F.setName(F.getName() + "$body"); + auto StubDecl = cloneFunctionDecl(*M, F); + StubDecl->setName(StubUnmangledName); + StubDecl->setPersonalityFn(nullptr); + StubDecl->setLinkage(GlobalValue::ExternalLinkage); + F.replaceAllUsesWith(StubDecl); + + auto StubName = Mangle(StubUnmangledName); + auto BodyName = Mangle(F.getName()); + if (auto CallbackAddr = CCMgr.getCompileCallback( + [BodyName, &TargetVSO, &ES]() -> JITTargetAddress { + if (auto Sym = lookup({&TargetVSO}, BodyName)) + return Sym->getAddress(); + else { + ES.reportError(Sym.takeError()); + return 0; + } + })) { + auto Flags = JITSymbolFlags::fromGlobalValue(F); + Flags &= ~JITSymbolFlags::Weak; + StubCallbacksAndLinkages[std::move(StubName)] = + std::make_pair(*CallbackAddr, Flags); + } else { + ES.reportError(CallbackAddr.takeError()); + R.failMaterialization(); + return; + } + } + + // Build the stub inits map. + IndirectStubsManager::StubInitsMap StubInits; + for (auto &KV : StubCallbacksAndLinkages) + StubInits[*KV.first] = KV.second; + + // Build the function-body-extracting materialization unit. + if (auto Err = R.getTargetVSO().define( + llvm::make_unique<ExtractingIRMaterializationUnit>(ES, *this, + std::move(M)))) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + // Build the stubs. + // FIXME: Remove function bodies materialization unit if stub creation fails. + auto &StubsMgr = getStubsManager(TargetVSO); + if (auto Err = StubsMgr.createStubs(StubInits)) { + ES.reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + // Resolve and finalize stubs. + SymbolMap ResolvedStubs; + for (auto &KV : StubCallbacksAndLinkages) { + if (auto Sym = StubsMgr.findStub(*KV.first, false)) + ResolvedStubs[KV.first] = Sym; + else + llvm_unreachable("Stub went missing"); + } + + R.resolve(ResolvedStubs); + + BaseLayer.emit(std::move(R), std::move(K), std::move(GlobalsModule)); +} + +IndirectStubsManager &CompileOnDemandLayer2::getStubsManager(const VSO &V) { + std::lock_guard<std::mutex> Lock(CODLayerMutex); + StubManagersMap::iterator I = StubsMgrs.find(&V); + if (I == StubsMgrs.end()) + I = StubsMgrs.insert(std::make_pair(&V, BuildIndirectStubsManager())).first; + return *I->second; +} + +void CompileOnDemandLayer2::emitExtractedFunctionsModule( + MaterializationResponsibility R, std::unique_ptr<Module> M) { + auto K = getExecutionSession().allocateVModule(); + BaseLayer.emit(std::move(R), std::move(K), std::move(M)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp new file mode 100644 index 000000000000..4325d57f73d0 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -0,0 +1,1690 @@ +//===----- Core.cpp - Core ORC APIs (MaterializationUnit, VSO, etc.) ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" + +#if LLVM_ENABLE_THREADS +#include <future> +#endif + +namespace llvm { +namespace orc { + +char FailedToMaterialize::ID = 0; +char SymbolsNotFound::ID = 0; + +RegisterDependenciesFunction NoDependenciesToRegister = + RegisterDependenciesFunction(); + +void MaterializationUnit::anchor() {} + +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.isWeak()) + OS << 'W'; + else if (Flags.isCommon()) + OS << 'C'; + else + OS << 'S'; + + if (Flags.isExported()) + OS << 'E'; + else + OS << 'H'; + + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { + OS << format("0x%016x", Sym.getAddress()) << " " << Sym.getFlags(); + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { + OS << "\"" << *KV.first << "\": " << KV.second; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + OS << "{"; + if (!Symbols.empty()) { + OS << " \"" << **Symbols.begin() << "\""; + for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) + OS << ", \"" << *Sym << "\""; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { + OS << "{"; + if (!Symbols.empty()) { + OS << " {" << *Symbols.begin() << "}"; + for (auto &Sym : make_range(std::next(Symbols.begin()), Symbols.end())) + OS << ", {" << Sym << "}"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + OS << "{"; + if (!SymbolFlags.empty()) { + OS << " {\"" << *SymbolFlags.begin()->first + << "\": " << SymbolFlags.begin()->second << "}"; + for (auto &KV : + make_range(std::next(SymbolFlags.begin()), SymbolFlags.end())) + OS << ", {\"" << *KV.first << "\": " << KV.second << "}"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { + OS << "{"; + if (!Deps.empty()) { + OS << " { " << Deps.begin()->first->getName() << ": " + << Deps.begin()->second << " }"; + for (auto &KV : make_range(std::next(Deps.begin()), Deps.end())) + OS << ", { " << KV.first->getName() << ": " << KV.second << " }"; + } + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const VSOList &VSOs) { + OS << "["; + if (!VSOs.empty()) { + assert(VSOs.front() && "VSOList entries must not be null"); + OS << " " << VSOs.front()->getName(); + for (auto *V : make_range(std::next(VSOs.begin()), VSOs.end())) { + assert(V && "VSOList entries must not be null"); + OS << ", " << V->getName(); + } + } + OS << " ]"; + return OS; +} + +FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code FailedToMaterialize::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void FailedToMaterialize::log(raw_ostream &OS) const { + OS << "Failed to materialize symbols: " << Symbols; +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsNotFound::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsNotFound::log(raw_ostream &OS) const { + OS << "Symbols not found: " << Symbols; +} + +void ExecutionSessionBase::legacyFailQuery(AsynchronousSymbolQuery &Q, + Error Err) { + assert(!!Err && "Error should be in failure state"); + + bool SendErrorToQuery; + runSessionLocked([&]() { + Q.detach(); + SendErrorToQuery = Q.canStillFail(); + }); + + if (SendErrorToQuery) + Q.handleFailed(std::move(Err)); + else + reportError(std::move(Err)); +} + +Expected<SymbolMap> ExecutionSessionBase::legacyLookup( + ExecutionSessionBase &ES, LegacyAsyncLookupFunction AsyncLookup, + SymbolNameSet Names, bool WaitUntilReady, + RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + ES.reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + ES.reportError(std::move(Err)); + }; + } +#endif + + auto Query = std::make_shared<AsynchronousSymbolQuery>( + Names, std::move(OnResolve), std::move(OnReady)); + // FIXME: This should be run session locked along with the registration code + // and error reporting below. + SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); + + // If the query was lodged successfully then register the dependencies, + // otherwise fail it with an error. + if (UnresolvedSymbols.empty()) + RegisterDependencies(Query->QueryRegistrations); + else { + bool DeliverError = runSessionLocked([&]() { + Query->detach(); + return Query->canStillFail(); + }); + auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); + if (DeliverError) + Query->handleFailed(std::move(Err)); + else + ES.reportError(std::move(Err)); + } + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif +} + +void ExecutionSessionBase::lookup( + const VSOList &VSOs, const SymbolNameSet &Symbols, + SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + RegisterDependenciesFunction RegisterDependencies) { + + // lookup can be re-entered recursively if running on a single thread. Run any + // outstanding MUs in case this query depends on them, otherwise the main + // thread will starve waiting for a result from an MU that it failed to run. + runOutstandingMUs(); + + auto Unresolved = std::move(Symbols); + std::map<VSO *, MaterializationUnitList> MUsMap; + auto Q = std::make_shared<AsynchronousSymbolQuery>( + Symbols, std::move(OnResolve), std::move(OnReady)); + bool QueryIsFullyResolved = false; + bool QueryIsFullyReady = false; + bool QueryFailed = false; + + runSessionLocked([&]() { + for (auto *V : VSOs) { + assert(V && "VSOList entries must not be null"); + assert(!MUsMap.count(V) && + "VSOList should not contain duplicate entries"); + V->lodgeQuery(Q, Unresolved, MUsMap[V]); + } + + if (Unresolved.empty()) { + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryIsFullyResolved = Q->isFullyResolved(); + QueryIsFullyReady = Q->isFullyReady(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + } else { + // Query failed due to unresolved symbols. + QueryFailed = true; + + // Disconnect the query from its dependencies. + Q->detach(); + + // Replace the MUs. + for (auto &KV : MUsMap) + for (auto &MU : KV.second) + KV.first->replace(std::move(MU)); + } + }); + + if (QueryFailed) { + Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); + return; + } else { + if (QueryIsFullyResolved) + Q->handleFullyResolved(); + if (QueryIsFullyReady) + Q->handleFullyReady(); + } + + // Move the MUs to the OutstandingMUs list, then materialize. + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + for (auto &KV : MUsMap) + for (auto &MU : KV.second) + OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); + } + + runOutstandingMUs(); +} + +Expected<SymbolMap> +ExecutionSessionBase::lookup(const VSOList &VSOs, const SymbolNameSet &Symbols, + RegisterDependenciesFunction RegisterDependencies, + bool WaitUntilReady) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + std::mutex ErrMutex; + Error ResolutionError = Error::success(); + std::promise<void> PromisedReady; + Error ReadyError = Error::success(); + auto OnResolve = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + { + ErrorAsOutParameter _(&ResolutionError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ResolutionError = R.takeError(); + } + PromisedResult.set_value(SymbolMap()); + } + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + if (Err) { + ErrorAsOutParameter _(&ReadyError); + std::lock_guard<std::mutex> Lock(ErrMutex); + ReadyError = std::move(Err); + } + PromisedReady.set_value(); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + Error ReadyError = Error::success(); + + auto OnResolve = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; + + std::function<void(Error)> OnReady; + if (WaitUntilReady) { + OnReady = [&](Error Err) { + ErrorAsOutParameter _(&ReadyError); + if (Err) + ReadyError = std::move(Err); + }; + } else { + OnReady = [&](Error Err) { + if (Err) + reportError(std::move(Err)); + }; + } +#endif + + // Perform the asynchronous lookup. + lookup(VSOs, Symbols, OnResolve, OnReady, RegisterDependencies); + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + } + + if (WaitUntilReady) { + auto ReadyFuture = PromisedReady.get_future(); + ReadyFuture.get(); + + { + std::lock_guard<std::mutex> Lock(ErrMutex); + if (ReadyError) + return std::move(ReadyError); + } + } else + cantFail(std::move(ReadyError)); + + return std::move(Result); + +#else + if (ResolutionError) { + // ReadyError will never be assigned. Consume the success value. + cantFail(std::move(ReadyError)); + return std::move(ResolutionError); + } + + if (ReadyError) + return std::move(ReadyError); + + return Result; +#endif +} + +void ExecutionSessionBase::runOutstandingMUs() { + while (1) { + std::pair<VSO *, std::unique_ptr<MaterializationUnit>> VSOAndMU; + + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + if (!OutstandingMUs.empty()) { + VSOAndMU = std::move(OutstandingMUs.back()); + OutstandingMUs.pop_back(); + } + } + + if (VSOAndMU.first) { + assert(VSOAndMU.second && "VSO, but no MU?"); + dispatchMaterialization(*VSOAndMU.first, std::move(VSOAndMU.second)); + } else + break; + } +} + +AsynchronousSymbolQuery::AsynchronousSymbolQuery( + const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved, + SymbolsReadyCallback NotifySymbolsReady) + : NotifySymbolsResolved(std::move(NotifySymbolsResolved)), + NotifySymbolsReady(std::move(NotifySymbolsReady)) { + NotYetResolvedCount = NotYetReadyCount = Symbols.size(); + + for (auto &S : Symbols) + ResolvedSymbols[S] = nullptr; +} + +void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, + JITEvaluatedSymbol Sym) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Resolving symbol outside the requested set"); + assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + I->second = std::move(Sym); + --NotYetResolvedCount; +} + +void AsynchronousSymbolQuery::handleFullyResolved() { + assert(NotYetResolvedCount == 0 && "Not fully resolved?"); + assert(NotifySymbolsResolved && + "NotifySymbolsResolved already called or error occurred"); + NotifySymbolsResolved(std::move(ResolvedSymbols)); + NotifySymbolsResolved = SymbolsResolvedCallback(); +} + +void AsynchronousSymbolQuery::notifySymbolReady() { + assert(NotYetReadyCount != 0 && "All symbols already finalized"); + --NotYetReadyCount; +} + +void AsynchronousSymbolQuery::handleFullyReady() { + assert(QueryRegistrations.empty() && + "Query is still registered with some symbols"); + assert(!NotifySymbolsResolved && "Resolution not applied yet"); + NotifySymbolsReady(Error::success()); + NotifySymbolsReady = SymbolsReadyCallback(); +} + +bool AsynchronousSymbolQuery::canStillFail() { + return (NotifySymbolsResolved || NotifySymbolsReady); +} + +void AsynchronousSymbolQuery::handleFailed(Error Err) { + assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && + NotYetResolvedCount == 0 && NotYetReadyCount == 0 && + "Query should already have been abandoned"); + if (NotifySymbolsResolved) { + NotifySymbolsResolved(std::move(Err)); + NotifySymbolsResolved = SymbolsResolvedCallback(); + } else { + assert(NotifySymbolsReady && "Failed after both callbacks issued?"); + NotifySymbolsReady(std::move(Err)); + } + NotifySymbolsReady = SymbolsReadyCallback(); +} + +void AsynchronousSymbolQuery::addQueryDependence(VSO &V, SymbolStringPtr Name) { + bool Added = QueryRegistrations[&V].insert(std::move(Name)).second; + (void)Added; + assert(Added && "Duplicate dependence notification?"); +} + +void AsynchronousSymbolQuery::removeQueryDependence( + VSO &V, const SymbolStringPtr &Name) { + auto QRI = QueryRegistrations.find(&V); + assert(QRI != QueryRegistrations.end() && "No dependencies registered for V"); + assert(QRI->second.count(Name) && "No dependency on Name in V"); + QRI->second.erase(Name); + if (QRI->second.empty()) + QueryRegistrations.erase(QRI); +} + +void AsynchronousSymbolQuery::detach() { + ResolvedSymbols.clear(); + NotYetResolvedCount = 0; + NotYetReadyCount = 0; + for (auto &KV : QueryRegistrations) + KV.first->detachQueryHelper(*this, KV.second); + QueryRegistrations.clear(); +} + +MaterializationResponsibility::MaterializationResponsibility( + VSO &V, SymbolFlagsMap SymbolFlags) + : V(V), SymbolFlags(std::move(SymbolFlags)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + +#ifndef NDEBUG + for (auto &KV : this->SymbolFlags) + KV.second |= JITSymbolFlags::Materializing; +#endif +} + +MaterializationResponsibility::~MaterializationResponsibility() { + assert(SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); +} + +SymbolNameSet MaterializationResponsibility::getRequestedSymbols() { + return V.getRequestedSymbols(SymbolFlags); +} + +void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto I = SymbolFlags.find(KV.first); + assert(I != SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + assert(I->second.isMaterializing() && "Duplicate resolution"); + I->second &= ~JITSymbolFlags::Materializing; + if (I->second.isWeak()) + assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && + "Resolving symbol with incorrect flags"); + else + assert(I->second == KV.second.getFlags() && + "Resolving symbol with incorrect flags"); + } +#endif + + V.resolve(Symbols); +} + +void MaterializationResponsibility::finalize() { +#ifndef NDEBUG + for (auto &KV : SymbolFlags) + assert(!KV.second.isMaterializing() && + "Failed to resolve symbol before finalization"); +#endif // NDEBUG + + V.finalize(SymbolFlags); + SymbolFlags.clear(); +} + +Error MaterializationResponsibility::defineMaterializing( + const SymbolFlagsMap &NewSymbolFlags) { + // Add the given symbols to this responsibility object. + // It's ok if we hit a duplicate here: In that case the new version will be + // discarded, and the VSO::defineMaterializing method will return a duplicate + // symbol error. + for (auto &KV : NewSymbolFlags) { + auto I = SymbolFlags.insert(KV).first; + (void)I; +#ifndef NDEBUG + I->second |= JITSymbolFlags::Materializing; +#endif + } + + return V.defineMaterializing(NewSymbolFlags); +} + +void MaterializationResponsibility::failMaterialization() { + + SymbolNameSet FailedSymbols; + for (auto &KV : SymbolFlags) + FailedSymbols.insert(KV.first); + + V.notifyFailed(FailedSymbols); + SymbolFlags.clear(); +} + +void MaterializationResponsibility::replace( + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) + SymbolFlags.erase(KV.first); + + V.replace(std::move(MU)); +} + +MaterializationResponsibility +MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) { + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = SymbolFlags.find(Name); + assert(I != SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + SymbolFlags.erase(I); + } + + return MaterializationResponsibility(V, std::move(DelegatedFlags)); +} + +void MaterializationResponsibility::addDependencies( + const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { + assert(SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + V.addDependencies(Name, Dependencies); +} + +void MaterializationResponsibility::addDependenciesForAll( + const SymbolDependenceMap &Dependencies) { + for (auto &KV : SymbolFlags) + V.addDependencies(KV.first, Dependencies); +} + +AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( + SymbolMap Symbols) + : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {} + +void AbsoluteSymbolsMaterializationUnit::materialize( + MaterializationResponsibility R) { + R.resolve(Symbols); + R.finalize(); +} + +void AbsoluteSymbolsMaterializationUnit::discard(const VSO &V, + SymbolStringPtr Name) { + assert(Symbols.count(Name) && "Symbol is not part of this MU"); + Symbols.erase(Name); +} + +SymbolFlagsMap +AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { + SymbolFlagsMap Flags; + for (const auto &KV : Symbols) + Flags[KV.first] = KV.second.getFlags(); + return Flags; +} + +ReExportsMaterializationUnit::ReExportsMaterializationUnit( + VSO *SourceVSO, SymbolAliasMap Aliases) + : MaterializationUnit(extractFlags(Aliases)), SourceVSO(SourceVSO), + Aliases(std::move(Aliases)) {} + +void ReExportsMaterializationUnit::materialize( + MaterializationResponsibility R) { + + auto &ES = R.getTargetVSO().getExecutionSession(); + VSO &TgtV = R.getTargetVSO(); + VSO &SrcV = SourceVSO ? *SourceVSO : TgtV; + + // Find the set of requested aliases and aliasees. Return any unrequested + // aliases back to the VSO so as to not prematurely materialize any aliasees. + auto RequestedSymbols = R.getRequestedSymbols(); + SymbolAliasMap RequestedAliases; + + for (auto &Name : RequestedSymbols) { + auto I = Aliases.find(Name); + assert(I != Aliases.end() && "Symbol not found in aliases map?"); + RequestedAliases[Name] = std::move(I->second); + Aliases.erase(I); + } + + if (!Aliases.empty()) { + if (SourceVSO) + R.replace(reexports(*SourceVSO, std::move(Aliases))); + else + R.replace(symbolAliases(std::move(Aliases))); + } + + // The OnResolveInfo struct will hold the aliases and responsibilty for each + // query in the list. + struct OnResolveInfo { + OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases) + : R(std::move(R)), Aliases(std::move(Aliases)) {} + + MaterializationResponsibility R; + SymbolAliasMap Aliases; + }; + + // Build a list of queries to issue. In each round we build the largest set of + // aliases that we can resolve without encountering a chain definition of the + // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would + // be waitin on a symbol that it itself had to resolve. Usually this will just + // involve one round and a single query. + + std::vector<std::pair<SymbolNameSet, std::shared_ptr<OnResolveInfo>>> + QueryInfos; + while (!RequestedAliases.empty()) { + SymbolNameSet ResponsibilitySymbols; + SymbolNameSet QuerySymbols; + SymbolAliasMap QueryAliases; + + for (auto I = RequestedAliases.begin(), E = RequestedAliases.end(); + I != E;) { + auto Tmp = I++; + + // Chain detected. Skip this symbol for this round. + if (&SrcV == &TgtV && (QueryAliases.count(Tmp->second.Aliasee) || + RequestedAliases.count(Tmp->second.Aliasee))) + continue; + + ResponsibilitySymbols.insert(Tmp->first); + QuerySymbols.insert(Tmp->second.Aliasee); + QueryAliases[Tmp->first] = std::move(Tmp->second); + RequestedAliases.erase(Tmp); + } + assert(!QuerySymbols.empty() && "Alias cycle detected!"); + + auto QueryInfo = std::make_shared<OnResolveInfo>( + R.delegate(ResponsibilitySymbols), std::move(QueryAliases)); + QueryInfos.push_back( + make_pair(std::move(QuerySymbols), std::move(QueryInfo))); + } + + // Issue the queries. + while (!QueryInfos.empty()) { + auto QuerySymbols = std::move(QueryInfos.back().first); + auto QueryInfo = std::move(QueryInfos.back().second); + + QueryInfos.pop_back(); + + auto RegisterDependencies = [QueryInfo, + &SrcV](const SymbolDependenceMap &Deps) { + // If there were no materializing symbols, just bail out. + if (Deps.empty()) + return; + + // Otherwise the only deps should be on SrcV. + assert(Deps.size() == 1 && Deps.count(&SrcV) && + "Unexpected dependencies for reexports"); + + auto &SrcVDeps = Deps.find(&SrcV)->second; + SymbolDependenceMap PerAliasDepsMap; + auto &PerAliasDeps = PerAliasDepsMap[&SrcV]; + + for (auto &KV : QueryInfo->Aliases) + if (SrcVDeps.count(KV.second.Aliasee)) { + PerAliasDeps = {KV.second.Aliasee}; + QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); + } + }; + + auto OnResolve = [QueryInfo](Expected<SymbolMap> Result) { + if (Result) { + SymbolMap ResolutionMap; + for (auto &KV : QueryInfo->Aliases) { + assert(Result->count(KV.second.Aliasee) && + "Result map missing entry?"); + ResolutionMap[KV.first] = JITEvaluatedSymbol( + (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + } + QueryInfo->R.resolve(ResolutionMap); + QueryInfo->R.finalize(); + } else { + auto &ES = QueryInfo->R.getTargetVSO().getExecutionSession(); + ES.reportError(Result.takeError()); + QueryInfo->R.failMaterialization(); + } + }; + + auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; + + ES.lookup({&SrcV}, QuerySymbols, std::move(OnResolve), std::move(OnReady), + std::move(RegisterDependencies)); + } +} + +void ReExportsMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { + assert(Aliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + Aliases.erase(Name); +} + +SymbolFlagsMap +ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) + SymbolFlags[KV.first] = KV.second.AliasFlags; + + return SymbolFlags; +} + +Expected<SymbolAliasMap> +buildSimpleReexportsAliasMap(VSO &SourceV, const SymbolNameSet &Symbols) { + auto Flags = SourceV.lookupFlags(Symbols); + + if (Flags.size() != Symbols.size()) { + SymbolNameSet Unresolved = Symbols; + for (auto &KV : Flags) + Unresolved.erase(KV.first); + return make_error<SymbolsNotFound>(std::move(Unresolved)); + } + + SymbolAliasMap Result; + for (auto &Name : Symbols) { + assert(Flags.count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]); + } + + return Result; +} + +Error VSO::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { + return ES.runSessionLocked([&]() -> Error { + std::vector<SymbolMap::iterator> AddedSyms; + + for (auto &KV : SymbolFlags) { + SymbolMap::iterator EntryItr; + bool Added; + + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Materializing; + + std::tie(EntryItr, Added) = Symbols.insert( + std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + + if (Added) + AddedSyms.push_back(EntryItr); + else { + // Remove any symbols already added. + for (auto &SI : AddedSyms) + Symbols.erase(SI); + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(*KV.first); + } + } + + return Error::success(); + }); +} + +void VSO::replace(std::unique_ptr<MaterializationUnit> MU) { + assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + + auto MustRunMU = + ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> { + +#ifndef NDEBUG + for (auto &KV : MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI != Symbols.end() && "Replacing unknown symbol"); + assert(!SymI->second.getFlags().isLazy() && + SymI->second.getFlags().isMaterializing() && + "Can not replace symbol that is not materializing"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Symbol being replaced should have no UnmaterializedInfo"); + } +#endif // NDEBUG + + // If any symbol has pending queries against it then we need to + // materialize MU immediately. + for (auto &KV : MU->getSymbols()) { + auto MII = MaterializingInfos.find(KV.first); + if (MII != MaterializingInfos.end()) { + if (!MII->second.PendingQueries.empty()) + return std::move(MU); + } + } + + // Otherwise, make MU responsible for all the symbols. + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + for (auto &KV : UMI->MU->getSymbols()) { + assert(!KV.second.isLazy() && + "Lazy flag should be managed internally."); + assert(!KV.second.isMaterializing() && + "Materializing flags should be managed internally."); + + auto SymI = Symbols.find(KV.first); + JITSymbolFlags ReplaceFlags = KV.second; + ReplaceFlags |= JITSymbolFlags::Lazy; + SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(), + std::move(ReplaceFlags)); + UnmaterializedInfos[KV.first] = UMI; + } + + return nullptr; + }); + + if (MustRunMU) + ES.dispatchMaterialization(*this, std::move(MustRunMU)); +} + +SymbolNameSet VSO::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) { + return ES.runSessionLocked([&]() { + SymbolNameSet RequestedSymbols; + + for (auto &KV : SymbolFlags) { + assert(Symbols.count(KV.first) && "VSO does not cover this symbol?"); + assert(Symbols[KV.first].getFlags().isMaterializing() && + "getRequestedSymbols can only be called for materializing " + "symbols"); + auto I = MaterializingInfos.find(KV.first); + if (I == MaterializingInfos.end()) + continue; + + if (!I->second.PendingQueries.empty()) + RequestedSymbols.insert(KV.first); + } + + return RequestedSymbols; + }); +} + +void VSO::addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + assert(Symbols.count(Name) && "Name not in symbol table"); + assert((Symbols[Name].getFlags().isLazy() || + Symbols[Name].getFlags().isMaterializing()) && + "Symbol is not lazy or materializing"); + + auto &MI = MaterializingInfos[Name]; + assert(!MI.IsFinalized && "Can not add dependencies to finalized symbol"); + + for (auto &KV : Dependencies) { + assert(KV.first && "Null VSO in dependency?"); + auto &OtherVSO = *KV.first; + auto &DepsOnOtherVSO = MI.UnfinalizedDependencies[&OtherVSO]; + + for (auto &OtherSymbol : KV.second) { +#ifndef NDEBUG + // Assert that this symbol exists and has not been finalized already. + auto SymI = OtherVSO.Symbols.find(OtherSymbol); + assert(SymI != OtherVSO.Symbols.end() && + (SymI->second.getFlags().isLazy() || + SymI->second.getFlags().isMaterializing()) && + "Dependency on finalized symbol"); +#endif + + auto &OtherMI = OtherVSO.MaterializingInfos[OtherSymbol]; + + if (OtherMI.IsFinalized) + transferFinalizedNodeDependencies(MI, Name, OtherMI); + else if (&OtherVSO != this || OtherSymbol != Name) { + OtherMI.Dependants[this].insert(Name); + DepsOnOtherVSO.insert(OtherSymbol); + } + } + + if (DepsOnOtherVSO.empty()) + MI.UnfinalizedDependencies.erase(&OtherVSO); + } +} + +void VSO::resolve(const SymbolMap &Resolved) { + auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet FullyResolvedQueries; + for (const auto &KV : Resolved) { + auto &Name = KV.first; + auto Sym = KV.second; + + assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() && + "Materializing flags should be managed internally"); + + auto I = Symbols.find(Name); + + assert(I != Symbols.end() && "Symbol not found"); + assert(!I->second.getFlags().isLazy() && + I->second.getFlags().isMaterializing() && + "Symbol should be materializing"); + assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); + + assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == + (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) & + ~JITSymbolFlags::Weak) && + "Resolved flags should match the declared flags"); + + // Once resolved, symbols can never be weak. + JITSymbolFlags ResolvedFlags = Sym.getFlags(); + ResolvedFlags &= ~JITSymbolFlags::Weak; + ResolvedFlags |= JITSymbolFlags::Materializing; + I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags); + + auto &MI = MaterializingInfos[Name]; + for (auto &Q : MI.PendingQueries) { + Q->resolve(Name, Sym); + if (Q->isFullyResolved()) + FullyResolvedQueries.insert(Q); + } + } + + return FullyResolvedQueries; + }); + + for (auto &Q : FullyResolvedQueries) { + assert(Q->isFullyResolved() && "Q not fully resolved"); + Q->handleFullyResolved(); + } +} + +void VSO::finalize(const SymbolFlagsMap &Finalized) { + auto FullyReadyQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet ReadyQueries; + + for (const auto &KV : Finalized) { + const auto &Name = KV.first; + + auto MII = MaterializingInfos.find(Name); + assert(MII != MaterializingInfos.end() && + "Missing MaterializingInfo entry"); + + auto &MI = MII->second; + + // For each dependant, transfer this node's unfinalized dependencies to + // it. If the dependant node is fully finalized then notify any pending + // queries. + for (auto &KV : MI.Dependants) { + auto &DependantVSO = *KV.first; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantVSO.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantVSO.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); + + auto &DependantMI = DependantMII->second; + + // Remove the dependant's dependency on this node. + assert(DependantMI.UnfinalizedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); + DependantMI.UnfinalizedDependencies[this].erase(Name); + if (DependantMI.UnfinalizedDependencies[this].empty()) + DependantMI.UnfinalizedDependencies.erase(this); + + // Transfer unfinalized dependencies from this node to the dependant. + DependantVSO.transferFinalizedNodeDependencies(DependantMI, + DependantName, MI); + + // If the dependant is finalized and this node was the last of its + // unfinalized dependencies then notify any pending queries on the + // dependant node. + if (DependantMI.IsFinalized && + DependantMI.UnfinalizedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); + for (auto &Q : DependantMI.PendingQueries) { + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ReadyQueries.insert(Q); + Q->removeQueryDependence(DependantVSO, DependantName); + } + + // If this dependant node was fully finalized we can erase its + // MaterializingInfo and update its materializing state. + assert(DependantVSO.Symbols.count(DependantName) && + "Dependant has no entry in the Symbols table"); + auto &DependantSym = DependantVSO.Symbols[DependantName]; + DependantSym.setFlags(static_cast<JITSymbolFlags::FlagNames>( + DependantSym.getFlags() & ~JITSymbolFlags::Materializing)); + DependantVSO.MaterializingInfos.erase(DependantMII); + } + } + } + MI.Dependants.clear(); + MI.IsFinalized = true; + + if (MI.UnfinalizedDependencies.empty()) { + for (auto &Q : MI.PendingQueries) { + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ReadyQueries.insert(Q); + Q->removeQueryDependence(*this, Name); + } + assert(Symbols.count(Name) && + "Symbol has no entry in the Symbols table"); + auto &Sym = Symbols[Name]; + Sym.setFlags(static_cast<JITSymbolFlags::FlagNames>( + Sym.getFlags() & ~JITSymbolFlags::Materializing)); + MaterializingInfos.erase(MII); + } + } + + return ReadyQueries; + }); + + for (auto &Q : FullyReadyQueries) { + assert(Q->isFullyReady() && "Q is not fully ready"); + Q->handleFullyReady(); + } +} + +void VSO::notifyFailed(const SymbolNameSet &FailedSymbols) { + + // FIXME: This should fail any transitively dependant symbols too. + + auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet FailedQueries; + + for (auto &Name : FailedSymbols) { + auto I = Symbols.find(Name); + assert(I != Symbols.end() && "Symbol not present in this VSO"); + Symbols.erase(I); + + auto MII = MaterializingInfos.find(Name); + + // If we have not created a MaterializingInfo for this symbol yet then + // there is nobody to notify. + if (MII == MaterializingInfos.end()) + continue; + + // Copy all the queries to the FailedQueries list, then abandon them. + // This has to be a copy, and the copy has to come before the abandon + // operation: Each Q.detach() call will reach back into this + // PendingQueries list to remove Q. + for (auto &Q : MII->second.PendingQueries) + FailedQueries.insert(Q); + + for (auto &Q : FailedQueries) + Q->detach(); + + assert(MII->second.PendingQueries.empty() && + "Queries remain after symbol was failed"); + + MaterializingInfos.erase(MII); + } + + return FailedQueries; + }); + + for (auto &Q : FailedQueriesToNotify) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +void VSO::setSearchOrder(VSOList NewSearchOrder, bool SearchThisVSOFirst) { + if (SearchThisVSOFirst && NewSearchOrder.front() != this) + NewSearchOrder.insert(NewSearchOrder.begin(), this); + + ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); +} + +void VSO::addToSearchOrder(VSO &V) { + ES.runSessionLocked([&]() { SearchOrder.push_back(&V); }); +} + +void VSO::replaceInSearchOrder(VSO &OldV, VSO &NewV) { + ES.runSessionLocked([&]() { + auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &OldV); + + if (I != SearchOrder.end()) + *I = &NewV; + }); +} + +void VSO::removeFromSearchOrder(VSO &V) { + ES.runSessionLocked([&]() { + auto I = std::find(SearchOrder.begin(), SearchOrder.end(), &V); + if (I != SearchOrder.end()) + SearchOrder.erase(I); + }); +} + +SymbolFlagsMap VSO::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() { + SymbolFlagsMap Result; + auto Unresolved = lookupFlagsImpl(Result, Names); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, FallbackDefs); + (void)Unresolved2; + assert(Unresolved2.empty() && + "All fallback defs should have been found by lookupFlagsImpl"); + } + }; + return Result; + }); +} + +SymbolNameSet VSO::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { + SymbolNameSet Unresolved; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + + if (I == Symbols.end()) { + Unresolved.insert(Name); + continue; + } + + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags()); + } + + return Unresolved; +} + +void VSO::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, MaterializationUnitList &MUs) { + assert(Q && "Query can not be null"); + + lodgeQueryImpl(Q, Unresolved, MUs); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + for (auto &D : FallbackDefs) + Unresolved.erase(D); + lodgeQueryImpl(Q, FallbackDefs, MUs); + assert(FallbackDefs.empty() && + "All fallback defs should have been found by lookupImpl"); + } + } +} + +void VSO::lodgeQueryImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs) { + for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { + auto TmpI = I++; + auto Name = *TmpI; + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If we found Name in V, remove it frome the Unresolved set and add it + // to the added set. + Unresolved.erase(TmpI); + + // If the symbol has an address then resolve it. + if (SymI->second.getAddress() != 0) + Q->resolve(Name, SymI->second); + + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.getFlags().isLazy()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + assert(!SymI->second.getFlags().isMaterializing() && + "Materializing and lazy should not both be set"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + auto Flags = SymK->second.getFlags(); + Flags &= ~JITSymbolFlags::Lazy; + Flags |= JITSymbolFlags::Materializing; + SymK->second.setFlags(Flags); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } else if (!SymI->second.getFlags().isMaterializing()) { + // The symbol is neither lazy nor materializing. Finalize it and + // continue. + Q->notifySymbolReady(); + continue; + } + + // Add the query to the PendingQueries list. + assert(SymI->second.getFlags().isMaterializing() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.PendingQueries.push_back(Q); + Q->addQueryDependence(*this, Name); + } +} + +SymbolNameSet VSO::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { + assert(Q && "Query can not be null"); + + ES.runOutstandingMUs(); + + LookupImplActionFlags ActionFlags = None; + std::vector<std::unique_ptr<MaterializationUnit>> MUs; + + SymbolNameSet Unresolved = std::move(Names); + ES.runSessionLocked([&, this]() { + ActionFlags = lookupImpl(Q, MUs, Unresolved); + if (FallbackDefinitionGenerator && !Unresolved.empty()) { + assert(ActionFlags == None && + "ActionFlags set but unresolved symbols remain?"); + auto FallbackDefs = FallbackDefinitionGenerator(*this, Unresolved); + if (!FallbackDefs.empty()) { + for (auto &D : FallbackDefs) + Unresolved.erase(D); + ActionFlags = lookupImpl(Q, MUs, FallbackDefs); + assert(FallbackDefs.empty() && + "All fallback defs should have been found by lookupImpl"); + } + } + }); + + assert((MUs.empty() || ActionFlags == None) && + "If action flags are set, there should be no work to do (so no MUs)"); + + if (ActionFlags & NotifyFullyResolved) + Q->handleFullyResolved(); + + if (ActionFlags & NotifyFullyReady) + Q->handleFullyReady(); + + // FIXME: Swap back to the old code below once RuntimeDyld works with + // callbacks from asynchronous queries. + // Add MUs to the OutstandingMUs list. + { + std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); + for (auto &MU : MUs) + ES.OutstandingMUs.push_back(make_pair(this, std::move(MU))); + } + ES.runOutstandingMUs(); + + // Dispatch any required MaterializationUnits for materialization. + // for (auto &MU : MUs) + // ES.dispatchMaterialization(*this, std::move(MU)); + + return Unresolved; +} + +VSO::LookupImplActionFlags +VSO::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { + LookupImplActionFlags ActionFlags = None; + + for (auto I = Unresolved.begin(), E = Unresolved.end(); I != E;) { + auto TmpI = I++; + auto Name = *TmpI; + + // Search for the name in Symbols. Skip it if not found. + auto SymI = Symbols.find(Name); + if (SymI == Symbols.end()) + continue; + + // If we found Name in V, remove it frome the Unresolved set and add it + // to the dependencies set. + Unresolved.erase(TmpI); + + // If the symbol has an address then resolve it. + if (SymI->second.getAddress() != 0) { + Q->resolve(Name, SymI->second); + if (Q->isFullyResolved()) + ActionFlags |= NotifyFullyResolved; + } + + // If the symbol is lazy, get the MaterialiaztionUnit for it. + if (SymI->second.getFlags().isLazy()) { + assert(SymI->second.getAddress() == 0 && + "Lazy symbol should not have a resolved address"); + assert(!SymI->second.getFlags().isMaterializing() && + "Materializing and lazy should not both be set"); + auto UMII = UnmaterializedInfos.find(Name); + assert(UMII != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + auto MU = std::move(UMII->second->MU); + assert(MU != nullptr && "Materializer should not be null"); + + // Kick all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : MU->getSymbols()) { + auto SymK = Symbols.find(KV.first); + auto Flags = SymK->second.getFlags(); + Flags &= ~JITSymbolFlags::Lazy; + Flags |= JITSymbolFlags::Materializing; + SymK->second.setFlags(Flags); + UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + MUs.push_back(std::move(MU)); + } else if (!SymI->second.getFlags().isMaterializing()) { + // The symbol is neither lazy nor materializing. Finalize it and + // continue. + Q->notifySymbolReady(); + if (Q->isFullyReady()) + ActionFlags |= NotifyFullyReady; + continue; + } + + // Add the query to the PendingQueries list. + assert(SymI->second.getFlags().isMaterializing() && + "By this line the symbol should be materializing"); + auto &MI = MaterializingInfos[Name]; + MI.PendingQueries.push_back(Q); + Q->addQueryDependence(*this, Name); + } + + return ActionFlags; +} + +void VSO::dump(raw_ostream &OS) { + ES.runSessionLocked([&, this]() { + OS << "VSO \"" << VSOName + << "\" (ES: " << format("0x%016x", reinterpret_cast<uintptr_t>(&ES)) + << "):\n" + << "Symbol table:\n"; + + for (auto &KV : Symbols) { + OS << " \"" << *KV.first + << "\": " << format("0x%016x", KV.second.getAddress()); + if (KV.second.getFlags().isLazy() || + KV.second.getFlags().isMaterializing()) { + OS << " ("; + if (KV.second.getFlags().isLazy()) { + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << " Lazy (MU=" << I->second->MU.get() << ")"; + } + if (KV.second.getFlags().isMaterializing()) + OS << " Materializing"; + OS << " )\n"; + } else + OS << "\n"; + } + + if (!MaterializingInfos.empty()) + OS << " MaterializingInfos entries:\n"; + for (auto &KV : MaterializingInfos) { + OS << " \"" << *KV.first << "\":\n" + << " IsFinalized = " << (KV.second.IsFinalized ? "true" : "false") + << "\n" + << " " << KV.second.PendingQueries.size() + << " pending queries: { "; + for (auto &Q : KV.second.PendingQueries) + OS << Q.get() << " "; + OS << "}\n Dependants:\n"; + for (auto &KV2 : KV.second.Dependants) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + OS << " Unfinalized Dependencies:\n"; + for (auto &KV2 : KV.second.UnfinalizedDependencies) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + } + }); +} + +VSO::VSO(ExecutionSessionBase &ES, std::string Name) + : ES(ES), VSOName(std::move(Name)) { + SearchOrder.push_back(this); +} + +Error VSO::defineImpl(MaterializationUnit &MU) { + SymbolNameSet Duplicates; + SymbolNameSet MUDefsOverridden; + + struct ExistingDefOverriddenEntry { + SymbolMap::iterator ExistingDefItr; + JITSymbolFlags NewFlags; + }; + std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden; + + for (auto &KV : MU.getSymbols()) { + assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); + assert(!KV.second.isMaterializing() && + "Materializing flags should be managed internally."); + + SymbolMap::iterator EntryItr; + bool Added; + + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Lazy; + + std::tie(EntryItr, Added) = Symbols.insert( + std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + + if (!Added) { + if (KV.second.isStrong()) { + if (EntryItr->second.getFlags().isStrong() || + (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + Duplicates.insert(KV.first); + else + ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + } else + MUDefsOverridden.insert(KV.first); + } + } + + if (!Duplicates.empty()) { + // We need to remove the symbols we added. + for (auto &KV : MU.getSymbols()) { + if (Duplicates.count(KV.first)) + continue; + + bool Found = false; + for (const auto &EDO : ExistingDefsOverridden) + if (EDO.ExistingDefItr->first == KV.first) + Found = true; + + if (!Found) + Symbols.erase(KV.first); + } + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(**Duplicates.begin()); + } + + // Update flags on existing defs and call discard on their materializers. + for (auto &EDO : ExistingDefsOverridden) { + assert(EDO.ExistingDefItr->second.getFlags().isLazy() && + !EDO.ExistingDefItr->second.getFlags().isMaterializing() && + "Overridden existing def should be in the Lazy state"); + + EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + + auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + assert(UMII != UnmaterializedInfos.end() && + "Overridden existing def should have an UnmaterializedInfo"); + + UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + } + + // Discard overridden symbols povided by MU. + for (auto &Sym : MUDefsOverridden) + MU.doDiscard(*this, Sym); + + return Error::success(); +} + +void VSO::detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols) { + for (auto &QuerySymbol : QuerySymbols) { + assert(MaterializingInfos.count(QuerySymbol) && + "QuerySymbol does not have MaterializingInfo"); + auto &MI = MaterializingInfos[QuerySymbol]; + + auto IdenticalQuery = + [&](const std::shared_ptr<AsynchronousSymbolQuery> &R) { + return R.get() == &Q; + }; + + auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(), + IdenticalQuery); + assert(I != MI.PendingQueries.end() && + "Query Q should be in the PendingQueries list for QuerySymbol"); + MI.PendingQueries.erase(I); + } +} + +void VSO::transferFinalizedNodeDependencies( + MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName, + MaterializingInfo &FinalizedMI) { + for (auto &KV : FinalizedMI.UnfinalizedDependencies) { + auto &DependencyVSO = *KV.first; + SymbolNameSet *UnfinalizedDependenciesOnDependencyVSO = nullptr; + + for (auto &DependencyName : KV.second) { + auto &DependencyMI = DependencyVSO.MaterializingInfos[DependencyName]; + + // Do not add self dependencies. + if (&DependencyMI == &DependantMI) + continue; + + // If we haven't looked up the dependencies for DependencyVSO yet, do it + // now and cache the result. + if (!UnfinalizedDependenciesOnDependencyVSO) + UnfinalizedDependenciesOnDependencyVSO = + &DependantMI.UnfinalizedDependencies[&DependencyVSO]; + + DependencyMI.Dependants[this].insert(DependantName); + UnfinalizedDependenciesOnDependencyVSO->insert(DependencyName); + } + } +} + +VSO &ExecutionSession::createVSO(std::string Name) { + return runSessionLocked([&, this]() -> VSO & { + VSOs.push_back(std::unique_ptr<VSO>(new VSO(*this, std::move(Name)))); + return *VSOs.back(); + }); +} + +Expected<SymbolMap> lookup(const VSOList &VSOs, SymbolNameSet Names) { + + if (VSOs.empty()) + return SymbolMap(); + + auto &ES = (*VSOs.begin())->getExecutionSession(); + + return ES.lookup(VSOs, Names, NoDependenciesToRegister, true); +} + +/// Look up a symbol by searching a list of VSOs. +Expected<JITEvaluatedSymbol> lookup(const VSOList &VSOs, SymbolStringPtr Name) { + SymbolNameSet Names({Name}); + if (auto ResultMap = lookup(VSOs, std::move(Names))) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +MangleAndInterner::MangleAndInterner(ExecutionSessionBase &ES, + const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.getSymbolStringPool().intern(MangledName); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index b7220dba88e9..6157677ce355 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -13,10 +13,51 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" namespace llvm { namespace orc { +JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) + : TT(std::move(TT)) {} + +Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { + return JITTargetMachineBuilder(Triple(sys::getProcessTriple())); +} + +Expected<std::unique_ptr<TargetMachine>> +JITTargetMachineBuilder::createTargetMachine() { + if (!Arch.empty()) { + Triple::ArchType Type = Triple::getArchTypeForLLVMName(Arch); + + if (Type == Triple::UnknownArch) + return make_error<StringError>(std::string("Unknown arch: ") + Arch, + inconvertibleErrorCode()); + } + + std::string ErrMsg; + auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); + if (!TheTarget) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + auto *TM = + TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), + Options, RM, CM, OptLevel, /*JIT*/ true); + if (!TM) + return make_error<StringError>("Could not allocate target machine", + inconvertibleErrorCode()); + + return std::unique_ptr<TargetMachine>(TM); +} + +JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( + const std::vector<std::string> &FeatureVec) { + for (const auto &F : FeatureVec) + Features.AddFeature(F); + return *this; +} + CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) : InitList( GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), @@ -67,7 +108,9 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const { } ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0)); - Value *Data = CS->getOperand(2); + Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr; + if (Data && !isa<GlobalValue>(Data)) + Data = nullptr; return Element(Priority->getZExtValue(), Func, Data); } @@ -83,20 +126,123 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) { CtorDtorIterator(DtorsList, true)); } -void LocalCXXRuntimeOverrides::runDestructors() { +void CtorDtorRunner2::add(iterator_range<CtorDtorIterator> CtorDtors) { + if (CtorDtors.begin() == CtorDtors.end()) + return; + + MangleAndInterner Mangle( + V.getExecutionSession(), + (*CtorDtors.begin()).Func->getParent()->getDataLayout()); + + for (const auto &CtorDtor : CtorDtors) { + assert(CtorDtor.Func && CtorDtor.Func->hasName() && + "Ctor/Dtor function must be named to be runnable under the JIT"); + + if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) { + dbgs() << " Skipping because why now?\n"; + continue; + } + + CtorDtorsByPriority[CtorDtor.Priority].push_back( + Mangle(CtorDtor.Func->getName())); + } +} + +Error CtorDtorRunner2::run() { + using CtorDtorTy = void (*)(); + + SymbolNameSet Names; + + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + auto Added = Names.insert(Name).second; + (void)Added; + assert(Added && "Ctor/Dtor names clashed"); + } + } + + if (auto CtorDtorMap = lookup({&V}, std::move(Names))) { + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + assert(CtorDtorMap->count(Name) && "No entry for Name"); + auto CtorDtor = reinterpret_cast<CtorDtorTy>( + static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + CtorDtor(); + } + } + return Error::success(); + } else + return CtorDtorMap.takeError(); + + CtorDtorsByPriority.clear(); + + return Error::success(); +} + +void LocalCXXRuntimeOverridesBase::runDestructors() { auto& CXXDestructorDataPairs = DSOHandleOverride; for (auto &P : CXXDestructorDataPairs) P.first(P.second); CXXDestructorDataPairs.clear(); } -int LocalCXXRuntimeOverrides::CXAAtExitOverride(DestructorPtr Destructor, - void *Arg, void *DSOHandle) { +int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, + void *Arg, + void *DSOHandle) { auto& CXXDestructorDataPairs = *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle); CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg)); return 0; } +Error LocalCXXRuntimeOverrides2::enable(VSO &V, MangleAndInterner &Mangle) { + SymbolMap RuntimeInterposes( + {{Mangle("__dso_handle"), + JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), + JITSymbolFlags::Exported)}, + {Mangle("__cxa_atexit"), + JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), + JITSymbolFlags::Exported)}}); + + return V.define(absoluteSymbols(std::move(RuntimeInterposes))); +} + +DynamicLibraryFallbackGenerator::DynamicLibraryFallbackGenerator( + sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + : Dylib(std::move(Dylib)), Allow(std::move(Allow)), + GlobalPrefix(DL.getGlobalPrefix()) {} + +SymbolNameSet DynamicLibraryFallbackGenerator:: +operator()(VSO &V, const SymbolNameSet &Names) { + orc::SymbolNameSet Added; + orc::SymbolMap NewSymbols; + + bool HasGlobalPrefix = (GlobalPrefix != '\0'); + + for (auto &Name : Names) { + if (!Allow(Name) || (*Name).empty()) + continue; + + if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) + continue; + + std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { + Added.insert(Name); + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), + JITSymbolFlags::Exported); + } + } + + // Add any new symbols to V. Since the fallback generator is only called for + // symbols that are not already defined, this will never trigger a duplicate + // definition error, so we can wrap this call in a 'cantFail'. + if (!NewSymbols.empty()) + cantFail(V.define(absoluteSymbols(std::move(NewSymbols)))); + + return Added; +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp new file mode 100644 index 000000000000..0c17f9b7ad49 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -0,0 +1,44 @@ +//===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" + +namespace llvm { +namespace orc { + +IRCompileLayer2::IRCompileLayer2(ExecutionSession &ES, ObjectLayer &BaseLayer, + CompileFunction Compile) + : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {} + +void IRCompileLayer2::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + this->NotifyCompiled = std::move(NotifyCompiled); +} + +void IRCompileLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + assert(M && "Module must not be null"); + + if (auto Obj = Compile(*M)) { + { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + if (NotifyCompiled) + NotifyCompiled(K, std::move(M)); + else + M = nullptr; + } + BaseLayer.emit(std::move(R), std::move(K), std::move(*Obj)); + } else { + R.failMaterialization(); + getExecutionSession().reportError(Obj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp new file mode 100644 index 000000000000..4dd3cfdfe387 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -0,0 +1,34 @@ +//===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRTransformLayer2::IRTransformLayer2(ExecutionSession &ES, + IRLayer &BaseLayer, + TransformFunction Transform) + : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void IRTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<Module> M) { + assert(M && "Module must not be null"); + + if (auto TransformedMod = Transform(std::move(M))) + BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedMod)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedMod.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 68397beae63a..9ca2c5cb4a55 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -13,38 +13,123 @@ #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> +using namespace llvm; +using namespace llvm::orc; + +namespace { + +class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { +public: + using CompileFunction = JITCompileCallbackManager::CompileFunction; + + CompileCallbackMaterializationUnit(SymbolStringPtr Name, + CompileFunction Compile) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}})), + Name(std::move(Name)), Compile(std::move(Compile)) {} + +private: + void materialize(MaterializationResponsibility R) { + SymbolMap Result; + Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + R.resolve(Result); + R.finalize(); + } + + void discard(const VSO &V, SymbolStringPtr Name) { + llvm_unreachable("Discard should never occur on a LMU?"); + } + + SymbolStringPtr Name; + CompileFunction Compile; +}; + +} // namespace + namespace llvm { namespace orc { void JITCompileCallbackManager::anchor() {} void IndirectStubsManager::anchor() {} +Expected<JITTargetAddress> +JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { + if (auto TrampolineAddr = getAvailableTrampolineAddr()) { + auto CallbackName = ES.getSymbolStringPool().intern( + std::string("cc") + std::to_string(++NextCallbackId)); + + std::lock_guard<std::mutex> Lock(CCMgrMutex); + AddrToSymbol[*TrampolineAddr] = CallbackName; + cantFail(CallbacksVSO.define( + llvm::make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile)))); + return *TrampolineAddr; + } else + return TrampolineAddr.takeError(); +} + +JITTargetAddress JITCompileCallbackManager::executeCompileCallback( + JITTargetAddress TrampolineAddr) { + SymbolStringPtr Name; + + { + std::unique_lock<std::mutex> Lock(CCMgrMutex); + auto I = AddrToSymbol.find(TrampolineAddr); + + // If this address is not associated with a compile callback then report an + // error to the execution session and return ErrorHandlerAddress to the + // callee. + if (I == AddrToSymbol.end()) { + Lock.unlock(); + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "No compile callback for trampoline at " + << format("0x%016x", TrampolineAddr); + } + ES.reportError( + make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + return ErrorHandlerAddress; + } else + Name = I->second; + } + + if (auto Sym = lookup({&CallbacksVSO}, Name)) + return Sym->getAddress(); + else { + // If anything goes wrong materializing Sym then report it to the session + // and return the ErrorHandlerAddress; + ES.reportError(Sym.takeError()); + return ErrorHandlerAddress; + } +} + std::unique_ptr<JITCompileCallbackManager> -createLocalCompileCallbackManager(const Triple &T, +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { switch (T.getArch()) { default: return nullptr; case Triple::aarch64: { typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86: { typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86_64: { if ( T.getOS() == Triple::OSType::Win32 ) { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } else { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } } @@ -54,7 +139,11 @@ createLocalCompileCallbackManager(const Triple &T, std::function<std::unique_ptr<IndirectStubsManager>()> createLocalIndirectStubsManagerBuilder(const Triple &T) { switch (T.getArch()) { - default: return nullptr; + default: + return [](){ + return llvm::make_unique< + orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); + }; case Triple::aarch64: return [](){ @@ -176,7 +265,6 @@ void makeAllSymbolsExternallyAccessible(Module &M) { Function* cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap) { - assert(F.getParent() != &Dst && "Can't copy decl over existing function."); Function *NewF = Function::Create(cast<FunctionType>(F.getValueType()), F.getLinkage(), F.getName(), &Dst); @@ -214,7 +302,6 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap) { - assert(GV.getParent() != &Dst && "Can't copy decl over existing global var."); GlobalVariable *NewGV = new GlobalVariable( Dst, GV.getValueType(), GV.isConstant(), GV.getLinkage(), nullptr, GV.getName(), nullptr, @@ -236,8 +323,8 @@ void moveGlobalVariableInitializer(GlobalVariable &OrigGV, assert(VMap[&OrigGV] == NewGV && "Incorrect global variable mapping in VMap."); assert(NewGV->getParent() != OrigGV.getParent() && - "moveGlobalVariable should only be used to move initializers between " - "modules"); + "moveGlobalVariableInitializer should only be used to move " + "initializers between modules"); NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, nullptr, Materializer)); diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp new file mode 100644 index 000000000000..52ff4efe56b2 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -0,0 +1,134 @@ +//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/Mangler.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<LLJIT>> +LLJIT::Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL) { + return std::unique_ptr<LLJIT>( + new LLJIT(std::move(ES), std::move(TM), std::move(DL))); +} + +Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { + auto InternedName = ES->getSymbolStringPool().intern(Name); + SymbolMap Symbols({{InternedName, Sym}}); + return Main.define(absoluteSymbols(std::move(Symbols))); +} + +Error LLJIT::addIRModule(VSO &V, std::unique_ptr<Module> M) { + assert(M && "Can not add null module"); + + if (auto Err = applyDataLayout(*M)) + return Err; + + auto K = ES->allocateVModule(); + return CompileLayer.add(V, K, std::move(M)); +} + +Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(VSO &V, + StringRef Name) { + return llvm::orc::lookup({&V}, ES->getSymbolStringPool().intern(Name)); +} + +LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL) + : ES(std::move(ES)), Main(this->ES->createVSO("main")), TM(std::move(TM)), + DL(std::move(DL)), + ObjLinkingLayer(*this->ES, + [this](VModuleKey K) { return getMemoryManager(K); }), + CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)), + CtorRunner(Main), DtorRunner(Main) {} + +std::shared_ptr<RuntimeDyld::MemoryManager> +LLJIT::getMemoryManager(VModuleKey K) { + return llvm::make_unique<SectionMemoryManager>(); +} + +std::string LLJIT::mangle(StringRef UnmangledName) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL); + } + return MangledName; +} + +Error LLJIT::applyDataLayout(Module &M) { + if (M.getDataLayout().isDefault()) + M.setDataLayout(DL); + + if (M.getDataLayout() != DL) + return make_error<StringError>( + "Added modules have incompatible data layouts", + inconvertibleErrorCode()); + + return Error::success(); +} + +void LLJIT::recordCtorDtors(Module &M) { + CtorRunner.add(getConstructors(M)); + DtorRunner.add(getDestructors(M)); +} + +Expected<std::unique_ptr<LLLazyJIT>> +LLLazyJIT::Create(std::unique_ptr<ExecutionSession> ES, + std::unique_ptr<TargetMachine> TM, DataLayout DL, + LLVMContext &Ctx) { + const Triple &TT = TM->getTargetTriple(); + + auto CCMgr = createLocalCompileCallbackManager(TT, *ES, 0); + if (!CCMgr) + return make_error<StringError>( + std::string("No callback manager available for ") + TT.str(), + inconvertibleErrorCode()); + + auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); + if (!ISMBuilder) + return make_error<StringError>( + std::string("No indirect stubs manager builder for ") + TT.str(), + inconvertibleErrorCode()); + + return std::unique_ptr<LLLazyJIT>( + new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL), Ctx, + std::move(CCMgr), std::move(ISMBuilder))); +} + +Error LLLazyJIT::addLazyIRModule(VSO &V, std::unique_ptr<Module> M) { + assert(M && "Can not add null module"); + + if (auto Err = applyDataLayout(*M)) + return Err; + + makeAllSymbolsExternallyAccessible(*M); + + recordCtorDtors(*M); + + auto K = ES->allocateVModule(); + return CODLayer.add(V, K, std::move(M)); +} + +LLLazyJIT::LLLazyJIT( + std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, + DataLayout DL, LLVMContext &Ctx, + std::unique_ptr<JITCompileCallbackManager> CCMgr, + std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) + : LLJIT(std::move(ES), std::move(TM), std::move(DL)), + CCMgr(std::move(CCMgr)), TransformLayer(*this->ES, CompileLayer), + CODLayer(*this->ES, TransformLayer, *this->CCMgr, std::move(ISMBuilder), + [&]() -> LLVMContext & { return Ctx; }) {} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp new file mode 100644 index 000000000000..b9da3b7fb8d5 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -0,0 +1,106 @@ +//===-------------------- Layer.cpp - Layer interfaces --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} +IRLayer::~IRLayer() {} + +Error IRLayer::add(VSO &V, VModuleKey K, std::unique_ptr<Module> M) { + return V.define(llvm::make_unique<BasicIRLayerMaterializationUnit>( + *this, std::move(K), std::move(M))); +} + +IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, + std::unique_ptr<Module> M) + : MaterializationUnit(SymbolFlagsMap()), M(std::move(M)) { + + MangleAndInterner Mangle(ES, this->M->getDataLayout()); + for (auto &G : this->M->global_values()) { + if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && + !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } + } +} + +IRMaterializationUnit::IRMaterializationUnit( + std::unique_ptr<Module> M, SymbolFlagsMap SymbolFlags, + SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags)), M(std::move(M)), + SymbolToDefinition(std::move(SymbolToDefinition)) {} + +void IRMaterializationUnit::discard(const VSO &V, SymbolStringPtr Name) { + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && + "Symbol not provided by this MU, or previously discarded"); + assert(!I->second->isDeclaration() && + "Discard should only apply to definitions"); + I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + SymbolToDefinition.erase(I); +} + +BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( + IRLayer &L, VModuleKey K, std::unique_ptr<Module> M) + : IRMaterializationUnit(L.getExecutionSession(), std::move(M)), + L(L), K(std::move(K)) {} + +void BasicIRLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + L.emit(std::move(R), std::move(K), std::move(M)); +} + +ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} + +ObjectLayer::~ObjectLayer() {} + +Error ObjectLayer::add(VSO &V, VModuleKey K, std::unique_ptr<MemoryBuffer> O) { + return V.define(llvm::make_unique<BasicObjectLayerMaterializationUnit>( + *this, std::move(K), std::move(O))); +} + +BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( + ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O) + : MaterializationUnit(SymbolFlagsMap()), L(L), K(std::move(K)), + O(std::move(O)) { + + auto &ES = L.getExecutionSession(); + auto Obj = cantFail( + object::ObjectFile::createObjectFile(this->O->getMemBufferRef())); + + for (auto &Sym : Obj->symbols()) { + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) && + (Sym.getFlags() & object::BasicSymbolRef::SF_Exported)) { + auto InternedName = + ES.getSymbolStringPool().intern(cantFail(Sym.getName())); + SymbolFlags[InternedName] = JITSymbolFlags::fromObjectSymbol(Sym); + } + } +} + +void BasicObjectLayerMaterializationUnit::materialize( + MaterializationResponsibility R) { + L.emit(std::move(R), std::move(K), std::move(O)); +} + +void BasicObjectLayerMaterializationUnit::discard(const VSO &V, + SymbolStringPtr Name) { + // FIXME: Support object file level discard. This could be done by building a + // filter to pass to the object layer along with the object itself. +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp new file mode 100644 index 000000000000..18be9a042f7f --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp @@ -0,0 +1,68 @@ +//===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Legacy.h" + +namespace llvm { +namespace orc { + +void SymbolResolver::anchor() {} + +JITSymbolResolverAdapter::JITSymbolResolverAdapter( + ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR) + : ES(ES), R(R), MR(MR) {} + +Expected<JITSymbolResolverAdapter::LookupResult> +JITSymbolResolverAdapter::lookup(const LookupSet &Symbols) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + auto LookupFn = [&, this](std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Unresolved) { + return R.lookup(std::move(Q), std::move(Unresolved)); + }; + + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + if (MR) + MR->addDependenciesForAll(Deps); + }; + + auto InternedResult = + ES.legacyLookup(ES, std::move(LookupFn), std::move(InternedSymbols), + false, RegisterDependencies); + + if (!InternedResult) + return InternedResult.takeError(); + + JITSymbolResolver::LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = KV.second; + + return Result; +} + +Expected<JITSymbolResolverAdapter::LookupFlagsResult> +JITSymbolResolverAdapter::lookupFlags(const LookupSet &Symbols) { + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + SymbolFlagsMap SymbolFlags = R.lookupFlags(InternedSymbols); + LookupFlagsResult Result; + for (auto &KV : SymbolFlags) { + ResolvedStrings.insert(KV.first); + Result[*KV.first] = KV.second; + } + + return Result; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp index 8f2d6fd6c32b..3796e3d37bc2 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -14,11 +14,23 @@ namespace llvm { namespace orc { -JITSymbol NullResolver::findSymbol(const std::string &Name) { +SymbolFlagsMap NullResolver::lookupFlags(const SymbolNameSet &Symbols) { + return SymbolFlagsMap(); +} + +SymbolNameSet +NullResolver::lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) { + assert(Symbols.empty() && "Null resolver: Symbols must be empty"); + return Symbols; +} + +JITSymbol NullLegacyResolver::findSymbol(const std::string &Name) { llvm_unreachable("Unexpected cross-object symbol reference"); } -JITSymbol NullResolver::findSymbolInLogicalDylib(const std::string &Name) { +JITSymbol +NullLegacyResolver::findSymbolInLogicalDylib(const std::string &Name) { llvm_unreachable("Unexpected cross-object symbol reference"); } diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp new file mode 100644 index 000000000000..6980c8140fd0 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -0,0 +1,34 @@ +//===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +ObjectTransformLayer2::ObjectTransformLayer2(ExecutionSession &ES, + ObjectLayer &BaseLayer, + TransformFunction Transform) + : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void ObjectTransformLayer2::emit(MaterializationResponsibility R, VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Module must not be null"); + + if (auto TransformedObj = Transform(std::move(O))) + BaseLayer.emit(std::move(R), std::move(K), std::move(*TransformedObj)); + else { + R.failMaterialization(); + getExecutionSession().reportError(TransformedObj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp index f945acaf95ee..d6005d24a648 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -9,28 +9,20 @@ #include "OrcCBindingsStack.h" #include "llvm-c/OrcBindings.h" +#include "llvm/ExecutionEngine/JITEventListener.h" using namespace llvm; -LLVMSharedModuleRef LLVMOrcMakeSharedModule(LLVMModuleRef Mod) { - return wrap(new std::shared_ptr<Module>(unwrap(Mod))); -} - -void LLVMOrcDisposeSharedModuleRef(LLVMSharedModuleRef SharedMod) { - delete unwrap(SharedMod); -} - LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { TargetMachine *TM2(unwrap(TM)); Triple T(TM2->getTargetTriple()); - auto CompileCallbackMgr = orc::createLocalCompileCallbackManager(T, 0); auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(T); - OrcCBindingsStack *JITStack = new OrcCBindingsStack( - *TM2, std::move(CompileCallbackMgr), IndirectStubsMgrBuilder); + OrcCBindingsStack *JITStack = + new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder)); return wrap(JITStack); } @@ -75,24 +67,24 @@ LLVMOrcErrorCode LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, LLVMOrcErrorCode LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMSharedModuleRef Mod, + LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - std::shared_ptr<Module> *M(unwrap(Mod)); - return J.addIRModuleEager(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); + std::unique_ptr<Module> M(unwrap(Mod)); + return J.addIRModuleEager(*RetHandle, std::move(M), SymbolResolver, + SymbolResolverCtx); } LLVMOrcErrorCode LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMSharedModuleRef Mod, + LLVMOrcModuleHandle *RetHandle, LLVMModuleRef Mod, LLVMOrcSymbolResolverFn SymbolResolver, void *SymbolResolverCtx) { OrcCBindingsStack &J = *unwrap(JITStack); - std::shared_ptr<Module> *M(unwrap(Mod)); - return J.addIRModuleLazy(*RetHandle, *M, SymbolResolver, SymbolResolverCtx); + std::unique_ptr<Module> M(unwrap(Mod)); + return J.addIRModuleLazy(*RetHandle, std::move(M), SymbolResolver, + SymbolResolverCtx); } LLVMOrcErrorCode @@ -120,9 +112,27 @@ LLVMOrcErrorCode LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, return J.findSymbolAddress(*RetAddr, SymbolName, true); } +LLVMOrcErrorCode LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, + LLVMOrcTargetAddress *RetAddr, + LLVMOrcModuleHandle H, + const char *SymbolName) { + OrcCBindingsStack &J = *unwrap(JITStack); + return J.findSymbolAddressIn(*RetAddr, H, SymbolName, true); +} + LLVMOrcErrorCode LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { auto *J = unwrap(JITStack); auto Err = J->shutdown(); delete J; return Err; } + +void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->RegisterJITEventListener(unwrap(L)); +} + +void LLVMOrcUnregisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) +{ + unwrap(JITStack)->UnregisterJITEventListener(unwrap(L)); +} diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 405970e063d8..b9f8a370d2f0 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -15,6 +15,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" @@ -33,6 +34,7 @@ #include <algorithm> #include <cstdint> #include <functional> +#include <map> #include <memory> #include <set> #include <string> @@ -42,68 +44,61 @@ namespace llvm { class OrcCBindingsStack; -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(std::shared_ptr<Module>, - LLVMSharedModuleRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) namespace detail { +// FIXME: Kill this off once the Layer concept becomes an interface. +class GenericLayer { +public: + virtual ~GenericLayer() = default; - class GenericHandle { - public: - virtual ~GenericHandle() = default; - - virtual JITSymbol findSymbolIn(const std::string &Name, - bool ExportedSymbolsOnly) = 0; - virtual Error removeModule() = 0; + virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, + bool ExportedSymbolsOnly) = 0; + virtual Error removeModule(orc::VModuleKey K) = 0; }; - template <typename LayerT> class GenericHandleImpl : public GenericHandle { + template <typename LayerT> class GenericLayerImpl : public GenericLayer { public: - GenericHandleImpl(LayerT &Layer, typename LayerT::ModuleHandleT Handle) - : Layer(Layer), Handle(std::move(Handle)) {} + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - JITSymbol findSymbolIn(const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - Error removeModule() override { return Layer.removeModule(Handle); } + Error removeModule(orc::VModuleKey K) override { + return Layer.removeModule(K); + } private: LayerT &Layer; - typename LayerT::ModuleHandleT Handle; }; template <> - class GenericHandleImpl<orc::RTDyldObjectLinkingLayer> - : public GenericHandle { + class GenericLayerImpl<orc::RTDyldObjectLinkingLayer> : public GenericLayer { private: using LayerT = orc::RTDyldObjectLinkingLayer; public: + GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - GenericHandleImpl(LayerT &Layer, typename LayerT::ObjHandleT Handle) - : Layer(Layer), Handle(std::move(Handle)) {} - - JITSymbol findSymbolIn(const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(Handle, Name, ExportedSymbolsOnly); + return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); } - Error removeModule() override { return Layer.removeObject(Handle); } + Error removeModule(orc::VModuleKey K) override { + return Layer.removeObject(K); + } private: LayerT &Layer; - typename LayerT::ObjHandleT Handle; }; - - template <typename LayerT, typename HandleT> - std::unique_ptr<GenericHandleImpl<LayerT>> - createGenericHandle(LayerT &Layer, HandleT Handle) { - return llvm::make_unique<GenericHandleImpl<LayerT>>(Layer, - std::move(Handle)); + template <typename LayerT> + std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { + return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer); } } // end namespace detail @@ -126,20 +121,123 @@ private: using OwningObject = object::OwningBinary<object::ObjectFile>; -public: - using ModuleHandleT = unsigned; + class CBindingsResolver : public orc::SymbolResolver { + public: + CBindingsResolver(OrcCBindingsStack &Stack, + LLVMOrcSymbolResolverFn ExternalResolver, + void *ExternalResolverCtx) + : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), + ExternalResolverCtx(std::move(ExternalResolverCtx)) {} + + orc::SymbolFlagsMap + lookupFlags(const orc::SymbolNameSet &Symbols) override { + orc::SymbolFlagsMap SymbolFlags; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) + SymbolFlags[S] = Sym.getFlags(); + else if (auto Err = Sym.takeError()) { + Stack.reportError(std::move(Err)); + return orc::SymbolFlagsMap(); + } + } + + return SymbolFlags; + } + + orc::SymbolNameSet + lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query, + orc::SymbolNameSet Symbols) override { + orc::SymbolNameSet UnresolvedSymbols; + + for (auto &S : Symbols) { + if (auto Sym = findSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + Query->notifySymbolReady(); + } else { + Stack.ES.legacyFailQuery(*Query, Addr.takeError()); + return orc::SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + Stack.ES.legacyFailQuery(*Query, std::move(Err)); + return orc::SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + + if (Query->isFullyResolved()) + Query->handleFullyResolved(); + + if (Query->isFullyReady()) + Query->handleFullyReady(); + + return UnresolvedSymbols; + } + + private: + JITSymbol findSymbol(const std::string &Name) { + // Search order: + // 1. JIT'd symbols. + // 2. Runtime overrides. + // 3. External resolver (if present). + + if (auto Sym = Stack.CODLayer.findSymbol(Name, true)) + return Sym; + else if (auto Err = Sym.takeError()) + return Sym.takeError(); + if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) + return Sym; + + if (ExternalResolver) + return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx), + JITSymbolFlags::Exported); + + return JITSymbol(nullptr); + } + + OrcCBindingsStack &Stack; + LLVMOrcSymbolResolverFn ExternalResolver; + void *ExternalResolverCtx = nullptr; + }; + +public: OrcCBindingsStack(TargetMachine &TM, - std::unique_ptr<CompileCallbackMgr> CCMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), - CCMgr(std::move(CCMgr)), - ObjectLayer( - []() { - return std::make_shared<SectionMemoryManager>(); - }), + : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)), + DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), + ObjectLayer(ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), - CODLayer(CompileLayer, + CODLayer(ES, CompileLayer, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + return ResolverI->second; + }, + [this](orc::VModuleKey K, + std::shared_ptr<orc::SymbolResolver> Resolver) { + assert(!Resolvers.count(K) && "Resolver already present"); + Resolvers[K] = std::move(Resolver); + }, [](Function &F) { return std::set<Function *>({&F}); }, *this->CCMgr, std::move(IndirectStubsMgrBuilder), false), CXXRuntimeOverrides( @@ -174,15 +272,15 @@ public: createLazyCompileCallback(JITTargetAddress &RetAddr, LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { - if (auto CCInfoOrErr = CCMgr->getCompileCallback()) { - auto &CCInfo = *CCInfoOrErr; - CCInfo.setCompileAction([=]() -> JITTargetAddress { - return Callback(wrap(this), CallbackCtx); - }); - RetAddr = CCInfo.getAddress(); + auto WrappedCallback = [=]() -> JITTargetAddress { + return Callback(wrap(this), CallbackCtx); + }; + + if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) { + RetAddr = *CCAddr; return LLVMOrcErrSuccess; } else - return mapError(CCInfoOrErr.takeError()); + return mapError(CCAddr.takeError()); } LLVMOrcErrorCode createIndirectStub(StringRef StubName, @@ -195,42 +293,9 @@ public: JITTargetAddress Addr) { return mapError(IndirectStubsMgr->updatePointer(Name, Addr)); } - - std::shared_ptr<JITSymbolResolver> - createResolver(LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return orc::createLambdaResolver( - [this, ExternalResolver, ExternalResolverCtx](const std::string &Name) - -> JITSymbol { - // Search order: - // 1. JIT'd symbols. - // 2. Runtime overrides. - // 3. External resolver (if present). - - if (auto Sym = CODLayer.findSymbol(Name, true)) - return Sym; - else if (auto Err = Sym.takeError()) - return Sym.takeError(); - - if (auto Sym = CXXRuntimeOverrides.searchOverrides(Name)) - return Sym; - - if (ExternalResolver) - return JITSymbol( - ExternalResolver(Name.c_str(), ExternalResolverCtx), - JITSymbolFlags::Exported); - - return JITSymbol(nullptr); - }, - [](const std::string &Name) -> JITSymbol { - return JITSymbol(nullptr); - }); - } - template <typename LayerT> LLVMOrcErrorCode - addIRModule(ModuleHandleT &RetHandle, LayerT &Layer, - std::shared_ptr<Module> M, + addIRModule(orc::VModuleKey &RetKey, LayerT &Layer, std::unique_ptr<Module> M, std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { @@ -247,79 +312,73 @@ public: for (auto Dtor : orc::getDestructors(*M)) DtorNames.push_back(mangle(Dtor.Func->getName())); - // Create the resolver. - auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); - // Add the module to the JIT. - ModuleHandleT H; - if (auto LHOrErr = Layer.addModule(std::move(M), std::move(Resolver))) - H = createHandle(Layer, *LHOrErr); - else - return mapError(LHOrErr.takeError()); + RetKey = ES.allocateVModule(); + Resolvers[RetKey] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); + if (auto Err = Layer.addModule(RetKey, std::move(M))) + return mapError(std::move(Err)); + + KeyLayers[RetKey] = detail::createGenericLayer(Layer); // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), H); + orc::CtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), + RetKey); if (auto Err = CtorRunner.runViaLayer(*this)) return mapError(std::move(Err)); - IRStaticDestructorRunners.emplace_back(std::move(DtorNames), H); + IRStaticDestructorRunners.emplace_back(std::move(DtorNames), RetKey); - RetHandle = H; return LLVMOrcErrSuccess; } - LLVMOrcErrorCode addIRModuleEager(ModuleHandleT &RetHandle, - std::shared_ptr<Module> M, + LLVMOrcErrorCode addIRModuleEager(orc::VModuleKey &RetKey, + std::unique_ptr<Module> M, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - return addIRModule(RetHandle, CompileLayer, std::move(M), + return addIRModule(RetKey, CompileLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode addIRModuleLazy(ModuleHandleT &RetHandle, - std::shared_ptr<Module> M, + LLVMOrcErrorCode addIRModuleLazy(orc::VModuleKey &RetKey, + std::unique_ptr<Module> M, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - return addIRModule(RetHandle, CODLayer, std::move(M), + return addIRModule(RetKey, CODLayer, std::move(M), llvm::make_unique<SectionMemoryManager>(), std::move(ExternalResolver), ExternalResolverCtx); } - LLVMOrcErrorCode removeModule(ModuleHandleT H) { - if (auto Err = GenericHandles[H]->removeModule()) + LLVMOrcErrorCode removeModule(orc::VModuleKey K) { + // FIXME: Should error release the module key? + if (auto Err = KeyLayers[K]->removeModule(K)) return mapError(std::move(Err)); - GenericHandles[H] = nullptr; - FreeHandleIndexes.push_back(H); + ES.releaseVModule(K); + KeyLayers.erase(K); return LLVMOrcErrSuccess; } - LLVMOrcErrorCode addObject(ModuleHandleT &RetHandle, + LLVMOrcErrorCode addObject(orc::VModuleKey &RetKey, std::unique_ptr<MemoryBuffer> ObjBuffer, LLVMOrcSymbolResolverFn ExternalResolver, void *ExternalResolverCtx) { - if (auto ObjOrErr = - object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef())) { - auto &Obj = *ObjOrErr; - auto OwningObj = - std::make_shared<OwningObject>(std::move(Obj), std::move(ObjBuffer)); + if (auto Obj = object::ObjectFile::createObjectFile( + ObjBuffer->getMemBufferRef())) { - // Create the resolver. - auto Resolver = createResolver(ExternalResolver, ExternalResolverCtx); + RetKey = ES.allocateVModule(); + Resolvers[RetKey] = std::make_shared<CBindingsResolver>( + *this, ExternalResolver, ExternalResolverCtx); - ModuleHandleT H; - if (auto HOrErr = ObjectLayer.addObject(std::move(OwningObj), - std::move(Resolver))) - H = createHandle(ObjectLayer, *HOrErr); - else - return mapError(HOrErr.takeError()); + if (auto Err = ObjectLayer.addObject(RetKey, std::move(ObjBuffer))) + return mapError(std::move(Err)); - RetHandle = H; + KeyLayers[RetKey] = detail::createGenericLayer(ObjectLayer); return LLVMOrcErrSuccess; } else - return mapError(ObjOrErr.takeError()); + return mapError(Obj.takeError()); } JITSymbol findSymbol(const std::string &Name, @@ -329,9 +388,10 @@ public: return CODLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); } - JITSymbol findSymbolIn(ModuleHandleT H, const std::string &Name, + JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, bool ExportedSymbolsOnly) { - return GenericHandles[H]->findSymbolIn(Name, ExportedSymbolsOnly); + assert(KeyLayers.count(K) && "looking up symbol in unknown module"); + return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); } LLVMOrcErrorCode findSymbolAddress(JITTargetAddress &RetAddr, @@ -354,26 +414,48 @@ public: return LLVMOrcErrSuccess; } + LLVMOrcErrorCode findSymbolAddressIn(JITTargetAddress &RetAddr, + orc::VModuleKey K, + const std::string &Name, + bool ExportedSymbolsOnly) { + RetAddr = 0; + if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { + // Successful lookup, non-null symbol: + if (auto AddrOrErr = Sym.getAddress()) { + RetAddr = *AddrOrErr; + return LLVMOrcErrSuccess; + } else + return mapError(AddrOrErr.takeError()); + } else if (auto Err = Sym.takeError()) { + // Lookup failure - report error. + return mapError(std::move(Err)); + } + // Otherwise we had a successful lookup but got a null result. We already + // set RetAddr to '0' above, so just return success. + return LLVMOrcErrSuccess; + } + const std::string &getErrorMessage() const { return ErrMsg; } -private: - template <typename LayerT, typename HandleT> - unsigned createHandle(LayerT &Layer, HandleT Handle) { - unsigned NewHandle; - if (!FreeHandleIndexes.empty()) { - NewHandle = FreeHandleIndexes.back(); - FreeHandleIndexes.pop_back(); - GenericHandles[NewHandle] = - detail::createGenericHandle(Layer, std::move(Handle)); - return NewHandle; - } else { - NewHandle = GenericHandles.size(); - GenericHandles.push_back( - detail::createGenericHandle(Layer, std::move(Handle))); + void RegisterJITEventListener(JITEventListener *L) { + if (!L) + return; + EventListeners.push_back(L); + } + + void UnregisterJITEventListener(JITEventListener *L) { + if (!L) + return; + + auto I = find(reverse(EventListeners), L); + if (I != EventListeners.rend()) { + std::swap(*I, EventListeners.back()); + EventListeners.pop_back(); } - return NewHandle; } +private: + LLVMOrcErrorCode mapError(Error Err) { LLVMOrcErrorCode Result = LLVMOrcErrSuccess; handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { @@ -386,22 +468,44 @@ private: return Result; } + void reportError(Error Err) { + // FIXME: Report errors on the execution session. + logAllUnhandledErrors(std::move(Err), errs(), "ORC error: "); + }; + + void notifyFinalized(orc::VModuleKey K, + const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + for (auto &Listener : EventListeners) + Listener->NotifyObjectEmitted(Obj, LoadedObjInfo); + } + + void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { + for (auto &Listener : EventListeners) + Listener->NotifyFreeingObject(Obj); + } + + orc::ExecutionSession ES; + std::unique_ptr<CompileCallbackMgr> CCMgr; + + std::vector<JITEventListener *> EventListeners; + DataLayout DL; SectionMemoryManager CCMgrMemMgr; std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; - std::unique_ptr<CompileCallbackMgr> CCMgr; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; CODLayerT CODLayer; - std::vector<std::unique_ptr<detail::GenericHandle>> GenericHandles; - std::vector<unsigned> FreeHandleIndexes; + std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; orc::LocalCXXRuntimeOverrides CXXRuntimeOverrides; std::vector<orc::CtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; std::string ErrMsg; + + std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>> Resolvers; }; } // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp index c218cb9a523c..f4102b359a6b 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp @@ -29,6 +29,12 @@ public: std::string message(int condition) const override { switch (static_cast<OrcErrorCode>(condition)) { + case OrcErrorCode::UnknownORCError: + return "Unknown ORC error"; + case OrcErrorCode::DuplicateDefinition: + return "Duplicate symbol definition"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; case OrcErrorCode::RemoteAllocatorDoesNotExist: return "Remote allocator does not exist"; case OrcErrorCode::RemoteAllocatorIdAlreadyInUse: @@ -45,8 +51,6 @@ public: return "Could not negotiate RPC function"; case OrcErrorCode::RPCResponseAbandoned: return "RPC response abandoned"; - case OrcErrorCode::JITSymbolNotFound: - return "JIT symbol not found"; case OrcErrorCode::UnexpectedRPCCall: return "Unexpected RPC call"; case OrcErrorCode::UnexpectedRPCResponse: @@ -67,6 +71,7 @@ static ManagedStatic<OrcErrorCategory> OrcErrCat; namespace llvm { namespace orc { +char DuplicateDefinition::ID = 0; char JITSymbolNotFound::ID = 0; std::error_code orcError(OrcErrorCode ErrCode) { @@ -74,6 +79,22 @@ std::error_code orcError(OrcErrorCode ErrCode) { return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat); } + +DuplicateDefinition::DuplicateDefinition(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code DuplicateDefinition::convertToErrorCode() const { + return orcError(OrcErrorCode::DuplicateDefinition); +} + +void DuplicateDefinition::log(raw_ostream &OS) const { + OS << "Duplicate definition of symbol '" << SymbolName << "'"; +} + +const std::string &DuplicateDefinition::getSymbolName() const { + return SymbolName; +} + JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) : SymbolName(std::move(SymbolName)) {} diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index f89f21adff41..4def579e7097 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -125,8 +125,13 @@ OrcMCJITReplacement::runFunction(Function *F, } void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { - for (auto &M : LocalModules) - ExecutionEngine::runStaticConstructorsDestructors(*M, isDtors); + auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; + + for (auto &KV : CtorDtorsMap) + cantFail(CtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) + .runViaLayer(LazyEmitLayer)); + + CtorDtorsMap.clear(); } } // End namespace orc. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 1dc8d4ac7bc5..abe89ce70af9 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -21,6 +21,7 @@ #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" @@ -54,6 +55,7 @@ class ObjectCache; namespace orc { class OrcMCJITReplacement : public ExecutionEngine { + // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are // expecting - see finalizeMemory. @@ -138,18 +140,75 @@ class OrcMCJITReplacement : public ExecutionEngine { std::shared_ptr<MCJITMemoryManager> ClientMM; }; - class LinkingResolver : public JITSymbolResolver { + class LinkingORCResolver : public orc::SymbolResolver { public: - LinkingResolver(OrcMCJITReplacement &M) : M(M) {} + LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} + + SymbolFlagsMap lookupFlags(const SymbolNameSet &Symbols) override { + SymbolFlagsMap SymbolFlags; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + SymbolFlags[S] = Sym.getFlags(); + } else if (auto Err = Sym.takeError()) { + M.reportError(std::move(Err)); + return SymbolFlagsMap(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) { + SymbolFlags[S] = Sym2.getFlags(); + } else if (auto Err = Sym2.takeError()) { + M.reportError(std::move(Err)); + return SymbolFlagsMap(); + } + } + } - JITSymbol findSymbol(const std::string &Name) override { - return M.ClientResolver->findSymbol(Name); + return SymbolFlags; } - JITSymbol findSymbolInLogicalDylib(const std::string &Name) override { - if (auto Sym = M.findMangledSymbol(Name)) - return Sym; - return M.ClientResolver->findSymbolInLogicalDylib(Name); + SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, + SymbolNameSet Symbols) override { + SymbolNameSet UnresolvedSymbols; + bool NewSymbolsResolved = false; + + for (auto &S : Symbols) { + if (auto Sym = M.findMangledSymbol(*S)) { + if (auto Addr = Sym.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); + Query->notifySymbolReady(); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else { + if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { + if (auto Addr = Sym2.getAddress()) { + Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); + Query->notifySymbolReady(); + NewSymbolsResolved = true; + } else { + M.ES.legacyFailQuery(*Query, Addr.takeError()); + return SymbolNameSet(); + } + } else if (auto Err = Sym2.takeError()) { + M.ES.legacyFailQuery(*Query, std::move(Err)); + return SymbolNameSet(); + } else + UnresolvedSymbols.insert(S); + } + } + + if (NewSymbolsResolved && Query->isFullyResolved()) + Query->handleFullyResolved(); + + if (NewSymbolsResolved && Query->isFullyReady()) + Query->handleFullyReady(); + + return UnresolvedSymbols; } private: @@ -160,26 +219,37 @@ private: static ExecutionEngine * createOrcMCJITReplacement(std::string *ErrorMsg, std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> Resolver, + std::shared_ptr<LegacyJITSymbolResolver> Resolver, std::unique_ptr<TargetMachine> TM) { return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), std::move(TM)); } + void reportError(Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: "); + } + public: - OrcMCJITReplacement( - std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<JITSymbolResolver> ClientResolver, - std::unique_ptr<TargetMachine> TM) - : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), - MemMgr(std::make_shared<MCJITReplacementMemMgr>(*this, - std::move(MemMgr))), - Resolver(std::make_shared<LinkingResolver>(*this)), + OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, + std::unique_ptr<TargetMachine> TM) + : ExecutionEngine(TM->createDataLayout()), + TM(std::move(TM)), + MemMgr( + std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), + Resolver(std::make_shared<LinkingORCResolver>(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), - ObjectLayer([this]() { return this->MemMgr; }, NotifyObjectLoaded, - NotifyFinalized), - CompileLayer(ObjectLayer, SimpleCompiler(*this->TM)), + ObjectLayer( + ES, + [this](VModuleKey K) { + return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; + }, + NotifyObjectLoaded, NotifyFinalized), + CompileLayer(ObjectLayer, SimpleCompiler(*this->TM), + [this](VModuleKey K, std::unique_ptr<Module> M) { + Modules.push_back(std::move(M)); + }), LazyEmitLayer(CompileLayer) {} static void Register() { @@ -194,43 +264,63 @@ public: } else { assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); } - auto *MPtr = M.release(); - ShouldDelete[MPtr] = true; - auto Deleter = [this](Module *Mod) { - auto I = ShouldDelete.find(Mod); - if (I != ShouldDelete.end() && I->second) - delete Mod; - }; - LocalModules.push_back(std::shared_ptr<Module>(MPtr, std::move(Deleter))); - cantFail(LazyEmitLayer.addModule(LocalModules.back(), Resolver)); + + // Rename, bump linkage and record static constructors and destructors. + // We have to do this before we hand over ownership of the module to the + // JIT. + std::vector<std::string> CtorNames, DtorNames; + { + unsigned CtorId = 0, DtorId = 0; + for (auto Ctor : orc::getConstructors(*M)) { + std::string NewCtorName = ("$static_ctor." + Twine(CtorId++)).str(); + Ctor.Func->setName(NewCtorName); + Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); + Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); + CtorNames.push_back(mangle(NewCtorName)); + } + for (auto Dtor : orc::getDestructors(*M)) { + std::string NewDtorName = ("$static_dtor." + Twine(DtorId++)).str(); + dbgs() << "Found dtor: " << NewDtorName << "\n"; + Dtor.Func->setName(NewDtorName); + Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); + Dtor.Func->setVisibility(GlobalValue::HiddenVisibility); + DtorNames.push_back(mangle(NewDtorName)); + } + } + + auto K = ES.allocateVModule(); + + UnexecutedConstructors[K] = std::move(CtorNames); + UnexecutedDestructors[K] = std::move(DtorNames); + + cantFail(LazyEmitLayer.addModule(K, std::move(M))); } void addObjectFile(std::unique_ptr<object::ObjectFile> O) override { - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O), - nullptr); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData()))); } void addObjectFile(object::OwningBinary<object::ObjectFile> O) override { - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>(std::move(O)); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + std::unique_ptr<object::ObjectFile> Obj; + std::unique_ptr<MemoryBuffer> ObjBuffer; + std::tie(Obj, ObjBuffer) = O.takeBinary(); + cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer))); } void addArchive(object::OwningBinary<object::Archive> A) override { Archives.push_back(std::move(A)); } - + bool removeModule(Module *M) override { - for (auto I = LocalModules.begin(), E = LocalModules.end(); I != E; ++I) { - if (I->get() == M) { - ShouldDelete[M] = false; - LocalModules.erase(I); - return true; - } - } - return false; + auto I = Modules.begin(); + for (auto E = Modules.end(); I != E; ++I) + if (I->get() == M) + break; + if (I == Modules.end()) + return false; + Modules.erase(I); + return true; } uint64_t getSymbolAddress(StringRef Name) { @@ -238,7 +328,7 @@ public: } JITSymbol findSymbol(StringRef Name) { - return findMangledSymbol(Mangle(Name)); + return findMangledSymbol(mangle(Name)); } void finalizeObject() override { @@ -318,12 +408,9 @@ private: } std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); if (ChildBin->isObject()) { - std::unique_ptr<object::ObjectFile> ChildObj( - static_cast<object::ObjectFile*>(ChildBinOrErr->release())); - auto Obj = - std::make_shared<object::OwningBinary<object::ObjectFile>>( - std::move(ChildObj), nullptr); - cantFail(ObjectLayer.addObject(std::move(Obj), Resolver)); + cantFail(ObjectLayer.addObject( + ES.allocateVModule(), + MemoryBuffer::getMemBufferCopy(ChildBin->getData()))); if (auto Sym = ObjectLayer.findSymbol(Name, true)) return Sym; } @@ -339,12 +426,11 @@ private: NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {} - void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H, - const RTDyldObjectLinkingLayer::ObjectPtr &Obj, + void operator()(VModuleKey K, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &Info) const { - M.UnfinalizedSections[H] = std::move(M.SectionsAllocatedSinceLastLoad); + M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad); M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); - M.MemMgr->notifyObjectLoaded(&M, *Obj->getBinary()); + M.MemMgr->notifyObjectLoaded(&M, Obj); } private: OrcMCJITReplacement &M; @@ -354,15 +440,16 @@ private: public: NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {} - void operator()(RTDyldObjectLinkingLayerBase::ObjHandleT H) { - M.UnfinalizedSections.erase(H); + void operator()(VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &Info) { + M.UnfinalizedSections.erase(K); } private: OrcMCJITReplacement &M; }; - std::string Mangle(StringRef Name) { + std::string mangle(StringRef Name) { std::string MangledName; { raw_string_ostream MangledNameStream(MangledName); @@ -375,17 +462,18 @@ private: using CompileLayerT = IRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; + ExecutionSession ES; + std::unique_ptr<TargetMachine> TM; std::shared_ptr<MCJITReplacementMemMgr> MemMgr; - std::shared_ptr<LinkingResolver> Resolver; - std::shared_ptr<JITSymbolResolver> ClientResolver; + std::shared_ptr<LinkingORCResolver> Resolver; + std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; Mangler Mang; // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr // delete blocks in LocalModules refer to the ShouldDelete map, so // LocalModules needs to be destructed before ShouldDelete. std::map<Module*, bool> ShouldDelete; - std::vector<std::shared_ptr<Module>> LocalModules; NotifyObjectLoadedT NotifyObjectLoaded; NotifyFinalizedT NotifyFinalized; @@ -394,19 +482,15 @@ private: CompileLayerT CompileLayer; LazyEmitLayerT LazyEmitLayer; + std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors; + std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors; + // We need to store ObjLayerT::ObjSetHandles for each of the object sets // that have been emitted but not yet finalized so that we can forward the // mapSectionAddress calls appropriately. using SectionAddrSet = std::set<const void *>; - struct ObjHandleCompare { - bool operator()(ObjectLayerT::ObjHandleT H1, - ObjectLayerT::ObjHandleT H2) const { - return &*H1 < &*H2; - } - }; SectionAddrSet SectionsAllocatedSinceLastLoad; - std::map<ObjectLayerT::ObjHandleT, SectionAddrSet, ObjHandleCompare> - UnfinalizedSections; + std::map<VModuleKey, SectionAddrSet> UnfinalizedSections; std::vector<object::OwningBinary<object::Archive>> Archives; }; diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp new file mode 100644 index 000000000000..71b4b73ca6d3 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -0,0 +1,177 @@ +//===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" + +namespace { + +using namespace llvm; +using namespace llvm::orc; + +class VSOSearchOrderResolver : public JITSymbolResolver { +public: + VSOSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} + + Expected<LookupResult> lookup(const LookupSet &Symbols) { + auto &ES = MR.getTargetVSO().getExecutionSession(); + SymbolNameSet InternedSymbols; + + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + MR.addDependenciesForAll(Deps); + }; + + auto InternedResult = + MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { + return ES.lookup(VSOs, InternedSymbols, RegisterDependencies, false); + }); + + if (!InternedResult) + return InternedResult.takeError(); + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + + return Result; + } + + Expected<LookupFlagsResult> lookupFlags(const LookupSet &Symbols) { + auto &ES = MR.getTargetVSO().getExecutionSession(); + + SymbolNameSet InternedSymbols; + + for (auto &S : Symbols) + InternedSymbols.insert(ES.getSymbolStringPool().intern(S)); + + SymbolFlagsMap InternedResult; + MR.getTargetVSO().withSearchOrderDo([&](const VSOList &VSOs) { + // An empty search order is pathalogical, but allowed. + if (VSOs.empty()) + return; + + assert(VSOs.front() && "VSOList entry can not be null"); + InternedResult = VSOs.front()->lookupFlags(InternedSymbols); + }); + + LookupFlagsResult Result; + for (auto &KV : InternedResult) + Result[*KV.first] = std::move(KV.second); + + return Result; + } + +private: + MaterializationResponsibility &MR; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +RTDyldObjectLinkingLayer2::RTDyldObjectLinkingLayer2( + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, + NotifyLoadedFunction NotifyLoaded, NotifyFinalizedFunction NotifyFinalized) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), ProcessAllSections(false) {} + +void RTDyldObjectLinkingLayer2::emit(MaterializationResponsibility R, + VModuleKey K, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + + auto &ES = getExecutionSession(); + + auto ObjFile = object::ObjectFile::createObjectFile(*O); + if (!ObjFile) { + getExecutionSession().reportError(ObjFile.takeError()); + R.failMaterialization(); + } + + auto MemoryManager = GetMemoryManager(K); + + VSOSearchOrderResolver Resolver(R); + auto RTDyld = llvm::make_unique<RuntimeDyld>(*MemoryManager, Resolver); + RTDyld->setProcessAllSections(ProcessAllSections); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + + assert(!ActiveRTDylds.count(K) && + "An active RTDyld already exists for this key?"); + ActiveRTDylds[K] = RTDyld.get(); + + assert(!MemMgrs.count(K) && + "A memory manager already exists for this key?"); + MemMgrs[K] = std::move(MemoryManager); + } + + auto Info = RTDyld->loadObject(**ObjFile); + + { + std::set<StringRef> InternalSymbols; + for (auto &Sym : (*ObjFile)->symbols()) { + if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) { + if (auto SymName = Sym.getName()) + InternalSymbols.insert(*SymName); + else { + ES.reportError(SymName.takeError()); + R.failMaterialization(); + return; + } + } + } + + SymbolMap Symbols; + for (auto &KV : RTDyld->getSymbolTable()) + if (!InternalSymbols.count(KV.first)) + Symbols[ES.getSymbolStringPool().intern(KV.first)] = KV.second; + + R.resolve(Symbols); + } + + if (NotifyLoaded) + NotifyLoaded(K, **ObjFile, *Info); + + RTDyld->finalizeWithMemoryManagerLocking(); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + ActiveRTDylds.erase(K); + } + + if (RTDyld->hasError()) { + ES.reportError(make_error<StringError>(RTDyld->getErrorString(), + inconvertibleErrorCode())); + R.failMaterialization(); + return; + } + + R.finalize(); + + if (NotifyFinalized) + NotifyFinalized(K); +} + +void RTDyldObjectLinkingLayer2::mapSectionAddress( + VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr) const { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto ActiveRTDyldItr = ActiveRTDylds.find(K); + + assert(ActiveRTDyldItr != ActiveRTDylds.end() && + "No active RTDyld instance found for key"); + ActiveRTDyldItr->second->mapSectionAddress(LocalAddress, TargetAddr); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp new file mode 100644 index 000000000000..7bf8120d23df --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -0,0 +1,497 @@ +//===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines a JITEventListener object that tells perf about JITted +// functions, including source line information. +// +// Documentation for perf jit integration is available at: +// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt +// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Twine.h" +#include "llvm/Config/config.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolSize.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errno.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/MutexGuard.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Threading.h" +#include "llvm/Support/raw_ostream.h" + +#include <sys/mman.h> // mmap() +#include <sys/types.h> // getpid() +#include <time.h> // clock_gettime(), time(), localtime_r() */ +#include <unistd.h> // for getpid(), read(), close() + +using namespace llvm; +using namespace llvm::object; +typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind; + +namespace { + +// language identifier (XXX: should we generate something better from debug +// info?) +#define JIT_LANG "llvm-IR" +#define LLVM_PERF_JIT_MAGIC \ + ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \ + (uint32_t)'D') +#define LLVM_PERF_JIT_VERSION 1 + +// bit 0: set if the jitdump file is using an architecture-specific timestamp +// clock source +#define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0) + +struct LLVMPerfJitHeader; + +class PerfJITEventListener : public JITEventListener { +public: + PerfJITEventListener(); + ~PerfJITEventListener() { + if (MarkerAddr) + CloseMarker(); + } + + void NotifyObjectEmitted(const ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) override; + void NotifyFreeingObject(const ObjectFile &Obj) override; + +private: + bool InitDebuggingDir(); + bool OpenMarker(); + void CloseMarker(); + static bool FillMachine(LLVMPerfJitHeader &hdr); + + void NotifyCode(Expected<llvm::StringRef> &Symbol, uint64_t CodeAddr, + uint64_t CodeSize); + void NotifyDebug(uint64_t CodeAddr, DILineInfoTable Lines); + + // cache lookups + pid_t Pid; + + // base directory for output data + std::string JitPath; + + // output data stream, closed via Dumpstream + int DumpFd = -1; + + // output data stream + std::unique_ptr<raw_fd_ostream> Dumpstream; + + // prevent concurrent dumps from messing up the output file + sys::Mutex Mutex; + + // perf mmap marker + void *MarkerAddr = NULL; + + // perf support ready + bool SuccessfullyInitialized = false; + + // identifier for functions, primarily to identify when moving them around + uint64_t CodeGeneration = 1; +}; + +// The following are POD struct definitions from the perf jit specification + +enum LLVMPerfJitRecordType { + JIT_CODE_LOAD = 0, + JIT_CODE_MOVE = 1, // not emitted, code isn't moved + JIT_CODE_DEBUG_INFO = 2, + JIT_CODE_CLOSE = 3, // not emitted, unnecessary + JIT_CODE_UNWINDING_INFO = 4, // not emitted + + JIT_CODE_MAX +}; + +struct LLVMPerfJitHeader { + uint32_t Magic; // characters "JiTD" + uint32_t Version; // header version + uint32_t TotalSize; // total size of header + uint32_t ElfMach; // elf mach target + uint32_t Pad1; // reserved + uint32_t Pid; + uint64_t Timestamp; // timestamp + uint64_t Flags; // flags +}; + +// record prefix (mandatory in each record) +struct LLVMPerfJitRecordPrefix { + uint32_t Id; // record type identifier + uint32_t TotalSize; + uint64_t Timestamp; +}; + +struct LLVMPerfJitRecordCodeLoad { + LLVMPerfJitRecordPrefix Prefix; + + uint32_t Pid; + uint32_t Tid; + uint64_t Vma; + uint64_t CodeAddr; + uint64_t CodeSize; + uint64_t CodeIndex; +}; + +struct LLVMPerfJitDebugEntry { + uint64_t Addr; + int Lineno; // source line number starting at 1 + int Discrim; // column discriminator, 0 is default + // followed by null terminated filename, \xff\0 if same as previous entry +}; + +struct LLVMPerfJitRecordDebugInfo { + LLVMPerfJitRecordPrefix Prefix; + + uint64_t CodeAddr; + uint64_t NrEntry; + // followed by NrEntry LLVMPerfJitDebugEntry records +}; + +static inline uint64_t timespec_to_ns(const struct timespec *ts) { + const uint64_t NanoSecPerSec = 1000000000; + return ((uint64_t)ts->tv_sec * NanoSecPerSec) + ts->tv_nsec; +} + +static inline uint64_t perf_get_timestamp(void) { + struct timespec ts; + int ret; + + ret = clock_gettime(CLOCK_MONOTONIC, &ts); + if (ret) + return 0; + + return timespec_to_ns(&ts); +} + +PerfJITEventListener::PerfJITEventListener() : Pid(::getpid()) { + // check if clock-source is supported + if (!perf_get_timestamp()) { + errs() << "kernel does not support CLOCK_MONOTONIC\n"; + return; + } + + if (!InitDebuggingDir()) { + errs() << "could not initialize debugging directory\n"; + return; + } + + std::string Filename; + raw_string_ostream FilenameBuf(Filename); + FilenameBuf << JitPath << "/jit-" << Pid << ".dump"; + + // Need to open ourselves, because we need to hand the FD to OpenMarker() and + // raw_fd_ostream doesn't expose the FD. + using sys::fs::openFileForWrite; + if (auto EC = + openFileForReadWrite(FilenameBuf.str(), DumpFd, + sys::fs::CD_CreateNew, sys::fs::OF_None)) { + errs() << "could not open JIT dump file " << FilenameBuf.str() << ": " + << EC.message() << "\n"; + return; + } + + Dumpstream = make_unique<raw_fd_ostream>(DumpFd, true); + + LLVMPerfJitHeader Header = {0}; + if (!FillMachine(Header)) + return; + + // signal this process emits JIT information + if (!OpenMarker()) + return; + + // emit dumpstream header + Header.Magic = LLVM_PERF_JIT_MAGIC; + Header.Version = LLVM_PERF_JIT_VERSION; + Header.TotalSize = sizeof(Header); + Header.Pid = Pid; + Header.Timestamp = perf_get_timestamp(); + Dumpstream->write(reinterpret_cast<const char *>(&Header), sizeof(Header)); + + // Everything initialized, can do profiling now. + if (!Dumpstream->has_error()) + SuccessfullyInitialized = true; +} + +void PerfJITEventListener::NotifyObjectEmitted( + const ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &L) { + + if (!SuccessfullyInitialized) + return; + + OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj); + const ObjectFile &DebugObj = *DebugObjOwner.getBinary(); + + // Get the address of the object image for use as a unique identifier + std::unique_ptr<DIContext> Context = DWARFContext::create(DebugObj); + + // Use symbol info to iterate over functions in the object. + for (const std::pair<SymbolRef, uint64_t> &P : computeSymbolSizes(DebugObj)) { + SymbolRef Sym = P.first; + std::string SourceFileName; + + Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType(); + if (!SymTypeOrErr) { + // There's not much we can with errors here + consumeError(SymTypeOrErr.takeError()); + continue; + } + SymbolRef::Type SymType = *SymTypeOrErr; + if (SymType != SymbolRef::ST_Function) + continue; + + Expected<StringRef> Name = Sym.getName(); + if (!Name) { + consumeError(Name.takeError()); + continue; + } + + Expected<uint64_t> AddrOrErr = Sym.getAddress(); + if (!AddrOrErr) { + consumeError(AddrOrErr.takeError()); + continue; + } + uint64_t Addr = *AddrOrErr; + uint64_t Size = P.second; + + // According to spec debugging info has to come before loading the + // corresonding code load. + DILineInfoTable Lines = Context->getLineInfoForAddressRange( + Addr, Size, FileLineInfoKind::AbsoluteFilePath); + + NotifyDebug(Addr, Lines); + NotifyCode(Name, Addr, Size); + } + + Dumpstream->flush(); +} + +void PerfJITEventListener::NotifyFreeingObject(const ObjectFile &Obj) { + // perf currently doesn't have an interface for unloading. But munmap()ing the + // code section does, so that's ok. +} + +bool PerfJITEventListener::InitDebuggingDir() { + time_t Time; + struct tm LocalTime; + char TimeBuffer[sizeof("YYYYMMDD")]; + SmallString<64> Path; + + // search for location to dump data to + if (const char *BaseDir = getenv("JITDUMPDIR")) + Path.append(BaseDir); + else if (!sys::path::home_directory(Path)) + Path = "."; + + // create debug directory + Path += "/.debug/jit/"; + if (auto EC = sys::fs::create_directories(Path)) { + errs() << "could not create jit cache directory " << Path << ": " + << EC.message() << "\n"; + return false; + } + + // create unique directory for dump data related to this process + time(&Time); + localtime_r(&Time, &LocalTime); + strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime); + Path += JIT_LANG "-jit-"; + Path += TimeBuffer; + + SmallString<128> UniqueDebugDir; + + using sys::fs::createUniqueDirectory; + if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) { + errs() << "could not create unique jit cache directory " << UniqueDebugDir + << ": " << EC.message() << "\n"; + return false; + } + + JitPath = UniqueDebugDir.str(); + + return true; +} + +bool PerfJITEventListener::OpenMarker() { + // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap + // is captured either live (perf record running when we mmap) or in deferred + // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump + // file for more meta data info about the jitted code. Perf report/annotate + // detect this special filename and process the jitdump file. + // + // Mapping must be PROT_EXEC to ensure it is captured by perf record + // even when not using -d option. + MarkerAddr = ::mmap(NULL, sys::Process::getPageSize(), PROT_READ | PROT_EXEC, + MAP_PRIVATE, DumpFd, 0); + + if (MarkerAddr == MAP_FAILED) { + errs() << "could not mmap JIT marker\n"; + return false; + } + return true; +} + +void PerfJITEventListener::CloseMarker() { + if (!MarkerAddr) + return; + + munmap(MarkerAddr, sys::Process::getPageSize()); + MarkerAddr = nullptr; +} + +bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader &hdr) { + char id[16]; + struct { + uint16_t e_type; + uint16_t e_machine; + } info; + + size_t RequiredMemory = sizeof(id) + sizeof(info); + + ErrorOr<std::unique_ptr<MemoryBuffer>> MB = + MemoryBuffer::getFileSlice("/proc/self/exe", + RequiredMemory, + 0); + + // This'll not guarantee that enough data was actually read from the + // underlying file. Instead the trailing part of the buffer would be + // zeroed. Given the ELF signature check below that seems ok though, + // it's unlikely that the file ends just after that, and the + // consequence would just be that perf wouldn't recognize the + // signature. + if (auto EC = MB.getError()) { + errs() << "could not open /proc/self/exe: " << EC.message() << "\n"; + return false; + } + + memcpy(&id, (*MB)->getBufferStart(), sizeof(id)); + memcpy(&info, (*MB)->getBufferStart() + sizeof(id), sizeof(info)); + + // check ELF signature + if (id[0] != 0x7f || id[1] != 'E' || id[2] != 'L' || id[3] != 'F') { + errs() << "invalid elf signature\n"; + return false; + } + + hdr.ElfMach = info.e_machine; + + return true; +} + +void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol, + uint64_t CodeAddr, uint64_t CodeSize) { + assert(SuccessfullyInitialized); + + // 0 length functions can't have samples. + if (CodeSize == 0) + return; + + LLVMPerfJitRecordCodeLoad rec; + rec.Prefix.Id = JIT_CODE_LOAD; + rec.Prefix.TotalSize = sizeof(rec) + // debug record itself + Symbol->size() + 1 + // symbol name + CodeSize; // and code + rec.Prefix.Timestamp = perf_get_timestamp(); + + rec.CodeSize = CodeSize; + rec.Vma = 0; + rec.CodeAddr = CodeAddr; + rec.Pid = Pid; + rec.Tid = get_threadid(); + + // avoid interspersing output + MutexGuard Guard(Mutex); + + rec.CodeIndex = CodeGeneration++; // under lock! + + Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec)); + Dumpstream->write(Symbol->data(), Symbol->size() + 1); + Dumpstream->write(reinterpret_cast<const char *>(CodeAddr), CodeSize); +} + +void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr, + DILineInfoTable Lines) { + assert(SuccessfullyInitialized); + + // Didn't get useful debug info. + if (Lines.empty()) + return; + + LLVMPerfJitRecordDebugInfo rec; + rec.Prefix.Id = JIT_CODE_DEBUG_INFO; + rec.Prefix.TotalSize = sizeof(rec); // will be increased further + rec.Prefix.Timestamp = perf_get_timestamp(); + rec.CodeAddr = CodeAddr; + rec.NrEntry = Lines.size(); + + // compute total size size of record (variable due to filenames) + DILineInfoTable::iterator Begin = Lines.begin(); + DILineInfoTable::iterator End = Lines.end(); + for (DILineInfoTable::iterator It = Begin; It != End; ++It) { + DILineInfo &line = It->second; + rec.Prefix.TotalSize += sizeof(LLVMPerfJitDebugEntry); + rec.Prefix.TotalSize += line.FileName.size() + 1; + } + + // The debug_entry describes the source line information. It is defined as + // follows in order: + // * uint64_t code_addr: address of function for which the debug information + // is generated + // * uint32_t line : source file line number (starting at 1) + // * uint32_t discrim : column discriminator, 0 is default + // * char name[n] : source file name in ASCII, including null termination + + // avoid interspersing output + MutexGuard Guard(Mutex); + + Dumpstream->write(reinterpret_cast<const char *>(&rec), sizeof(rec)); + + for (DILineInfoTable::iterator It = Begin; It != End; ++It) { + LLVMPerfJitDebugEntry LineInfo; + DILineInfo &Line = It->second; + + LineInfo.Addr = It->first; + // The function re-created by perf is preceded by a elf + // header. Need to adjust for that, otherwise the results are + // wrong. + LineInfo.Addr += 0x40; + LineInfo.Lineno = Line.Line; + LineInfo.Discrim = Line.Discriminator; + + Dumpstream->write(reinterpret_cast<const char *>(&LineInfo), + sizeof(LineInfo)); + Dumpstream->write(Line.FileName.c_str(), Line.FileName.size() + 1); + } +} + +// There should be only a single event listener per process, otherwise perf gets +// confused. +llvm::ManagedStatic<PerfJITEventListener> PerfListener; + +} // end anonymous namespace + +namespace llvm { +JITEventListener *JITEventListener::createPerfJITEventListener() { + return &*PerfListener; +} + +} // namespace llvm + +LLVMJITEventListenerRef LLVMCreatePerfJITEventListener(void) +{ + return wrap(JITEventListener::createPerfJITEventListener()); +} diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp index 87059ef2b88f..18eb0e461921 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp @@ -47,3 +47,53 @@ ARMJITSymbolFlags llvm::ARMJITSymbolFlags::fromObjectSymbol( Flags |= ARMJITSymbolFlags::Thumb; return Flags; } + +/// Performs lookup by, for each symbol, first calling +/// findSymbolInLogicalDylib and if that fails calling +/// findSymbol. +Expected<JITSymbolResolver::LookupResult> +LegacyJITSymbolResolver::lookup(const LookupSet &Symbols) { + JITSymbolResolver::LookupResult Result; + for (auto &Symbol : Symbols) { + std::string SymName = Symbol.str(); + if (auto Sym = findSymbolInLogicalDylib(SymName)) { + if (auto AddrOrErr = Sym.getAddress()) + Result[Symbol] = JITEvaluatedSymbol(*AddrOrErr, Sym.getFlags()); + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) + return std::move(Err); + else { + // findSymbolInLogicalDylib failed. Lets try findSymbol. + if (auto Sym = findSymbol(SymName)) { + if (auto AddrOrErr = Sym.getAddress()) + Result[Symbol] = JITEvaluatedSymbol(*AddrOrErr, Sym.getFlags()); + else + return AddrOrErr.takeError(); + } else if (auto Err = Sym.takeError()) + return std::move(Err); + else + return make_error<StringError>("Symbol not found: " + Symbol, + inconvertibleErrorCode()); + } + } + + return std::move(Result); +} + +/// Performs flags lookup by calling findSymbolInLogicalDylib and +/// returning the flags value for that symbol. +Expected<JITSymbolResolver::LookupFlagsResult> +LegacyJITSymbolResolver::lookupFlags(const LookupSet &Symbols) { + JITSymbolResolver::LookupFlagsResult Result; + + for (auto &Symbol : Symbols) { + std::string SymName = Symbol.str(); + if (auto Sym = findSymbolInLogicalDylib(SymName)) + Result[Symbol] = Sym.getFlags(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + } + + return std::move(Result); +} diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp index 99e84b7496d4..e774af05ebdd 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -298,4 +298,6 @@ void *RTDyldMemoryManager::getPointerToNamedFunction(const std::string &Name, return (void*)Addr; } +void RTDyldMemoryManager::anchor() {} +void MCJITMemoryManager::anchor() {} } // namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index c5e4dfa1e536..1189be599edd 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -122,10 +122,8 @@ void RuntimeDyldImpl::resolveRelocations() { MutexGuard locked(lock); // Print out the sections prior to relocation. - DEBUG( - for (int i = 0, e = Sections.size(); i != e; ++i) - dumpSectionMemory(Sections[i], "before relocations"); - ); + LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i) + dumpSectionMemory(Sections[i], "before relocations");); // First, resolve relocations associated with external symbols. if (auto Err = resolveExternalSymbols()) { @@ -140,18 +138,15 @@ void RuntimeDyldImpl::resolveRelocations() { // entry provides the section to which the relocation will be applied. int Idx = it->first; uint64_t Addr = Sections[Idx].getLoadAddress(); - DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t" - << format("%p", (uintptr_t)Addr) << "\n"); + LLVM_DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t" + << format("%p", (uintptr_t)Addr) << "\n"); resolveRelocationList(it->second, Addr); } Relocations.clear(); // Print out sections after relocation. - DEBUG( - for (int i = 0, e = Sections.size(); i != e; ++i) - dumpSectionMemory(Sections[i], "after relocations"); - ); - + LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i) + dumpSectionMemory(Sections[i], "after relocations");); } void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress, @@ -202,10 +197,35 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { ObjSectionToIDMap LocalSections; // Common symbols requiring allocation, with their sizes and alignments - CommonSymbolList CommonSymbols; + CommonSymbolList CommonSymbolsToAllocate; + + uint64_t CommonSize = 0; + uint32_t CommonAlign = 0; + + // First, collect all weak and common symbols. We need to know if stronger + // definitions occur elsewhere. + JITSymbolResolver::LookupFlagsResult SymbolFlags; + { + JITSymbolResolver::LookupSet Symbols; + for (auto &Sym : Obj.symbols()) { + uint32_t Flags = Sym.getFlags(); + if ((Flags & SymbolRef::SF_Common) || (Flags & SymbolRef::SF_Weak)) { + // Get symbol name. + if (auto NameOrErr = Sym.getName()) + Symbols.insert(*NameOrErr); + else + return NameOrErr.takeError(); + } + } + + if (auto FlagsResultOrErr = Resolver.lookupFlags(Symbols)) + SymbolFlags = std::move(*FlagsResultOrErr); + else + return FlagsResultOrErr.takeError(); + } // Parse symbols - DEBUG(dbgs() << "Parse symbols:\n"); + LLVM_DEBUG(dbgs() << "Parse symbols:\n"); for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; ++I) { uint32_t Flags = I->getFlags(); @@ -214,106 +234,112 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { if (Flags & SymbolRef::SF_Undefined) continue; - if (Flags & SymbolRef::SF_Common) - CommonSymbols.push_back(*I); - else { + // Get the symbol type. + object::SymbolRef::Type SymType; + if (auto SymTypeOrErr = I->getType()) + SymType = *SymTypeOrErr; + else + return SymTypeOrErr.takeError(); - // Get the symbol type. - object::SymbolRef::Type SymType; - if (auto SymTypeOrErr = I->getType()) - SymType = *SymTypeOrErr; - else - return SymTypeOrErr.takeError(); + // Get symbol name. + StringRef Name; + if (auto NameOrErr = I->getName()) + Name = *NameOrErr; + else + return NameOrErr.takeError(); - // Get symbol name. - StringRef Name; - if (auto NameOrErr = I->getName()) - Name = *NameOrErr; - else - return NameOrErr.takeError(); - - // Compute JIT symbol flags. - JITSymbolFlags JITSymFlags = getJITSymbolFlags(*I); - - // If this is a weak definition, check to see if there's a strong one. - // If there is, skip this symbol (we won't be providing it: the strong - // definition will). If there's no strong definition, make this definition - // strong. - if (JITSymFlags.isWeak()) { - // First check whether there's already a definition in this instance. - // FIXME: Override existing weak definitions with strong ones. - if (GlobalSymbolTable.count(Name)) - continue; - // Then check the symbol resolver to see if there's a definition - // elsewhere in this logical dylib. - if (auto Sym = Resolver.findSymbolInLogicalDylib(Name)) { - if (Sym.getFlags().isStrongDefinition()) - continue; - } else if (auto Err = Sym.takeError()) - return std::move(Err); - // else - JITSymFlags &= ~JITSymbolFlags::Weak; - } + // Compute JIT symbol flags. + JITSymbolFlags JITSymFlags = getJITSymbolFlags(*I); + + // If this is a weak definition, check to see if there's a strong one. + // If there is, skip this symbol (we won't be providing it: the strong + // definition will). If there's no strong definition, make this definition + // strong. + if (JITSymFlags.isWeak() || JITSymFlags.isCommon()) { + // First check whether there's already a definition in this instance. + // FIXME: Override existing weak definitions with strong ones. + if (GlobalSymbolTable.count(Name)) + continue; - if (Flags & SymbolRef::SF_Absolute && - SymType != object::SymbolRef::ST_File) { - uint64_t Addr = 0; - if (auto AddrOrErr = I->getAddress()) - Addr = *AddrOrErr; - else - return AddrOrErr.takeError(); - - unsigned SectionID = AbsoluteSymbolSection; - - DEBUG(dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name - << " SID: " << SectionID << " Offset: " - << format("%p", (uintptr_t)Addr) - << " flags: " << Flags << "\n"); - GlobalSymbolTable[Name] = - SymbolTableEntry(SectionID, Addr, JITSymFlags); - } else if (SymType == object::SymbolRef::ST_Function || - SymType == object::SymbolRef::ST_Data || - SymType == object::SymbolRef::ST_Unknown || - SymType == object::SymbolRef::ST_Other) { - - section_iterator SI = Obj.section_end(); - if (auto SIOrErr = I->getSection()) - SI = *SIOrErr; - else - return SIOrErr.takeError(); + // Then check whether we found flags for an existing symbol during the + // flags lookup earlier. + auto FlagsI = SymbolFlags.find(Name); + if (FlagsI == SymbolFlags.end() || + (JITSymFlags.isWeak() && !FlagsI->second.isStrong()) || + (JITSymFlags.isCommon() && FlagsI->second.isCommon())) { + if (JITSymFlags.isWeak()) + JITSymFlags &= ~JITSymbolFlags::Weak; + if (JITSymFlags.isCommon()) { + JITSymFlags &= ~JITSymbolFlags::Common; + uint32_t Align = I->getAlignment(); + uint64_t Size = I->getCommonSize(); + if (!CommonAlign) + CommonAlign = Align; + CommonSize += alignTo(CommonSize, Align) + Size; + CommonSymbolsToAllocate.push_back(*I); + } + } else + continue; + } - if (SI == Obj.section_end()) - continue; + if (Flags & SymbolRef::SF_Absolute && + SymType != object::SymbolRef::ST_File) { + uint64_t Addr = 0; + if (auto AddrOrErr = I->getAddress()) + Addr = *AddrOrErr; + else + return AddrOrErr.takeError(); + + unsigned SectionID = AbsoluteSymbolSection; + + LLVM_DEBUG(dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name + << " SID: " << SectionID + << " Offset: " << format("%p", (uintptr_t)Addr) + << " flags: " << Flags << "\n"); + GlobalSymbolTable[Name] = SymbolTableEntry(SectionID, Addr, JITSymFlags); + } else if (SymType == object::SymbolRef::ST_Function || + SymType == object::SymbolRef::ST_Data || + SymType == object::SymbolRef::ST_Unknown || + SymType == object::SymbolRef::ST_Other) { + + section_iterator SI = Obj.section_end(); + if (auto SIOrErr = I->getSection()) + SI = *SIOrErr; + else + return SIOrErr.takeError(); - // Get symbol offset. - uint64_t SectOffset; - if (auto Err = getOffset(*I, *SI, SectOffset)) - return std::move(Err); + if (SI == Obj.section_end()) + continue; - bool IsCode = SI->isText(); - unsigned SectionID; - if (auto SectionIDOrErr = findOrEmitSection(Obj, *SI, IsCode, - LocalSections)) - SectionID = *SectionIDOrErr; - else - return SectionIDOrErr.takeError(); + // Get symbol offset. + uint64_t SectOffset; + if (auto Err = getOffset(*I, *SI, SectOffset)) + return std::move(Err); + + bool IsCode = SI->isText(); + unsigned SectionID; + if (auto SectionIDOrErr = + findOrEmitSection(Obj, *SI, IsCode, LocalSections)) + SectionID = *SectionIDOrErr; + else + return SectionIDOrErr.takeError(); - DEBUG(dbgs() << "\tType: " << SymType << " Name: " << Name - << " SID: " << SectionID << " Offset: " - << format("%p", (uintptr_t)SectOffset) - << " flags: " << Flags << "\n"); - GlobalSymbolTable[Name] = + LLVM_DEBUG(dbgs() << "\tType: " << SymType << " Name: " << Name + << " SID: " << SectionID + << " Offset: " << format("%p", (uintptr_t)SectOffset) + << " flags: " << Flags << "\n"); + GlobalSymbolTable[Name] = SymbolTableEntry(SectionID, SectOffset, JITSymFlags); - } } } // Allocate common symbols - if (auto Err = emitCommonSymbols(Obj, CommonSymbols)) + if (auto Err = emitCommonSymbols(Obj, CommonSymbolsToAllocate, CommonSize, + CommonAlign)) return std::move(Err); // Parse and process relocations - DEBUG(dbgs() << "Parse relocations:\n"); + LLVM_DEBUG(dbgs() << "Parse relocations:\n"); for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); SI != SE; ++SI) { StubMap Stubs; @@ -336,7 +362,7 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { else return SectionIDOrErr.takeError(); - DEBUG(dbgs() << "\tSectionID: " << SectionID << "\n"); + LLVM_DEBUG(dbgs() << "\tSectionID: " << SectionID << "\n"); for (; I != E;) if (auto IOrErr = processRelocationRef(SectionID, I, Obj, LocalSections, Stubs)) @@ -621,45 +647,12 @@ JITSymbolFlags RuntimeDyldImpl::getJITSymbolFlags(const BasicSymbolRef &SR) { } Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, - CommonSymbolList &CommonSymbols) { - if (CommonSymbols.empty()) + CommonSymbolList &SymbolsToAllocate, + uint64_t CommonSize, + uint32_t CommonAlign) { + if (SymbolsToAllocate.empty()) return Error::success(); - uint64_t CommonSize = 0; - uint32_t CommonAlign = CommonSymbols.begin()->getAlignment(); - CommonSymbolList SymbolsToAllocate; - - DEBUG(dbgs() << "Processing common symbols...\n"); - - for (const auto &Sym : CommonSymbols) { - StringRef Name; - if (auto NameOrErr = Sym.getName()) - Name = *NameOrErr; - else - return NameOrErr.takeError(); - - // Skip common symbols already elsewhere. - if (GlobalSymbolTable.count(Name)) { - DEBUG(dbgs() << "\tSkipping already emitted common symbol '" << Name - << "'\n"); - continue; - } - - if (auto Sym = Resolver.findSymbolInLogicalDylib(Name)) { - if (!Sym.getFlags().isCommon()) { - DEBUG(dbgs() << "\tSkipping common symbol '" << Name - << "' in favor of stronger definition.\n"); - continue; - } - } - uint32_t Align = Sym.getAlignment(); - uint64_t Size = Sym.getCommonSize(); - - CommonSize = alignTo(CommonSize, Align) + Size; - - SymbolsToAllocate.push_back(Sym); - } - // Allocate memory for the section unsigned SectionID = Sections.size(); uint8_t *Addr = MemMgr.allocateDataSection(CommonSize, CommonAlign, SectionID, @@ -671,8 +664,9 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, SectionEntry("<common symbols>", Addr, CommonSize, CommonSize, 0)); memset(Addr, 0, CommonSize); - DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " - << format("%p", Addr) << " DataSize: " << CommonSize << "\n"); + LLVM_DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID + << " new addr: " << format("%p", Addr) + << " DataSize: " << CommonSize << "\n"); // Assign the address of each symbol for (auto &Sym : SymbolsToAllocate) { @@ -690,8 +684,8 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, Offset += AlignOffset; } JITSymbolFlags JITSymFlags = getJITSymbolFlags(Sym); - DEBUG(dbgs() << "Allocating common symbol " << Name << " address " - << format("%p", Addr) << "\n"); + LLVM_DEBUG(dbgs() << "Allocating common symbol " << Name << " address " + << format("%p", Addr) << "\n"); GlobalSymbolTable[Name] = SymbolTableEntry(SectionID, Offset, JITSymFlags); Offset += Size; @@ -787,21 +781,22 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, DataSize &= ~(getStubAlignment() - 1); } - DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name - << " obj addr: " << format("%p", pData) - << " new addr: " << format("%p", Addr) - << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize - << " Allocate: " << Allocate << "\n"); + LLVM_DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " + << Name << " obj addr: " << format("%p", pData) + << " new addr: " << format("%p", Addr) << " DataSize: " + << DataSize << " StubBufSize: " << StubBufSize + << " Allocate: " << Allocate << "\n"); } else { // Even if we didn't load the section, we need to record an entry for it // to handle later processing (and by 'handle' I mean don't do anything // with these sections). Allocate = 0; Addr = nullptr; - DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name - << " obj addr: " << format("%p", data.data()) << " new addr: 0" - << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize - << " Allocate: " << Allocate << "\n"); + LLVM_DEBUG( + dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name + << " obj addr: " << format("%p", data.data()) << " new addr: 0" + << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize + << " Allocate: " << Allocate << "\n"); } Sections.push_back( @@ -978,10 +973,11 @@ void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID, // Addr is a uint64_t because we can't assume the pointer width // of the target is the same as that of the host. Just use a generic // "big enough" type. - DEBUG(dbgs() << "Reassigning address for section " << SectionID << " (" - << Sections[SectionID].getName() << "): " - << format("0x%016" PRIx64, Sections[SectionID].getLoadAddress()) - << " -> " << format("0x%016" PRIx64, Addr) << "\n"); + LLVM_DEBUG( + dbgs() << "Reassigning address for section " << SectionID << " (" + << Sections[SectionID].getName() << "): " + << format("0x%016" PRIx64, Sections[SectionID].getLoadAddress()) + << " -> " << format("0x%016" PRIx64, Addr) << "\n"); Sections[SectionID].setLoadAddress(Addr); } @@ -997,14 +993,50 @@ void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, } Error RuntimeDyldImpl::resolveExternalSymbols() { + StringMap<JITEvaluatedSymbol> ExternalSymbolMap; + + // Resolution can trigger emission of more symbols, so iterate until + // we've resolved *everything*. + { + JITSymbolResolver::LookupSet ResolvedSymbols; + + while (true) { + JITSymbolResolver::LookupSet NewSymbols; + + for (auto &RelocKV : ExternalSymbolRelocations) { + StringRef Name = RelocKV.first(); + if (!Name.empty() && !GlobalSymbolTable.count(Name) && + !ResolvedSymbols.count(Name)) + NewSymbols.insert(Name); + } + + if (NewSymbols.empty()) + break; + + auto NewResolverResults = Resolver.lookup(NewSymbols); + if (!NewResolverResults) + return NewResolverResults.takeError(); + + assert(NewResolverResults->size() == NewSymbols.size() && + "Should have errored on unresolved symbols"); + + for (auto &RRKV : *NewResolverResults) { + assert(!ResolvedSymbols.count(RRKV.first) && "Redundant resolution?"); + ExternalSymbolMap.insert(RRKV); + ResolvedSymbols.insert(RRKV.first); + } + } + } + while (!ExternalSymbolRelocations.empty()) { + StringMap<RelocationList>::iterator i = ExternalSymbolRelocations.begin(); StringRef Name = i->first(); if (Name.size() == 0) { // This is an absolute symbol, use an address of zero. - DEBUG(dbgs() << "Resolving absolute relocations." - << "\n"); + LLVM_DEBUG(dbgs() << "Resolving absolute relocations." + << "\n"); RelocationList &Relocs = i->second; resolveRelocationList(Relocs, 0); } else { @@ -1012,29 +1044,10 @@ Error RuntimeDyldImpl::resolveExternalSymbols() { JITSymbolFlags Flags; RTDyldSymbolTable::const_iterator Loc = GlobalSymbolTable.find(Name); if (Loc == GlobalSymbolTable.end()) { - // This is an external symbol, try to get its address from the symbol - // resolver. - // First search for the symbol in this logical dylib. - if (auto Sym = Resolver.findSymbolInLogicalDylib(Name.data())) { - if (auto AddrOrErr = Sym.getAddress()) { - Addr = *AddrOrErr; - Flags = Sym.getFlags(); - } else - return AddrOrErr.takeError(); - } else if (auto Err = Sym.takeError()) - return Err; - - // If that fails, try searching for an external symbol. - if (!Addr) { - if (auto Sym = Resolver.findSymbol(Name.data())) { - if (auto AddrOrErr = Sym.getAddress()) { - Addr = *AddrOrErr; - Flags = Sym.getFlags(); - } else - return AddrOrErr.takeError(); - } else if (auto Err = Sym.takeError()) - return Err; - } + auto RRI = ExternalSymbolMap.find(Name); + assert(RRI != ExternalSymbolMap.end() && "No result for symbol"); + Addr = RRI->second.getAddress(); + Flags = RRI->second.getFlags(); // The call to getSymbolAddress may have caused additional modules to // be loaded, which may have added new entries to the // ExternalSymbolRelocations map. Consquently, we need to update our @@ -1065,8 +1078,8 @@ Error RuntimeDyldImpl::resolveExternalSymbols() { // if the target symbol is Thumb. Addr = modifyAddressBasedOnFlags(Addr, Flags); - DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t" - << format("0x%lx", Addr) << "\n"); + LLVM_DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t" + << format("0x%lx", Addr) << "\n"); // This list may have been updated when we called getSymbolAddress, so // don't change this code to get the list earlier. RelocationList &Relocs = i->second; @@ -1095,6 +1108,7 @@ uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress( void RuntimeDyld::MemoryManager::anchor() {} void JITSymbolResolver::anchor() {} +void LegacyJITSymbolResolver::anchor() {} RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver) @@ -1185,6 +1199,12 @@ JITEvaluatedSymbol RuntimeDyld::getSymbol(StringRef Name) const { return Dyld->getSymbol(Name); } +std::map<StringRef, JITEvaluatedSymbol> RuntimeDyld::getSymbolTable() const { + if (!Dyld) + return std::map<StringRef, JITEvaluatedSymbol>(); + return Dyld->getSymbolTable(); +} + void RuntimeDyld::resolveRelocations() { Dyld->resolveRelocations(); } void RuntimeDyld::reassignSectionAddress(unsigned SectionID, uint64_t Addr) { diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 5bc7434e703f..fa8906869b3a 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -688,12 +688,13 @@ RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); - DEBUG(dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr << "'...\n"); + LLVM_DEBUG(dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr + << "'...\n"); RuntimeDyldCheckerExprEval P(*this, ErrStream); bool Result = P.evaluate(CheckExpr); (void)Result; - DEBUG(dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " - << (Result ? "passed" : "FAILED") << ".\n"); + LLVM_DEBUG(dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " + << (Result ? "passed" : "FAILED") << ".\n"); return Result; } @@ -731,7 +732,14 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { if (getRTDyld().getSymbol(Symbol)) return true; - return !!getRTDyld().Resolver.findSymbol(Symbol); + JITSymbolResolver::LookupSet Symbols({Symbol}); + auto Result = getRTDyld().Resolver.lookup(Symbols); + if (!Result) { + logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); + return false; + } + assert(Result->count(Symbol) && "Missing symbol result"); + return true; } uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { @@ -742,7 +750,16 @@ uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) return InternalSymbol.getAddress(); - return cantFail(getRTDyld().Resolver.findSymbol(Symbol).getAddress()); + + JITSymbolResolver::LookupSet Symbols({Symbol}); + auto Result = getRTDyld().Resolver.lookup(Symbols); + if (!Result) { + logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); + return 0; + } + auto I = Result->find(Symbol); + assert(I != Result->end() && "Missing symbol result"); + return I->second.getAddress(); } uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 2c57eee191db..cc6729d21320 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -65,7 +65,7 @@ template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> { typedef Elf_Ehdr_Impl<ELFT> Elf_Ehdr; - typedef typename ELFDataTypeTypedefHelper<ELFT>::value_type addr_type; + typedef typename ELFT::uint addr_type; DyldELFObject(ELFObjectFile<ELFT> &&Obj); @@ -148,8 +148,8 @@ template <typename ELFT> static Expected<std::unique_ptr<DyldELFObject<ELFT>>> createRTDyldELFObject(MemoryBufferRef Buffer, const ObjectFile &SourceObject, const LoadedELFObjectInfo &L) { - typedef typename ELFFile<ELFT>::Elf_Shdr Elf_Shdr; - typedef typename ELFDataTypeTypedefHelper<ELFT>::value_type addr_type; + typedef typename ELFT::Shdr Elf_Shdr; + typedef typename ELFT::uint addr_type; Expected<std::unique_ptr<DyldELFObject<ELFT>>> ObjOrErr = DyldELFObject<ELFT>::create(Buffer); @@ -273,8 +273,8 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, case ELF::R_X86_64_64: { support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = Value + Addend; - DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " - << format("%p\n", Section.getAddressWithOffset(Offset))); + LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); break; } case ELF::R_X86_64_32: @@ -286,8 +286,8 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, uint32_t TruncatedAddr = (Value & 0xFFFFFFFF); support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) = TruncatedAddr; - DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " - << format("%p\n", Section.getAddressWithOffset(Offset))); + LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); break; } case ELF::R_X86_64_PC8: { @@ -312,6 +312,22 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, int64_t RealOffset = Value + Addend - FinalAddress; support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = RealOffset; + LLVM_DEBUG(dbgs() << "Writing " << format("%p", RealOffset) << " at " + << format("%p\n", FinalAddress)); + break; + } + case ELF::R_X86_64_GOTOFF64: { + // Compute Value - GOTBase. + uint64_t GOTBase = 0; + for (const auto &Section : Sections) { + if (Section.getName() == ".got") { + GOTBase = Section.getLoadAddressWithOffset(0); + break; + } + } + assert(GOTBase != 0 && "missing GOT"); + int64_t GOTOffset = Value - GOTBase + Addend; + support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = GOTOffset; break; } } @@ -326,6 +342,9 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, Value + Addend; break; } + // Handle R_386_PLT32 like R_386_PC32 since it should be able to + // reach any 32 bit address. + case ELF::R_386_PLT32: case ELF::R_386_PC32: { uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF; @@ -351,12 +370,12 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, // Data should use target endian. Code should always use little endian. bool isBE = Arch == Triple::aarch64_be; - DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x" - << format("%llx", Section.getAddressWithOffset(Offset)) - << " FinalAddress: 0x" << format("%llx", FinalAddress) - << " Value: 0x" << format("%llx", Value) << " Type: 0x" - << format("%x", Type) << " Addend: 0x" << format("%llx", Addend) - << "\n"); + LLVM_DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x" + << format("%llx", Section.getAddressWithOffset(Offset)) + << " FinalAddress: 0x" << format("%llx", FinalAddress) + << " Value: 0x" << format("%llx", Value) << " Type: 0x" + << format("%x", Type) << " Addend: 0x" + << format("%llx", Addend) << "\n"); switch (Type) { default: @@ -471,11 +490,12 @@ void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section, uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF; Value += Addend; - DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: " - << Section.getAddressWithOffset(Offset) - << " FinalAddress: " << format("%p", FinalAddress) << " Value: " - << format("%x", Value) << " Type: " << format("%x", Type) - << " Addend: " << format("%x", Addend) << "\n"); + LLVM_DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: " + << Section.getAddressWithOffset(Offset) + << " FinalAddress: " << format("%p", FinalAddress) + << " Value: " << format("%x", Value) + << " Type: " << format("%x", Type) + << " Addend: " << format("%x", Addend) << "\n"); switch (Type) { default: @@ -526,10 +546,11 @@ void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) { IsMipsN64ABI = false; return; } - unsigned AbiVariant; - Obj.getPlatformFlags(AbiVariant); - IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32; - IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2; + if (auto *E = dyn_cast<ELFObjectFileBase>(&Obj)) { + unsigned AbiVariant = E->getPlatformFlags(); + IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32; + IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2; + } IsMipsN64ABI = Obj.getFileFormatName().equals("ELF64-mips"); } @@ -718,9 +739,11 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, writeInt16BE(LocalAddress, applyPPClo(Value + Addend) & ~3); break; case ELF::R_PPC64_ADDR16_HI: + case ELF::R_PPC64_ADDR16_HIGH: writeInt16BE(LocalAddress, applyPPChi(Value + Addend)); break; case ELF::R_PPC64_ADDR16_HA: + case ELF::R_PPC64_ADDR16_HIGHA: writeInt16BE(LocalAddress, applyPPCha(Value + Addend)); break; case ELF::R_PPC64_ADDR16_HIGHER: @@ -767,8 +790,9 @@ void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section, int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend); if (SignExtend64<26>(delta) != delta) llvm_unreachable("Relocation R_PPC64_REL24 overflow"); - // Generates a 'bl <address>' instruction - writeInt32BE(LocalAddress, 0x48000001 | (delta & 0x03FFFFFC)); + // We preserve bits other than LI field, i.e. PO and AA/LK fields. + uint32_t Inst = readBytesUnaligned(LocalAddress, 4); + writeInt32BE(LocalAddress, (Inst & 0xFC000003) | (delta & 0x03FFFFFC)); } break; case ELF::R_PPC64_REL32: { uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset); @@ -855,16 +879,16 @@ void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section, break; case ELF::R_BPF_64_64: { write(isBE, Section.getAddressWithOffset(Offset), Value + Addend); - DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " - << format("%p\n", Section.getAddressWithOffset(Offset))); + LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); break; } case ELF::R_BPF_64_32: { Value += Addend; assert(Value <= UINT32_MAX); write(isBE, Section.getAddressWithOffset(Offset), static_cast<uint32_t>(Value)); - DEBUG(dbgs() << "Writing " << format("%p", Value) << " at " - << format("%p\n", Section.getAddressWithOffset(Offset))); + LLVM_DEBUG(dbgs() << "Writing " << format("%p", Value) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); break; } } @@ -1021,7 +1045,7 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID, relocation_iterator RelI, StubMap &Stubs) { - DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation."); + LLVM_DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation."); SectionEntry &Section = Sections[SectionID]; uint64_t Offset = RelI->getOffset(); @@ -1032,10 +1056,10 @@ void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID, resolveRelocation(Section, Offset, (uint64_t)Section.getAddressWithOffset(i->second), RelType, 0); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.getStubOffset(); uint8_t *StubTargetAddr = createStubFunction( Section.getAddressWithOffset(Section.getStubOffset())); @@ -1092,8 +1116,8 @@ RuntimeDyldELF::processRelocationRef( else return TargetNameOrErr.takeError(); } - DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend - << " TargetName: " << TargetName << "\n"); + LLVM_DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend + << " TargetName: " << TargetName << "\n"); RelocationValueRef Value; // First search for the symbol in the local symbol table SymbolRef::Type SymType = SymbolRef::ST_Unknown; @@ -1134,7 +1158,7 @@ RuntimeDyldELF::processRelocationRef( section_iterator si = *SectionOrErr; if (si == Obj.section_end()) llvm_unreachable("Symbol section not found, bad object file format!"); - DEBUG(dbgs() << "\t\tThis is section symbol\n"); + LLVM_DEBUG(dbgs() << "\t\tThis is section symbol\n"); bool isCode = si->isText(); if (auto SectionIDOrErr = findOrEmitSection(Obj, (*si), isCode, ObjSectionToID)) @@ -1166,8 +1190,8 @@ RuntimeDyldELF::processRelocationRef( uint64_t Offset = RelI->getOffset(); - DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset - << "\n"); + LLVM_DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset + << "\n"); if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be)) { if (RelType == ELF::R_AARCH64_CALL26 || RelType == ELF::R_AARCH64_JUMP26) { resolveAArch64Branch(SectionID, Value, RelI, Stubs); @@ -1189,7 +1213,7 @@ RuntimeDyldELF::processRelocationRef( if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL || RelType == ELF::R_ARM_JUMP24) { // This is an ARM branch relocation, need to use a stub function. - DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.\n"); + LLVM_DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.\n"); SectionEntry &Section = Sections[SectionID]; // Look for an existing stub. @@ -1199,10 +1223,10 @@ RuntimeDyldELF::processRelocationRef( Section, Offset, reinterpret_cast<uint64_t>(Section.getAddressWithOffset(i->second)), RelType, 0); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.getStubOffset(); uint8_t *StubTargetAddr = createStubFunction( Section.getAddressWithOffset(Section.getStubOffset())); @@ -1237,7 +1261,7 @@ RuntimeDyldELF::processRelocationRef( uint32_t Opcode = readBytesUnaligned(Placeholder, 4); if (RelType == ELF::R_MIPS_26) { // This is an Mips branch relocation, need to use a stub function. - DEBUG(dbgs() << "\t\tThis is a Mips branch relocation."); + LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation."); SectionEntry &Section = Sections[SectionID]; // Extract the addend from the instruction. @@ -1252,14 +1276,13 @@ RuntimeDyldELF::processRelocationRef( if (i != Stubs.end()) { RelocationEntry RE(SectionID, Offset, RelType, i->second); addRelocationForSection(RE, SectionID); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.getStubOffset(); - unsigned AbiVariant; - O.getPlatformFlags(AbiVariant); + unsigned AbiVariant = Obj.getPlatformFlags(); uint8_t *StubTargetAddr = createStubFunction( Section.getAddressWithOffset(Section.getStubOffset()), AbiVariant); @@ -1340,7 +1363,7 @@ RuntimeDyldELF::processRelocationRef( addRelocationForSection(RE, Value.SectionID); } else if (RelType == ELF::R_MIPS_26) { // This is an Mips branch relocation, need to use a stub function. - DEBUG(dbgs() << "\t\tThis is a Mips branch relocation."); + LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation."); SectionEntry &Section = Sections[SectionID]; // Look up for existing stub. @@ -1348,14 +1371,13 @@ RuntimeDyldELF::processRelocationRef( if (i != Stubs.end()) { RelocationEntry RE(SectionID, Offset, RelType, i->second); addRelocationForSection(RE, SectionID); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.getStubOffset(); - unsigned AbiVariant; - O.getPlatformFlags(AbiVariant); + unsigned AbiVariant = Obj.getPlatformFlags(); uint8_t *StubTargetAddr = createStubFunction( Section.getAddressWithOffset(Section.getStubOffset()), AbiVariant); @@ -1412,8 +1434,7 @@ RuntimeDyldELF::processRelocationRef( } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) { if (RelType == ELF::R_PPC64_REL24) { // Determine ABI variant in use for this object. - unsigned AbiVariant; - Obj.getPlatformFlags(AbiVariant); + unsigned AbiVariant = Obj.getPlatformFlags(); AbiVariant &= ELF::EF_PPC64_ABI; // A PPC branch relocation will need a stub function if the target is // an external symbol (either Value.SymbolName is set, or SymType is @@ -1461,10 +1482,10 @@ RuntimeDyldELF::processRelocationRef( reinterpret_cast<uint64_t>( Section.getAddressWithOffset(i->second)), RelType, 0); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.getStubOffset(); uint8_t *StubTargetAddr = createStubFunction( Section.getAddressWithOffset(Section.getStubOffset()), @@ -1581,7 +1602,7 @@ RuntimeDyldELF::processRelocationRef( // parts of the stub separately. However, as things stand, we allocate // a stub for every relocation, so using a GOT in JIT code should be // no less space efficient than using an explicit constant pool. - DEBUG(dbgs() << "\t\tThis is a SystemZ indirect relocation."); + LLVM_DEBUG(dbgs() << "\t\tThis is a SystemZ indirect relocation."); SectionEntry &Section = Sections[SectionID]; // Look for an existing stub. @@ -1589,10 +1610,10 @@ RuntimeDyldELF::processRelocationRef( uintptr_t StubAddress; if (i != Stubs.end()) { StubAddress = uintptr_t(Section.getAddressWithOffset(i->second)); - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function. - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); uintptr_t BaseAddress = uintptr_t(Section.getAddress()); uintptr_t StubAlignment = getStubAlignment(); @@ -1643,10 +1664,10 @@ RuntimeDyldELF::processRelocationRef( uintptr_t StubAddress; if (i != Stubs.end()) { StubAddress = uintptr_t(Section.getAddress()) + i->second; - DEBUG(dbgs() << " Stub function found\n"); + LLVM_DEBUG(dbgs() << " Stub function found\n"); } else { // Create a new stub function (equivalent to a PLT entry). - DEBUG(dbgs() << " Create a new stub function\n"); + LLVM_DEBUG(dbgs() << " Create a new stub function\n"); uintptr_t BaseAddress = uintptr_t(Section.getAddress()); uintptr_t StubAlignment = getStubAlignment(); @@ -1695,6 +1716,29 @@ RuntimeDyldELF::processRelocationRef( addRelocationForSymbol(RE, Value.SymbolName); else addRelocationForSection(RE, Value.SectionID); + } else if (RelType == ELF::R_X86_64_GOT64) { + // Fill in a 64-bit GOT offset. + uint64_t GOTOffset = allocateGOTEntries(1); + resolveRelocation(Sections[SectionID], Offset, GOTOffset, + ELF::R_X86_64_64, 0); + + // Fill in the value of the symbol we're targeting into the GOT + RelocationEntry RE = + computeGOTOffsetRE(GOTOffset, Value.Offset, ELF::R_X86_64_64); + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + } else if (RelType == ELF::R_X86_64_GOTPC64) { + // Materialize the address of the base of the GOT relative to the PC. + // This doesn't create a GOT entry, but it does mean we need a GOT + // section. + (void)allocateGOTEntries(0); + resolveGOTOffsetRelocation(SectionID, Offset, Addend, ELF::R_X86_64_PC64); + } else if (RelType == ELF::R_X86_64_GOTOFF64) { + // GOTOFF relocations ultimately require a section difference relocation. + (void)allocateGOTEntries(0); + processSimpleRelocation(SectionID, Offset, RelType, Value); } else if (RelType == ELF::R_X86_64_PC32) { Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset)); processSimpleRelocation(SectionID, Offset, RelType, Value); @@ -1866,6 +1910,7 @@ bool RuntimeDyldELF::relocationNeedsGot(const RelocationRef &R) const { if (Arch == Triple::x86_64) return RelTy == ELF::R_X86_64_GOTPCREL || RelTy == ELF::R_X86_64_GOTPCRELX || + RelTy == ELF::R_X86_64_GOT64 || RelTy == ELF::R_X86_64_REX_GOTPCRELX; return false; } @@ -1882,6 +1927,9 @@ bool RuntimeDyldELF::relocationNeedsStub(const RelocationRef &R) const { case ELF::R_X86_64_GOTPCREL: case ELF::R_X86_64_GOTPCRELX: case ELF::R_X86_64_REX_GOTPCRELX: + case ELF::R_X86_64_GOTPC64: + case ELF::R_X86_64_GOT64: + case ELF::R_X86_64_GOTOFF64: case ELF::R_X86_64_PC32: case ELF::R_X86_64_PC64: case ELF::R_X86_64_64: diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index e046a8504e9f..4d7cc36d0666 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -87,7 +87,7 @@ public: uint8_t *getAddress() const { return Address; } - /// \brief Return the address of this section with an offset. + /// Return the address of this section with an offset. uint8_t *getAddressWithOffset(unsigned OffsetBytes) const { assert(OffsetBytes <= AllocationSize && "Offset out of bounds!"); return Address + OffsetBytes; @@ -98,7 +98,7 @@ public: uint64_t getLoadAddress() const { return LoadAddress; } void setLoadAddress(uint64_t LA) { LoadAddress = LA; } - /// \brief Return the load address of this section with an offset. + /// Return the load address of this section with an offset. uint64_t getLoadAddressWithOffset(unsigned OffsetBytes) const { assert(OffsetBytes <= AllocationSize && "Offset out of bounds!"); return LoadAddress + OffsetBytes; @@ -217,7 +217,7 @@ public: } }; -/// @brief Symbol info for RuntimeDyld. +/// Symbol info for RuntimeDyld. class SymbolTableEntry { public: SymbolTableEntry() = default; @@ -381,13 +381,14 @@ protected: return Addr; } - /// \brief Given the common symbols discovered in the object file, emit a + /// Given the common symbols discovered in the object file, emit a /// new section for them and update the symbol mappings in the object and /// symbol table. Error emitCommonSymbols(const ObjectFile &Obj, - CommonSymbolList &CommonSymbols); + CommonSymbolList &CommonSymbols, uint64_t CommonSize, + uint32_t CommonAlign); - /// \brief Emits section data from the object file to the MemoryManager. + /// Emits section data from the object file to the MemoryManager. /// \param IsCode if it's true then allocateCodeSection() will be /// used for emits, else allocateDataSection() will be used. /// \return SectionID. @@ -395,7 +396,7 @@ protected: const SectionRef &Section, bool IsCode); - /// \brief Find Section in LocalSections. If the secton is not found - emit + /// Find Section in LocalSections. If the secton is not found - emit /// it and store in LocalSections. /// \param IsCode if it's true then allocateCodeSection() will be /// used for emmits, else allocateDataSection() will be used. @@ -404,26 +405,26 @@ protected: const SectionRef &Section, bool IsCode, ObjSectionToIDMap &LocalSections); - // \brief Add a relocation entry that uses the given section. + // Add a relocation entry that uses the given section. void addRelocationForSection(const RelocationEntry &RE, unsigned SectionID); - // \brief Add a relocation entry that uses the given symbol. This symbol may + // Add a relocation entry that uses the given symbol. This symbol may // be found in the global symbol table, or it may be external. void addRelocationForSymbol(const RelocationEntry &RE, StringRef SymbolName); - /// \brief Emits long jump instruction to Addr. + /// Emits long jump instruction to Addr. /// \return Pointer to the memory area for emitting target address. uint8_t *createStubFunction(uint8_t *Addr, unsigned AbiVariant = 0); - /// \brief Resolves relocations from Relocs list with address from Value. + /// Resolves relocations from Relocs list with address from Value. void resolveRelocationList(const RelocationList &Relocs, uint64_t Value); - /// \brief A object file specific relocation resolver + /// A object file specific relocation resolver /// \param RE The relocation to be resolved /// \param Value Target symbol address to apply the relocation action virtual void resolveRelocation(const RelocationEntry &RE, uint64_t Value) = 0; - /// \brief Parses one or more object file relocations (some object files use + /// Parses one or more object file relocations (some object files use /// relocation pairs) and stores it to Relocations or SymbolRelocations /// (this depends on the object file type). /// \return Iterator to the next relocation that needs to be parsed. @@ -432,35 +433,35 @@ protected: const ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) = 0; - /// \brief Resolve relocations to external symbols. + /// Resolve relocations to external symbols. Error resolveExternalSymbols(); - // \brief Compute an upper bound of the memory that is required to load all + // Compute an upper bound of the memory that is required to load all // sections Error computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, uint32_t &CodeAlign, uint64_t &RODataSize, uint32_t &RODataAlign, uint64_t &RWDataSize, uint32_t &RWDataAlign); - // \brief Compute GOT size + // Compute GOT size unsigned computeGOTSize(const ObjectFile &Obj); - // \brief Compute the stub buffer size required for a section + // Compute the stub buffer size required for a section unsigned computeSectionStubBufSize(const ObjectFile &Obj, const SectionRef &Section); - // \brief Implementation of the generic part of the loadObject algorithm. + // Implementation of the generic part of the loadObject algorithm. Expected<ObjSectionToIDMap> loadObjectImpl(const object::ObjectFile &Obj); - // \brief Return size of Global Offset Table (GOT) entry + // Return size of Global Offset Table (GOT) entry virtual size_t getGOTEntrySize() { return 0; } - // \brief Return true if the relocation R may require allocating a GOT entry. + // Return true if the relocation R may require allocating a GOT entry. virtual bool relocationNeedsGot(const RelocationRef &R) const { return false; } - // \brief Return true if the relocation R may require allocating a stub. + // Return true if the relocation R may require allocating a stub. virtual bool relocationNeedsStub(const RelocationRef &R) const { return true; // Conservative answer } @@ -518,6 +519,21 @@ public: return JITEvaluatedSymbol(TargetAddr, SymEntry.getFlags()); } + std::map<StringRef, JITEvaluatedSymbol> getSymbolTable() const { + std::map<StringRef, JITEvaluatedSymbol> Result; + + for (auto &KV : GlobalSymbolTable) { + auto SectionID = KV.second.getSectionID(); + uint64_t SectionAddr = 0; + if (SectionID != AbsoluteSymbolSection) + SectionAddr = getSectionLoadAddress(SectionID); + Result[KV.first()] = + JITEvaluatedSymbol(SectionAddr + KV.second.getOffset(), KV.second.getFlags()); + } + + return Result; + } + void resolveRelocations(); void reassignSectionAddress(unsigned SectionID, uint64_t Addr); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index b0561f68edb3..c5a215c83331 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -196,10 +196,10 @@ Error RuntimeDyldMachO::populateIndirectSymbolPointersSection( assert((PTSectionSize % PTEntrySize) == 0 && "Pointers section does not contain a whole number of stubs?"); - DEBUG(dbgs() << "Populating pointer table section " - << Sections[PTSectionID].getName() << ", Section ID " - << PTSectionID << ", " << NumPTEntries << " entries, " - << PTEntrySize << " bytes each:\n"); + LLVM_DEBUG(dbgs() << "Populating pointer table section " + << Sections[PTSectionID].getName() << ", Section ID " + << PTSectionID << ", " << NumPTEntries << " entries, " + << PTEntrySize << " bytes each:\n"); for (unsigned i = 0; i < NumPTEntries; ++i) { unsigned SymbolIndex = @@ -210,8 +210,8 @@ Error RuntimeDyldMachO::populateIndirectSymbolPointersSection( IndirectSymbolName = *IndirectSymbolNameOrErr; else return IndirectSymbolNameOrErr.takeError(); - DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex - << ", PT offset: " << PTEntryOffset << "\n"); + LLVM_DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex + << ", PT offset: " << PTEntryOffset << "\n"); RelocationEntry RE(PTSectionID, PTEntryOffset, MachO::GENERIC_RELOC_VANILLA, 0, false, 2); addRelocationForSymbol(RE, IndirectSymbolName); @@ -275,8 +275,8 @@ unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(uint8_t *P, int64_t DeltaForEH) { typedef typename Impl::TargetPtrT TargetPtrT; - DEBUG(dbgs() << "Processing FDE: Delta for text: " << DeltaForText - << ", Delta for EH: " << DeltaForEH << "\n"); + LLVM_DEBUG(dbgs() << "Processing FDE: Delta for text: " << DeltaForText + << ", Delta for EH: " << DeltaForEH << "\n"); uint32_t Length = readBytesUnaligned(P, 4); P += 4; uint8_t *Ret = P + Length; diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h index 04678f224466..dd65051edad7 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h @@ -80,9 +80,9 @@ public: SmallString<32> RelTypeName; RelI->getTypeName(RelTypeName); #endif - DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset - << " RelType: " << RelTypeName << " TargetName: " << TargetName - << " Addend " << Addend << "\n"); + LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelTypeName << " TargetName: " + << TargetName << " Addend " << Addend << "\n"); unsigned TargetSectionID = -1; if (Section == Obj.section_end()) { @@ -145,10 +145,11 @@ public: : Sections[RE.Sections.SectionA].getLoadAddressWithOffset( RE.Addend); assert(Result <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_I386_DIR32" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_DIR32" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); writeBytesUnaligned(Result, Target, 4); break; } @@ -159,10 +160,11 @@ public: Sections[RE.Sections.SectionA].getLoadAddressWithOffset(RE.Addend) - Sections[0].getLoadAddress(); assert(Result <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_I386_DIR32NB" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_DIR32NB" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); writeBytesUnaligned(Result, Target, 4); break; } @@ -176,10 +178,11 @@ public: "relocation overflow"); assert(static_cast<int64_t>(Result) >= INT32_MIN && "relocation underflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_I386_REL32" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_REL32" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); writeBytesUnaligned(Result, Target, 4); break; } @@ -187,18 +190,18 @@ public: // 16-bit section index of the section that contains the target. assert(static_cast<uint32_t>(RE.SectionID) <= UINT16_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_I386_SECTION Value: " << RE.SectionID - << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_SECTION Value: " + << RE.SectionID << '\n'); writeBytesUnaligned(RE.SectionID, Target, 2); break; case COFF::IMAGE_REL_I386_SECREL: // 32-bit offset of the target from the beginning of its section. assert(static_cast<uint64_t>(RE.Addend) <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_I386_SECREL Value: " << RE.Addend - << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_I386_SECREL Value: " + << RE.Addend << '\n'); writeBytesUnaligned(RE.Addend, Target, 4); break; default: diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index 9000435764df..729ea1ec48a4 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -97,9 +97,9 @@ public: SmallString<32> RelTypeName; RelI->getTypeName(RelTypeName); #endif - DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset - << " RelType: " << RelTypeName << " TargetName: " << TargetName - << " Addend " << Addend << "\n"); + LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelTypeName << " TargetName: " + << TargetName << " Addend " << Addend << "\n"); unsigned TargetSectionID = -1; if (Section == Obj.section_end()) { @@ -187,10 +187,11 @@ public: : Sections[RE.Sections.SectionA].getLoadAddressWithOffset(RE.Addend); Result |= ISASelectionBit; assert(Result <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_ADDR32" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_ADDR32" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); writeBytesUnaligned(Result, Target, 4); break; } @@ -200,10 +201,11 @@ public: uint64_t Result = Sections[RE.Sections.SectionA].getLoadAddress() - Sections[0].getLoadAddress() + RE.Addend; assert(Result <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_ADDR32NB" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_ADDR32NB" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); Result |= ISASelectionBit; writeBytesUnaligned(Result, Target, 4); break; @@ -212,18 +214,18 @@ public: // 16-bit section index of the section that contains the target. assert(static_cast<uint32_t>(RE.SectionID) <= UINT16_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_SECTION Value: " << RE.SectionID - << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_SECTION Value: " + << RE.SectionID << '\n'); writeBytesUnaligned(RE.SectionID, Target, 2); break; case COFF::IMAGE_REL_ARM_SECREL: // 32-bit offset of the target from the beginning of its section. assert(static_cast<uint64_t>(RE.Addend) <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_SECREL Value: " << RE.Addend - << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_SECREL Value: " << RE.Addend + << '\n'); writeBytesUnaligned(RE.Addend, Target, 2); break; case COFF::IMAGE_REL_ARM_MOV32T: { @@ -231,10 +233,11 @@ public: uint64_t Result = Sections[RE.Sections.SectionA].getLoadAddressWithOffset(RE.Addend); assert(Result <= UINT32_MAX && "relocation overflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_MOV32T" - << " TargetSection: " << RE.Sections.SectionA - << " Value: " << format("0x%08" PRIx32, Result) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_MOV32T" + << " TargetSection: " << RE.Sections.SectionA + << " Value: " << format("0x%08" PRIx32, Result) + << '\n'); // MOVW(T3): |11110|i|10|0|1|0|0|imm4|0|imm3|Rd|imm8| // imm32 = zext imm4:i:imm3:imm8 @@ -262,9 +265,9 @@ public: "relocation overflow"); assert(static_cast<int64_t>(RE.Addend) >= INT32_MIN && "relocation underflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_BRANCH20T" - << " Value: " << static_cast<int32_t>(Value) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_BRANCH20T" + << " Value: " << static_cast<int32_t>(Value) << '\n'); static_cast<void>(Value); llvm_unreachable("unimplemented relocation"); break; @@ -277,9 +280,9 @@ public: "relocation overflow"); assert(static_cast<int64_t>(RE.Addend) >= INT32_MIN && "relocation underflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_BRANCH24T" - << " Value: " << static_cast<int32_t>(Value) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_BRANCH24T" + << " Value: " << static_cast<int32_t>(Value) << '\n'); static_cast<void>(Value); llvm_unreachable("unimplemented relocation"); break; @@ -292,9 +295,9 @@ public: "relocation overflow"); assert(static_cast<int64_t>(RE.Addend) >= INT32_MIN && "relocation underflow"); - DEBUG(dbgs() << "\t\tOffset: " << RE.Offset - << " RelType: IMAGE_REL_ARM_BLX23T" - << " Value: " << static_cast<int32_t>(Value) << '\n'); + LLVM_DEBUG(dbgs() << "\t\tOffset: " << RE.Offset + << " RelType: IMAGE_REL_ARM_BLX23T" + << " Value: " << static_cast<int32_t>(Value) << '\n'); static_cast<void>(Value); llvm_unreachable("unimplemented relocation"); break; diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index 7cbb43854151..2d6e5c4aea67 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -30,15 +30,33 @@ private: // unregisteredEH frame sections with the memory manager. SmallVector<SID, 2> UnregisteredEHFrameSections; SmallVector<SID, 2> RegisteredEHFrameSections; + uint64_t ImageBase; + + // Fake an __ImageBase pointer by returning the section with the lowest adress + uint64_t getImageBase() { + if (!ImageBase) { + ImageBase = std::numeric_limits<uint64_t>::max(); + for (const SectionEntry &Section : Sections) + ImageBase = std::min(ImageBase, Section.getLoadAddress()); + } + return ImageBase; + } + + void write32BitOffset(uint8_t *Target, int64_t Addend, uint64_t Delta) { + uint64_t Result = Addend + Delta; + assert(Result <= UINT32_MAX && "Relocation overflow"); + writeBytesUnaligned(Result, Target, 4); + } public: RuntimeDyldCOFFX86_64(RuntimeDyld::MemoryManager &MM, JITSymbolResolver &Resolver) - : RuntimeDyldCOFF(MM, Resolver) {} + : RuntimeDyldCOFF(MM, Resolver), ImageBase(0) {} - unsigned getMaxStubSize() override { - return 6; // 2-byte jmp instruction + 32-bit relative address - } + unsigned getStubAlignment() override { return 1; } + + // 2-byte jmp instruction + 32-bit relative address + 64-bit absolute jump + unsigned getMaxStubSize() override { return 14; } // The target location for the relocation is described by RE.SectionID and // RE.Offset. RE.SectionID can be used to find the SectionEntry. Each @@ -85,13 +103,17 @@ public: } case COFF::IMAGE_REL_AMD64_ADDR32NB: { - // Note ADDR32NB requires a well-established notion of - // image base. This address must be less than or equal - // to every section's load address, and all sections must be - // within a 32 bit offset from the base. - // - // For now we just set these to zero. - writeBytesUnaligned(0, Target, 4); + // ADDR32NB requires an offset less than 2GB from 'ImageBase'. + // The MemoryManager can make sure this is always true by forcing the + // memory layout to be: CodeSection < ReadOnlySection < ReadWriteSection. + const uint64_t ImageBase = getImageBase(); + if (Value < ImageBase || ((Value - ImageBase) > UINT32_MAX)) { + llvm::errs() << "IMAGE_REL_AMD64_ADDR32NB relocation requires an" + << "ordered section layout.\n"; + write32BitOffset(Target, 0, 0); + } else { + write32BitOffset(Target, RE.Addend, Value - ImageBase); + } break; } @@ -106,6 +128,52 @@ public: } } + std::tuple<uint64_t, uint64_t, uint64_t> + generateRelocationStub(unsigned SectionID, StringRef TargetName, + uint64_t Offset, uint64_t RelType, uint64_t Addend, + StubMap &Stubs) { + uintptr_t StubOffset; + SectionEntry &Section = Sections[SectionID]; + + RelocationValueRef OriginalRelValueRef; + OriginalRelValueRef.SectionID = SectionID; + OriginalRelValueRef.Offset = Offset; + OriginalRelValueRef.Addend = Addend; + OriginalRelValueRef.SymbolName = TargetName.data(); + + auto Stub = Stubs.find(OriginalRelValueRef); + if (Stub == Stubs.end()) { + LLVM_DEBUG(dbgs() << " Create a new stub function for " + << TargetName.data() << "\n"); + + StubOffset = Section.getStubOffset(); + Stubs[OriginalRelValueRef] = StubOffset; + createStubFunction(Section.getAddressWithOffset(StubOffset)); + Section.advanceStubOffset(getMaxStubSize()); + } else { + LLVM_DEBUG(dbgs() << " Stub function found for " << TargetName.data() + << "\n"); + StubOffset = Stub->second; + } + + // FIXME: If RelType == COFF::IMAGE_REL_AMD64_ADDR32NB we should be able + // to ignore the __ImageBase requirement and just forward to the stub + // directly as an offset of this section: + // write32BitOffset(Section.getAddressWithOffset(Offset), 0, StubOffset); + // .xdata exception handler's aren't having this though. + + // Resolve original relocation to stub function. + const RelocationEntry RE(SectionID, Offset, RelType, Addend); + resolveRelocation(RE, Section.getLoadAddressWithOffset(StubOffset)); + + // adjust relocation info so resolution writes to the stub function + Addend = 0; + Offset = StubOffset + 6; + RelType = COFF::IMAGE_REL_AMD64_ADDR64; + + return std::make_tuple(Offset, RelType, Addend); + } + Expected<relocation_iterator> processRelocationRef(unsigned SectionID, relocation_iterator RelI, @@ -131,6 +199,11 @@ public: SectionEntry &Section = Sections[SectionID]; uintptr_t ObjTarget = Section.getObjAddress() + Offset; + Expected<StringRef> TargetNameOrErr = Symbol->getName(); + if (!TargetNameOrErr) + return TargetNameOrErr.takeError(); + StringRef TargetName = *TargetNameOrErr; + switch (RelType) { case COFF::IMAGE_REL_AMD64_REL32: @@ -142,6 +215,11 @@ public: case COFF::IMAGE_REL_AMD64_ADDR32NB: { uint8_t *Displacement = (uint8_t *)ObjTarget; Addend = readBytesUnaligned(Displacement, 4); + + if (IsExtern) + std::tie(Offset, RelType, Addend) = generateRelocationStub( + SectionID, TargetName, Offset, RelType, Addend, Stubs); + break; } @@ -155,14 +233,9 @@ public: break; } - Expected<StringRef> TargetNameOrErr = Symbol->getName(); - if (!TargetNameOrErr) - return TargetNameOrErr.takeError(); - StringRef TargetName = *TargetNameOrErr; - - DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset - << " RelType: " << RelType << " TargetName: " << TargetName - << " Addend " << Addend << "\n"); + LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelType << " TargetName: " + << TargetName << " Addend " << Addend << "\n"); if (IsExtern) { RelocationEntry RE(SectionID, Offset, RelType, Addend); @@ -183,7 +256,6 @@ public: return ++RelI; } - unsigned getStubAlignment() override { return 1; } void registerEHFrames() override { for (auto const &EHFrameSID : UnregisteredEHFrameSections) { uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress(); @@ -194,6 +266,7 @@ public: } UnregisteredEHFrameSections.clear(); } + Error finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) override { // Look for and record the EH frame section IDs. @@ -202,11 +275,12 @@ public: StringRef Name; if (auto EC = Section.getName(Name)) return errorCodeToError(EC); - // Note unwind info is split across .pdata and .xdata, so this - // may not be sufficiently general for all users. - if (Name == ".xdata") { + + // Note unwind info is stored in .pdata but often points to .xdata + // with an IMAGE_REL_AMD64_ADDR32NB relocation. Using a memory manager + // that keeps sections ordered in relation to __ImageBase is necessary. + if (Name == ".pdata") UnregisteredEHFrameSections.push_back(SectionPair.second); - } } return Error::success(); } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp index fe0f48e66a81..3a166b40af2d 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp @@ -55,12 +55,12 @@ RuntimeDyldELFMips::evaluateMIPS32Relocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type) { - DEBUG(dbgs() << "evaluateMIPS32Relocation, LocalAddress: 0x" - << format("%llx", Section.getAddressWithOffset(Offset)) - << " FinalAddress: 0x" - << format("%llx", Section.getLoadAddressWithOffset(Offset)) - << " Value: 0x" << format("%llx", Value) << " Type: 0x" - << format("%x", Type) << "\n"); + LLVM_DEBUG(dbgs() << "evaluateMIPS32Relocation, LocalAddress: 0x" + << format("%llx", Section.getAddressWithOffset(Offset)) + << " FinalAddress: 0x" + << format("%llx", Section.getLoadAddressWithOffset(Offset)) + << " Value: 0x" << format("%llx", Value) << " Type: 0x" + << format("%x", Type) << "\n"); switch (Type) { default: @@ -110,15 +110,16 @@ int64_t RuntimeDyldELFMips::evaluateMIPS64Relocation( const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend, uint64_t SymOffset, SID SectionID) { - DEBUG(dbgs() << "evaluateMIPS64Relocation, LocalAddress: 0x" - << format("%llx", Section.getAddressWithOffset(Offset)) - << " FinalAddress: 0x" - << format("%llx", Section.getLoadAddressWithOffset(Offset)) - << " Value: 0x" << format("%llx", Value) << " Type: 0x" - << format("%x", Type) << " Addend: 0x" << format("%llx", Addend) - << " Offset: " << format("%llx" PRIx64, Offset) - << " SID: " << format("%d", SectionID) - << " SymOffset: " << format("%x", SymOffset) << "\n"); + LLVM_DEBUG(dbgs() << "evaluateMIPS64Relocation, LocalAddress: 0x" + << format("%llx", Section.getAddressWithOffset(Offset)) + << " FinalAddress: 0x" + << format("%llx", Section.getLoadAddressWithOffset(Offset)) + << " Value: 0x" << format("%llx", Value) << " Type: 0x" + << format("%x", Type) << " Addend: 0x" + << format("%llx", Addend) + << " Offset: " << format("%llx" PRIx64, Offset) + << " SID: " << format("%d", SectionID) + << " SymOffset: " << format("%x", SymOffset) << "\n"); switch (Type) { default: @@ -307,13 +308,12 @@ void RuntimeDyldELFMips::resolveMIPSO32Relocation(const SectionEntry &Section, uint8_t *TargetPtr = Section.getAddressWithOffset(Offset); Value += Addend; - DEBUG(dbgs() << "resolveMIPSO32Relocation, LocalAddress: " - << Section.getAddressWithOffset(Offset) << " FinalAddress: " - << format("%p", Section.getLoadAddressWithOffset(Offset)) - << " Value: " << format("%x", Value) - << " Type: " << format("%x", Type) - << " Addend: " << format("%x", Addend) - << " SymOffset: " << format("%x", Offset) << "\n"); + LLVM_DEBUG(dbgs() << "resolveMIPSO32Relocation, LocalAddress: " + << Section.getAddressWithOffset(Offset) << " FinalAddress: " + << format("%p", Section.getLoadAddressWithOffset(Offset)) + << " Value: " << format("%x", Value) << " Type: " + << format("%x", Type) << " Addend: " << format("%x", Addend) + << " SymOffset: " << format("%x", Offset) << "\n"); Value = evaluateMIPS32Relocation(Section, Offset, Value, Type); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h index ce54a2717673..f53b9e6bd75a 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h @@ -39,13 +39,13 @@ protected: uint64_t SymOffset, SID SectionID); private: - /// \brief A object file specific relocation resolver + /// A object file specific relocation resolver /// \param RE The relocation to be resolved /// \param Value Target symbol address to apply the relocation action uint64_t evaluateRelocation(const RelocationEntry &RE, uint64_t Value, uint64_t Addend); - /// \brief A object file specific relocation resolver + /// A object file specific relocation resolver /// \param RE The relocation to be resolved /// \param Value Target symbol address to apply the relocation action void applyRelocation(const RelocationEntry &RE, uint64_t Value); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 97cbc153b227..2a619c549cfa 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -32,18 +32,37 @@ public: unsigned getStubAlignment() override { return 8; } /// Extract the addend encoded in the instruction / memory location. - int64_t decodeAddend(const RelocationEntry &RE) const { + Expected<int64_t> decodeAddend(const RelocationEntry &RE) const { const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); unsigned NumBytes = 1 << RE.Size; int64_t Addend = 0; // Verify that the relocation has the correct size and alignment. switch (RE.RelType) { - default: - llvm_unreachable("Unsupported relocation type!"); - case MachO::ARM64_RELOC_UNSIGNED: - assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); + default: { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Unsupported relocation type: " + << getRelocName(RE.RelType); + } + return make_error<StringError>(std::move(ErrMsg), + inconvertibleErrorCode()); + } + case MachO::ARM64_RELOC_POINTER_TO_GOT: + case MachO::ARM64_RELOC_UNSIGNED: { + if (NumBytes != 4 && NumBytes != 8) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Invalid relocation size for relocation " + << getRelocName(RE.RelType); + } + return make_error<StringError>(std::move(ErrMsg), + inconvertibleErrorCode()); + } break; + } case MachO::ARM64_RELOC_BRANCH26: case MachO::ARM64_RELOC_PAGE21: case MachO::ARM64_RELOC_PAGEOFF12: @@ -58,6 +77,7 @@ public: switch (RE.RelType) { default: llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_POINTER_TO_GOT: case MachO::ARM64_RELOC_UNSIGNED: // This could be an unaligned memory location. if (NumBytes == 4) @@ -66,9 +86,11 @@ public: Addend = *reinterpret_cast<support::ulittle64_t *>(LocalAddress); break; case MachO::ARM64_RELOC_BRANCH26: { - // Verify that the relocation points to the expected branch instruction. + // Verify that the relocation points to a B/BL instruction. auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); - assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + assert(((*p & 0xFC000000) == 0x14000000 || + (*p & 0xFC000000) == 0x94000000) && + "Expected branch instruction."); // Get the 26 bit addend encoded in the branch instruction and sign-extend // to 64 bit. The lower 2 bits are always zeros and are therefore implicit @@ -137,6 +159,7 @@ public: switch (RelType) { default: llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_POINTER_TO_GOT: case MachO::ARM64_RELOC_UNSIGNED: assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); break; @@ -154,6 +177,7 @@ public: switch (RelType) { default: llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_POINTER_TO_GOT: case MachO::ARM64_RELOC_UNSIGNED: // This could be an unaligned memory location. if (NumBytes == 4) @@ -164,7 +188,9 @@ public: case MachO::ARM64_RELOC_BRANCH26: { auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); // Verify that the relocation points to the expected branch instruction. - assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + assert(((*p & 0xFC000000) == 0x14000000 || + (*p & 0xFC000000) == 0x94000000) && + "Expected branch instruction."); // Verify addend value. assert((Addend & 0x3) == 0 && "Branch target is not aligned"); @@ -278,7 +304,20 @@ public: return processSubtractRelocation(SectionID, RelI, Obj, ObjSectionToID); RelocationEntry RE(getRelocationEntry(SectionID, Obj, RelI)); - RE.Addend = decodeAddend(RE); + + if (RE.RelType == MachO::ARM64_RELOC_POINTER_TO_GOT) { + bool Valid = + (RE.Size == 2 && RE.IsPCRel) || (RE.Size == 3 && !RE.IsPCRel); + if (!Valid) + return make_error<StringError>("ARM64_RELOC_POINTER_TO_GOT supports " + "32-bit pc-rel or 64-bit absolute only", + inconvertibleErrorCode()); + } + + if (auto Addend = decodeAddend(RE)) + RE.Addend = *Addend; + else + return Addend.takeError(); assert((ExplicitAddend == 0 || RE.Addend == 0) && "Relocation has "\ "ARM64_RELOC_ADDEND and embedded addend in the instruction."); @@ -292,13 +331,17 @@ public: return ValueOrErr.takeError(); bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); - if (!IsExtern && RE.IsPCRel) + if (RE.RelType == MachO::ARM64_RELOC_POINTER_TO_GOT) { + // We'll take care of the offset in processGOTRelocation. + Value.Offset = 0; + } else if (!IsExtern && RE.IsPCRel) makeValueAddendPCRel(Value, RelI, 1 << RE.Size); RE.Addend = Value.Offset; if (RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGE21 || - RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12) + RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12 || + RE.RelType == MachO::ARM64_RELOC_POINTER_TO_GOT) processGOTRelocation(RE, Value, Stubs); else { if (Value.SymbolName) @@ -311,7 +354,7 @@ public: } void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { - DEBUG(dumpRelocationToResolve(RE, Value)); + LLVM_DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); @@ -331,6 +374,19 @@ public: encodeAddend(LocalAddress, 1 << RE.Size, RelType, Value + RE.Addend); break; } + + case MachO::ARM64_RELOC_POINTER_TO_GOT: { + assert(((RE.Size == 2 && RE.IsPCRel) || (RE.Size == 3 && !RE.IsPCRel)) && + "ARM64_RELOC_POINTER_TO_GOT only supports 32-bit pc-rel or 64-bit " + "absolute"); + // Addend is the GOT entry address and RE.Offset the target of the + // relocation. + uint64_t Result = + RE.IsPCRel ? (RE.Addend - RE.Offset) : (Value + RE.Addend); + encodeAddend(LocalAddress, 1 << RE.Size, RelType, Result); + break; + } + case MachO::ARM64_RELOC_BRANCH26: { assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_BRANCH26 not supported"); // Check if branch is in range. @@ -368,7 +424,7 @@ public: writeBytesUnaligned(Value, LocalAddress, 1 << RE.Size); break; } - case MachO::ARM64_RELOC_POINTER_TO_GOT: + case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: llvm_unreachable("Relocation type not yet implemented!"); @@ -386,7 +442,9 @@ public: private: void processGOTRelocation(const RelocationEntry &RE, RelocationValueRef &Value, StubMap &Stubs) { - assert(RE.Size == 2); + assert((RE.RelType == MachO::ARM64_RELOC_POINTER_TO_GOT && + (RE.Size == 2 || RE.Size == 3)) || + RE.Size == 2); SectionEntry &Section = Sections[RE.SectionID]; StubMap::const_iterator i = Stubs.find(Value); int64_t Offset; @@ -459,6 +517,23 @@ private: return ++RelI; } + static const char *getRelocName(uint32_t RelocType) { + switch (RelocType) { + case MachO::ARM64_RELOC_UNSIGNED: return "ARM64_RELOC_UNSIGNED"; + case MachO::ARM64_RELOC_SUBTRACTOR: return "ARM64_RELOC_SUBTRACTOR"; + case MachO::ARM64_RELOC_BRANCH26: return "ARM64_RELOC_BRANCH26"; + case MachO::ARM64_RELOC_PAGE21: return "ARM64_RELOC_PAGE21"; + case MachO::ARM64_RELOC_PAGEOFF12: return "ARM64_RELOC_PAGEOFF12"; + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: return "ARM64_RELOC_GOT_LOAD_PAGE21"; + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: return "ARM64_RELOC_GOT_LOAD_PAGEOFF12"; + case MachO::ARM64_RELOC_POINTER_TO_GOT: return "ARM64_RELOC_POINTER_TO_GOT"; + case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: return "ARM64_RELOC_TLVP_LOAD_PAGE21"; + case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: return "ARM64_RELOC_TLVP_LOAD_PAGEOFF12"; + case MachO::ARM64_RELOC_ADDEND: return "ARM64_RELOC_ADDEND"; + } + return "Unrecognized arm64 addend"; + } + }; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index 990629de2f1d..64a6b2901819 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -47,6 +47,18 @@ public: return Addr; } + bool isAddrTargetThumb(unsigned SectionID, uint64_t Offset) { + auto TargetObjAddr = Sections[SectionID].getObjAddress() + Offset; + for (auto &KV : GlobalSymbolTable) { + auto &Entry = KV.second; + auto SymbolObjAddr = + Sections[Entry.getSectionID()].getObjAddress() + Entry.getOffset(); + if (TargetObjAddr == SymbolObjAddr) + return (Entry.getFlags().getTargetFlags() & ARMJITSymbolFlags::Thumb); + } + return false; + } + Expected<int64_t> decodeAddend(const RelocationEntry &RE) const { const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); @@ -161,12 +173,18 @@ public: // the value as being a thumb stub: we don't want to mix it up with an ARM // stub targeting the same function. if (RE.RelType == MachO::ARM_THUMB_RELOC_BR22) - Value.IsStubThumb = TargetIsLocalThumbFunc; + Value.IsStubThumb = true; if (RE.IsPCRel) makeValueAddendPCRel(Value, RelI, (RE.RelType == MachO::ARM_THUMB_RELOC_BR22) ? 4 : 8); + // If this is a non-external branch target check whether Value points to a + // thumb func. + if (!Value.SymbolName && (RelType == MachO::ARM_RELOC_BR24 || + RelType == MachO::ARM_THUMB_RELOC_BR22)) + RE.IsTargetThumbFunc = isAddrTargetThumb(Value.SectionID, Value.Offset); + if (RE.RelType == MachO::ARM_RELOC_BR24 || RE.RelType == MachO::ARM_THUMB_RELOC_BR22) processBranchRelocation(RE, Value, Stubs); @@ -182,7 +200,7 @@ public: } void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { - DEBUG(dumpRelocationToResolve(RE, Value)); + LLVM_DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); @@ -388,11 +406,11 @@ private: // addend = Encoded - Expected // = Encoded - (AddrA - AddrB) - DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB - << ", Addend: " << Addend << ", SectionA ID: " << SectionAID - << ", SectionAOffset: " << SectionAOffset - << ", SectionB ID: " << SectionBID - << ", SectionBOffset: " << SectionBOffset << "\n"); + LLVM_DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA + << ", AddrB: " << AddrB << ", Addend: " << Addend + << ", SectionA ID: " << SectionAID << ", SectionAOffset: " + << SectionAOffset << ", SectionB ID: " << SectionBID + << ", SectionBOffset: " << SectionBOffset << "\n"); RelocationEntry R(SectionID, Offset, RelocType, Addend, SectionAID, SectionAOffset, SectionBID, SectionBOffset, IsPCRel, HalfDiffKindBits); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index c42f1751a181..d384d70b8b0f 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -97,7 +97,7 @@ public: } void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { - DEBUG(dumpRelocationToResolve(RE, Value)); + LLVM_DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); @@ -192,11 +192,11 @@ private: // Compute the addend 'C' from the original expression 'A - B + C'. Addend -= AddrA - AddrB; - DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB - << ", Addend: " << Addend << ", SectionA ID: " << SectionAID - << ", SectionAOffset: " << SectionAOffset - << ", SectionB ID: " << SectionBID - << ", SectionBOffset: " << SectionBOffset << "\n"); + LLVM_DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA + << ", AddrB: " << AddrB << ", Addend: " << Addend + << ", SectionA ID: " << SectionAID << ", SectionAOffset: " + << SectionAOffset << ", SectionB ID: " << SectionBID + << ", SectionBOffset: " << SectionBOffset << "\n"); RelocationEntry R(SectionID, Offset, RelocType, Addend, SectionAID, SectionAOffset, SectionBID, SectionBOffset, IsPCRel, Size); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index 32fd3efddd0d..9732ea6a0cd2 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -85,7 +85,7 @@ public: } void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { - DEBUG(dumpRelocationToResolve(RE, Value)); + LLVM_DEBUG(dumpRelocationToResolve(RE, Value)); const SectionEntry &Section = Sections[RE.SectionID]; uint8_t *LocalAddress = Section.getAddressWithOffset(RE.Offset); diff --git a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index 2dc66a1502f8..05ab4a074e37 100644 --- a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -232,6 +232,8 @@ SectionMemoryManager::~SectionMemoryManager() { SectionMemoryManager::MemoryMapper::~MemoryMapper() {} +void SectionMemoryManager::anchor() {} + namespace { // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls // into sys::Memory. diff --git a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp index 18dfa4e3c319..9626b8d3ffa3 100644 --- a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp +++ b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp @@ -97,6 +97,8 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple, Options, RelocModel, CMModel, OptLevel, /*JIT*/ true); Target->Options.EmulatedTLS = EmulatedTLS; + Target->Options.ExplicitEmulatedTLS = true; + assert(Target && "Could not allocate target machine!"); return Target; } |