diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2019-08-21 18:13:02 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2019-08-21 18:13:02 +0000 |
commit | 54db30ce18663e6c2991958f3b5d18362e8e93c4 (patch) | |
tree | 4aa6442802570767398cc83ba484e97b1309bdc2 /contrib/llvm/lib/ExecutionEngine | |
parent | 35284c22e9c8348159b7ce032ea45f2cdeb65298 (diff) | |
parent | e6d1592492a3a379186bfb02bd0f4eda0669c0d5 (diff) |
Merge llvm trunk r366426, resolve conflicts, and update FREEBSD-Xlist.
Notes
Notes:
svn path=/projects/clang900-import/; revision=351344
Diffstat (limited to 'contrib/llvm/lib/ExecutionEngine')
76 files changed, 4819 insertions, 1450 deletions
diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index ae96c7f5955f..1c6c0406d048 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -1,9 +1,8 @@ //===-- ExecutionEngine.cpp - Common Implementation shared by EEs ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1020,32 +1019,6 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { return Result; } -/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst -/// with the integer held in IntVal. -static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, - unsigned StoreBytes) { - assert((IntVal.getBitWidth()+7)/8 >= StoreBytes && "Integer too small!"); - const uint8_t *Src = (const uint8_t *)IntVal.getRawData(); - - if (sys::IsLittleEndianHost) { - // Little-endian host - the source is ordered from LSB to MSB. Order the - // destination from LSB to MSB: Do a straight copy. - memcpy(Dst, Src, StoreBytes); - } else { - // Big-endian host - the source is an array of 64 bit words ordered from - // LSW to MSW. Each word is ordered from MSB to LSB. Order the destination - // from MSB to LSB: Reverse the word order, but not the bytes in a word. - while (StoreBytes > sizeof(uint64_t)) { - StoreBytes -= sizeof(uint64_t); - // May not be aligned so use memcpy. - memcpy(Dst + StoreBytes, Src, sizeof(uint64_t)); - Src += sizeof(uint64_t); - } - - memcpy(Dst, Src + sizeof(uint64_t) - StoreBytes, StoreBytes); - } -} - void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr, Type *Ty) { const unsigned StoreBytes = getDataLayout().getTypeStoreSize(Ty); @@ -1093,33 +1066,6 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, std::reverse((uint8_t*)Ptr, StoreBytes + (uint8_t*)Ptr); } -/// LoadIntFromMemory - Loads the integer stored in the LoadBytes bytes starting -/// from Src into IntVal, which is assumed to be wide enough and to hold zero. -static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) { - assert((IntVal.getBitWidth()+7)/8 >= LoadBytes && "Integer too small!"); - uint8_t *Dst = reinterpret_cast<uint8_t *>( - const_cast<uint64_t *>(IntVal.getRawData())); - - if (sys::IsLittleEndianHost) - // Little-endian host - the destination must be ordered from LSB to MSB. - // The source is ordered from LSB to MSB: Do a straight copy. - memcpy(Dst, Src, LoadBytes); - else { - // Big-endian - the destination is an array of 64 bit words ordered from - // LSW to MSW. Each word must be ordered from MSB to LSB. The source is - // ordered from MSB to LSB: Reverse the word order, but not the bytes in - // a word. - while (LoadBytes > sizeof(uint64_t)) { - LoadBytes -= sizeof(uint64_t); - // May not be aligned so use memcpy. - memcpy(Dst, Src + LoadBytes, sizeof(uint64_t)); - Dst += sizeof(uint64_t); - } - - memcpy(Dst + sizeof(uint64_t) - LoadBytes, Src, LoadBytes); - } -} - /// FIXME: document /// void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, diff --git a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 3be4bec566a0..c741fe2b3778 100644 --- a/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -1,9 +1,8 @@ //===-- ExecutionEngineBindings.cpp - C bindings for EEs ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp b/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp index 8204f5a90268..08d20156a590 100644 --- a/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -1,9 +1,8 @@ //===----- GDBRegistrationListener.cpp - Registers objects with GDB -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index e9051c198506..1ebc820a8b49 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- IntelJITEventListener.cpp - Tell Intel profiler about JITed code --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -142,13 +141,25 @@ void IntelJITEventListener::notifyObjectLoaded( uint64_t Addr = *AddrOrErr; uint64_t Size = P.second; + auto SecOrErr = Sym.getSection(); + if (!SecOrErr) { + // TODO: Actually report errors helpfully. + consumeError(SecOrErr.takeError()); + continue; + } + object::section_iterator Sec = *SecOrErr; + if (Sec == Obj.section_end()) + continue; + uint64_t Index = Sec->getIndex(); + // Record this address in a local vector Functions.push_back((void*)Addr); // Build the function loaded notification message iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper, Name->data(), Addr, Size); - DILineInfoTable Lines = Context->getLineInfoForAddressRange(Addr, Size); + DILineInfoTable Lines = + Context->getLineInfoForAddressRange({Addr, Index}, Size); DILineInfoTable::iterator Begin = Lines.begin(); DILineInfoTable::iterator End = Lines.end(); for (DILineInfoTable::iterator It = Begin; It != End; ++It) { diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h index 777d0f179cb5..68699c6a2200 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h @@ -1,9 +1,8 @@ //===-- IntelJITEventsWrapper.h - Intel JIT Events API Wrapper --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h index 61d8cc75d9f2..16ce672150cc 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h @@ -1,9 +1,8 @@ /*===-- ittnotify_config.h - JIT Profiling API internal config-----*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h index 5df752f66f10..15008fe93e60 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h @@ -1,9 +1,8 @@ /*===-- ittnotify_types.h - JIT Profiling API internal types--------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c index bc8fea148749..074e0735628a 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c @@ -1,9 +1,8 @@ /*===-- jitprofiling.c - JIT (Just-In-Time) Profiling API----------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h index efd2b1a33f75..ba627b430ff1 100644 --- a/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h +++ b/contrib/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h @@ -1,9 +1,8 @@ /*===-- jitprofiling.h - JIT Profiling API-------------------------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp index 98dca1102759..51f31d3d5d8f 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -1,9 +1,8 @@ //===-- Execution.cpp - Implement code to simulate the program ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,6 +43,60 @@ static void SetValue(Value *V, GenericValue Val, ExecutionContext &SF) { } //===----------------------------------------------------------------------===// +// Unary Instruction Implementations +//===----------------------------------------------------------------------===// + +static void executeFNegInst(GenericValue &Dest, GenericValue Src, Type *Ty) { + switch (Ty->getTypeID()) { + case Type::FloatTyID: + Dest.FloatVal = -Src.FloatVal; + break; + case Type::DoubleTyID: + Dest.DoubleVal = -Src.DoubleVal; + break; + default: + llvm_unreachable("Unhandled type for FNeg instruction"); + } +} + +void Interpreter::visitUnaryOperator(UnaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src = getOperandValue(I.getOperand(0), SF); + GenericValue R; // Result + + // First process vector operation + if (Ty->isVectorTy()) { + R.AggregateVal.resize(Src.AggregateVal.size()); + + switch(I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].FloatVal = -Src.AggregateVal[i].FloatVal; + } else if (cast<VectorType>(Ty)->getElementType()->isDoubleTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].DoubleVal = -Src.AggregateVal[i].DoubleVal; + } else { + llvm_unreachable("Unhandled type for FNeg instruction"); + } + break; + } + } else { + switch (I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: executeFNegInst(R, Src, Ty); break; + } + } + SetValue(&I, R, SF); +} + +//===----------------------------------------------------------------------===// // Binary Instruction Implementations //===----------------------------------------------------------------------===// @@ -2113,7 +2166,7 @@ void Interpreter::run() { // Track the number of dynamic instructions executed. ++NumDynamicInsts; - LLVM_DEBUG(dbgs() << "About to interpret: " << I); + LLVM_DEBUG(dbgs() << "About to interpret: " << I << "\n"); visit(I); // Dispatch to one of the visit* methods... } } diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 334fcacf8078..c3a2ccc582c9 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -1,9 +1,8 @@ //===-- ExternalFunctions.cpp - Implement External Functions --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp index 9818adfff82e..5727f7adb49c 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -1,9 +1,8 @@ //===- Interpreter.cpp - Top-Level LLVM Interpreter Implementation --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h index 33542e7e43ad..e72d778317d6 100644 --- a/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/contrib/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -1,9 +1,8 @@ //===-- Interpreter.h ------------------------------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -125,6 +124,7 @@ public: void visitSwitchInst(SwitchInst &I); void visitIndirectBrInst(IndirectBrInst &I); + void visitUnaryOperator(UnaryOperator &I); void visitBinaryOperator(BinaryOperator &I); void visitICmpInst(ICmpInst &I); void visitFCmpInst(FCmpInst &I); diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/contrib/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h new file mode 100644 index 000000000000..1271ad962b38 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -0,0 +1,82 @@ +//===--- BasicGOTAndStubsBuilder.h - Generic GOT/Stub creation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A base for simple GOT and stub creation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +template <typename BuilderImpl> class BasicGOTAndStubsBuilder { +public: + BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + + void run() { + // We're going to be adding new atoms, but we don't want to iterate over + // the newly added ones, so just copy the existing atoms out. + std::vector<DefinedAtom *> DAs(G.defined_atoms().begin(), + G.defined_atoms().end()); + + for (auto *DA : DAs) + for (auto &E : DA->edges()) + if (impl().isGOTEdge(E)) + impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget())); + else if (impl().isExternalBranchEdge(E)) + impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget())); + } + +protected: + Atom &getGOTEntryAtom(Atom &Target) { + assert(Target.hasName() && "GOT edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + auto &GOTEntry = impl().createGOTEntry(Target); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + return *GOTEntryI->second; + } + + Atom &getStubAtom(Atom &Target) { + assert(Target.hasName() && + "External branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + auto &StubAtom = impl().createStub(Target); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub atom"); + return *StubI->second; + } + + AtomGraph &G; + +private: + BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); } + + DenseMap<StringRef, DefinedAtom *> GOTEntries; + DenseMap<StringRef, DefinedAtom *> Stubs; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp new file mode 100644 index 000000000000..25f0e9040ffe --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -0,0 +1,544 @@ +//===-------- JITLink_EHFrameSupport.cpp - JITLink eh-frame utils ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "EHFrameSupportImpl.h" + +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/Support/DynamicLibrary.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), + EHFrameAddress(EHFrameAddress), + EHFrameReader(EHFrameContent, G.getEndianness()), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + +Error EHFrameParser::atomize() { + while (!EHFrameReader.empty()) { + size_t RecordOffset = EHFrameReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << "Processing eh-frame record at " + << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) + << " (offset " << RecordOffset << ")\n"; + }); + + size_t CIELength = 0; + uint32_t CIELengthField; + if (auto Err = EHFrameReader.readInteger(CIELengthField)) + return Err; + + // Process CIE length/extended-length fields to build the atom. + // + // The value of these fields describe the length of the *rest* of the CIE + // (not including data up to the end of the field itself) so we have to + // bump CIELength to include the data up to the end of the field: 4 bytes + // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. + if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + break; + + // If the regular length field's value is 0xffffffff, use extended length. + if (CIELengthField == 0xffffffff) { + uint64_t CIEExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + return Err; + if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + return make_error<JITLinkError>("CIE record extends past the end of " + "the __eh_frame section"); + if (CIEExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) + return make_error<JITLinkError>("CIE record too large to process"); + CIELength = CIEExtendedLengthField + 12; + } else { + if (CIELengthField > EHFrameReader.bytesRemaining()) + return make_error<JITLinkError>("CIE record extends past the end of " + "the __eh_frame section"); + CIELength = CIELengthField + 4; + } + + LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); + + // Add an atom for this record. + CurRecordAtom = &G.addAnonymousAtom( + EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); + CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + + // Read the CIE Pointer. + size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); + uint32_t CIEPointer; + if (auto Err = EHFrameReader.readInteger(CIEPointer)) + return Err; + + // Based on the CIE pointer value, parse this as a CIE or FDE record. + if (CIEPointer == 0) { + if (auto Err = processCIE()) + return Err; + } else { + if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + return Err; + } + + EHFrameReader.setOffset(RecordOffset + CIELength); + } + + return Error::success(); +} + +Expected<EHFrameParser::AugmentationInfo> +EHFrameParser::parseAugmentationString() { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error<JITLinkError>("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error<JITLinkError>("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + } + + return std::move(AugInfo); +} + +Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = EHFrameReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = EHFrameReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Error EHFrameParser::processCIE() { + // Use the dwarf namespace for convenient access to pointer encoding + // constants. + using namespace dwarf; + + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + + CIEInformation CIEInfo(*CurRecordAtom); + + uint8_t Version = 0; + if (auto Err = EHFrameReader.readInteger(Version)) + return Err; + + if (Version != 0x01) + return make_error<JITLinkError>("Bad CIE version " + Twine(Version) + + " (should be 0x01) in eh-frame"); + + auto AugInfo = parseAugmentationString(); + if (!AugInfo) + return AugInfo.takeError(); + + // Skip the EH Data field if present. + if (AugInfo->EHDataFieldPresent) + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + // Read and sanity check the code alignment factor. + { + uint64_t CodeAlignmentFactor = 0; + if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + return Err; + if (CodeAlignmentFactor != 1) + return make_error<JITLinkError>("Unsupported CIE code alignment factor " + + Twine(CodeAlignmentFactor) + + " (expected 1)"); + } + + // Read and sanity check the data alignment factor. + { + int64_t DataAlignmentFactor = 0; + if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + return Err; + if (DataAlignmentFactor != -8) + return make_error<JITLinkError>("Unsupported CIE data alignment factor " + + Twine(DataAlignmentFactor) + + " (expected -8)"); + } + + // Skip the return address register field. + if (auto Err = EHFrameReader.skip(1)) + return Err; + + uint64_t AugmentationDataLength = 0; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': { + CIEInfo.FDEsHaveLSDAField = true; + uint8_t LSDAPointerEncoding; + if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + return Err; + if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error<JITLinkError>( + "Unsupported LSDA pointer encoding " + + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + case 'P': { + uint8_t PersonalityPointerEncoding = 0; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + return Err; + if (PersonalityPointerEncoding != + (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) + return make_error<JITLinkError>( + "Unspported personality pointer " + "encoding " + + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + uint32_t PersonalityPointerAddress; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + return Err; + break; + } + case 'R': { + uint8_t FDEPointerEncoding; + if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + return Err; + if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error<JITLinkError>( + "Unsupported FDE address pointer " + "encoding " + + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + default: + llvm_unreachable("Invalid augmentation string field"); + } + } + + if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error<JITLinkError>("Read past the end of the augmentation " + "data while parsing fields"); + + assert(!CIEInfos.count(CurRecordAtom->getAddress()) && + "Multiple CIEs recorded at the same address?"); + CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo); + + return Error::success(); +} + +Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + LLVM_DEBUG({ + dbgs() << " CIE pointer: " + << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; + }); + + auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); + if (CIEInfoItr == CIEInfos.end()) + return make_error<JITLinkError>( + "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) + + " points to non-existant CIE at " + + formatv("{0:x16}", CIEPointerAddress - CIEPointer)); + auto &CIEInfo = CIEInfoItr->second; + + // The CIEPointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToCIERelocKind, + CIEPointerAddress - CurRecordAtom->getAddress(), + *CIEInfo.CIEAtom, 0); + + // Read and sanity check the PC-start pointer and size. + JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + + auto PCBeginDelta = readAbsolutePointer(); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + + JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + }); + + auto *TargetAtom = G.getAtomByAddress(PCBegin); + + if (!TargetAtom) + return make_error<JITLinkError>("FDE PC-begin " + + formatv("{0:x16}", PCBegin) + + " does not point at atom"); + + if (TargetAtom->getAddress() != PCBegin) + return make_error<JITLinkError>( + "FDE PC-begin " + formatv("{0:x16}", PCBegin) + + " does not point to start of atom at " + + formatv("{0:x16}", TargetAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + + // The PC-start pointer and size look good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + PCBeginAddress - CurRecordAtom->getAddress(), + *TargetAtom, 0); + + // Add a keep-alive relocation from the function to the FDE to ensure it is + // not dead stripped. + TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + + // Skip over the PC range size field. + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + if (CIEInfo.FDEsHaveLSDAField) { + uint64_t AugmentationDataSize; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + return Err; + if (AugmentationDataSize != G.getPointerSize()) + return make_error<JITLinkError>( + "Unexpected FDE augmentation data size (expected " + + Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) + + ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress())); + JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto LSDADelta = readAbsolutePointer(); + if (!LSDADelta) + return LSDADelta.takeError(); + + JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + + auto *LSDAAtom = G.getAtomByAddress(LSDA); + + if (!LSDAAtom) + return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point at atom"); + + if (LSDAAtom->getAddress() != LSDA) + return make_error<JITLinkError>( + "FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point to start of atom at " + + formatv("{0:x16}", LSDAAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + + // LSDA looks good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, + 0); + } + + return Error::success(); +} + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) { + return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, + FDEToCIERelocKind, FDEToTargetRelocKind) + .atomize(); +} + +// Determine whether we can register EH tables. +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !(defined(_AIX) && defined(__ibmxl__)) && !defined(__SEH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__)) +#define HAVE_EHTABLE_SUPPORT 1 +#else +#define HAVE_EHTABLE_SUPPORT 0 +#endif + +#if HAVE_EHTABLE_SUPPORT +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error<JITLinkError>("could not register eh-frame: " + "__register_frame function not found"); +} + +static Error deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error<JITLinkError>("could not deregister eh-frame: " + "__deregister_frame function not found"); +} +#endif + +#ifdef __APPLE__ + +template <typename HandleFDEFn> +Error walkAppleEHFrameSection(const char *const SectionStart, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + CurCFIRecord += Size; + + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +EHFrameRegistrar::~EHFrameRegistrar() {} + +InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { + static InProcessEHFrameRegistrar Instance; + return Instance; +} + +InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} + +AtomGraphPassFunction +createEHFrameRecorderPass(const Triple &TT, + StoreFrameAddressFunction StoreFrameAddress) { + const char *EHFrameSectionName = nullptr; + if (TT.getObjectFormat() == Triple::MachO) + EHFrameSectionName = "__eh_frame"; + else + EHFrameSectionName = ".eh_frame"; + + auto RecordEHFrame = [EHFrameSectionName, + StoreFrameAddress](AtomGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first atom + // in it. + JITTargetAddress Addr = 0; + if (auto *S = G.findSectionByName(EHFrameSectionName)) + Addr = S->getRange().getStart(); + StoreFrameAddress(Addr); + return Error::success(); + }; + + return RecordEHFrame; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h new file mode 100644 index 000000000000..d679edef7ea6 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -0,0 +1,72 @@ +//===------- EHFrameSupportImpl.h - JITLink eh-frame utils ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H + +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/BinaryStreamReader.h" + +namespace llvm { +namespace jitlink { + +/// A generic parser for eh-frame sections. +/// +/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and +/// FDEToTarget relocation kinds. +class EHFrameParser { +public: + EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind); + Error atomize(); + +private: + struct AugmentationInfo { + bool AugmentationDataPresent = false; + bool EHDataFieldPresent = false; + uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; + }; + + Expected<AugmentationInfo> parseAugmentationString(); + Expected<JITTargetAddress> readAbsolutePointer(); + Error processCIE(); + Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + + struct CIEInformation { + CIEInformation() = default; + CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {} + DefinedAtom *CIEAtom = nullptr; + bool FDEsHaveLSDAField = false; + }; + + AtomGraph &G; + Section &EHFrameSection; + StringRef EHFrameContent; + JITTargetAddress EHFrameAddress; + BinaryStreamReader EHFrameReader; + DefinedAtom *CurRecordAtom = nullptr; + DenseMap<JITTargetAddress, CIEInformation> CIEInfos; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; +}; + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp new file mode 100644 index 000000000000..9d0a7459dc09 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,172 @@ +//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast<JITLinkErrorCode>(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic<JITLinkerErrorCategory> JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +const StringRef getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + case Edge::LayoutNext: + return "Layout-Next"; + default: + llvm_unreachable("Unrecognized relocation kind"); + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { + OS << "<"; + if (A.getName().empty()) + OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + else + OS << A.getName(); + OS << " ["; + if (A.isDefined()) { + auto &DA = static_cast<const DefinedAtom &>(A); + OS << " section=" << DA.getSection().getName(); + if (DA.isLive()) + OS << " live"; + if (DA.shouldDiscard()) + OS << " should-discard"; + } else + OS << " external"; + OS << " ]>"; + return OS; +} + +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) + << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName + << " -> " << E.getTarget() << " + " << E.getAddend(); +} + +Section::~Section() { + for (auto *DA : DefinedAtoms) + DA->~DefinedAtom(); +} + +void AtomGraph::dump(raw_ostream &OS, + std::function<StringRef(Edge::Kind)> EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Defined atoms:\n"; + for (auto *DA : defined_atoms()) { + OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + << "\n"; + for (auto &E : DA->edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, *DA, E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, *DA, E, EdgeNumberString); + } + OS << "\n"; + } + } + + OS << "Absolute atoms:\n"; + for (auto *A : absolute_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; + + OS << "External atoms:\n"; + for (auto *A : external_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; +} + +JITLinkContext::~JITLinkContext() {} + +bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { + return true; +} + +AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return AtomGraphPassFunction(); +} + +Error JITLinkContext::modifyPassConfig(const Triple &TT, + PassConfiguration &Config) { + return Error::success(); +} + +Error markAllAtomsLive(AtomGraph &G) { + for (auto *DA : G.defined_atoms()) + DA->setLive(true); + return Error::success(); +} + +void jitLink(std::unique_ptr<JITLinkContext> Ctx) { + auto Magic = identify_magic(Ctx->getObjectBuffer().getBuffer()); + switch (Magic) { + case file_magic::macho_object: + return jitLink_MachO(std::move(Ctx)); + default: + Ctx->notifyFailed(make_error<JITLinkError>("Unsupported file format")); + }; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp new file mode 100644 index 000000000000..96e074da122b --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,481 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" +#include "EHFrameSupportImpl.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { + + // Build the atom graph. + if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) + G = std::move(*GraphOrErr); + else + return Ctx->notifyFailed(GraphOrErr.takeError()); + assert(G && "Graph should have been created by buildGraph above"); + + // Prune and optimize the graph. + if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dumpGraph(dbgs()); + }); + + prune(*G); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dumpGraph(dbgs()); + }); + + // Run post-pruning passes. + if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // Sort atoms into segments. + layOutAtoms(); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return Ctx->notifyFailed(std::move(Err)); + + // Notify client that the defined atoms have been assigned addresses. + Ctx->notifyResolved(*G); + + auto ExternalSymbols = getExternalSymbolNames(); + + // We're about to hand off ownership of ourself to the continuation. Grab a + // pointer to the context so that we can call it to initiate the lookup. + // + // FIXME: Once callee expressions are defined to be sequenced before argument + // expressions (c++17) we can simplify all this to: + // + // Ctx->lookup(std::move(UnresolvedExternals), + // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { + // Self->linkPhase2(std::move(Self), std::move(Result)); + // }); + // + // FIXME: Use move capture once we have c++14. + auto *TmpCtx = Ctx.get(); + auto *UnownedSelf = Self.release(); + auto Phase2Continuation = + [UnownedSelf](Expected<AsyncLookupResult> LookupResult) { + std::unique_ptr<JITLinkerBase> Self(UnownedSelf); + UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); + }; + TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LR) { + // If the lookup failed, bail out. + if (!LR) + return deallocateAndBailOut(LR.takeError()); + + // Assign addresses to external atoms. + applyLookupResult(*LR); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + // Copy atom content to working memory and fix up. + if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + return deallocateAndBailOut(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + return deallocateAndBailOut(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr<JITLinkerBase> Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { + if (Err) + return deallocateAndBailOut(std::move(Err)); + Ctx->notifyFinalized(std::move(Alloc)); +} + +Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { + for (auto &P : Passes) + if (auto Err = P(G)) + return Err; + return Error::success(); +} + +void JITLinkerBase::layOutAtoms() { + // Group sections by protections, and whether or not they're zero-fill. + for (auto &S : G->sections()) { + + // Skip empty sections. + if (S.atoms_empty()) + continue; + + auto &SL = Layout[S.getProtectionFlags()]; + if (S.isZeroFill()) + SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + else + SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + } + + // Sort sections within the layout by ordinal. + { + auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, + const SegmentLayout::SectionLayout &RHS) { + return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + }; + for (auto &KV : Layout) { + auto &SL = KV.second; + std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), + CompareByOrdinal); + std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), + CompareByOrdinal); + } + } + + // Add atoms to the sections. + for (auto &KV : Layout) { + auto &SL = KV.second; + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { + for (auto &SI : *SIList) { + // First build the set of layout-heads (i.e. "heads" of layout-next + // chains) by copying the section atoms, then eliminating any that + // appear as layout-next targets. + DenseSet<DefinedAtom *> LayoutHeads; + for (auto *DA : SI.S->atoms()) + LayoutHeads.insert(DA); + + for (auto *DA : SI.S->atoms()) + if (DA->hasLayoutNext()) + LayoutHeads.erase(&DA->getLayoutNext()); + + // Next, sort the layout heads by address order. + std::vector<DefinedAtom *> OrderedLayoutHeads; + OrderedLayoutHeads.reserve(LayoutHeads.size()); + for (auto *DA : LayoutHeads) + OrderedLayoutHeads.push_back(DA); + + // Now sort the list of layout heads by address. + std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Now populate the SI.Atoms field by appending each of the chains. + for (auto *DA : OrderedLayoutHeads) { + SI.Atoms.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + SI.Atoms.push_back(&Next); + DA = &Next; + } + } + } + } + } + + LLVM_DEBUG({ + dbgs() << "Segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; + auto &SL = KV.second; + for (auto &SIEntry : + {std::make_pair(&SL.ContentSections, "content sections"), + std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { + auto &SIList = *SIEntry.first; + dbgs() << " " << SIEntry.second << ":\n"; + for (auto &SI : SIList) { + dbgs() << " " << SI.S->getName() << ":\n"; + for (auto *DA : SI.Atoms) + dbgs() << " " << *DA << "\n"; + } + } + } + }); +} + +Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + // Calculate segment content size. + size_t SegContentSize = 0; + for (auto &SI : SegLayout.ContentSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + + // Bump to section alignment before processing atoms. + SegContentSize = alignTo(SegContentSize, SI.S->getAlignment()); + + for (auto *DA : SI.Atoms) { + SegContentSize = alignTo(SegContentSize, DA->getAlignment()); + SegContentSize += DA->getSize(); + } + } + + // Get segment content alignment. + unsigned SegContentAlign = 1; + if (!SegLayout.ContentSections.empty()) { + auto &FirstContentSection = SegLayout.ContentSections.front(); + SegContentAlign = + std::max(FirstContentSection.S->getAlignment(), + FirstContentSection.Atoms.front()->getAlignment()); + } + + // Calculate segment zero-fill size. + uint64_t SegZeroFillSize = 0; + for (auto &SI : SegLayout.ZeroFillSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + + // Bump to section alignment before processing atoms. + SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment()); + + for (auto *DA : SI.Atoms) { + SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); + SegZeroFillSize += DA->getSize(); + } + } + + // Calculate segment zero-fill alignment. + uint32_t SegZeroFillAlign = 1; + + if (!SegLayout.ZeroFillSections.empty()) { + auto &FirstZeroFillSection = SegLayout.ZeroFillSections.front(); + SegZeroFillAlign = + std::max(FirstZeroFillSection.S->getAlignment(), + FirstZeroFillSection.Atoms.front()->getAlignment()); + } + + if (SegContentSize == 0) + SegContentAlign = SegZeroFillAlign; + + if (SegContentAlign % SegZeroFillAlign != 0) + return make_error<JITLinkError>("First content atom alignment does not " + "accommodate first zero-fill atom " + "alignment"); + + Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, + SegZeroFillAlign}; + + LLVM_DEBUG({ + dbgs() << (&KV == &*Layout.begin() ? "" : "; ") + << static_cast<sys::Memory::ProtectionFlags>(Prot) << ": " + << SegContentSize << " content bytes (alignment " + << SegContentAlign << ") + " << SegZeroFillSize + << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + }); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << "JIT linker got working memory:\n"; + for (auto &KV : Layout) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first); + dbgs() << " " << Prot << ": " + << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + } + }); + + // Update atom target addresses. + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SL = KV.second; + + JITTargetAddress AtomTargetAddr = + Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); + + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) + for (auto &SI : *SIList) { + AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment()); + for (auto *DA : SI.Atoms) { + AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); + DA->setAddress(AtomTargetAddr); + AtomTargetAddr += DA->getSize(); + } + } + } + + return Error::success(); +} + +DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const { + // Identify unresolved external atoms. + DenseSet<StringRef> UnresolvedExternals; + for (auto *DA : G->external_atoms()) { + assert(DA->getAddress() == 0 && + "External has already been assigned an address"); + assert(DA->getName() != StringRef() && DA->getName() != "" && + "Externals must be named"); + UnresolvedExternals.insert(DA->getName()); + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto &KV : Result) { + Atom &A = G->getAtomByName(KV.first); + assert(A.getAddress() == 0 && "Atom already resolved"); + A.setAddress(KV.second.getAddress()); + } + + LLVM_DEBUG({ + dbgs() << "Externals after applying lookup result:\n"; + for (auto *A : G->external_atoms()) + dbgs() << " " << A->getName() << ": " + << formatv("{0:x16}", A->getAddress()) << "\n"; + }); + assert(llvm::all_of(G->external_atoms(), + [](Atom *A) { return A->getAddress() != 0; }) && + "All atoms should have been resolved by this point"); +} + +void JITLinkerBase::deallocateAndBailOut(Error Err) { + assert(Err && "Should not be bailing out on success value"); + assert(Alloc && "can not call deallocateAndBailOut before allocation"); + Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate())); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void prune(AtomGraph &G) { + std::vector<DefinedAtom *> Worklist; + DenseMap<DefinedAtom *, std::vector<Edge *>> EdgesToUpdate; + + // Build the initial worklist from all atoms initially live. + for (auto *DA : G.defined_atoms()) { + if (!DA->isLive() || DA->shouldDiscard()) + continue; + + for (auto &E : DA->edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Propagate live flags to all atoms reachable from the initial live set. + while (!Worklist.empty()) { + DefinedAtom &NextLive = *Worklist.back(); + Worklist.pop_back(); + + assert(!NextLive.shouldDiscard() && + "should-discard nodes should never make it into the worklist"); + + // If this atom has already been marked as live, or is marked to be + // discarded, then skip it. + if (NextLive.isLive()) + continue; + + // Otherwise set it as live and add any non-live atoms that it points to + // to the worklist. + NextLive.setLive(true); + + for (auto &E : NextLive.edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Collect atoms to remove, then remove them from the graph. + std::vector<DefinedAtom *> AtomsToRemove; + for (auto *DA : G.defined_atoms()) + if (DA->shouldDiscard() || !DA->isLive()) + AtomsToRemove.push_back(DA); + + LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); + for (auto *DA : AtomsToRemove) { + LLVM_DEBUG(dbgs() << " " << *DA << "... "); + + // Check whether we need to replace this atom with an external atom. + // + // We replace if all of the following hold: + // (1) The atom is marked should-discard, + // (2) it has live edges (i.e. edges from live atoms) pointing to it. + // + // Otherwise we simply delete the atom. + + G.removeDefinedAtom(*DA); + + auto EdgesToUpdateItr = EdgesToUpdate.find(DA); + if (EdgesToUpdateItr != EdgesToUpdate.end()) { + auto &ExternalReplacement = G.addExternalAtom(DA->getName()); + for (auto *EdgeToUpdate : EdgesToUpdateItr->second) + EdgeToUpdate->setTarget(ExternalReplacement); + LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); + } else + LLVM_DEBUG(dbgs() << "deleted\n"); + } + + // Finally, discard any absolute symbols that were marked should-discard. + { + std::vector<Atom *> AbsoluteAtomsToRemove; + for (auto *A : G.absolute_atoms()) + if (A->shouldDiscard() || A->isLive()) + AbsoluteAtomsToRemove.push_back(A); + for (auto *A : AbsoluteAtomsToRemove) + G.removeAbsoluteAtom(*A); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h new file mode 100644 index 000000000000..e6fd6e38f7a6 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,256 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr<JITLinkContext> Ctx, PassConfiguration Passes) + : Ctx(std::move(Ctx)), Passes(std::move(Passes)) { + assert(this->Ctx && "Ctx can not be null"); + } + + virtual ~JITLinkerBase(); + +protected: + struct SegmentLayout { + using SectionAtomsList = std::vector<DefinedAtom *>; + struct SectionLayout { + SectionLayout(Section &S) : S(&S) {} + + Section *S; + SectionAtomsList Atoms; + }; + + using SectionLayoutList = std::vector<SectionLayout>; + + SectionLayoutList ContentSections; + SectionLayoutList ZeroFillSections; + }; + + using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; + + // Phase 1: + // 1.1: Build atom graph + // 1.2: Run pre-prune passes + // 1.2: Prune graph + // 1.3: Run post-prune passes + // 1.4: Sort atoms into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr<JITLinkerBase> Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up atom contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LookupResult); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err); + + // Build a graph from the given object buffer. + // To be implemented by the client. + virtual Expected<std::unique_ptr<AtomGraph>> + buildGraph(MemoryBufferRef ObjBuffer) = 0; + + // For debug dumping of the atom graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + +private: + // Run all passes in the given pass list, bailing out immediately if any pass + // returns an error. + Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + + // Copy atom contents and apply relocations. + // Implemented in JITLinker. + virtual Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; + + void layOutAtoms(); + Error allocateSegments(const SegmentLayoutMap &Layout); + DenseSet<StringRef> getExternalSymbolNames() const; + void applyLookupResult(AsyncLookupResult LR); + void deallocateAndBailOut(Error Err); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr<JITLinkContext> Ctx; + PassConfiguration Passes; + std::unique_ptr<AtomGraph> G; + SegmentLayoutMap Layout; + std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; +}; + +template <typename LinkerImpl> class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template <typename... ArgTs> static void link(ArgTs &&... Args) { + auto L = llvm::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast<const LinkerImpl &>(*this); + } + + Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = Alloc.getWorkingMemory( + static_cast<sys::Memory::ProtectionFlags>(Prot)); + char *LastAtomEnd = SegMem.data(); + char *AtomDataPtr = LastAtomEnd; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto &SI : SegLayout.ContentSections) { + LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + + AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment()); + + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr + << " to meet section alignment " + << " of " << SI.S->getAlignment() << "\n"; + }); + + for (auto *DA : SI.Atoms) { + + // Align. + AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " + << (const void *)AtomDataPtr << " to meet alignment of " + << DA->getAlignment() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastAtomEnd != AtomDataPtr) + dbgs() << " Zero padding from " << (const void *)LastAtomEnd + << " to " << (const void *)AtomDataPtr << "\n"; + }); + while (LastAtomEnd != AtomDataPtr) + *LastAtomEnd++ = 0; + + // Copy initial atom content. + LLVM_DEBUG({ + dbgs() << " Copying atom " << *DA << " content, " + << DA->getContent().size() << " bytes, from " + << (const void *)DA->getContent().data() << " to " + << (const void *)AtomDataPtr << "\n"; + }); + memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); + + // Copy atom data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : DA->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) + return Err; + } + + // Point the atom's content to the fixed up buffer. + DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); + + // Update atom end pointer. + LastAtomEnd = AtomDataPtr + DA->getContent().size(); + AtomDataPtr = LastAtomEnd; + } + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastAtomEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastAtomEnd != SegMem.data() + SegMem.size()) + *LastAtomEnd++ = 0; + } + + return Error::success(); + } +}; + +/// Dead strips and replaces discarded definitions with external atoms. +/// +/// Finds the set of nodes reachable from any node initially marked live +/// (nodes marked should-discard are treated as not live, even if they are +/// reachable). All nodes not marked as live at the end of this process, +/// are deleted. Nodes that are live, but marked should-discard are replaced +/// with external atoms and all edges to them are re-written. +void prune(AtomGraph &G); + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp new file mode 100644 index 000000000000..267307cfde05 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -0,0 +1,105 @@ +//===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/Support/Process.h" + +namespace llvm { +namespace jitlink { + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; +JITLinkMemoryManager::Allocation::~Allocation() = default; + +Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>> +InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast<char *>(SegBlocks[Seg].base()), + SegBlocks[Seg].allocatedSize()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return reinterpret_cast<JITTargetAddress>(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + for (auto &KV : SegBlocks) + if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + for (auto &KV : Request) { + auto &Seg = KV.second; + + if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate()) + return make_error<StringError>("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0) + return make_error<StringError>("Page size is not a multiple of " + "alignment", + inconvertibleErrorCode()); + + uint64_t ZeroFillStart = + alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); + uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + + std::error_code EC; + auto SegMem = + sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Zero out the zero-fill memory. + memset(static_cast<char *>(SegMem.base()) + ZeroFillStart, 0, + Seg.getZeroFillSize()); + + // Record the block for this segment. + Blocks[KV.first] = std::move(SegMem); + } + return std::unique_ptr<InProcessMemoryManager::Allocation>( + new IPMMAlloc(std::move(Blocks))); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/MachO.cpp new file mode 100644 index 000000000000..15995b8ce98f --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/MachO.cpp @@ -0,0 +1,78 @@ +//===-------------- MachO.cpp - JIT linker function for MachO -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { + + // We don't want to do full MachO validation here. Just parse enough of the + // header to find out what MachO linker to use. + + StringRef Data = Ctx->getObjectBuffer().getBuffer(); + if (Data.size() < 4) { + Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer")); + return; + } + + uint32_t Magic; + memcpy(&Magic, Data.data(), sizeof(uint32_t)); + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) + << ", identifier = \"" + << Ctx->getObjectBuffer().getBufferIdentifier() << "\"\n"; + }); + + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) { + Ctx->notifyFailed( + make_error<JITLinkError>("MachO 32-bit platforms not supported")); + return; + } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + MachO::mach_header_64 Header; + + memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64)); + if (Magic == MachO::MH_CIGAM_64) + swapStruct(Header); + + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: cputype = " + << format("0x%08" PRIx32, Header.cputype) + << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype) + << "\n"; + }); + + switch (Header.cputype) { + case MachO::CPU_TYPE_X86_64: + return jitLink_MachO_x86_64(std::move(Ctx)); + } + Ctx->notifyFailed(make_error<JITLinkError>("MachO-64 CPU type not valid")); + return; + } + + Ctx->notifyFailed(make_error<JITLinkError>("MachO magic not valid")); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp new file mode 100644 index 000000000000..1501c7ad0bc5 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp @@ -0,0 +1,411 @@ +//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} + +Expected<std::unique_ptr<AtomGraph>> MachOAtomGraphBuilder::buildGraph() { + if (auto Err = parseSections()) + return std::move(Err); + + if (auto Err = addAtoms()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(llvm::make_unique<AtomGraph>(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, + CustomAtomizeFunction Atomizer) { + assert(!CustomAtomizeFunctions.count(SectionName) && + "Custom atomizer for this section already exists"); + CustomAtomizeFunctions[SectionName] = std::move(Atomizer); +} + +bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) { + // If these atoms are the same then they're trivially "locked". + if (&A == &B) + return true; + + // If A and B are different, check whether either is undefined. (in which + // case they are not locked). + if (!A.isDefined() || !B.isDefined()) + return false; + + // A and B are different, but they're both defined atoms. We need to check + // whether they're part of the same alt_entry chain. + auto &DA = static_cast<const DefinedAtom &>(A); + auto &DB = static_cast<const DefinedAtom &>(B); + + auto AStartItr = AltEntryStarts.find(&DA); + if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out. + return false; + + auto BStartItr = AltEntryStarts.find(&DB); + if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out. + return false; + + // A and B are layout locked if they're in the same chain. + return AStartItr->second == BStartItr->second; +} + +unsigned +MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { + if (!CommonSymbolsSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + auto &GenericSection = G->createSection("<common>", 1, Prot, true); + CommonSymbolsSection = MachOSection(GenericSection); + } + return *CommonSymbolsSection; +} + +Error MachOAtomGraphBuilder::parseSections() { + for (auto &SecRef : Obj.sections()) { + assert((SecRef.getAlignment() <= std::numeric_limits<uint32_t>::max()) && + "Section alignment does not fit in 32 bits"); + + StringRef Name; + if (auto EC = SecRef.getName(Name)) + return errorCodeToError(EC); + + unsigned SectionIndex = SecRef.getIndex() + 1; + + uint32_t Align = SecRef.getAlignment(); + if (!isPowerOf2_32(Align)) + return make_error<JITLinkError>("Section " + Name + + " has non-power-of-2 " + "alignment"); + + // FIXME: Get real section permissions + // How, exactly, on MachO? + sys::Memory::ProtectionFlags Prot; + if (SecRef.isText()) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS()); + + LLVM_DEBUG({ + dbgs() << "Adding section " << Name << ": " + << format("0x%016" PRIx64, SecRef.getAddress()) + << ", align: " << SecRef.getAlignment() << "\n"; + }); + + assert(!Sections.count(SectionIndex) && "Section index already in use"); + + auto &MachOSec = + Sections + .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(), + SecRef.getAlignment()) + .first->second; + + if (!SecRef.isVirtual()) { + // If this section has content then record it. + Expected<StringRef> Content = SecRef.getContents(); + if (!Content) + return Content.takeError(); + if (Content->size() != SecRef.getSize()) + return make_error<JITLinkError>("Section content size does not match " + "declared size for " + + Name); + MachOSec.setContent(*Content); + } else { + // If this is a zero-fill section then just record the size. + MachOSec.setZeroFill(SecRef.getSize()); + } + + uint32_t SectionFlags = + Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags + : Obj.getSection(SecRef.getRawDataRefImpl()).flags; + + MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP); + } + + return Error::success(); +} + +// Adds atoms with identified start addresses (but not lengths) for all named +// atoms. +// Also, for every section that contains named atoms, but does not have an +// atom at offset zero of that section, constructs an anonymous atom covering +// that range. +Error MachOAtomGraphBuilder::addNonCustomAtoms() { + using AddrToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; + DenseMap<MachOSection *, AddrToAtomMap> SecToAtoms; + + DenseMap<MachOSection *, unsigned> FirstOrdinal; + std::vector<DefinedAtom *> AltEntryAtoms; + + DenseSet<StringRef> ProcessedSymbols; // Used to check for duplicate defs. + + for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; + ++SymI) { + object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + + // Bail out on duplicate definitions: There should never be more than one + // definition for a symbol in a given object file. + if (ProcessedSymbols.count(*Name)) + return make_error<JITLinkError>("Duplicate definition within object: " + + *Name); + else + ProcessedSymbols.insert(*Name); + + auto Addr = Sym.getAddress(); + if (!Addr) + return Addr.takeError(); + + auto SymType = Sym.getType(); + if (!SymType) + return SymType.takeError(); + + auto Flags = Sym.getFlags(); + + if (Flags & object::SymbolRef::SF_Undefined) { + LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); + G->addExternalAtom(*Name); + continue; + } else if (Flags & object::SymbolRef::SF_Absolute) { + LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " + << format("0x%016" PRIx64, *Addr) << "\n"); + auto &A = G->addAbsoluteAtom(*Name, *Addr); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + continue; + } else if (Flags & object::SymbolRef::SF_Common) { + LLVM_DEBUG({ + dbgs() << "Adding common \"" << *Name + << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; + }); + auto &A = + G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U), + Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + continue; + } + + LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); + + // This atom is neither undefined nor absolute, so it must be defined in + // this object. Get its section index. + auto SecItr = Sym.getSection(); + if (!SecItr) + return SecItr.takeError(); + + uint64_t SectionIndex = (*SecItr)->getIndex() + 1; + + LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); + + auto SecByIndexItr = Sections.find(SectionIndex); + if (SecByIndexItr == Sections.end()) + return make_error<JITLinkError>("Unrecognized section index in macho"); + + auto &Sec = SecByIndexItr->second; + + auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U)); + + DA.setGlobal(Flags & object::SymbolRef::SF_Global); + DA.setExported(Flags & object::SymbolRef::SF_Exported); + DA.setWeak(Flags & object::SymbolRef::SF_Weak); + + DA.setCallable(*SymType & object::SymbolRef::ST_Function); + + // Check NDesc flags. + { + uint16_t NDesc = 0; + if (Obj.is64Bit()) + NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc; + else + NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc; + + // Record atom for alt-entry post-processing (where the layout-next + // constraints will be added). + if (NDesc & MachO::N_ALT_ENTRY) + AltEntryAtoms.push_back(&DA); + + // If this atom has a no-dead-strip attr attached then mark it live. + if (NDesc & MachO::N_NO_DEAD_STRIP) + DA.setLive(true); + } + + LLVM_DEBUG({ + dbgs() << " Added " << *Name + << " addr: " << format("0x%016" PRIx64, *Addr) + << ", align: " << DA.getAlignment() + << ", section: " << Sec.getGenericSection().getName() << "\n"; + }); + + auto &SecAtoms = SecToAtoms[&Sec]; + SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA; + } + + // Add anonymous atoms. + for (auto &KV : Sections) { + auto &S = KV.second; + + // Skip empty sections. + if (S.empty()) + continue; + + // Skip sections with custom handling. + if (CustomAtomizeFunctions.count(S.getName())) + continue; + + auto SAI = SecToAtoms.find(&S); + + // If S is not in the SecToAtoms map then it contained no named atom. Add + // one anonymous atom to cover the whole section. + if (SAI == SecToAtoms.end()) { + SecToAtoms[&S][0] = &G->addAnonymousAtom( + S.getGenericSection(), S.getAddress(), S.getAlignment()); + continue; + } + + // Otherwise, check whether this section had an atom covering offset zero. + // If not, add one. + auto &SecAtoms = SAI->second; + if (!SecAtoms.count(0)) + SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getAlignment()); + } + + LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); + + // Set atom contents and any section-based flags. + for (auto &KV : SecToAtoms) { + auto &S = *KV.first; + auto &SecAtoms = KV.second; + + // Iterate the atoms in reverse order and set up their contents. + JITTargetAddress LastAtomAddr = S.getSize(); + for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { + auto Offset = I->first; + auto &A = *I->second; + LLVM_DEBUG({ + dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " + << S.getAddress() + LastAtomAddr << " ]\n"; + }); + + if (S.isZeroFill()) + A.setZeroFill(LastAtomAddr - Offset); + else + A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); + + // If the section has no-dead-strip set then mark the atom as live. + if (S.isNoDeadStrip()) + A.setLive(true); + + LastAtomAddr = Offset; + } + } + + LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n"); + + // Sort alt-entry atoms by address in ascending order. + llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Process alt-entry atoms in address order to build the table of alt-entry + // atoms to alt-entry chain starts. + for (auto *DA : AltEntryAtoms) { + assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts"); + + // DA is an alt-entry atom. Look for the predecessor atom that it is locked + // to, bailing out if we do not find one. + auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1); + if (!AltEntryPred) + return AltEntryPred.takeError(); + + // Add a LayoutNext edge from the predecessor to this atom. + AltEntryPred->setLayoutNext(*DA); + + // Check to see whether the predecessor itself is an alt-entry atom. + auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred); + if (AltEntryStartItr != AltEntryStarts.end()) { + // If the predecessor was an alt-entry atom then re-use its value. + LLVM_DEBUG({ + dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second + << " (based on existing entry for " << *AltEntryPred << ")\n"; + }); + AltEntryStarts[DA] = AltEntryStartItr->second; + } else { + // If the predecessor does not have an entry then add an entry for this + // atom (i.e. the alt_entry atom) and a self-reference entry for the + /// predecessory atom that is the start of this chain. + LLVM_DEBUG({ + dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n" + << " " << *DA << " -> " << *AltEntryPred << "\n"; + }); + AltEntryStarts[&*AltEntryPred] = &*AltEntryPred; + AltEntryStarts[DA] = &*AltEntryPred; + } + } + + return Error::success(); +} + +Error MachOAtomGraphBuilder::addAtoms() { + // Add all named atoms. + if (auto Err = addNonCustomAtoms()) + return Err; + + // Process special sections. + for (auto &KV : Sections) { + auto &S = KV.second; + auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); + if (HI != CustomAtomizeFunctions.end()) { + auto &Atomize = HI->second; + if (auto Err = Atomize(S)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h new file mode 100644 index 000000000000..72d441b24d06 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h @@ -0,0 +1,138 @@ +//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "JITLinkGeneric.h" + +#include "llvm/Object/MachO.h" + +namespace llvm { +namespace jitlink { + +class MachOAtomGraphBuilder { +public: + virtual ~MachOAtomGraphBuilder(); + Expected<std::unique_ptr<AtomGraph>> buildGraph(); + +protected: + using OffsetToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; + + class MachOSection { + public: + MachOSection() = default; + + /// Create a MachO section with the given address and alignment. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment) + : Address(Address), GenericSection(&GenericSection), + Alignment(Alignment) {} + + /// Create a section without address, content or size (used for common + /// symbol sections). + MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} + + Section &getGenericSection() const { + assert(GenericSection && "Section is null"); + return *GenericSection; + } + + StringRef getName() const { + assert(GenericSection && "No generic section attached"); + return GenericSection->getName(); + } + + MachOSection &setContent(StringRef Content) { + assert(!ContentPtr && !Size && "Content/zeroFill already set"); + ContentPtr = Content.data(); + Size = Content.size(); + return *this; + } + + MachOSection &setZeroFill(uint64_t Size) { + assert(!ContentPtr && !this->Size && "Content/zeroFill already set"); + this->Size = Size; + return *this; + } + + bool isZeroFill() const { return !ContentPtr; } + + bool empty() const { return getSize() == 0; } + + size_t getSize() const { return Size; } + + StringRef getContent() const { + assert(ContentPtr && "getContent() called on zero-fill section"); + return {ContentPtr, static_cast<size_t>(Size)}; + } + + JITTargetAddress getAddress() const { return Address; } + + unsigned getAlignment() const { return Alignment; } + + MachOSection &setNoDeadStrip(bool NoDeadStrip) { + this->NoDeadStrip = NoDeadStrip; + return *this; + } + + bool isNoDeadStrip() const { return NoDeadStrip; } + + private: + JITTargetAddress Address = 0; + Section *GenericSection = nullptr; + const char *ContentPtr = nullptr; + uint64_t Size = 0; + unsigned Alignment = 0; + bool NoDeadStrip = false; + }; + + using CustomAtomizeFunction = std::function<Error(MachOSection &S)>; + + MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); + + AtomGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); + + virtual Error addRelocations() = 0; + + /// Returns true if Atom A and Atom B are at a fixed offset from one another + /// (i.e. if they're part of the same alt-entry chain). + bool areLayoutLocked(const Atom &A, const Atom &B); + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + MachOSection &getCommonSection(); + + Error parseSections(); + Error addNonCustomAtoms(); + Error addAtoms(); + + const object::MachOObjectFile &Obj; + std::unique_ptr<AtomGraph> G; + DenseMap<const DefinedAtom *, const DefinedAtom *> AltEntryStarts; + DenseMap<unsigned, MachOSection> Sections; + StringMap<CustomAtomizeFunction> CustomAtomizeFunctions; + Optional<MachOSection> CommonSymbolsSection; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/contrib/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/contrib/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp new file mode 100644 index 000000000000..4010678c6d33 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -0,0 +1,608 @@ +//===---- MachO_x86_64.cpp -JIT linker implementation for MachO/x86-64 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO/x86-64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +public: + MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOAtomGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { + return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), + EHFrameSection.getContent(), + EHFrameSection.getAddress(), NegDelta32, Delta64); + }); + } + +private: + static Expected<MachOX86RelocationKind> + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel && RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta<W>'. They may + // be turned into NegDelta<W> by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error<JITLinkError>( + "Unsupported x86-64 relocation: address=" + + formatv("{0:x8}", RI.r_address) + + ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + + ", kind=" + formatv("{0:x1}", RI.r_type) + + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + + ", extern= " + (RI.r_extern ? "true" : "false") + + ", length=" + formatv("{0:d}", RI.r_length)); + } + + Expected<Atom &> findAtomBySymbolIndex(const MachO::relocation_info &RI) { + auto &Obj = getObject(); + if (RI.r_symbolnum >= NumSymbols) + return make_error<JITLinkError>("Symbol index out of range"); + auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); + auto Name = SymI->getName(); + if (!Name) + return Name.takeError(); + return getGraph().getAtomByName(*Name); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = std::tuple<MachOX86RelocationKind, Atom *, uint64_t>; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected<PairRelocInfo> + parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error<JITLinkError>("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error<JITLinkError>("x86_64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + auto FromAtom = findAtomBySymbolIndex(SubRI); + if (!FromAtom) + return FromAtom.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const little64_t *)FixupContent; + else + FixupValue = *(const little32_t *)FixupContent; + + // Find 'ToAtom' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Atom *ToAtom = nullptr; + if (UnsignedRI.r_extern) { + // Find target atom by symbol index. + if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + } else { + if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + FixupValue -= ToAtom->getAddress(); + } + + MachOX86RelocationKind DeltaKind; + Atom *TargetAtom; + uint64_t Addend; + if (areLayoutLocked(AtomToFix, *FromAtom)) { + TargetAtom = ToAtom; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + // FIXME: handle extern 'from'. + } else if (areLayoutLocked(AtomToFix, *ToAtom)) { + TargetAtom = &*FromAtom; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + } else { + // AtomToFix was neither FromAtom nor ToAtom. + return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " + "either 'A' or 'B' (or an atom in one " + "of their alt-entry groups)"); + } + + return PairRelocInfo(DeltaKind, TargetAtom, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &G = getGraph(); + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + + // Find the atom that the fixup points to. + DefinedAtom *AtomToFix = nullptr; + { + auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); + if (!AtomToFixOrErr) + return AtomToFixOrErr.takeError(); + AtomToFix = &*AtomToFixOrErr; + } + + if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + AtomToFix->getAddress() + AtomToFix->getContent().size()) + return make_error<JITLinkError>( + "Relocation content extends past end of fixup atom"); + + // Get a pointer to the fixup content. + const char *FixupContent = AtomToFix->getContent().data() + + (FixupAddress - AtomToFix->getAddress()); + + // The target atom and addend will be populated by the switch below. + Atom *TargetAtom = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = + static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetAtom, Addend) = *PairInfo; + assert(TargetAtom && "No target atom from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Addend); + printEdge(dbgs(), *AtomToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachO_x86_64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { +public: + MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G) + : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + } + + DefinedAtom &createGOTEntry(Atom &Target) { + auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8); + GOTEntryAtom.setContent( + StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 8)); + GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); + return GOTEntryAtom; + } + + void fixGOTEdge(Edge &E, Atom &GOTEntry) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + E.setKind(PCRel32); + E.setTarget(GOTEntry); + // Leave the edge addend as-is. + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch32 && !E.getTarget().isDefined(); + } + + DefinedAtom &createStub(Atom &Target) { + auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2); + StubAtom.setContent( + StringRef(reinterpret_cast<const char *>(StubContent), 6)); + + // Re-use GOT entries for stub targets. + auto &GOTEntryAtom = getGOTEntryAtom(Target); + StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); + + return StubAtom; + } + + void fixExternalBranchEdge(Edge &E, Atom &Stub) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false); + } + return *StubsSection; + } + + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachO_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker<MachOJITLinker_x86_64> { + friend class JITLinker<MachOJITLinker_x86_64>; + +public: + MachOJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + Expected<std::unique_ptr<AtomGraph>> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Atom &A, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + using namespace support; + + char *FixupPtr = AtomWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch32: + case PCRel32: + case PCRel32Anon: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + Triple TT("x86_64-apple-macosx"); + + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllAtomsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + MachO_x86_64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); +} + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp index ffc6707e1488..08815b7a80ae 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -1,9 +1,8 @@ //===-- MCJIT.cpp - MC-based Just-in-Time Compiler ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h index 1119e138720f..77097fc0d17e 100644 --- a/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/contrib/llvm/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -1,9 +1,8 @@ //===-- MCJIT.h - Class definition for the MCJIT ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index 21af6b585c41..2ad9d24555f3 100644 --- a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- OProfileJITEventListener.cpp - Tell OProfile about JITted code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp index b473ac3faf4c..1a2667736926 100644 --- a/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp +++ b/contrib/llvm/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp @@ -1,9 +1,8 @@ //===-- OProfileWrapper.cpp - OProfile JIT API Wrapper implementation -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 241eb3600da7..99bf53bc3afa 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -1,9 +1,8 @@ //===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp new file mode 100644 index 000000000000..d46b6fcf9a5f --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -0,0 +1,86 @@ +//===------ CompileUtils.cpp - Utilities for compiling IR in the JIT ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include <algorithm> + +namespace llvm { +namespace orc { + +/// Compile a Module to an ObjectFile. +SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { + CompileResult CachedObject = tryToLoadFromObjectCache(M); + if (CachedObject) + return CachedObject; + + SmallVector<char, 0> ObjBufferSV; + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + PM.run(M); + } + + auto ObjBuffer = llvm::make_unique<SmallVectorMemoryBuffer>( + std::move(ObjBufferSV), + "<in memory object compiled from " + M.getModuleIdentifier() + ">"); + + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (Obj) { + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); + } + + // TODO: Actually report errors helpfully. + consumeError(Obj.takeError()); + return nullptr; +} + +SimpleCompiler::CompileResult +SimpleCompiler::tryToLoadFromObjectCache(const Module &M) { + if (!ObjCache) + return CompileResult(); + + return ObjCache->getObject(&M); +} + +void SimpleCompiler::notifyObjectCompiled(const Module &M, + const MemoryBuffer &ObjBuffer) { + if (ObjCache) + ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); +} + +ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache) + : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + +std::unique_ptr<MemoryBuffer> ConcurrentIRCompiler::operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp index 73c0bcdf7d28..dac37e030e0c 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -1,9 +1,8 @@ //===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,17 +26,17 @@ namespace { #ifndef NDEBUG -cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(false), +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), cl::desc("debug print hidden symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(false), +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), cl::desc("debug print callable symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintData("debug-orc-print-data", cl::init(false), +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), cl::desc("debug print data symbols defined by " "materialization units"), cl::Hidden); @@ -134,8 +133,6 @@ struct PrintSymbolMapElemsMatchingCLOpts { namespace llvm { namespace orc { - SymbolStringPool::PoolMapEntry SymbolStringPtr::Tombstone(0); - char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; @@ -222,6 +219,31 @@ raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { return OS; } +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }\n"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) : Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); @@ -262,85 +284,46 @@ void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { } AsynchronousSymbolQuery::AsynchronousSymbolQuery( - const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved, - SymbolsReadyCallback NotifySymbolsReady) - : NotifySymbolsResolved(std::move(NotifySymbolsResolved)), - NotifySymbolsReady(std::move(NotifySymbolsReady)) { - NotYetResolvedCount = NotYetReadyCount = Symbols.size(); + const SymbolNameSet &Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete) + : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { + assert(RequiredState >= SymbolState::Resolved && + "Cannot query for a symbols that have not reached the resolve state " + "yet"); + + OutstandingSymbolsCount = Symbols.size(); for (auto &S : Symbols) ResolvedSymbols[S] = nullptr; } -void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, - JITEvaluatedSymbol Sym) { +void AsynchronousSymbolQuery::notifySymbolMetRequiredState( + const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { auto I = ResolvedSymbols.find(Name); assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); I->second = std::move(Sym); - --NotYetResolvedCount; -} - -void AsynchronousSymbolQuery::handleFullyResolved() { - assert(NotYetResolvedCount == 0 && "Not fully resolved?"); - - if (!NotifySymbolsResolved) { - // handleFullyResolved may be called by handleFullyReady (see comments in - // that method), in which case this is a no-op, so bail out. - assert(!NotifySymbolsReady && - "NotifySymbolsResolved already called or an error occurred"); - return; - } - - auto TmpNotifySymbolsResolved = std::move(NotifySymbolsResolved); - NotifySymbolsResolved = SymbolsResolvedCallback(); - TmpNotifySymbolsResolved(std::move(ResolvedSymbols)); -} - -void AsynchronousSymbolQuery::notifySymbolReady() { - assert(NotYetReadyCount != 0 && "All symbols already emitted"); - --NotYetReadyCount; + --OutstandingSymbolsCount; } -void AsynchronousSymbolQuery::handleFullyReady() { - assert(NotifySymbolsReady && - "NotifySymbolsReady already called or an error occurred"); +void AsynchronousSymbolQuery::handleComplete() { + assert(OutstandingSymbolsCount == 0 && + "Symbols remain, handleComplete called prematurely"); - auto TmpNotifySymbolsReady = std::move(NotifySymbolsReady); - NotifySymbolsReady = SymbolsReadyCallback(); - - if (NotYetResolvedCount == 0 && NotifySymbolsResolved) { - // The NotifyResolved callback of one query must have caused this query to - // become ready (i.e. there is still a handleFullyResolved callback waiting - // to be made back up the stack). Fold the handleFullyResolved call into - // this one before proceeding. This will cause the call further up the - // stack to become a no-op. - handleFullyResolved(); - } - - assert(QueryRegistrations.empty() && - "Query is still registered with some symbols"); - assert(!NotifySymbolsResolved && "Resolution not applied yet"); - TmpNotifySymbolsReady(Error::success()); + auto TmpNotifyComplete = std::move(NotifyComplete); + NotifyComplete = SymbolsResolvedCallback(); + TmpNotifyComplete(std::move(ResolvedSymbols)); } -bool AsynchronousSymbolQuery::canStillFail() { - return (NotifySymbolsResolved || NotifySymbolsReady); -} +bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } void AsynchronousSymbolQuery::handleFailed(Error Err) { assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && - NotYetResolvedCount == 0 && NotYetReadyCount == 0 && + OutstandingSymbolsCount == 0 && "Query should already have been abandoned"); - if (NotifySymbolsResolved) { - NotifySymbolsResolved(std::move(Err)); - NotifySymbolsResolved = SymbolsResolvedCallback(); - } else { - assert(NotifySymbolsReady && "Failed after both callbacks issued?"); - NotifySymbolsReady(std::move(Err)); - } - NotifySymbolsReady = SymbolsReadyCallback(); + NotifyComplete(std::move(Err)); + NotifyComplete = SymbolsResolvedCallback(); } void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, @@ -363,8 +346,7 @@ void AsynchronousSymbolQuery::removeQueryDependence( void AsynchronousSymbolQuery::detach() { ResolvedSymbols.clear(); - NotYetResolvedCount = 0; - NotYetReadyCount = 0; + OutstandingSymbolsCount = 0; for (auto &KV : QueryRegistrations) KV.first->detachQueryHelper(*this, KV.second); QueryRegistrations.clear(); @@ -374,11 +356,6 @@ MaterializationResponsibility::MaterializationResponsibility( JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { assert(!this->SymbolFlags.empty() && "Materializing nothing?"); - -#ifndef NDEBUG - for (auto &KV : this->SymbolFlags) - KV.second |= JITSymbolFlags::Materializing; -#endif } MaterializationResponsibility::~MaterializationResponsibility() { @@ -390,16 +367,15 @@ SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { return JD.getRequestedSymbols(SymbolFlags); } -void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { - LLVM_DEBUG(dbgs() << "In " << JD.getName() << " resolving " << Symbols - << "\n"); +void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; + }); #ifndef NDEBUG for (auto &KV : Symbols) { auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); - assert(I->second.isMaterializing() && "Duplicate resolution"); - I->second &= ~JITSymbolFlags::Materializing; if (I->second.isWeak()) assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && "Resolving symbol with incorrect flags"); @@ -412,12 +388,11 @@ void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { JD.resolve(Symbols); } -void MaterializationResponsibility::emit() { -#ifndef NDEBUG - for (auto &KV : SymbolFlags) - assert(!KV.second.isMaterializing() && - "Failed to resolve symbol before emission"); -#endif // NDEBUG +void MaterializationResponsibility::notifyEmitted() { + + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; + }); JD.emit(SymbolFlags); SymbolFlags.clear(); @@ -429,19 +404,19 @@ Error MaterializationResponsibility::defineMaterializing( // It's ok if we hit a duplicate here: In that case the new version will be // discarded, and the JITDylib::defineMaterializing method will return a // duplicate symbol error. - for (auto &KV : NewSymbolFlags) { - auto I = SymbolFlags.insert(KV).first; - (void)I; -#ifndef NDEBUG - I->second |= JITSymbolFlags::Materializing; -#endif - } + for (auto &KV : NewSymbolFlags) + SymbolFlags.insert(KV); return JD.defineMaterializing(NewSymbolFlags); } void MaterializationResponsibility::failMaterialization() { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " failing materialization for " + << SymbolFlags << "\n"; + }); + SymbolNameSet FailedSymbols; for (auto &KV : SymbolFlags) FailedSymbols.insert(KV.first); @@ -510,8 +485,8 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { void AbsoluteSymbolsMaterializationUnit::materialize( MaterializationResponsibility R) { - R.resolve(Symbols); - R.emit(); + R.notifyResolved(Symbols); + R.notifyEmitted(); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -559,6 +534,14 @@ void ReExportsMaterializationUnit::materialize( Aliases.erase(I); } + LLVM_DEBUG({ + ES.runSessionLocked([&]() { + dbgs() << "materializing reexports: target = " << TgtJD.getName() + << ", source = " << SrcJD.getName() << " " << RequestedAliases + << "\n"; + }); + }); + if (!Aliases.empty()) { if (SourceJD) R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); @@ -641,7 +624,7 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnResolve = [QueryInfo](Expected<SymbolMap> Result) { + auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -650,8 +633,8 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - QueryInfo->R.resolve(ResolutionMap); - QueryInfo->R.emit(); + QueryInfo->R.notifyResolved(ResolutionMap); + QueryInfo->R.notifyEmitted(); } else { auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); ES.reportError(Result.takeError()); @@ -659,10 +642,8 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, - std::move(OnResolve), std::move(OnReady), + SymbolState::Resolved, std::move(OnComplete), std::move(RegisterDependencies)); } } @@ -687,17 +668,20 @@ Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { auto Flags = SourceJD.lookupFlags(Symbols); - if (Flags.size() != Symbols.size()) { + if (!Flags) + return Flags.takeError(); + + if (Flags->size() != Symbols.size()) { SymbolNameSet Unresolved = Symbols; - for (auto &KV : Flags) + for (auto &KV : *Flags) Unresolved.erase(KV.first); return make_error<SymbolsNotFound>(std::move(Unresolved)); } SymbolAliasMap Result; for (auto &Name : Symbols) { - assert(Flags.count(Name) && "Missing entry in flags map"); - Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]); + assert(Flags->count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]); } return Result; @@ -709,14 +693,17 @@ ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, : SourceJD(SourceJD), MatchNonExported(MatchNonExported), Allow(std::move(Allow)) {} -SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, - const SymbolNameSet &Names) { +Expected<SymbolNameSet> +ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolAliasMap AliasMap; auto Flags = SourceJD.lookupFlags(Names); - for (auto &KV : Flags) { + if (!Flags) + return Flags.takeError(); + + for (auto &KV : *Flags) { if (Allow && !Allow(KV.first)) continue; AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); @@ -731,21 +718,19 @@ SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { return ES.runSessionLocked([&]() -> Error { - std::vector<SymbolMap::iterator> AddedSyms; + std::vector<SymbolTable::iterator> AddedSyms; for (auto &KV : SymbolFlags) { - SymbolMap::iterator EntryItr; + SymbolTable::iterator EntryItr; bool Added; - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Materializing; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + std::tie(EntryItr, Added) = + Symbols.insert(std::make_pair(KV.first, SymbolTableEntry(KV.second))); - if (Added) + if (Added) { AddedSyms.push_back(EntryItr); - else { + EntryItr->second.setState(SymbolState::Materializing); + } else { // Remove any symbols already added. for (auto &SI : AddedSyms) Symbols.erase(SI); @@ -769,9 +754,10 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { for (auto &KV : MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI != Symbols.end() && "Replacing unknown symbol"); - assert(!SymI->second.getFlags().isLazy() && - SymI->second.getFlags().isMaterializing() && - "Can not replace symbol that is not materializing"); + assert(SymI->second.isInMaterializationPhase() && + "Can not call replace on a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Symbol should not have materializer attached already"); assert(UnmaterializedInfos.count(KV.first) == 0 && "Symbol being replaced should have no UnmaterializedInfo"); } @@ -782,7 +768,7 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { for (auto &KV : MU->getSymbols()) { auto MII = MaterializingInfos.find(KV.first); if (MII != MaterializingInfos.end()) { - if (!MII->second.PendingQueries.empty()) + if (MII->second.hasQueriesPending()) return std::move(MU); } } @@ -790,16 +776,15 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { // Otherwise, make MU responsible for all the symbols. auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) { - assert(!KV.second.isLazy() && - "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); - auto SymI = Symbols.find(KV.first); - JITSymbolFlags ReplaceFlags = KV.second; - ReplaceFlags |= JITSymbolFlags::Lazy; - SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(), - std::move(ReplaceFlags)); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Can not replace a symbol that has a materializer attached"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Unexpected materializer entry in map"); + SymI->second.setAddress(SymI->second.getAddress()); + SymI->second.setMaterializerAttached(true); UnmaterializedInfos[KV.first] = UMI; } @@ -817,14 +802,14 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { for (auto &KV : SymbolFlags) { assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); - assert(Symbols.find(KV.first)->second.getFlags().isMaterializing() && - "getRequestedSymbols can only be called for materializing " - "symbols"); + assert(Symbols.find(KV.first)->second.isInMaterializationPhase() && + "getRequestedSymbols can only be called for symbols that have " + "started materializing"); auto I = MaterializingInfos.find(KV.first); if (I == MaterializingInfos.end()) continue; - if (!I->second.PendingQueries.empty()) + if (I->second.hasQueriesPending()) RequestedSymbols.insert(KV.first); } @@ -835,9 +820,8 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { void JITDylib::addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { assert(Symbols.count(Name) && "Name not in symbol table"); - assert((Symbols[Name].getFlags().isLazy() || - Symbols[Name].getFlags().isMaterializing()) && - "Symbol is not lazy or materializing"); + assert(Symbols[Name].isInMaterializationPhase() && + "Can not add dependencies for a symbol that is not materializing"); auto &MI = MaterializingInfos[Name]; assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); @@ -852,9 +836,8 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // Assert that this symbol exists and has not been emitted already. auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); assert(SymI != OtherJITDylib.Symbols.end() && - (SymI->second.getFlags().isLazy() || - SymI->second.getFlags().isMaterializing()) && - "Dependency on emitted symbol"); + (SymI->second.getState() != SymbolState::Ready && + "Dependency on emitted symbol")); #endif auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; @@ -873,54 +856,52 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, } void JITDylib::resolve(const SymbolMap &Resolved) { - auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet FullyResolvedQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Resolved) { auto &Name = KV.first; auto Sym = KV.second; - assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() && - "Materializing flags should be managed internally"); - auto I = Symbols.find(Name); assert(I != Symbols.end() && "Symbol not found"); - assert(!I->second.getFlags().isLazy() && - I->second.getFlags().isMaterializing() && + assert(!I->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(I->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == - (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) & - ~JITSymbolFlags::Weak) && + (I->second.getFlags() & ~JITSymbolFlags::Weak) && "Resolved flags should match the declared flags"); // Once resolved, symbols can never be weak. JITSymbolFlags ResolvedFlags = Sym.getFlags(); ResolvedFlags &= ~JITSymbolFlags::Weak; - ResolvedFlags |= JITSymbolFlags::Materializing; - I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags); + I->second.setAddress(Sym.getAddress()); + I->second.setFlags(ResolvedFlags); + I->second.setState(SymbolState::Resolved); auto &MI = MaterializingInfos[Name]; - for (auto &Q : MI.PendingQueries) { - Q->resolve(Name, Sym); - if (Q->isFullyResolved()) - FullyResolvedQueries.insert(Q); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, Sym); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); } } - return FullyResolvedQueries; + return CompletedQueries; }); - for (auto &Q : FullyResolvedQueries) { - assert(Q->isFullyResolved() && "Q not fully resolved"); - Q->handleFullyResolved(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q not completed"); + Q->handleComplete(); } } void JITDylib::emit(const SymbolFlagsMap &Emitted) { - auto FullyReadyQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet ReadyQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Emitted) { const auto &Name = KV.first; @@ -962,20 +943,22 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { DependantMI.UnemittedDependencies.empty()) { assert(DependantMI.Dependants.empty() && "Dependants should be empty by now"); - for (auto &Q : DependantMI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); - Q->removeQueryDependence(DependantJD, DependantName); - } // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. - assert(DependantJD.Symbols.count(DependantName) && + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && "Dependant has no entry in the Symbols table"); - auto &DependantSym = DependantJD.Symbols[DependantName]; - DependantSym.setFlags(DependantSym.getFlags() & - ~JITSymbolFlags::Materializing); + DependantSymI->second.setState(SymbolState::Ready); + + for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + DependantJD.MaterializingInfos.erase(DependantMII); } } @@ -984,26 +967,25 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { MI.IsEmitted = true; if (MI.UnemittedDependencies.empty()) { - for (auto &Q : MI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table"); + SymI->second.setState(SymbolState::Ready); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - assert(Symbols.count(Name) && - "Symbol has no entry in the Symbols table"); - auto &Sym = Symbols[Name]; - Sym.setFlags(Sym.getFlags() & ~JITSymbolFlags::Materializing); MaterializingInfos.erase(MII); } } - return ReadyQueries; + return CompletedQueries; }); - for (auto &Q : FullyReadyQueries) { - assert(Q->isFullyReady() && "Q is not fully ready"); - Q->handleFullyReady(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q is not complete"); + Q->handleComplete(); } } @@ -1013,6 +995,7 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { AsynchronousSymbolQuerySet FailedQueries; + std::vector<MaterializingInfosMap::iterator> MIIsToRemove; for (auto &Name : FailedSymbols) { auto I = Symbols.find(Name); @@ -1026,17 +1009,40 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { if (MII == MaterializingInfos.end()) continue; + // Remove this symbol from the dependants list of any dependencies. + for (auto &KV : MII->second.UnemittedDependencies) { + auto *DependencyJD = KV.first; + auto &Dependencies = KV.second; + for (auto &DependencyName : Dependencies) { + auto DependencyMII = + DependencyJD->MaterializingInfos.find(DependencyName); + assert(DependencyMII != DependencyJD->MaterializingInfos.end() && + "Unemitted dependency must have a MaterializingInfo entry"); + assert(DependencyMII->second.Dependants.count(this) && + "Dependency's dependants list does not contain this JITDylib"); + assert(DependencyMII->second.Dependants[this].count(Name) && + "Dependency's dependants list does not contain dependant"); + DependencyMII->second.Dependants[this].erase(Name); + } + } + // Copy all the queries to the FailedQueries list, then abandon them. // This has to be a copy, and the copy has to come before the abandon // operation: Each Q.detach() call will reach back into this // PendingQueries list to remove Q. - for (auto &Q : MII->second.PendingQueries) + for (auto &Q : MII->second.pendingQueries()) FailedQueries.insert(Q); - for (auto &Q : FailedQueries) - Q->detach(); + MIIsToRemove.push_back(std::move(MII)); + } + + // Detach failed queries. + for (auto &Q : FailedQueries) + Q->detach(); - assert(MII->second.PendingQueries.empty() && + // Remove the MaterializingInfos. + for (auto &MII : MIIsToRemove) { + assert(!MII->second.hasQueriesPending() && "Queries remain after symbol was failed"); MaterializingInfos.erase(MII); @@ -1052,9 +1058,11 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, bool SearchThisJITDylibFirst, bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } @@ -1092,7 +1100,7 @@ void JITDylib::removeFromSearchOrder(JITDylib &JD) { Error JITDylib::remove(const SymbolNameSet &Names) { return ES.runSessionLocked([&]() -> Error { using SymbolMaterializerItrPair = - std::pair<SymbolMap::iterator, UnmaterializedInfosMap::iterator>; + std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>; std::vector<SymbolMaterializerItrPair> SymbolsToRemove; SymbolNameSet Missing; SymbolNameSet Materializing; @@ -1107,13 +1115,14 @@ Error JITDylib::remove(const SymbolNameSet &Names) { } // Note symbol materializing. - if (I->second.getFlags().isMaterializing()) { + if (I->second.isInMaterializationPhase()) { Materializing.insert(Name); continue; } - auto UMII = I->second.getFlags().isLazy() ? UnmaterializedInfos.find(Name) - : UnmaterializedInfos.end(); + auto UMII = I->second.hasMaterializerAttached() + ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); SymbolsToRemove.push_back(std::make_pair(I, UMII)); } @@ -1143,16 +1152,23 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { - return ES.runSessionLocked([&, this]() { +Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { SymbolFlagsMap Result; auto Unresolved = lookupFlagsImpl(Result, Names); - if (DefGenerator && !Unresolved.empty()) { - auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - auto Unresolved2 = lookupFlagsImpl(Result, NewDefs); + if (!Unresolved) + return Unresolved.takeError(); + + if (DefGenerator && !Unresolved->empty()) { + auto NewDefs = DefGenerator(*this, *Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); + if (!Unresolved2) + return Unresolved2.takeError(); (void)Unresolved2; - assert(Unresolved2.empty() && + assert(Unresolved2->empty() && "All fallback defs should have been found by lookupFlagsImpl"); } }; @@ -1160,41 +1176,42 @@ SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { }); } -SymbolNameSet JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names) { +Expected<SymbolNameSet> JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { SymbolNameSet Unresolved; for (auto &Name : Names) { auto I = Symbols.find(Name); - - if (I == Symbols.end()) { + if (I != Symbols.end()) { + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = I->second.getFlags(); + } else Unresolved.insert(Name); - continue; - } - - assert(!Flags.count(Name) && "Symbol already present in Flags map"); - Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags()); } return Unresolved; } -void JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs) { +Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs) { assert(Q && "Query can not be null"); lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); if (DefGenerator && !Unresolved.empty()) { auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - lodgeQueryImpl(Q, NewDefs, MatchNonExported, MUs); - assert(NewDefs.empty() && + lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + + return Error::success(); } void JITDylib::lodgeQueryImpl( @@ -1204,6 +1221,7 @@ void JITDylib::lodgeQueryImpl( std::vector<SymbolStringPtr> ToRemove; for (auto Name : Unresolved) { + // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); if (SymI == Symbols.end()) @@ -1213,20 +1231,22 @@ void JITDylib::lodgeQueryImpl( if (!SymI->second.getFlags().isExported() && !MatchNonExported) continue; - // If we matched against Name in JD, mark it to be removed from the Unresolved - // set. + // If we matched against Name in JD, mark it to be removed from the + // Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) - Q->resolve(Name, SymI->second); + // If this symbol already meets the required state for then notify the + // query and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + continue; + } - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + // Otherwise this symbol does not yet meet the required state. Check whether + // it has a materializer attached, and if so prepare to run it. + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); + "Symbol not resolved but already has address?"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1237,27 +1257,20 @@ void JITDylib::lodgeQueryImpl( // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be - // ready. Notify the query and continue. - Q->notifySymbolReady(); - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1266,40 +1279,43 @@ void JITDylib::lodgeQueryImpl( Unresolved.erase(Name); } -SymbolNameSet JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Names) { +Expected<SymbolNameSet> +JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { assert(Q && "Query can not be null"); ES.runOutstandingMUs(); - LookupImplActionFlags ActionFlags = None; + bool QueryComplete = false; std::vector<std::unique_ptr<MaterializationUnit>> MUs; SymbolNameSet Unresolved = std::move(Names); - ES.runSessionLocked([&, this]() { - ActionFlags = lookupImpl(Q, MUs, Unresolved); + auto Err = ES.runSessionLocked([&, this]() -> Error { + QueryComplete = lookupImpl(Q, MUs, Unresolved); if (DefGenerator && !Unresolved.empty()) { - assert(ActionFlags == None && - "ActionFlags set but unresolved symbols remain?"); + assert(!QueryComplete && "query complete but unresolved symbols remain?"); auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - ActionFlags = lookupImpl(Q, MUs, NewDefs); - assert(NewDefs.empty() && + QueryComplete = lookupImpl(Q, MUs, *NewDefs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + return Error::success(); }); - assert((MUs.empty() || ActionFlags == None) && - "If action flags are set, there should be no work to do (so no MUs)"); + if (Err) + return std::move(Err); - if (ActionFlags & NotifyFullyResolved) - Q->handleFullyResolved(); + assert((MUs.empty() || !QueryComplete) && + "If action flags are set, there should be no work to do (so no MUs)"); - if (ActionFlags & NotifyFullyReady) - Q->handleFullyReady(); + if (QueryComplete) + Q->handleComplete(); // FIXME: Swap back to the old code below once RuntimeDyld works with // callbacks from asynchronous queries. @@ -1318,13 +1334,13 @@ SymbolNameSet JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, return Unresolved; } -JITDylib::LookupImplActionFlags -JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolNameSet &Unresolved) { - LookupImplActionFlags ActionFlags = None; - std::vector<SymbolStringPtr> ToRemove; +bool JITDylib::lookupImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { + bool QueryComplete = false; + std::vector<SymbolStringPtr> ToRemove; for (auto Name : Unresolved) { // Search for the name in Symbols. Skip it if not found. @@ -1335,19 +1351,17 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, // If we found Name, mark it to be removed from the Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) { - Q->resolve(Name, SymI->second); - if (Q->isFullyResolved()) - ActionFlags |= NotifyFullyResolved; + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + QueryComplete = true; + continue; } // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1358,29 +1372,21 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + assert(SymK != Symbols.end() && "Missing symbol table entry"); + SymK->second.setState(SymbolState::Materializing); + SymK->second.setMaterializerAttached(false); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be ready. - // Notify the query and continue. - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ActionFlags |= NotifyFullyReady; - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1388,7 +1394,7 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, for (auto &Name : ToRemove) Unresolved.erase(Name); - return ActionFlags; + return QueryComplete; } void JITDylib::dump(raw_ostream &OS) { @@ -1405,21 +1411,19 @@ void JITDylib::dump(raw_ostream &OS) { for (auto &KV : Symbols) { OS << " \"" << *KV.first << "\": "; if (auto Addr = KV.second.getAddress()) - OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags(); + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags() + << " "; else - OS << "<not resolved>"; - if (KV.second.getFlags().isLazy() || - KV.second.getFlags().isMaterializing()) { - OS << " ("; - if (KV.second.getFlags().isLazy()) { - auto I = UnmaterializedInfos.find(KV.first); - assert(I != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - OS << " Lazy (MU=" << I->second->MU.get() << ")"; - } - if (KV.second.getFlags().isMaterializing()) - OS << " Materializing"; - OS << ", " << KV.second.getFlags() << " )\n"; + OS << "<not resolved> "; + + OS << KV.second.getState(); + + if (KV.second.hasMaterializerAttached()) { + OS << " (Materializer "; + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << I->second->MU.get() << ")\n"; } else OS << "\n"; } @@ -1430,10 +1434,10 @@ void JITDylib::dump(raw_ostream &OS) { OS << " \"" << *KV.first << "\":\n" << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") << "\n" - << " " << KV.second.PendingQueries.size() + << " " << KV.second.pendingQueries().size() << " pending queries: { "; - for (auto &Q : KV.second.PendingQueries) - OS << Q.get() << " "; + for (const auto &Q : KV.second.pendingQueries()) + OS << Q.get() << " (" << Q->getRequiredState() << ") "; OS << "}\n Dependants:\n"; for (auto &KV2 : KV.second.Dependants) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; @@ -1444,6 +1448,51 @@ void JITDylib::dump(raw_ostream &OS) { }); } +void JITDylib::MaterializingInfo::addQuery( + std::shared_ptr<AsynchronousSymbolQuery> Q) { + + auto I = std::lower_bound( + PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) { + return V->getRequiredState() <= S; + }); + PendingQueries.insert(I.base(), std::move(Q)); +} + +void JITDylib::MaterializingInfo::removeQuery( + const AsynchronousSymbolQuery &Q) { + // FIXME: Implement 'find_as' for shared_ptr<T>/T*. + auto I = + std::find_if(PendingQueries.begin(), PendingQueries.end(), + [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); + assert(I != PendingQueries.end() && + "Query is not attached to this MaterializingInfo"); + PendingQueries.erase(I); +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { + AsynchronousSymbolQueryList Result; + while (!PendingQueries.empty()) { + if (PendingQueries.back()->getRequiredState() > RequiredState) + break; + + Result.push_back(std::move(PendingQueries.back())); + PendingQueries.pop_back(); + } + + return Result; +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeAllQueries() { + AsynchronousSymbolQueryList Result; + std::swap(Result, PendingQueries); + return Result; +} + JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { SearchOrder.push_back({this, true}); @@ -1451,77 +1500,52 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; - SymbolNameSet MUDefsOverridden; - - struct ExistingDefOverriddenEntry { - SymbolMap::iterator ExistingDefItr; - JITSymbolFlags NewFlags; - }; - std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden; - - for (auto &KV : MU.getSymbols()) { - assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); + std::vector<SymbolStringPtr> ExistingDefsOverridden; + std::vector<SymbolStringPtr> MUDefsOverridden; - SymbolMap::iterator EntryItr; - bool Added; + for (const auto &KV : MU.getSymbols()) { + auto I = Symbols.find(KV.first); - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Lazy; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); - - if (!Added) { + if (I != Symbols.end()) { if (KV.second.isStrong()) { - if (EntryItr->second.getFlags().isStrong() || - (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + if (I->second.getFlags().isStrong() || + I->second.getState() > SymbolState::NeverSearched) Duplicates.insert(KV.first); - else - ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + else { + assert(I->second.getState() == SymbolState::NeverSearched && + "Overridden existing def should be in the never-searched " + "state"); + ExistingDefsOverridden.push_back(KV.first); + } } else - MUDefsOverridden.insert(KV.first); + MUDefsOverridden.push_back(KV.first); } } - if (!Duplicates.empty()) { - // We need to remove the symbols we added. - for (auto &KV : MU.getSymbols()) { - if (Duplicates.count(KV.first)) - continue; - - bool Found = false; - for (const auto &EDO : ExistingDefsOverridden) - if (EDO.ExistingDefItr->first == KV.first) - Found = true; - - if (!Found) - Symbols.erase(KV.first); - } - - // FIXME: Return all duplicates. + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) return make_error<DuplicateDefinition>(**Duplicates.begin()); - } - // Update flags on existing defs and call discard on their materializers. - for (auto &EDO : ExistingDefsOverridden) { - assert(EDO.ExistingDefItr->second.getFlags().isLazy() && - !EDO.ExistingDefItr->second.getFlags().isMaterializing() && - "Overridden existing def should be in the Lazy state"); + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); - EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { - auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + auto UMII = UnmaterializedInfos.find(S); assert(UMII != UnmaterializedInfos.end() && "Overridden existing def should have an UnmaterializedInfo"); - - UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + UMII->second->MU->doDiscard(*this, S); } - // Discard overridden symbols povided by MU. - for (auto &Sym : MUDefsOverridden) - MU.doDiscard(*this, Sym); + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto &SymEntry = Symbols[KV.first]; + SymEntry.setFlags(KV.second); + SymEntry.setState(SymbolState::NeverSearched); + SymEntry.setMaterializerAttached(true); + } return Error::success(); } @@ -1532,17 +1556,7 @@ void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, assert(MaterializingInfos.count(QuerySymbol) && "QuerySymbol does not have MaterializingInfo"); auto &MI = MaterializingInfos[QuerySymbol]; - - auto IdenticalQuery = - [&](const std::shared_ptr<AsynchronousSymbolQuery> &R) { - return R.get() == &Q; - }; - - auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(), - IdenticalQuery); - assert(I != MI.PendingQueries.end() && - "Query Q should be in the PendingQueries list for QuerySymbol"); - MI.PendingQueries.erase(I); + MI.removeQuery(Q); } } @@ -1582,8 +1596,18 @@ JITDylib &ExecutionSession::getMainJITDylib() { return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); } +JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { + return runSessionLocked([&, this]() -> JITDylib * { + for (auto &JD : JDs) + if (JD->getName() == Name) + return JD.get(); + return nullptr; + }); +} + JITDylib &ExecutionSession::createJITDylib(std::string Name, bool AddToMainDylibSearchOrder) { + assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); @@ -1610,74 +1634,36 @@ void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { Expected<SymbolMap> ExecutionSession::legacyLookup( LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - bool WaitUntilReady, RegisterDependenciesFunction RegisterDependencies) { + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif auto Query = std::make_shared<AsynchronousSymbolQuery>( - Names, std::move(OnResolve), std::move(OnReady)); + Names, RequiredState, std::move(NotifyComplete)); // FIXME: This should be run session locked along with the registration code // and error reporting below. SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); @@ -1701,39 +1687,13 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); - + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1741,9 +1701,16 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( void ExecutionSession::lookup( const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies) { + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Looking up " << Symbols << " in " << SearchOrder + << " (required state: " << RequiredState << ")\n"; + }); + }); + // lookup can be re-entered recursively if running on a single thread. Run any // outstanding MUs in case this query depends on them, otherwise this lookup // will starve waiting for a result from an MU that is stuck in the queue. @@ -1751,38 +1718,32 @@ void ExecutionSession::lookup( auto Unresolved = std::move(Symbols); std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; - auto Q = std::make_shared<AsynchronousSymbolQuery>( - Unresolved, std::move(OnResolve), std::move(OnReady)); - bool QueryIsFullyResolved = false; - bool QueryIsFullyReady = false; - bool QueryFailed = false; - - runSessionLocked([&]() { - for (auto &KV : SearchOrder) { - assert(KV.first && "JITDylibList entries must not be null"); - assert(!CollectedMUsMap.count(KV.first) && - "JITDylibList should not contain duplicate entries"); - - auto &JD = *KV.first; - auto MatchNonExported = KV.second; - JD.lodgeQuery(Q, Unresolved, MatchNonExported, CollectedMUsMap[&JD]); - } + auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, + std::move(NotifyComplete)); + bool QueryComplete = false; + + auto LodgingErr = runSessionLocked([&]() -> Error { + auto LodgeQuery = [&]() -> Error { + for (auto &KV : SearchOrder) { + assert(KV.first && "JITDylibList entries must not be null"); + assert(!CollectedMUsMap.count(KV.first) && + "JITDylibList should not contain duplicate entries"); + + auto &JD = *KV.first; + auto MatchNonExported = KV.second; + if (auto Err = JD.lodgeQuery(Q, Unresolved, MatchNonExported, + CollectedMUsMap[&JD])) + return Err; + } - if (Unresolved.empty()) { - // Query lodged successfully. + if (!Unresolved.empty()) + return make_error<SymbolsNotFound>(std::move(Unresolved)); - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryIsFullyResolved = Q->isFullyResolved(); - QueryIsFullyReady = Q->isFullyReady(); + return Error::success(); + }; - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - } else { - // Query failed due to unresolved symbols. - QueryFailed = true; + if (auto Err = LodgeQuery()) { + // Query failed. // Disconnect the query from its dependencies. Q->detach(); @@ -1791,19 +1752,32 @@ void ExecutionSession::lookup( for (auto &KV : CollectedMUsMap) for (auto &MU : KV.second) KV.first->replace(std::move(MU)); + + return Err; } + + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryComplete = Q->isComplete(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + + return Error::success(); }); - if (QueryFailed) { - Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); + if (LodgingErr) { + Q->handleFailed(std::move(LodgingErr)); return; - } else { - if (QueryIsFullyResolved) - Q->handleFullyResolved(); - if (QueryIsFullyReady) - Q->handleFullyReady(); } + if (QueryComplete) + Q->handleComplete(); + // Move the MUs to the OutstandingMUs list, then materialize. { std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); @@ -1816,113 +1790,55 @@ void ExecutionSession::lookup( runOutstandingMUs(); } -Expected<SymbolMap> ExecutionSession::lookup( - const JITDylibSearchList &SearchOrder, const SymbolNameSet &Symbols, - RegisterDependenciesFunction RegisterDependencies, bool WaitUntilReady) { +Expected<SymbolMap> +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + const SymbolNameSet &Symbols, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + + auto NotifyComplete = [&](Expected<SymbolMap> R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif // Perform the asynchronous lookup. - lookup(SearchOrder, Symbols, OnResolve, OnReady, RegisterDependencies); + lookup(SearchOrder, Symbols, RequiredState, NotifyComplete, + RegisterDependencies); #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1933,8 +1849,8 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, SymbolStringPtr Name) { SymbolNameSet Names({Name}); - if (auto ResultMap = lookup(SearchOrder, std::move(Names), - NoDependenciesToRegister, true)) { + if (auto ResultMap = lookup(SearchOrder, std::move(Names), SymbolState::Ready, + NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 7c3c50b4d6e5..f7fc5f8f1797 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -1,9 +1,8 @@ //===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -130,8 +129,7 @@ Error CtorDtorRunner::run() { auto &ES = JD.getExecutionSession(); if (auto CtorDtorMap = - ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names), - NoDependenciesToRegister, true)) { + ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names))) { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); @@ -140,13 +138,10 @@ Error CtorDtorRunner::run() { CtorDtor(); } } + CtorDtorsByPriority.clear(); return Error::success(); } else return CtorDtorMap.takeError(); - - CtorDtorsByPriority.clear(); - - return Error::success(); } void LocalCXXRuntimeOverridesBase::runDestructors() { @@ -179,22 +174,24 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), - GlobalPrefix(DL.getGlobalPrefix()) {} + GlobalPrefix(GlobalPrefix) {} Expected<DynamicLibrarySearchGenerator> -DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); } -SymbolNameSet DynamicLibrarySearchGenerator:: -operator()(JITDylib &JD, const SymbolNameSet &Names) { +Expected<SymbolNameSet> +DynamicLibrarySearchGenerator::operator()(JITDylib &JD, + const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolMap NewSymbols; @@ -210,7 +207,8 @@ operator()(JITDylib &JD, const SymbolNameSet &Names) { if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) continue; - std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index d952d1be70da..81dfc02f55b2 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -1,9 +1,8 @@ //===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 7bc0d696e3ac..e3519284613e 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -1,9 +1,8 @@ //===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 82000ec5b32b..cc3656fe5dc5 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -1,9 +1,8 @@ //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -38,8 +37,8 @@ private: void materialize(MaterializationResponsibility R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); - R.resolve(Result); - R.emit(); + R.notifyResolved(Result); + R.notifyEmitted(); } void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { @@ -238,11 +237,11 @@ void makeStub(Function &F, Value &ImplPointer) { Module &M = *F.getParent(); BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); IRBuilder<> Builder(EntryBlock); - LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer); + LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); std::vector<Value*> CallArgs; for (auto &A : F.args()) CallArgs.push_back(&A); - CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs); + CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); Call->setTailCall(); Call->setAttributes(F.getAttributes()); if (F.getReturnType()->isVoidTy()) diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 4af09d196ff9..df23547a9de3 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -1,9 +1,8 @@ //===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index e2089f9106bd..b120691faf07 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -1,58 +1,37 @@ //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/Mangler.h" -namespace { +namespace llvm { +namespace orc { - // A SimpleCompiler that owns its TargetMachine. - class TMOwningSimpleCompiler : public llvm::orc::SimpleCompiler { - public: - TMOwningSimpleCompiler(std::unique_ptr<llvm::TargetMachine> TM) - : llvm::orc::SimpleCompiler(*TM), TM(std::move(TM)) {} - private: - // FIXME: shared because std::functions (and thus - // IRCompileLayer::CompileFunction) are not moveable. - std::shared_ptr<llvm::TargetMachine> TM; - }; +Error LLJITBuilderState::prepareForConstruction() { -} // end anonymous namespace + if (!JTMB) { + if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) + JTMB = std::move(*JTMBOrErr); + else + return JTMBOrErr.takeError(); + } -namespace llvm { -namespace orc { + return Error::success(); +} LLJIT::~LLJIT() { if (CompileThreads) CompileThreads->wait(); } -Expected<std::unique_ptr<LLJIT>> -LLJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - unsigned NumCompileThreads) { - - if (NumCompileThreads == 0) { - // If NumCompileThreads == 0 then create a single-threaded LLJIT instance. - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), - std::move(*TM), std::move(DL))); - } - - return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), - std::move(JTMB), std::move(DL), - NumCompileThreads)); -} - Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); @@ -65,13 +44,13 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { if (auto Err = applyDataLayout(*TSM.getModule())) return Err; - return CompileLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { assert(Obj && "Can not add null object"); - return ObjLinkingLayer.add(JD, std::move(Obj), ES->allocateVModule()); + return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule()); } Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, @@ -79,42 +58,76 @@ Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); } -LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, - std::unique_ptr<TargetMachine> TM, DataLayout DL) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique<SectionMemoryManager>(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - TMOwningSimpleCompiler(std::move(TM))), - CtorRunner(Main), DtorRunner(Main) {} - -LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique<SectionMemoryManager>(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - ConcurrentIRCompiler(std::move(JTMB))), - CtorRunner(Main), DtorRunner(Main) { - assert(NumCompileThreads != 0 && - "Multithreaded LLJIT instance can not be created with 0 threads"); - - // Move modules to new contexts when they're emitted so that we can compile - // them in parallel. - CompileLayer.setCloneToNewContextOnEmit(true); - - // Create a thread pool to compile on and set the execution session - // dispatcher to use the thread pool. - CompileThreads = llvm::make_unique<ThreadPool>(NumCompileThreads); - this->ES->setDispatchMaterialization( - [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { - // FIXME: Switch to move capture once we have c++14. - auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); - auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; - CompileThreads->async(std::move(Work)); - }); +std::unique_ptr<ObjectLayer> +LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { + + // If the config state provided an ObjectLinkingLayer factory then use it. + if (S.CreateObjectLinkingLayer) + return S.CreateObjectLinkingLayer(ES); + + // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs + // a new SectionMemoryManager for each object. + auto GetMemMgr = []() { return llvm::make_unique<SectionMemoryManager>(); }; + return llvm::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); +} + +Expected<IRCompileLayer::CompileFunction> +LLJIT::createCompileFunction(LLJITBuilderState &S, + JITTargetMachineBuilder JTMB) { + + /// If there is a custom compile function creator set then use it. + if (S.CreateCompileFunction) + return S.CreateCompileFunction(std::move(JTMB)); + + // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, + // depending on the number of threads requested. + if (S.NumCompileThreads > 0) + return ConcurrentIRCompiler(std::move(JTMB)); + + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + + return TMOwningSimpleCompiler(std::move(*TM)); +} + +LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) + : ES(S.ES ? std::move(S.ES) : llvm::make_unique<ExecutionSession>()), + Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), + DtorRunner(Main) { + + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer = createObjectLinkingLayer(S, *ES); + + if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else { + Err = DLOrErr.takeError(); + return; + } + + { + auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); + if (!CompileFunction) { + Err = CompileFunction.takeError(); + return; + } + CompileLayer = llvm::make_unique<IRCompileLayer>( + *ES, *ObjLinkingLayer, std::move(*CompileFunction)); + } + + if (S.NumCompileThreads > 0) { + CompileLayer->setCloneToNewContextOnEmit(true); + CompileThreads = llvm::make_unique<ThreadPool>(S.NumCompileThreads); + ES->setDispatchMaterialization( + [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { + // FIXME: Switch to move capture once we have c++14. + auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); + auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + CompileThreads->async(std::move(Work)); + }); + } } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -143,35 +156,11 @@ void LLJIT::recordCtorDtors(Module &M) { DtorRunner.add(getDestructors(M)); } -Expected<std::unique_ptr<LLLazyJIT>> -LLLazyJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - JITTargetAddress ErrorAddr, unsigned NumCompileThreads) { - auto ES = llvm::make_unique<ExecutionSession>(); - - const Triple &TT = JTMB.getTargetTriple(); - - auto LCTMgr = createLocalLazyCallThroughManager(TT, *ES, ErrorAddr); - if (!LCTMgr) - return LCTMgr.takeError(); - - auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); - if (!ISMBuilder) - return make_error<StringError>( - std::string("No indirect stubs manager builder for ") + TT.str(), - inconvertibleErrorCode()); - - if (NumCompileThreads == 0) { - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr<LLLazyJIT>( - new LLLazyJIT(std::move(ES), std::move(*TM), std::move(DL), - std::move(*LCTMgr), std::move(ISMBuilder))); - } - - return std::unique_ptr<LLLazyJIT>(new LLLazyJIT( - std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads, - std::move(*LCTMgr), std::move(ISMBuilder))); +Error LLLazyJITBuilderState::prepareForConstruction() { + if (auto Err = LLJITBuilderState::prepareForConstruction()) + return Err; + TT = JTMB->getTargetTriple(); + return Error::success(); } Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { @@ -182,28 +171,55 @@ Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { recordCtorDtors(*TSM.getModule()); - return CODLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); } -LLLazyJIT::LLLazyJIT( - std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, - DataLayout DL, std::unique_ptr<LazyCallThroughManager> LCTMgr, - std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) - : LLJIT(std::move(ES), std::move(TM), std::move(DL)), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) {} - -LLLazyJIT::LLLazyJIT( - std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads, - std::unique_ptr<LazyCallThroughManager> LCTMgr, - std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) - : LLJIT(std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) { - CODLayer.setCloneToNewContextOnEmit(true); +LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { + + // If LLJIT construction failed then bail out. + if (Err) + return; + + ErrorAsOutParameter _(&Err); + + /// Take/Create the lazy-compile callthrough manager. + if (S.LCTMgr) + LCTMgr = std::move(S.LCTMgr); + else { + if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( + S.TT, *ES, S.LazyCompileFailureAddr)) + LCTMgr = std::move(*LCTMgrOrErr); + else { + Err = LCTMgrOrErr.takeError(); + return; + } + } + + // Take/Create the indirect stubs manager builder. + auto ISMBuilder = std::move(S.ISMBuilder); + + // If none was provided, try to build one. + if (!ISMBuilder) + ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); + + // No luck. Bail out. + if (!ISMBuilder) { + Err = make_error<StringError>("Could not construct " + "IndirectStubsManagerBuilder for target " + + S.TT.str(), + inconvertibleErrorCode()); + return; + } + + // Create the transform layer. + TransformLayer = llvm::make_unique<IRTransformLayer>(*ES, *CompileLayer); + + // Create the COD layer. + CODLayer = llvm::make_unique<CompileOnDemandLayer>( + *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + + if (S.NumCompileThreads > 0) + CODLayer->setCloneToNewContextOnEmit(true); } } // End namespace orc. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 11af76825e9f..3ed2dabf4545 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -1,9 +1,8 @@ //===-------------------- Layer.cpp - Layer interfaces --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -88,17 +87,15 @@ void BasicIRLayerMaterializationUnit::materialize( #ifndef NDEBUG auto &ES = R.getTargetJITDylib().getExecutionSession(); + auto &N = R.getTargetJITDylib().getName(); #endif // NDEBUG auto Lock = TSM.getContextLock(); - LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Emitting, for " << R.getTargetJITDylib().getName() << ", " - << *this << "\n"; - });); + LLVM_DEBUG(ES.runSessionLocked( + [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); L.emit(std::move(R), std::move(TSM)); LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Finished emitting, for " << R.getTargetJITDylib().getName() - << ", " << *this << "\n"; + dbgs() << "Finished emitting, for " << N << ", " << *this << "\n"; });); } diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index 55f4a7c5afce..fc8205845654 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -1,9 +1,8 @@ //===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -52,18 +51,15 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { SymbolName = I->second.second; } - auto LookupResult = ES.lookup(JITDylibSearchList({{SourceJD, true}}), - {SymbolName}, NoDependenciesToRegister, true); + auto LookupResult = + ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); if (!LookupResult) { ES.reportError(LookupResult.takeError()); return ErrorHandlerAddr; } - assert(LookupResult->size() == 1 && "Unexpected number of results"); - assert(LookupResult->count(SymbolName) && "Unexpected result"); - - auto ResolvedAddr = LookupResult->begin()->second.getAddress(); + auto ResolvedAddr = LookupResult->getAddress(); std::shared_ptr<NotifyResolvedFunction> NotifyResolved = nullptr; { @@ -182,8 +178,8 @@ void LazyReexportsMaterializationUnit::materialize( for (auto &Alias : RequestedAliases) Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); - R.resolve(Stubs); - R.emit(); + R.notifyResolved(Stubs); + R.notifyEmitted(); } void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp index ddb72544b770..ce6368b57a89 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/Legacy.cpp @@ -1,9 +1,8 @@ //===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -37,8 +36,7 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, }; auto Q = std::make_shared<AsynchronousSymbolQuery>( - InternedSymbols, OnResolvedWithUnwrap, - [this](Error Err) { ES.reportError(std::move(Err)); }); + InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap); auto Unresolved = R.lookup(Q, InternedSymbols); if (Unresolved.empty()) { diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp index 922fc6f021ce..5b4345b870bb 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -1,9 +1,8 @@ //===---------- NullResolver.cpp - Reject symbol lookup requests ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 000000000000..def0b300eca1 --- /dev/null +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,483 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include <vector> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet<StringRef> &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared<JITLinkAsyncLookupContinuation>( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved, + std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.notifyResolved(InternedResult); + + Layer.notifyLoaded(MR); + } + + void notifyFinalized( + std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + + if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + + return; + } + MR.notifyEmitted(); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Layer.modifyPassConfig(MR, TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap<const DefinedAtom *, SymbolNameSet>; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast<DefinedAtom &>(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet<DefinedAtom *> DAAnonDeps; + }; + std::vector<WorklistEntry> Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet<DefinedAtom *> DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr<MemoryBuffer> ObjBuffer; + DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps; +}; + +ObjectLinkingLayer::Plugin::~Plugin() {} + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) {} + +ObjectLinkingLayer::~ObjectLinkingLayer() { + if (auto Err = removeAllModules()) + getExecutionSession().reportError(std::move(Err)); +} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O))); +} + +void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + PassConfiguration &PassConfig) { + for (auto &P : Plugins) + P->modifyPassConfig(MR, TT, PassConfig); +} + +void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { + for (auto &P : Plugins) + P->notifyLoaded(MR); +} + +Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, + AllocPtr Alloc) { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); + + if (Err) + return Err; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + UntrackedAllocs.push_back(std::move(Alloc)); + } + + return Error::success(); +} + +Error ObjectLinkingLayer::removeModule(VModuleKey K) { + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingModule(K)); + + AllocPtr Alloc; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + auto AllocItr = TrackedAllocs.find(K); + Alloc = std::move(AllocItr->second); + TrackedAllocs.erase(AllocItr); + } + + assert(Alloc && "No allocation for key K"); + + return joinErrors(std::move(Err), Alloc->deallocate()); +} + +Error ObjectLinkingLayer::removeAllModules() { + + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingAllModules()); + + std::vector<AllocPtr> Allocs; + { + std::lock_guard<std::mutex> Lock(LayerMutex); + Allocs = std::move(UntrackedAllocs); + + for (auto &KV : TrackedAllocs) + Allocs.push_back(std::move(KV.second)); + + TrackedAllocs.clear(); + } + + while (!Allocs.empty()) { + Err = joinErrors(std::move(Err), Allocs.back()->deallocate()); + Allocs.pop_back(); + } + + return Err; +} + +EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( + jitlink::EHFrameRegistrar &Registrar) + : Registrar(Registrar) {} + +void EHFrameRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + PassConfiguration &PassConfig) { + assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); + + PassConfig.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) { + if (Addr) + InProcessLinks[&MR] = Addr; + })); +} + +Error EHFrameRegistrationPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + + auto EHFrameAddrItr = InProcessLinks.find(&MR); + if (EHFrameAddrItr == InProcessLinks.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "eh-frame addr to register can not be null"); + + InProcessLinks.erase(EHFrameAddrItr); + if (auto Key = MR.getVModuleKey()) + TrackedEHFrameAddrs[Key] = EHFrameAddr; + else + UntrackedEHFrameAddrs.push_back(EHFrameAddr); + + return Registrar.registerEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { + auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K); + if (EHFrameAddrItr == TrackedEHFrameAddrs.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "Tracked eh-frame addr must not be null"); + + TrackedEHFrameAddrs.erase(EHFrameAddrItr); + + return Registrar.deregisterEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { + + std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs); + EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size()); + + for (auto &KV : TrackedEHFrameAddrs) + EHFrameAddrs.push_back(KV.second); + + TrackedEHFrameAddrs.clear(); + + Error Err = Error::success(); + + while (!EHFrameAddrs.empty()) { + auto EHFrameAddr = EHFrameAddrs.back(); + assert(EHFrameAddr && "Untracked eh-frame addr must not be null"); + EHFrameAddrs.pop_back(); + Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr)); + } + + return Err; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index 825f53204736..815517321b76 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -1,9 +1,8 @@ //===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp index aa4055542426..8ed23de419d1 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -1,9 +1,8 @@ //===------------- OrcABISupport.cpp - ABI specific support code ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -148,7 +147,7 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -230,7 +229,7 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -498,7 +497,7 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -684,7 +683,7 @@ Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -930,7 +929,7 @@ Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp index 6dea64a6e78f..28c8479abba4 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -1,9 +1,8 @@ //===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 817a4b89bfb0..98129e1690d2 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -1,9 +1,8 @@ //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -154,8 +153,8 @@ private: for (auto &S : Symbols) { if (auto Sym = findSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); } else { Stack.ES.legacyFailQuery(*Query, Addr.takeError()); return orc::SymbolNameSet(); @@ -167,11 +166,8 @@ private: UnresolvedSymbols.insert(S); } - if (Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (Query->isFullyReady()) - Query->handleFullyReady(); + if (Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -215,28 +211,31 @@ public: IndirectStubsManagerBuilder IndirectStubsMgrBuilder) : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), - ObjectLayer(ES, - [this](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && - "No resolver for module K"); - auto Resolver = std::move(ResolverI->second); - Resolvers.erase(ResolverI); - return ObjLayerT::Resources{ - std::make_shared<SectionMemoryManager>(), Resolver}; - }, - nullptr, - [this](orc::VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { - this->notifyFinalized(K, Obj, LoadedObjInfo); - }, - [this](orc::VModuleKey K, const object::ObjectFile &Obj) { - this->notifyFreed(K, Obj); - }), - CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + orc::SimpleCompiler(TM)), CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), std::move(IndirectStubsMgrBuilder), Resolvers)), CXXRuntimeOverrides( + AcknowledgeORCv1Deprecation, [this](const std::string &S) { return mangle(S); }) {} Error shutdown() { @@ -312,7 +311,8 @@ public: // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), K); + orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner( + AcknowledgeORCv1Deprecation, std::move(CtorNames), K); if (auto Err = CtorRunner.runViaLayer(*this)) return std::move(Err); @@ -469,7 +469,7 @@ private: return nullptr; return llvm::make_unique<CODLayerT>( - ES, CompileLayer, + AcknowledgeORCv1Deprecation, ES, CompileLayer, [&Resolvers](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); assert(ResolverI != Resolvers.end() && "No resolver for module K"); diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp index f4102b359a6b..e6e9a095319c 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcError.cpp @@ -1,9 +1,8 @@ //===---------------- OrcError.cpp - Error codes for ORC ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index 617bc2fc64b5..772a9c2c4ab2 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -1,9 +1,8 @@ //===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -128,7 +127,8 @@ void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; for (auto &KV : CtorDtorsMap) - cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) + cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>( + AcknowledgeORCv1Deprecation, std::move(KV.second), KV.first) .runViaLayer(LazyEmitLayer)); CtorDtorsMap.clear(); diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 36e7e83a8bab..169dc8f1d02b 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/contrib/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -1,9 +1,8 @@ //===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -177,8 +176,8 @@ class OrcMCJITReplacement : public ExecutionEngine { for (auto &S : Symbols) { if (auto Sym = M.findMangledSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -190,8 +189,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } else { if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { if (auto Addr = Sym2.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -205,11 +204,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } } - if (NewSymbolsResolved && Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (NewSymbolsResolved && Query->isFullyReady()) - Query->handleFullyReady(); + if (NewSymbolsResolved && Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -236,24 +232,24 @@ public: OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, std::unique_ptr<TargetMachine> TM) - : ExecutionEngine(TM->createDataLayout()), - TM(std::move(TM)), + : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), MemMgr( std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), Resolver(std::make_shared<LinkingORCResolver>(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), ObjectLayer( - ES, + AcknowledgeORCv1Deprecation, ES, [this](VModuleKey K) { return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; }, NotifyObjectLoaded, NotifyFinalized), - CompileLayer(ObjectLayer, SimpleCompiler(*this->TM), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + SimpleCompiler(*this->TM), [this](VModuleKey K, std::unique_ptr<Module> M) { Modules.push_back(std::move(M)); }), - LazyEmitLayer(CompileLayer) {} + LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} static void Register() { OrcMCJITReplacementCtor = createOrcMCJITReplacement; diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp index 2a7ab5ca8180..367b3639f841 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/RPCUtils.cpp @@ -1,9 +1,8 @@ //===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 299d76183cd4..b22ecd5f80a1 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -1,9 +1,8 @@ //===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,9 +41,6 @@ public: OnResolved(Result); }; - // We're not waiting for symbols to be ready. Just log any errors. - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - // Register dependencies for all symbols contained in this set. auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { MR.addDependenciesForAll(Deps); @@ -53,8 +49,8 @@ public: JITDylibSearchList SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); - ES.lookup(SearchOrder, InternedSymbols, OnResolvedWithUnwrap, OnReady, - RegisterDependencies); + ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, + OnResolvedWithUnwrap, RegisterDependencies); } Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { @@ -78,11 +74,8 @@ namespace llvm { namespace orc { RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( - ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, - NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted) - : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), - NotifyLoaded(std::move(NotifyLoaded)), - NotifyEmitted(std::move(NotifyEmitted)) {} + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) { @@ -96,7 +89,13 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, auto &ES = getExecutionSession(); - auto Obj = object::ObjectFile::createObjectFile(*O); + // Create a MemoryBufferRef backed MemoryBuffer (i.e. shallow) copy of the + // the underlying buffer to pass into RuntimeDyld. This allows us to hold + // ownership of the real underlying buffer and return it to the user once + // the object has been emitted. + auto ObjBuffer = MemoryBuffer::getMemBuffer(O->getMemBufferRef(), false); + + auto Obj = object::ObjectFile::createObjectFile(*ObjBuffer); if (!Obj) { getExecutionSession().reportError(Obj.takeError()); @@ -134,13 +133,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, JITDylibSearchOrderResolver Resolver(*SharedR); - /* Thoughts on proper cross-dylib weak symbol handling: - * - * Change selection of canonical defs to be a manually triggered process, and - * add a 'canonical' bit to symbol definitions. When canonical def selection - * is triggered, sweep the JITDylibs to mark defs as canonical, discard - * duplicate defs. - */ + // FIXME: Switch to move-capture for the 'O' buffer once we have c++14. + MemoryBuffer *UnownedObjBuffer = O.release(); jitLinkForORC( **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, [this, K, SharedR, &Obj, InternalSymbols]( @@ -149,8 +143,9 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR](Error Err) { - onObjEmit(K, *SharedR, std::move(Err)); + [this, K, SharedR, UnownedObjBuffer](Error Err) { + std::unique_ptr<MemoryBuffer> ObjBuffer(UnownedObjBuffer); + onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err)); }); } @@ -177,7 +172,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( auto I = R.getSymbols().find(InternedName); if (OverrideObjectFlags && I != R.getSymbols().end()) - Flags = JITSymbolFlags::stripTransientFlags(I->second); + Flags = I->second; else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) ExtraSymbolsToClaim[InternedName] = Flags; } @@ -189,7 +184,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) return Err; - R.resolve(Symbols); + R.notifyResolved(Symbols); if (NotifyLoaded) NotifyLoaded(K, Obj, *LoadedObjInfo); @@ -197,20 +192,29 @@ Error RTDyldObjectLinkingLayer::onObjLoad( return Error::success(); } -void RTDyldObjectLinkingLayer::onObjEmit(VModuleKey K, - MaterializationResponsibility &R, - Error Err) { +void RTDyldObjectLinkingLayer::onObjEmit( + VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer, + MaterializationResponsibility &R, Error Err) { if (Err) { getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); return; } - R.emit(); + R.notifyEmitted(); if (NotifyEmitted) - NotifyEmitted(K); + NotifyEmitted(K, std::move(ObjBuffer)); } +LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer( + ExecutionSession &ES, ResourcesGetter GetResources, + NotifyLoadedFtor NotifyLoaded, NotifyFinalizedFtor NotifyFinalized, + NotifyFreedFtor NotifyFreed) + : ES(ES), GetResources(std::move(GetResources)), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), + NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/contrib/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp index 9525b168fbd3..4cb7376758a7 100644 --- a/contrib/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp +++ b/contrib/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -1,10 +1,9 @@ //===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities //h-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index f195d0282998..5606421a3cb0 100644 --- a/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/contrib/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -266,16 +265,22 @@ void PerfJITEventListener::notifyObjectLoaded( consumeError(AddrOrErr.takeError()); continue; } - uint64_t Addr = *AddrOrErr; uint64_t Size = P.second; + object::SectionedAddress Address; + Address.Address = *AddrOrErr; + + uint64_t SectionIndex = object::SectionedAddress::UndefSection; + if (auto SectOrErr = Sym.getSection()) + if (*SectOrErr != Obj.section_end()) + SectionIndex = SectOrErr.get()->getIndex(); // According to spec debugging info has to come before loading the // corresonding code load. DILineInfoTable Lines = Context->getLineInfoForAddressRange( - Addr, Size, FileLineInfoKind::AbsoluteFilePath); + {*AddrOrErr, SectionIndex}, Size, FileLineInfoKind::AbsoluteFilePath); - NotifyDebug(Addr, Lines); - NotifyCode(Name, Addr, Size); + NotifyDebug(*AddrOrErr, Lines); + NotifyCode(Name, *AddrOrErr, Size); } Dumpstream->flush(); @@ -336,8 +341,8 @@ bool PerfJITEventListener::OpenMarker() { // // Mapping must be PROT_EXEC to ensure it is captured by perf record // even when not using -d option. - MarkerAddr = ::mmap(NULL, sys::Process::getPageSize(), PROT_READ | PROT_EXEC, - MAP_PRIVATE, DumpFd, 0); + MarkerAddr = ::mmap(NULL, sys::Process::getPageSizeEstimate(), + PROT_READ | PROT_EXEC, MAP_PRIVATE, DumpFd, 0); if (MarkerAddr == MAP_FAILED) { errs() << "could not mmap JIT marker\n"; @@ -350,7 +355,7 @@ void PerfJITEventListener::CloseMarker() { if (!MarkerAddr) return; - munmap(MarkerAddr, sys::Process::getPageSize()); + munmap(MarkerAddr, sys::Process::getPageSizeEstimate()); MarkerAddr = nullptr; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp index 0553c217c2a2..4e2d0f422f39 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp @@ -1,9 +1,8 @@ //===----------- JITSymbol.cpp - JITSymbol class implementation -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp index 75d4c2b5134e..46604ff4000c 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -1,9 +1,8 @@ //===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,8 +32,9 @@ namespace llvm { RTDyldMemoryManager::~RTDyldMemoryManager() {} // Determine whether we can register EH tables. -#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ - !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)) +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !(defined(_AIX) && defined(__ibmxl__)) && !defined(__SEH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__)) #define HAVE_EHTABLE_SUPPORT 1 #else #define HAVE_EHTABLE_SUPPORT 0 @@ -48,7 +48,7 @@ extern "C" void __deregister_frame(void *); // it may be found at runtime in a dynamically-loaded library. // For example, this happens when building LLVM with Visual C++ // but using the MingW runtime. -void __register_frame(void *p) { +static void __register_frame(void *p) { static bool Searched = false; static void((*rf)(void *)) = 0; @@ -61,7 +61,7 @@ void __register_frame(void *p) { rf(p); } -void __deregister_frame(void *p) { +static void __deregister_frame(void *p) { static bool Searched = false; static void((*df)(void *)) = 0; diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 53cb782c55c4..e26e6ce45db4 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyld.cpp - Run-time dynamic linker for MC-JIT ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,7 +12,6 @@ #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "RuntimeDyldCOFF.h" -#include "RuntimeDyldCheckerImpl.h" #include "RuntimeDyldELF.h" #include "RuntimeDyldImpl.h" #include "RuntimeDyldMachO.h" @@ -376,10 +374,55 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { else return IOrErr.takeError(); - // If there is an attached checker, notify it about the stubs for this - // section so that they can be verified. - if (Checker) - Checker->registerStubMap(Obj.getFileName(), SectionID, Stubs); + // If there is a NotifyStubEmitted callback set, call it to register any + // stubs created for this section. + if (NotifyStubEmitted) { + StringRef FileName = Obj.getFileName(); + StringRef SectionName = Sections[SectionID].getName(); + for (auto &KV : Stubs) { + + auto &VR = KV.first; + uint64_t StubAddr = KV.second; + + // If this is a named stub, just call NotifyStubEmitted. + if (VR.SymbolName) { + NotifyStubEmitted(FileName, SectionName, VR.SymbolName, SectionID, + StubAddr); + continue; + } + + // Otherwise we will have to try a reverse lookup on the globla symbol table. + for (auto &GSTMapEntry : GlobalSymbolTable) { + StringRef SymbolName = GSTMapEntry.first(); + auto &GSTEntry = GSTMapEntry.second; + if (GSTEntry.getSectionID() == VR.SectionID && + GSTEntry.getOffset() == VR.Offset) { + NotifyStubEmitted(FileName, SectionName, SymbolName, SectionID, + StubAddr); + break; + } + } + } + } + } + + // Process remaining sections + if (ProcessAllSections) { + LLVM_DEBUG(dbgs() << "Process remaining sections:\n"); + for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); + SI != SE; ++SI) { + + /* Ignore already loaded sections */ + if (LocalSections.find(*SI) != LocalSections.end()) + continue; + + bool IsCode = SI->isText(); + if (auto SectionIDOrErr = + findOrEmitSection(Obj, *SI, IsCode, LocalSections)) + LLVM_DEBUG(dbgs() << "\tSectionID: " << (*SectionIDOrErr) << "\n"); + else + return SectionIDOrErr.takeError(); + } } // Give the subclasses a chance to tie-up any loose ends. @@ -497,7 +540,14 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, return errorCodeToError(EC); uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section); - uint64_t SectionSize = DataSize + StubBufSize; + + uint64_t PaddingSize = 0; + if (Name == ".eh_frame") + PaddingSize += 4; + if (StubBufSize != 0) + PaddingSize += getStubAlignment() - 1; + + uint64_t SectionSize = DataSize + PaddingSize + StubBufSize; // The .eh_frame section (at least on Linux) needs an extra four bytes // padded @@ -703,9 +753,6 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, Addr += Size; } - if (Checker) - Checker->registerSection(Obj.getFileName(), SectionID); - return Error::success(); } @@ -725,6 +772,11 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, bool IsReadOnly = isReadOnlyData(Section); uint64_t DataSize = Section.getSize(); + // An alignment of 0 (at least with ELF) is identical to an alignment of 1, + // while being more "polite". Other formats do not support 0-aligned sections + // anyway, so we should guarantee that the alignment is always at least 1. + Alignment = std::max(1u, Alignment); + StringRef Name; if (auto EC = Section.getName(Name)) return errorCodeToError(EC); @@ -747,18 +799,19 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (!IsVirtual && !IsZeroInit) { // In either case, set the location of the unrelocated section in memory, // since we still process relocations for it even if we're not applying them. - if (auto EC = Section.getContents(data)) - return errorCodeToError(EC); + if (Expected<StringRef> E = Section.getContents()) + data = *E; + else + return E.takeError(); pData = data.data(); } - // Code section alignment needs to be at least as high as stub alignment or - // padding calculations may by incorrect when the section is remapped to a - // higher alignment. - if (IsCode) { + // If there are any stubs then the section alignment needs to be at least as + // high as stub alignment or padding calculations may by incorrect when the + // section is remapped. + if (StubBufSize != 0) { Alignment = std::max(Alignment, getStubAlignment()); - if (StubBufSize > 0) - PaddingSize += getStubAlignment() - 1; + PaddingSize += getStubAlignment() - 1; } // Some sections, such as debug info, don't need to be loaded for execution. @@ -789,7 +842,7 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // Align DataSize to stub alignment if we have any stubs (PaddingSize will // have been increased above to account for this). if (StubBufSize > 0) - DataSize &= ~(getStubAlignment() - 1); + DataSize &= -(uint64_t)getStubAlignment(); } LLVM_DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " @@ -817,9 +870,6 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (!IsRequired) Sections.back().setLoadAddress(0); - if (Checker) - Checker->registerSection(Obj.getFileName(), SectionID); - return SectionID; } @@ -1202,42 +1252,43 @@ RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, // permissions are applied. Dyld = nullptr; ProcessAllSections = false; - Checker = nullptr; } RuntimeDyld::~RuntimeDyld() {} static std::unique_ptr<RuntimeDyldCOFF> -createRuntimeDyldCOFF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, - JITSymbolResolver &Resolver, bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { +createRuntimeDyldCOFF( + Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver, bool ProcessAllSections, + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr<RuntimeDyldCOFF> Dyld = RuntimeDyldCOFF::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } static std::unique_ptr<RuntimeDyldELF> createRuntimeDyldELF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, JITSymbolResolver &Resolver, bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr<RuntimeDyldELF> Dyld = RuntimeDyldELF::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } static std::unique_ptr<RuntimeDyldMachO> -createRuntimeDyldMachO(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, - JITSymbolResolver &Resolver, - bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { +createRuntimeDyldMachO( + Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver, + bool ProcessAllSections, + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr<RuntimeDyldMachO> Dyld = RuntimeDyldMachO::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } @@ -1247,15 +1298,16 @@ RuntimeDyld::loadObject(const ObjectFile &Obj) { if (Obj.isELF()) Dyld = createRuntimeDyldELF(static_cast<Triple::ArchType>(Obj.getArch()), - MemMgr, Resolver, ProcessAllSections, Checker); + MemMgr, Resolver, ProcessAllSections, + std::move(NotifyStubEmitted)); else if (Obj.isMachO()) Dyld = createRuntimeDyldMachO( static_cast<Triple::ArchType>(Obj.getArch()), MemMgr, Resolver, - ProcessAllSections, Checker); + ProcessAllSections, std::move(NotifyStubEmitted)); else if (Obj.isCOFF()) Dyld = createRuntimeDyldCOFF( static_cast<Triple::ArchType>(Obj.getArch()), MemMgr, Resolver, - ProcessAllSections, Checker); + ProcessAllSections, std::move(NotifyStubEmitted)); else report_fatal_error("Incompatible object format!"); } @@ -1274,6 +1326,11 @@ void *RuntimeDyld::getSymbolLocalAddress(StringRef Name) const { return Dyld->getSymbolLocalAddress(Name); } +unsigned RuntimeDyld::getSymbolSectionID(StringRef Name) const { + assert(Dyld && "No RuntimeDyld instance attached"); + return Dyld->getSymbolSectionID(Name); +} + JITEvaluatedSymbol RuntimeDyld::getSymbol(StringRef Name) const { if (!Dyld) return nullptr; @@ -1312,6 +1369,16 @@ void RuntimeDyld::finalizeWithMemoryManagerLocking() { } } +StringRef RuntimeDyld::getSectionContent(unsigned SectionID) const { + assert(Dyld && "No Dyld instance attached"); + return Dyld->getSectionContent(SectionID); +} + +uint64_t RuntimeDyld::getSectionLoadAddress(unsigned SectionID) const { + assert(Dyld && "No Dyld instance attached"); + return Dyld->getSectionLoadAddress(SectionID); +} + void RuntimeDyld::registerEHFrames() { if (Dyld) Dyld->registerEHFrames(); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 340ddaab186d..d4e3b0ba7670 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h index 729a358fa0ea..4efd18a2e6c5 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 6eb6256080ff..ec31ea4e573c 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -1,23 +1,21 @@ //===--- RuntimeDyldChecker.cpp - RuntimeDyld tester framework --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "RuntimeDyldCheckerImpl.h" -#include "RuntimeDyldImpl.h" #include "llvm/ADT/STLExtras.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInst.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MSVCErrorWorkarounds.h" #include "llvm/Support/Path.h" #include <cctype> -#include <future> #include <memory> #include <utility> @@ -321,22 +319,22 @@ private: return std::make_pair(EvalResult(NextPC), RemainingExpr); } - // Evaluate a call to stub_addr. + // Evaluate a call to stub_addr/got_addr. // Look up and return the address of the stub for the given // (<file name>, <section name>, <symbol name>) tuple. // On success, returns a pair containing the stub address, plus the expression // remaining to be evaluated. - std::pair<EvalResult, StringRef> evalStubAddr(StringRef Expr, - ParseContext PCtx) const { + std::pair<EvalResult, StringRef> + evalStubOrGOTAddr(StringRef Expr, ParseContext PCtx, bool IsStubAddr) const { if (!Expr.startswith("(")) return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); StringRef RemainingExpr = Expr.substr(1).ltrim(); // Handle file-name specially, as it may contain characters that aren't // legal for symbols. - StringRef FileName; + StringRef StubContainerName; size_t ComaIdx = RemainingExpr.find(','); - FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + StubContainerName = RemainingExpr.substr(0, ComaIdx).rtrim(); RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); if (!RemainingExpr.startswith(",")) @@ -344,14 +342,6 @@ private: unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); RemainingExpr = RemainingExpr.substr(1).ltrim(); - StringRef SectionName; - std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!RemainingExpr.startswith(",")) - return std::make_pair( - unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - StringRef Symbol; std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); @@ -362,8 +352,8 @@ private: uint64_t StubAddr; std::string ErrorMsg = ""; - std::tie(StubAddr, ErrorMsg) = Checker.getStubAddrFor( - FileName, SectionName, Symbol, PCtx.IsInsideLoad); + std::tie(StubAddr, ErrorMsg) = Checker.getStubOrGOTAddrFor( + StubContainerName, Symbol, PCtx.IsInsideLoad, IsStubAddr); if (ErrorMsg != "") return std::make_pair(EvalResult(ErrorMsg), ""); @@ -423,7 +413,9 @@ private: else if (Symbol == "next_pc") return evalNextPC(RemainingExpr, PCtx); else if (Symbol == "stub_addr") - return evalStubAddr(RemainingExpr, PCtx); + return evalStubOrGOTAddr(RemainingExpr, PCtx, true); + else if (Symbol == "got_addr") + return evalStubOrGOTAddr(RemainingExpr, PCtx, false); else if (Symbol == "section_addr") return evalSectionAddr(RemainingExpr, PCtx); @@ -534,6 +526,11 @@ private: uint64_t LoadAddr = LoadAddrExprResult.getValue(); + // If there is no error but the content pointer is null then this is a + // zero-fill symbol/section. + if (LoadAddr == 0) + return std::make_pair(0, RemainingExpr); + return std::make_pair( EvalResult(Checker.readMemoryAtAddr(LoadAddr, ReadSize)), RemainingExpr); @@ -666,27 +663,29 @@ private: bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { MCDisassembler *Dis = Checker.Disassembler; - StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); - ArrayRef<uint8_t> SectionBytes( - reinterpret_cast<const uint8_t *>(SectionMem.data()), - SectionMem.size()); + StringRef SymbolMem = Checker.getSymbolContent(Symbol); + ArrayRef<uint8_t> SymbolBytes(SymbolMem.bytes_begin(), SymbolMem.size()); MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + Dis->getInstruction(Inst, Size, SymbolBytes, 0, nulls(), nulls()); return (S == MCDisassembler::Success); } }; } -RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, - MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) - : RTDyld(RTDyld), Disassembler(Disassembler), InstPrinter(InstPrinter), - ErrStream(ErrStream) { - RTDyld.Checker = this; -} +RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : IsSymbolValid(std::move(IsSymbolValid)), + GetSymbolInfo(std::move(GetSymbolInfo)), + GetSectionInfo(std::move(GetSectionInfo)), + GetStubInfo(std::move(GetStubInfo)), GetGOTInfo(std::move(GetGOTInfo)), + Endianness(Endianness), Disassembler(Disassembler), + InstPrinter(InstPrinter), ErrStream(ErrStream) {} bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); @@ -731,242 +730,134 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, return DidAllTestsPass && (NumRules != 0); } -Expected<JITSymbolResolver::LookupResult> RuntimeDyldCheckerImpl::lookup( - const JITSymbolResolver::LookupSet &Symbols) const { - -#ifdef _MSC_VER - using ExpectedLookupResult = MSVCPExpected<JITSymbolResolver::LookupResult>; -#else - using ExpectedLookupResult = Expected<JITSymbolResolver::LookupResult>; -#endif - - auto ResultP = std::make_shared<std::promise<ExpectedLookupResult>>(); - auto ResultF = ResultP->get_future(); - - getRTDyld().Resolver.lookup( - Symbols, [=](Expected<JITSymbolResolver::LookupResult> Result) { - ResultP->set_value(std::move(Result)); - }); - return ResultF.get(); -} - bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { - if (getRTDyld().getSymbol(Symbol)) - return true; - auto Result = lookup({Symbol}); + return IsSymbolValid(Symbol); +} - if (!Result) { - logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); - return false; +uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); + return 0; } - assert(Result->count(Symbol) && "Missing symbol result"); - return true; -} + if (SymInfo->isZeroFill()) + return 0; -uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { return static_cast<uint64_t>( - reinterpret_cast<uintptr_t>(getRTDyld().getSymbolLocalAddress(Symbol))); + reinterpret_cast<uintptr_t>(SymInfo->getContent().data())); } uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { - if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) - return InternalSymbol.getAddress(); - - auto Result = lookup({Symbol}); - if (!Result) { - logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); return 0; } - auto I = Result->find(Symbol); - assert(I != Result->end() && "Missing symbol result"); - return I->second.getAddress(); + + return SymInfo->getTargetAddress(); } uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, unsigned Size) const { uintptr_t PtrSizedAddr = static_cast<uintptr_t>(SrcAddr); assert(PtrSizedAddr == SrcAddr && "Linker memory pointer out-of-range."); - uint8_t *Src = reinterpret_cast<uint8_t*>(PtrSizedAddr); - return getRTDyld().readBytesUnaligned(Src, Size); + void *Ptr = reinterpret_cast<void*>(PtrSizedAddr); + + switch (Size) { + case 1: + return support::endian::read<uint8_t>(Ptr, Endianness); + case 2: + return support::endian::read<uint16_t>(Ptr, Endianness); + case 4: + return support::endian::read<uint32_t>(Ptr, Endianness); + case 8: + return support::endian::read<uint64_t>(Ptr, Endianness); + } + llvm_unreachable("Unsupported read size"); } - -std::pair<const RuntimeDyldCheckerImpl::SectionAddressInfo*, std::string> -RuntimeDyldCheckerImpl::findSectionAddrInfo(StringRef FileName, - StringRef SectionName) const { - - auto SectionMapItr = Stubs.find(FileName); - if (SectionMapItr == Stubs.end()) { - std::string ErrorMsg = "File '"; - ErrorMsg += FileName; - ErrorMsg += "' not found. "; - if (Stubs.empty()) - ErrorMsg += "No stubs registered."; - else { - ErrorMsg += "Available files are:"; - for (const auto& StubEntry : Stubs) { - ErrorMsg += " '"; - ErrorMsg += StubEntry.first; - ErrorMsg += "'"; - } - } - ErrorMsg += "\n"; - return std::make_pair(nullptr, ErrorMsg); +StringRef RuntimeDyldCheckerImpl::getSymbolContent(StringRef Symbol) const { + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); + return StringRef(); } - - auto SectionInfoItr = SectionMapItr->second.find(SectionName); - if (SectionInfoItr == SectionMapItr->second.end()) - return std::make_pair(nullptr, - ("Section '" + SectionName + "' not found in file '" + - FileName + "'\n").str()); - - return std::make_pair(&SectionInfoItr->second, std::string("")); + return SymInfo->getContent(); } std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( StringRef FileName, StringRef SectionName, bool IsInsideLoad) const { - const SectionAddressInfo *SectionInfo = nullptr; - { - std::string ErrorMsg; - std::tie(SectionInfo, ErrorMsg) = - findSectionAddrInfo(FileName, SectionName); - if (ErrorMsg != "") - return std::make_pair(0, ErrorMsg); - } - - unsigned SectionID = SectionInfo->SectionID; - uint64_t Addr; - if (IsInsideLoad) - Addr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>( - getRTDyld().Sections[SectionID].getAddress())); - else - Addr = getRTDyld().Sections[SectionID].getLoadAddress(); - - return std::make_pair(Addr, std::string("")); -} - -std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubAddrFor( - StringRef FileName, StringRef SectionName, StringRef SymbolName, - bool IsInsideLoad) const { - - const SectionAddressInfo *SectionInfo = nullptr; - { - std::string ErrorMsg; - std::tie(SectionInfo, ErrorMsg) = - findSectionAddrInfo(FileName, SectionName); - if (ErrorMsg != "") - return std::make_pair(0, ErrorMsg); + auto SecInfo = GetSectionInfo(FileName, SectionName); + if (!SecInfo) { + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + logAllUnhandledErrors(SecInfo.takeError(), ErrMsgStream, + "RTDyldChecker: "); + } + return std::make_pair(0, std::move(ErrMsg)); } - unsigned SectionID = SectionInfo->SectionID; - const StubOffsetsMap &SymbolStubs = SectionInfo->StubOffsets; - auto StubOffsetItr = SymbolStubs.find(SymbolName); - if (StubOffsetItr == SymbolStubs.end()) - return std::make_pair(0, - ("Stub for symbol '" + SymbolName + "' not found. " - "If '" + SymbolName + "' is an internal symbol this " - "may indicate that the stub target offset is being " - "computed incorrectly.\n").str()); + // If this address is being looked up in "load" mode, return the content + // pointer, otherwise return the target address. - uint64_t StubOffset = StubOffsetItr->second; + uint64_t Addr = 0; - uint64_t Addr; if (IsInsideLoad) { - uintptr_t SectionBase = reinterpret_cast<uintptr_t>( - getRTDyld().Sections[SectionID].getAddress()); - Addr = static_cast<uint64_t>(SectionBase) + StubOffset; - } else { - uint64_t SectionBase = getRTDyld().Sections[SectionID].getLoadAddress(); - Addr = SectionBase + StubOffset; - } - - return std::make_pair(Addr, std::string("")); -} - -StringRef -RuntimeDyldCheckerImpl::getSubsectionStartingAt(StringRef Name) const { - RTDyldSymbolTable::const_iterator pos = - getRTDyld().GlobalSymbolTable.find(Name); - if (pos == getRTDyld().GlobalSymbolTable.end()) - return StringRef(); - const auto &SymInfo = pos->second; - uint8_t *SectionAddr = getRTDyld().getSectionAddress(SymInfo.getSectionID()); - return StringRef(reinterpret_cast<const char *>(SectionAddr) + - SymInfo.getOffset(), - getRTDyld().Sections[SymInfo.getSectionID()].getSize() - - SymInfo.getOffset()); -} - -Optional<uint64_t> -RuntimeDyldCheckerImpl::getSectionLoadAddress(void *LocalAddress) const { - for (auto &S : getRTDyld().Sections) { - if (S.getAddress() == LocalAddress) - return S.getLoadAddress(); - } - return Optional<uint64_t>(); -} - -void RuntimeDyldCheckerImpl::registerSection( - StringRef FilePath, unsigned SectionID) { - StringRef FileName = sys::path::filename(FilePath); - const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.getName(); + if (SecInfo->isZeroFill()) + Addr = 0; + else + Addr = pointerToJITTargetAddress(SecInfo->getContent().data()); + } else + Addr = SecInfo->getTargetAddress(); - Stubs[FileName][SectionName].SectionID = SectionID; + return std::make_pair(Addr, ""); } -void RuntimeDyldCheckerImpl::registerStubMap( - StringRef FilePath, unsigned SectionID, - const RuntimeDyldImpl::StubMap &RTDyldStubs) { - StringRef FileName = sys::path::filename(FilePath); - const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.getName(); - - Stubs[FileName][SectionName].SectionID = SectionID; +std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubOrGOTAddrFor( + StringRef StubContainerName, StringRef SymbolName, bool IsInsideLoad, + bool IsStubAddr) const { - for (auto &StubMapEntry : RTDyldStubs) { - std::string SymbolName = ""; + auto StubInfo = IsStubAddr ? GetStubInfo(StubContainerName, SymbolName) + : GetGOTInfo(StubContainerName, SymbolName); - if (StubMapEntry.first.SymbolName) - SymbolName = StubMapEntry.first.SymbolName; - else { - // If this is a (Section, Offset) pair, do a reverse lookup in the - // global symbol table to find the name. - for (auto &GSTEntry : getRTDyld().GlobalSymbolTable) { - const auto &SymInfo = GSTEntry.second; - if (SymInfo.getSectionID() == StubMapEntry.first.SectionID && - SymInfo.getOffset() == - static_cast<uint64_t>(StubMapEntry.first.Offset)) { - SymbolName = GSTEntry.first(); - break; - } - } + if (!StubInfo) { + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + logAllUnhandledErrors(StubInfo.takeError(), ErrMsgStream, + "RTDyldChecker: "); } - - if (SymbolName != "") - Stubs[FileName][SectionName].StubOffsets[SymbolName] = - StubMapEntry.second; + return std::make_pair((uint64_t)0, std::move(ErrMsg)); } -} -RuntimeDyldChecker::RuntimeDyldChecker(RuntimeDyld &RTDyld, - MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) - : Impl(make_unique<RuntimeDyldCheckerImpl>(RTDyld, Disassembler, - InstPrinter, ErrStream)) {} + uint64_t Addr = 0; -RuntimeDyldChecker::~RuntimeDyldChecker() {} + if (IsInsideLoad) { + if (StubInfo->isZeroFill()) + return std::make_pair((uint64_t)0, "Detected zero-filled stub/GOT entry"); + Addr = pointerToJITTargetAddress(StubInfo->getContent().data()); + } else + Addr = StubInfo->getTargetAddress(); -RuntimeDyld& RuntimeDyldChecker::getRTDyld() { - return Impl->RTDyld; + return std::make_pair(Addr, ""); } -const RuntimeDyld& RuntimeDyldChecker::getRTDyld() const { - return Impl->RTDyld; -} +RuntimeDyldChecker::RuntimeDyldChecker( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : Impl(::llvm::make_unique<RuntimeDyldCheckerImpl>( + std::move(IsSymbolValid), std::move(GetSymbolInfo), + std::move(GetSectionInfo), std::move(GetStubInfo), + std::move(GetGOTInfo), Endianness, Disassembler, InstPrinter, + ErrStream)) {} + +RuntimeDyldChecker::~RuntimeDyldChecker() {} bool RuntimeDyldChecker::check(StringRef CheckExpr) const { return Impl->check(CheckExpr); @@ -982,8 +873,3 @@ RuntimeDyldChecker::getSectionAddr(StringRef FileName, StringRef SectionName, bool LocalAddress) { return Impl->getSectionAddr(FileName, SectionName, LocalAddress); } - -Optional<uint64_t> -RuntimeDyldChecker::getSectionLoadAddress(void *LocalAddress) const { - return Impl->getSectionLoadAddress(LocalAddress); -} diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index 6da1a68d06d6..ac9d4d460217 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCheckerImpl.h -- RuntimeDyld test framework --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,14 +15,22 @@ namespace llvm { class RuntimeDyldCheckerImpl { friend class RuntimeDyldChecker; - friend class RuntimeDyldImpl; friend class RuntimeDyldCheckerExprEval; - friend class RuntimeDyldELF; + + using IsSymbolValidFunction = + RuntimeDyldChecker::IsSymbolValidFunction; + using GetSymbolInfoFunction = RuntimeDyldChecker::GetSymbolInfoFunction; + using GetSectionInfoFunction = RuntimeDyldChecker::GetSectionInfoFunction; + using GetStubInfoFunction = RuntimeDyldChecker::GetStubInfoFunction; + using GetGOTInfoFunction = RuntimeDyldChecker::GetGOTInfoFunction; public: - RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - llvm::raw_ostream &ErrStream); + RuntimeDyldCheckerImpl( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + llvm::raw_ostream &ErrStream); bool check(StringRef CheckExpr) const; bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; @@ -31,15 +38,6 @@ public: private: // StubMap typedefs. - typedef std::map<std::string, uint64_t> StubOffsetsMap; - struct SectionAddressInfo { - uint64_t SectionID; - StubOffsetsMap StubOffsets; - }; - typedef std::map<std::string, SectionAddressInfo> SectionMap; - typedef std::map<std::string, SectionMap> StubMap; - - RuntimeDyldImpl &getRTDyld() const { return *RTDyld.Dyld; } Expected<JITSymbolResolver::LookupResult> lookup(const JITSymbolResolver::LookupSet &Symbols) const; @@ -49,32 +47,27 @@ private: uint64_t getSymbolRemoteAddr(StringRef Symbol) const; uint64_t readMemoryAtAddr(uint64_t Addr, unsigned Size) const; - std::pair<const SectionAddressInfo*, std::string> findSectionAddrInfo( - StringRef FileName, - StringRef SectionName) const; + StringRef getSymbolContent(StringRef Symbol) const; std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName, StringRef SectionName, bool IsInsideLoad) const; - std::pair<uint64_t, std::string> getStubAddrFor(StringRef FileName, - StringRef SectionName, - StringRef Symbol, - bool IsInsideLoad) const; - StringRef getSubsectionStartingAt(StringRef Name) const; + std::pair<uint64_t, std::string> + getStubOrGOTAddrFor(StringRef StubContainerName, StringRef Symbol, + bool IsInsideLoad, bool IsStubAddr) const; Optional<uint64_t> getSectionLoadAddress(void *LocalAddr) const; - void registerSection(StringRef FilePath, unsigned SectionID); - void registerStubMap(StringRef FilePath, unsigned SectionID, - const RuntimeDyldImpl::StubMap &RTDyldStubs); - - RuntimeDyld &RTDyld; + IsSymbolValidFunction IsSymbolValid; + GetSymbolInfoFunction GetSymbolInfo; + GetSectionInfoFunction GetSectionInfo; + GetStubInfoFunction GetStubInfo; + GetGOTInfoFunction GetGOTInfo; + support::endianness Endianness; MCDisassembler *Disassembler; MCInstPrinter *InstPrinter; llvm::raw_ostream &ErrStream; - - StubMap Stubs; }; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 226ee715e18b..60041a45e2b8 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1857,9 +1856,6 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, Sections[GOTSectionID] = SectionEntry(".got", Addr, TotalSize, TotalSize, 0); - if (Checker) - Checker->registerSection(Obj.getFileName(), GOTSectionID); - // For now, initialize all GOT entries to zero. We'll fill them in as // needed when GOT-based relocations are applied. memset(Addr, 0, TotalSize); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index f37bd0bbaea6..ef0784e2273b 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldELF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -61,7 +60,7 @@ class RuntimeDyldELF : public RuntimeDyldImpl { void resolveBPFRelocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend); - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) return 20; // movz; movk; movk; movk; br if (Arch == Triple::arm || Arch == Triple::thumb) diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 4c650e09ac1f..68b3468fbc9d 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldImpl.h - Run-time dynamic linker for MC-JIT --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -241,7 +240,6 @@ typedef StringMap<SymbolTableEntry> RTDyldSymbolTable; class RuntimeDyldImpl { friend class RuntimeDyld::LoadedObjectInfo; - friend class RuntimeDyldCheckerImpl; protected: static const unsigned AbsoluteSymbolSection = ~0U; @@ -251,9 +249,6 @@ protected: // The symbol resolver to use for external symbols. JITSymbolResolver &Resolver; - // Attached RuntimeDyldChecker instance. Null if no instance attached. - RuntimeDyldCheckerImpl *Checker; - // A list of all sections emitted by the dynamic linker. These sections are // referenced in the code by means of their index in this list - SectionID. typedef SmallVector<SectionEntry, 64> SectionList; @@ -313,20 +308,16 @@ protected: // the end of the list while the list is being processed. sys::Mutex lock; - virtual unsigned getMaxStubSize() = 0; + using NotifyStubEmittedFunction = + RuntimeDyld::NotifyStubEmittedFunction; + NotifyStubEmittedFunction NotifyStubEmitted; + + virtual unsigned getMaxStubSize() const = 0; virtual unsigned getStubAlignment() = 0; bool HasError; std::string ErrorStr; - uint64_t getSectionLoadAddress(unsigned SectionID) const { - return Sections[SectionID].getLoadAddress(); - } - - uint8_t *getSectionAddress(unsigned SectionID) const { - return Sections[SectionID].getAddress(); - } - void writeInt16BE(uint8_t *Addr, uint16_t Value) { if (IsTargetLittleEndian) sys::swapByteOrder(Value); @@ -472,7 +463,7 @@ protected: public: RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver) - : MemMgr(MemMgr), Resolver(Resolver), Checker(nullptr), + : MemMgr(MemMgr), Resolver(Resolver), ProcessAllSections(false), HasError(false) { } @@ -482,13 +473,22 @@ public: this->ProcessAllSections = ProcessAllSections; } - void setRuntimeDyldChecker(RuntimeDyldCheckerImpl *Checker) { - this->Checker = Checker; - } - virtual std::unique_ptr<RuntimeDyld::LoadedObjectInfo> loadObject(const object::ObjectFile &Obj) = 0; + uint64_t getSectionLoadAddress(unsigned SectionID) const { + return Sections[SectionID].getLoadAddress(); + } + + uint8_t *getSectionAddress(unsigned SectionID) const { + return Sections[SectionID].getAddress(); + } + + StringRef getSectionContent(unsigned SectionID) const { + return StringRef(reinterpret_cast<char *>(Sections[SectionID].getAddress()), + Sections[SectionID].getStubOffset() + getMaxStubSize()); + } + uint8_t* getSymbolLocalAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. @@ -502,6 +502,13 @@ public: return getSectionAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); } + unsigned getSymbolSectionID(StringRef Name) const { + auto GSTItr = GlobalSymbolTable.find(Name); + if (GSTItr == GlobalSymbolTable.end()) + return ~0U; + return GSTItr->second.getSectionID(); + } + JITEvaluatedSymbol getSymbol(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. @@ -560,6 +567,10 @@ public: virtual bool isCompatibleFile(const ObjectFile &Obj) const = 0; + void setNotifyStubEmitted(NotifyStubEmittedFunction NotifyStubEmitted) { + this->NotifyStubEmitted = std::move(NotifyStubEmitted); + } + virtual void registerEHFrames(); void deregisterEHFrames(); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index d47fcd45be88..202c3ca1c507 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachO.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index d71ca4e54953..650e7b79fbb8 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachO.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h index dd65051edad7..40910bea0c36 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h @@ -1,9 +1,8 @@ //===--- RuntimeDyldCOFFI386.h --- COFF/X86_64 specific code ---*- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,16 +27,16 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldCOFF(MM, Resolver) {} - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { return 8; // 2-byte jmp instruction + 32-bit relative address + 2 byte pad } unsigned getStubAlignment() override { return 1; } - Expected<relocation_iterator> + Expected<object::relocation_iterator> processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index 8723dd0fd0ea..bb2e9626e0b0 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -1,9 +1,8 @@ //===--- RuntimeDyldCOFFThumb.h --- COFF/Thumb specific code ---*- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,9 +21,10 @@ namespace llvm { -static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj, - section_iterator Section) { - Expected<SymbolRef::Type> SymTypeOrErr = Symbol->getType(); +static bool isThumbFunc(object::symbol_iterator Symbol, + const object::ObjectFile &Obj, + object::section_iterator Section) { + Expected<object::SymbolRef::Type> SymTypeOrErr = Symbol->getType(); if (!SymTypeOrErr) { std::string Buf; raw_string_ostream OS(Buf); @@ -33,12 +33,14 @@ static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj, report_fatal_error(Buf); } - if (*SymTypeOrErr != SymbolRef::ST_Function) + if (*SymTypeOrErr != object::SymbolRef::ST_Function) return false; // We check the IMAGE_SCN_MEM_16BIT flag in the section of the symbol to tell // if it's thumb or not - return cast<COFFObjectFile>(Obj).getCOFFSection(*Section)->Characteristics & + return cast<object::COFFObjectFile>(Obj) + .getCOFFSection(*Section) + ->Characteristics & COFF::IMAGE_SCN_MEM_16BIT; } @@ -48,16 +50,16 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldCOFF(MM, Resolver) {} - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { return 16; // 8-byte load instructions, 4-byte jump, 4-byte padding } unsigned getStubAlignment() override { return 1; } - Expected<relocation_iterator> + Expected<object::relocation_iterator> processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { auto Symbol = RelI->getSymbol(); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index aee5f6dc3746..d2d74534cf90 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFFX86_64.h --- COFF/X86_64 specific code ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,7 +61,7 @@ public: unsigned getStubAlignment() override { return 1; } // 2-byte jmp instruction + 32-bit relative address + 64-bit absolute jump - unsigned getMaxStubSize() override { return 14; } + unsigned getMaxStubSize() const override { return 14; } // The target location for the relocation is described by RE.SectionID and // RE.Offset. RE.SectionID can be used to find the SectionEntry. Each @@ -187,21 +186,21 @@ public: return std::make_tuple(Offset, RelType, Addend); } - Expected<relocation_iterator> + Expected<object::relocation_iterator> processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { // If possible, find the symbol referred to in the relocation, // and the section that contains it. - symbol_iterator Symbol = RelI->getSymbol(); + object::symbol_iterator Symbol = RelI->getSymbol(); if (Symbol == Obj.symbol_end()) report_fatal_error("Unknown symbol in relocation"); auto SectionOrError = Symbol->getSection(); if (!SectionOrError) return SectionOrError.takeError(); - section_iterator SecI = *SectionOrError; + object::section_iterator SecI = *SectionOrError; // If there is no section, this must be an external reference. const bool IsExtern = SecI == Obj.section_end(); @@ -280,11 +279,11 @@ public: UnregisteredEHFrameSections.clear(); } - Error finalizeLoad(const ObjectFile &Obj, + Error finalizeLoad(const object::ObjectFile &Obj, ObjSectionToIDMap &SectionMap) override { // Look for and record the EH frame section IDs. for (const auto &SectionPair : SectionMap) { - const SectionRef &Section = SectionPair.first; + const object::SectionRef &Section = SectionPair.first; StringRef Name; if (auto EC = Section.getName(Name)) return errorCodeToError(EC); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp index 3a166b40af2d..17cbe612fb43 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldELFMips.cpp ---- ELF/Mips specific code. -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h index f53b9e6bd75a..14fb36f070f8 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldELFMips.h ---- ELF/Mips specific code. -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 2a619c549cfa..f2ee1b06d494 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachOAArch64.h -- MachO/AArch64 specific code. -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,7 +26,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } unsigned getStubAlignment() override { return 8; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index ab7cd2bdae15..3bec8b979f7d 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -1,9 +1,8 @@ //===----- RuntimeDyldMachOARM.h ---- MachO/ARM specific code. ----*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -30,7 +29,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } unsigned getStubAlignment() override { return 4; } @@ -225,7 +224,7 @@ public: HighInsn = (HighInsn & 0xf800) | ((Value >> 12) & 0x7ff); uint16_t LowInsn = readBytesUnaligned(LocalAddress + 2, 2); - assert((LowInsn & 0xf800) != 0xf8000 && + assert((LowInsn & 0xf800) == 0xf800 && "Unrecognized thumb branch encoding (BR22 low bits)"); LowInsn = (LowInsn & 0xf800) | ((Value >> 1) & 0x7ff); diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index d384d70b8b0f..f0de27ba14bb 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -1,9 +1,8 @@ //===---- RuntimeDyldMachOI386.h ---- MachO/I386 specific code. ---*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,7 +26,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 0; } + unsigned getMaxStubSize() const override { return 0; } unsigned getStubAlignment() override { return 1; } diff --git a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index 9732ea6a0cd2..28febbdb948c 100644 --- a/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/contrib/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachOX86_64.h ---- MachO/X86_64 specific code. -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,9 +26,9 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } - unsigned getStubAlignment() override { return 1; } + unsigned getStubAlignment() override { return 8; } Expected<relocation_iterator> processRelocationRef(unsigned SectionID, relocation_iterator RelI, diff --git a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index 05ab4a074e37..925049b2a1b4 100644 --- a/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/contrib/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -1,9 +1,8 @@ //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -65,9 +64,9 @@ uint8_t *SectionMemoryManager::allocateSection( // Look in the list of free memory regions and use a block there if one // is available. for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { - if (FreeMB.Free.size() >= RequiredSize) { + if (FreeMB.Free.allocatedSize() >= RequiredSize) { Addr = (uintptr_t)FreeMB.Free.base(); - uintptr_t EndOfBlock = Addr + FreeMB.Free.size(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); @@ -116,7 +115,7 @@ uint8_t *SectionMemoryManager::allocateSection( // Remember that we allocated this memory MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.size(); + uintptr_t EndOfBlock = Addr + MB.allocatedSize(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); @@ -173,12 +172,12 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { } static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { - static const size_t PageSize = sys::Process::getPageSize(); + static const size_t PageSize = sys::Process::getPageSizeEstimate(); size_t StartOverlap = (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; - size_t TrimmedSize = M.size(); + size_t TrimmedSize = M.allocatedSize(); TrimmedSize -= StartOverlap; TrimmedSize -= TrimmedSize % PageSize; @@ -186,8 +185,9 @@ static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { TrimmedSize); assert(((uintptr_t)Trimmed.base() % PageSize) == 0); - assert((Trimmed.size() % PageSize) == 0); - assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size()); + assert((Trimmed.allocatedSize() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && + Trimmed.allocatedSize() <= M.allocatedSize()); return Trimmed; } @@ -210,17 +210,19 @@ SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, } // Remove all blocks which are now empty - MemGroup.FreeMem.erase( - remove_if(MemGroup.FreeMem, - [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }), - MemGroup.FreeMem.end()); + MemGroup.FreeMem.erase(remove_if(MemGroup.FreeMem, + [](FreeMemBlock &FreeMB) { + return FreeMB.Free.allocatedSize() == 0; + }), + MemGroup.FreeMem.end()); return std::error_code(); } void SectionMemoryManager::invalidateInstructionCache() { for (sys::MemoryBlock &Block : CodeMem.PendingMem) - sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); } SectionMemoryManager::~SectionMemoryManager() { diff --git a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp index 9626b8d3ffa3..0d9c6cfa0908 100644 --- a/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp +++ b/contrib/llvm/lib/ExecutionEngine/TargetSelect.cpp @@ -1,9 +1,8 @@ //===-- TargetSelect.cpp - Target Chooser Code ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // |