diff options
Diffstat (limited to 'ELF')
51 files changed, 2685 insertions, 1503 deletions
diff --git a/ELF/AArch64ErrataFix.cpp b/ELF/AArch64ErrataFix.cpp index b2eda4dcbc4e..7fb3e02e7ee4 100644 --- a/ELF/AArch64ErrataFix.cpp +++ b/ELF/AArch64ErrataFix.cpp @@ -6,7 +6,10 @@ // //===----------------------------------------------------------------------===// // This file implements Section Patching for the purpose of working around -// errata in CPUs. The general principle is that an erratum sequence of one or +// the AArch64 Cortex-53 errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 +// versions of the core. +// +// The general principle is that an erratum sequence of one or // more instructions is detected in the instruction stream, one of the // instructions in the sequence is replaced with a branch to a patch sequence // of replacement instructions. At the end of the replacement sequence the @@ -20,12 +23,6 @@ // - We can overwrite an instruction in the erratum sequence with a branch to // the replacement sequence. // - We can place the replacement sequence within range of the branch. - -// FIXME: -// - The implementation here only supports one patch, the AArch64 Cortex-53 -// errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 versions of the core. -// To keep the initial version simple there is no support for multiple -// architectures or selection of different patches. //===----------------------------------------------------------------------===// #include "AArch64ErrataFix.h" @@ -48,8 +45,8 @@ using namespace llvm::object; using namespace llvm::support; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { // Helper functions to identify instructions and conditions needed to trigger // the Cortex-A53-843419 erratum. @@ -333,16 +330,16 @@ static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2, } // Scan the instruction sequence starting at Offset Off from the base of -// InputSection IS. We update Off in this function rather than in the caller as -// we can skip ahead much further into the section when we know how many +// InputSection isec. We update Off in this function rather than in the caller +// as we can skip ahead much further into the section when we know how many // instructions we've scanned. -// Return the offset of the load or store instruction in IS that we want to +// Return the offset of the load or store instruction in isec that we want to // patch or 0 if no patch required. static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off, uint64_t limit) { uint64_t isecAddr = isec->getVA(0); - // Advance Off so that (ISAddr + Off) modulo 0x1000 is at least 0xff8. + // Advance Off so that (isecAddr + Off) modulo 0x1000 is at least 0xff8. uint64_t initialPageOff = (isecAddr + off) & 0xfff; if (initialPageOff < 0xff8) off += 0xff8 - initialPageOff; @@ -374,7 +371,7 @@ static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off, return patchOff; } -class lld::elf::Patch843419Section : public SyntheticSection { +class Patch843419Section : public SyntheticSection { public: Patch843419Section(InputSection *p, uint64_t off); @@ -386,13 +383,13 @@ public: // The Section we are patching. const InputSection *patchee; - // The offset of the instruction in the Patchee section we are patching. + // The offset of the instruction in the patchee section we are patching. uint64_t patcheeOffset; // A label for the start of the Patch that we can use as a relocation target. Symbol *patchSym; }; -lld::elf::Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) +Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4, ".text.patch"), patchee(p), patcheeOffset(off) { @@ -403,16 +400,16 @@ lld::elf::Patch843419Section::Patch843419Section(InputSection *p, uint64_t off) addSyntheticLocal(saver.save("$x"), STT_NOTYPE, 0, 0, *this); } -uint64_t lld::elf::Patch843419Section::getLDSTAddr() const { +uint64_t Patch843419Section::getLDSTAddr() const { return patchee->getVA(patcheeOffset); } -void lld::elf::Patch843419Section::writeTo(uint8_t *buf) { +void Patch843419Section::writeTo(uint8_t *buf) { // Copy the instruction that we will be replacing with a branch in the - // Patchee Section. + // patchee Section. write32le(buf, read32le(patchee->data().begin() + patcheeOffset)); - // Apply any relocation transferred from the original PatcheeSection. + // Apply any relocation transferred from the original patchee section. // For a SyntheticSection Buf already has outSecOff added, but relocateAlloc // also adds outSecOff so we need to subtract to avoid double counting. this->relocateAlloc(buf - outSecOff, buf - outSecOff + getSize()); @@ -461,18 +458,18 @@ void AArch64Err843419Patcher::init() { // $d.0 $d.1 $x.1. for (auto &kv : sectionMap) { std::vector<const Defined *> &mapSyms = kv.second; - if (mapSyms.size() <= 1) - continue; llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) { return a->value < b->value; }); mapSyms.erase( std::unique(mapSyms.begin(), mapSyms.end(), [=](const Defined *a, const Defined *b) { - return (isCodeMapSymbol(a) && isCodeMapSymbol(b)) || - (isDataMapSymbol(a) && isDataMapSymbol(b)); + return isCodeMapSymbol(a) == isCodeMapSymbol(b); }), mapSyms.end()); + // Always start with a Code Mapping Symbol. + if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front())) + mapSyms.erase(mapSyms.begin()); } initialized = true; } @@ -511,19 +508,16 @@ void AArch64Err843419Patcher::insertPatches( (*patchIt)->outSecOff = isecLimit; } - // merge all patch sections. We use the outSecOff assigned above to + // Merge all patch sections. We use the outSecOff assigned above to // determine the insertion point. This is ok as we only merge into an // InputSectionDescription once per pass, and at the end of the pass // assignAddresses() will recalculate all the outSecOff values. std::vector<InputSection *> tmp; tmp.reserve(isd.sections.size() + patches.size()); auto mergeCmp = [](const InputSection *a, const InputSection *b) { - if (a->outSecOff < b->outSecOff) - return true; - if (a->outSecOff == b->outSecOff && isa<Patch843419Section>(a) && - !isa<Patch843419Section>(b)) - return true; - return false; + if (a->outSecOff != b->outSecOff) + return a->outSecOff < b->outSecOff; + return isa<Patch843419Section>(a) && !isa<Patch843419Section>(b); }; std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(), patches.end(), std::back_inserter(tmp), mergeCmp); @@ -532,7 +526,7 @@ void AArch64Err843419Patcher::insertPatches( // Given an erratum sequence that starts at address adrpAddr, with an // instruction that we need to patch at patcheeOffset from the start of -// InputSection IS, create a Patch843419 Section and add it to the +// InputSection isec, create a Patch843419 Section and add it to the // Patches that we need to insert. static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset, InputSection *isec, @@ -578,7 +572,7 @@ static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset, // Scan all the instructions in InputSectionDescription, for each instance of // the erratum sequence create a Patch843419Section. We return the list of -// Patch843419Sections that need to be applied to ISD. +// Patch843419Sections that need to be applied to the InputSectionDescription. std::vector<Patch843419Section *> AArch64Err843419Patcher::patchInputSectionDescription( InputSectionDescription &isd) { @@ -594,10 +588,7 @@ AArch64Err843419Patcher::patchInputSectionDescription( // section size). std::vector<const Defined *> &mapSyms = sectionMap[isec]; - auto codeSym = llvm::find_if(mapSyms, [&](const Defined *ms) { - return ms->getName().startswith("$x"); - }); - + auto codeSym = mapSyms.begin(); while (codeSym != mapSyms.end()) { auto dataSym = std::next(codeSym); uint64_t off = (*codeSym)->value; @@ -606,7 +597,8 @@ AArch64Err843419Patcher::patchInputSectionDescription( while (off < limit) { uint64_t startAddr = isec->getVA(off); - if (uint64_t patcheeOffset = scanCortexA53Errata843419(isec, off, limit)) + if (uint64_t patcheeOffset = + scanCortexA53Errata843419(isec, off, limit)) implementPatch(startAddr, patcheeOffset, isec, patches); } if (dataSym == mapSyms.end()) @@ -630,7 +622,7 @@ AArch64Err843419Patcher::patchInputSectionDescription( // Ouptut and Input Sections may have been changed. // Returns false if no patches were required and no changes were made. bool AArch64Err843419Patcher::createFixes() { - if (initialized == false) + if (!initialized) init(); bool addressesChanged = false; @@ -649,3 +641,5 @@ bool AArch64Err843419Patcher::createFixes() { } return addressesChanged; } +} // namespace elf +} // namespace lld diff --git a/ELF/ARMErrataFix.cpp b/ELF/ARMErrataFix.cpp new file mode 100644 index 000000000000..493fafc6a0b2 --- /dev/null +++ b/ELF/ARMErrataFix.cpp @@ -0,0 +1,528 @@ +//===- ARMErrataFix.cpp ---------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This file implements Section Patching for the purpose of working around the +// Cortex-a8 erratum 657417 "A 32bit branch instruction that spans 2 4K regions +// can result in an incorrect instruction fetch or processor deadlock." The +// erratum affects all but r1p7, r2p5, r2p6, r3p1 and r3p2 revisions of the +// Cortex-A8. A high level description of the patching technique is given in +// the opening comment of AArch64ErrataFix.cpp. +//===----------------------------------------------------------------------===// + +#include "ARMErrataFix.h" + +#include "Config.h" +#include "LinkerScript.h" +#include "OutputSections.h" +#include "Relocations.h" +#include "Symbols.h" +#include "SyntheticSections.h" +#include "Target.h" +#include "lld/Common/Memory.h" +#include "lld/Common/Strings.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> + +using namespace llvm; +using namespace llvm::ELF; +using namespace llvm::object; +using namespace llvm::support; +using namespace llvm::support::endian; + +namespace lld { +namespace elf { + +// The documented title for Erratum 657417 is: +// "A 32bit branch instruction that spans two 4K regions can result in an +// incorrect instruction fetch or processor deadlock". Graphically using a +// 32-bit B.w instruction encoded as a pair of halfwords 0xf7fe 0xbfff +// xxxxxx000 // Memory region 1 start +// target: +// ... +// xxxxxxffe f7fe // First halfword of branch to target: +// xxxxxx000 // Memory region 2 start +// xxxxxx002 bfff // Second halfword of branch to target: +// +// The specific trigger conditions that can be detected at link time are: +// - There is a 32-bit Thumb-2 branch instruction with an address of the form +// xxxxxxFFE. The first 2 bytes of the instruction are in 4KiB region 1, the +// second 2 bytes are in region 2. +// - The branch instruction is one of BLX, BL, B.w BCC.w +// - The instruction preceding the branch is a 32-bit non-branch instruction. +// - The target of the branch is in region 1. +// +// The linker mitigation for the fix is to redirect any branch that meets the +// erratum conditions to a patch section containing a branch to the target. +// +// As adding patch sections may move branches onto region boundaries the patch +// must iterate until no more patches are added. +// +// Example, before: +// 00000FFA func: NOP.w // 32-bit Thumb function +// 00000FFE B.W func // 32-bit branch spanning 2 regions, dest in 1st. +// Example, after: +// 00000FFA func: NOP.w // 32-bit Thumb function +// 00000FFE B.w __CortexA8657417_00000FFE +// 00001002 2 - bytes padding +// 00001004 __CortexA8657417_00000FFE: B.w func + +class Patch657417Section : public SyntheticSection { +public: + Patch657417Section(InputSection *p, uint64_t off, uint32_t instr, bool isARM); + + void writeTo(uint8_t *buf) override; + + size_t getSize() const override { return 4; } + + // Get the virtual address of the branch instruction at patcheeOffset. + uint64_t getBranchAddr() const; + + // The Section we are patching. + const InputSection *patchee; + // The offset of the instruction in the Patchee section we are patching. + uint64_t patcheeOffset; + // A label for the start of the Patch that we can use as a relocation target. + Symbol *patchSym; + // A decoding of the branch instruction at patcheeOffset. + uint32_t instr; + // True If the patch is to be written in ARM state, otherwise the patch will + // be written in Thumb state. + bool isARM; +}; + +// Return true if the half-word, when taken as the first of a pair of halfwords +// is the first half of a 32-bit instruction. +// Reference from ARM Architecure Reference Manual ARMv7-A and ARMv7-R edition +// section A6.3: 32-bit Thumb instruction encoding +// | HW1 | HW2 | +// | 1 1 1 | op1 (2) | op2 (7) | x (4) |op| x (15) | +// With op1 == 0b00, a 16-bit instruction is encoded. +// +// We test only the first halfword, looking for op != 0b00. +static bool is32bitInstruction(uint16_t hw) { + return (hw & 0xe000) == 0xe000 && (hw & 0x1800) != 0x0000; +} + +// Reference from ARM Architecure Reference Manual ARMv7-A and ARMv7-R edition +// section A6.3.4 Branches and miscellaneous control. +// | HW1 | HW2 | +// | 1 1 1 | 1 0 | op (7) | x (4) | 1 | op1 (3) | op2 (4) | imm8 (8) | +// op1 == 0x0 op != x111xxx | Conditional branch (Bcc.W) +// op1 == 0x1 | Branch (B.W) +// op1 == 1x0 | Branch with Link and Exchange (BLX.w) +// op1 == 1x1 | Branch with Link (BL.W) + +static bool isBcc(uint32_t instr) { + return (instr & 0xf800d000) == 0xf0008000 && + (instr & 0x03800000) != 0x03800000; +} + +static bool isB(uint32_t instr) { return (instr & 0xf800d000) == 0xf0009000; } + +static bool isBLX(uint32_t instr) { return (instr & 0xf800d000) == 0xf000c000; } + +static bool isBL(uint32_t instr) { return (instr & 0xf800d000) == 0xf000d000; } + +static bool is32bitBranch(uint32_t instr) { + return isBcc(instr) || isB(instr) || isBL(instr) || isBLX(instr); +} + +Patch657417Section::Patch657417Section(InputSection *p, uint64_t off, + uint32_t instr, bool isARM) + : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4, + ".text.patch"), + patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) { + parent = p->getParent(); + patchSym = addSyntheticLocal( + saver.save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC, + isARM ? 0 : 1, getSize(), *this); + addSyntheticLocal(saver.save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this); +} + +uint64_t Patch657417Section::getBranchAddr() const { + return patchee->getVA(patcheeOffset); +} + +// Given a branch instruction instr at sourceAddr work out its destination +// address. This is only used when the branch instruction has no relocation. +static uint64_t getThumbDestAddr(uint64_t sourceAddr, uint32_t instr) { + uint8_t buf[4]; + write16le(buf, instr >> 16); + write16le(buf + 2, instr & 0x0000ffff); + int64_t offset; + if (isBcc(instr)) + offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP19); + else if (isB(instr)) + offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP24); + else + offset = target->getImplicitAddend(buf, R_ARM_THM_CALL); + return sourceAddr + offset + 4; +} + +void Patch657417Section::writeTo(uint8_t *buf) { + // The base instruction of the patch is always a 32-bit unconditional branch. + if (isARM) + write32le(buf, 0xea000000); + else + write32le(buf, 0x9000f000); + // If we have a relocation then apply it. For a SyntheticSection buf already + // has outSecOff added, but relocateAlloc also adds outSecOff so we need to + // subtract to avoid double counting. + if (!relocations.empty()) { + relocateAlloc(buf - outSecOff, buf - outSecOff + getSize()); + return; + } + + // If we don't have a relocation then we must calculate and write the offset + // ourselves. + // Get the destination offset from the addend in the branch instruction. + // We cannot use the instruction in the patchee section as this will have + // been altered to point to us! + uint64_t s = getThumbDestAddr(getBranchAddr(), instr); + uint64_t p = getVA(4); + target->relocateOne(buf, isARM ? R_ARM_JUMP24 : R_ARM_THM_JUMP24, s - p); +} + +// Given a branch instruction spanning two 4KiB regions, at offset off from the +// start of isec, return true if the destination of the branch is within the +// first of the two 4Kib regions. +static bool branchDestInFirstRegion(const InputSection *isec, uint64_t off, + uint32_t instr, const Relocation *r) { + uint64_t sourceAddr = isec->getVA(0) + off; + assert((sourceAddr & 0xfff) == 0xffe); + uint64_t destAddr = sourceAddr; + // If there is a branch relocation at the same offset we must use this to + // find the destination address as the branch could be indirected via a thunk + // or the PLT. + if (r) { + uint64_t dst = (r->expr == R_PLT_PC) ? r->sym->getPltVA() : r->sym->getVA(); + // Account for Thumb PC bias, usually cancelled to 0 by addend of -4. + destAddr = dst + r->addend + 4; + } else { + // If there is no relocation, we must have an intra-section branch + // We must extract the offset from the addend manually. + destAddr = getThumbDestAddr(sourceAddr, instr); + } + + return (destAddr & 0xfffff000) == (sourceAddr & 0xfffff000); +} + +// Return true if a branch can reach a patch section placed after isec. +// The Bcc.w instruction has a range of 1 MiB, all others have 16 MiB. +static bool patchInRange(const InputSection *isec, uint64_t off, + uint32_t instr) { + + // We need the branch at source to reach a patch section placed immediately + // after isec. As there can be more than one patch in the patch section we + // add 0x100 as contingency to account for worst case of 1 branch every 4KiB + // for a 1 MiB range. + return target->inBranchRange( + isBcc(instr) ? R_ARM_THM_JUMP19 : R_ARM_THM_JUMP24, isec->getVA(off), + isec->getVA() + isec->getSize() + 0x100); +} + +struct ScanResult { + // Offset of branch within its InputSection. + uint64_t off; + // Cached decoding of the branch instruction. + uint32_t instr; + // Branch relocation at off. Will be nullptr if no relocation exists. + Relocation *rel; +}; + +// Detect the erratum sequence, returning the offset of the branch instruction +// and a decoding of the branch. If the erratum sequence is not found then +// return an offset of 0 for the branch. 0 is a safe value to use for no patch +// as there must be at least one 32-bit non-branch instruction before the +// branch so the minimum offset for a patch is 4. +static ScanResult scanCortexA8Errata657417(InputSection *isec, uint64_t &off, + uint64_t limit) { + uint64_t isecAddr = isec->getVA(0); + // Advance Off so that (isecAddr + off) modulo 0x1000 is at least 0xffa. We + // need to check for a 32-bit instruction immediately before a 32-bit branch + // at 0xffe modulo 0x1000. + off = alignTo(isecAddr + off, 0x1000, 0xffa) - isecAddr; + if (off >= limit || limit - off < 8) { + // Need at least 2 4-byte sized instructions to trigger erratum. + off = limit; + return {0, 0, nullptr}; + } + + ScanResult scanRes = {0, 0, nullptr}; + const uint8_t *buf = isec->data().begin(); + // ARMv7-A Thumb 32-bit instructions are encoded 2 consecutive + // little-endian halfwords. + const ulittle16_t *instBuf = reinterpret_cast<const ulittle16_t *>(buf + off); + uint16_t hw11 = *instBuf++; + uint16_t hw12 = *instBuf++; + uint16_t hw21 = *instBuf++; + uint16_t hw22 = *instBuf++; + if (is32bitInstruction(hw11) && is32bitInstruction(hw21)) { + uint32_t instr1 = (hw11 << 16) | hw12; + uint32_t instr2 = (hw21 << 16) | hw22; + if (!is32bitBranch(instr1) && is32bitBranch(instr2)) { + // Find a relocation for the branch if it exists. This will be used + // to determine the target. + uint64_t branchOff = off + 4; + auto relIt = llvm::find_if(isec->relocations, [=](const Relocation &r) { + return r.offset == branchOff && + (r.type == R_ARM_THM_JUMP19 || r.type == R_ARM_THM_JUMP24 || + r.type == R_ARM_THM_CALL); + }); + if (relIt != isec->relocations.end()) + scanRes.rel = &(*relIt); + if (branchDestInFirstRegion(isec, branchOff, instr2, scanRes.rel)) { + if (patchInRange(isec, branchOff, instr2)) { + scanRes.off = branchOff; + scanRes.instr = instr2; + } else { + warn(toString(isec->file) + + ": skipping cortex-a8 657417 erratum sequence, section " + + isec->name + " is too large to patch"); + } + } + } + } + off += 0x1000; + return scanRes; +} + +void ARMErr657417Patcher::init() { + // The Arm ABI permits a mix of ARM, Thumb and Data in the same + // InputSection. We must only scan Thumb instructions to avoid false + // matches. We use the mapping symbols in the InputObjects to identify this + // data, caching the results in sectionMap so we don't have to recalculate + // it each pass. + + // The ABI Section 4.5.5 Mapping symbols; defines local symbols that describe + // half open intervals [Symbol Value, Next Symbol Value) of code and data + // within sections. If there is no next symbol then the half open interval is + // [Symbol Value, End of section). The type, code or data, is determined by + // the mapping symbol name, $a for Arm code, $t for Thumb code, $d for data. + auto isArmMapSymbol = [](const Symbol *s) { + return s->getName() == "$a" || s->getName().startswith("$a."); + }; + auto isThumbMapSymbol = [](const Symbol *s) { + return s->getName() == "$t" || s->getName().startswith("$t."); + }; + auto isDataMapSymbol = [](const Symbol *s) { + return s->getName() == "$d" || s->getName().startswith("$d."); + }; + + // Collect mapping symbols for every executable InputSection. + for (InputFile *file : objectFiles) { + auto *f = cast<ObjFile<ELF32LE>>(file); + for (Symbol *s : f->getLocalSymbols()) { + auto *def = dyn_cast<Defined>(s); + if (!def) + continue; + if (!isArmMapSymbol(def) && !isThumbMapSymbol(def) && + !isDataMapSymbol(def)) + continue; + if (auto *sec = dyn_cast_or_null<InputSection>(def->section)) + if (sec->flags & SHF_EXECINSTR) + sectionMap[sec].push_back(def); + } + } + // For each InputSection make sure the mapping symbols are in sorted in + // ascending order and are in alternating Thumb, non-Thumb order. + for (auto &kv : sectionMap) { + std::vector<const Defined *> &mapSyms = kv.second; + llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) { + return a->value < b->value; + }); + mapSyms.erase(std::unique(mapSyms.begin(), mapSyms.end(), + [=](const Defined *a, const Defined *b) { + return (isThumbMapSymbol(a) == + isThumbMapSymbol(b)); + }), + mapSyms.end()); + // Always start with a Thumb Mapping Symbol + if (!mapSyms.empty() && !isThumbMapSymbol(mapSyms.front())) + mapSyms.erase(mapSyms.begin()); + } + initialized = true; +} + +void ARMErr657417Patcher::insertPatches( + InputSectionDescription &isd, std::vector<Patch657417Section *> &patches) { + uint64_t spacing = 0x100000 - 0x7500; + uint64_t isecLimit; + uint64_t prevIsecLimit = isd.sections.front()->outSecOff; + uint64_t patchUpperBound = prevIsecLimit + spacing; + uint64_t outSecAddr = isd.sections.front()->getParent()->addr; + + // Set the outSecOff of patches to the place where we want to insert them. + // We use a similar strategy to initial thunk placement, using 1 MiB as the + // range of the Thumb-2 conditional branch with a contingency accounting for + // thunk generation. + auto patchIt = patches.begin(); + auto patchEnd = patches.end(); + for (const InputSection *isec : isd.sections) { + isecLimit = isec->outSecOff + isec->getSize(); + if (isecLimit > patchUpperBound) { + for (; patchIt != patchEnd; ++patchIt) { + if ((*patchIt)->getBranchAddr() - outSecAddr >= prevIsecLimit) + break; + (*patchIt)->outSecOff = prevIsecLimit; + } + patchUpperBound = prevIsecLimit + spacing; + } + prevIsecLimit = isecLimit; + } + for (; patchIt != patchEnd; ++patchIt) + (*patchIt)->outSecOff = isecLimit; + + // Merge all patch sections. We use the outSecOff assigned above to + // determine the insertion point. This is ok as we only merge into an + // InputSectionDescription once per pass, and at the end of the pass + // assignAddresses() will recalculate all the outSecOff values. + std::vector<InputSection *> tmp; + tmp.reserve(isd.sections.size() + patches.size()); + auto mergeCmp = [](const InputSection *a, const InputSection *b) { + if (a->outSecOff != b->outSecOff) + return a->outSecOff < b->outSecOff; + return isa<Patch657417Section>(a) && !isa<Patch657417Section>(b); + }; + std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(), + patches.end(), std::back_inserter(tmp), mergeCmp); + isd.sections = std::move(tmp); +} + +// Given a branch instruction described by ScanRes redirect it to a patch +// section containing an unconditional branch instruction to the target. +// Ensure that this patch section is 4-byte aligned so that the branch cannot +// span two 4 KiB regions. Place the patch section so that it is always after +// isec so the branch we are patching always goes forwards. +static void implementPatch(ScanResult sr, InputSection *isec, + std::vector<Patch657417Section *> &patches) { + + log("detected cortex-a8-657419 erratum sequence starting at " + + utohexstr(isec->getVA(sr.off)) + " in unpatched output."); + Patch657417Section *psec; + // We have two cases to deal with. + // Case 1. There is a relocation at patcheeOffset to a symbol. The + // unconditional branch in the patch must have a relocation so that any + // further redirection via the PLT or a Thunk happens as normal. At + // patcheeOffset we redirect the existing relocation to a Symbol defined at + // the start of the patch section. + // + // Case 2. There is no relocation at patcheeOffset. We are unlikely to have + // a symbol that we can use as a target for a relocation in the patch section. + // Luckily we know that the destination cannot be indirected via the PLT or + // a Thunk so we can just write the destination directly. + if (sr.rel) { + // Case 1. We have an existing relocation to redirect to patch and a + // Symbol target. + + // Create a branch relocation for the unconditional branch in the patch. + // This can be redirected via the PLT or Thunks. + RelType patchRelType = R_ARM_THM_JUMP24; + int64_t patchRelAddend = sr.rel->addend; + bool destIsARM = false; + if (isBL(sr.instr) || isBLX(sr.instr)) { + // The final target of the branch may be ARM or Thumb, if the target + // is ARM then we write the patch in ARM state to avoid a state change + // Thunk from the patch to the target. + uint64_t dstSymAddr = (sr.rel->expr == R_PLT_PC) ? sr.rel->sym->getPltVA() + : sr.rel->sym->getVA(); + destIsARM = (dstSymAddr & 1) == 0; + } + psec = make<Patch657417Section>(isec, sr.off, sr.instr, destIsARM); + if (destIsARM) { + // The patch will be in ARM state. Use an ARM relocation and account for + // the larger ARM PC-bias of 8 rather than Thumb's 4. + patchRelType = R_ARM_JUMP24; + patchRelAddend -= 4; + } + psec->relocations.push_back( + Relocation{sr.rel->expr, patchRelType, 0, patchRelAddend, sr.rel->sym}); + // Redirect the existing branch relocation to the patch. + sr.rel->expr = R_PC; + sr.rel->addend = -4; + sr.rel->sym = psec->patchSym; + } else { + // Case 2. We do not have a relocation to the patch. Add a relocation of the + // appropriate type to the patch at patcheeOffset. + + // The destination is ARM if we have a BLX. + psec = make<Patch657417Section>(isec, sr.off, sr.instr, isBLX(sr.instr)); + RelType type; + if (isBcc(sr.instr)) + type = R_ARM_THM_JUMP19; + else if (isB(sr.instr)) + type = R_ARM_THM_JUMP24; + else + type = R_ARM_THM_CALL; + isec->relocations.push_back( + Relocation{R_PC, type, sr.off, -4, psec->patchSym}); + } + patches.push_back(psec); +} + +// Scan all the instructions in InputSectionDescription, for each instance of +// the erratum sequence create a Patch657417Section. We return the list of +// Patch657417Sections that need to be applied to the InputSectionDescription. +std::vector<Patch657417Section *> +ARMErr657417Patcher::patchInputSectionDescription( + InputSectionDescription &isd) { + std::vector<Patch657417Section *> patches; + for (InputSection *isec : isd.sections) { + // LLD doesn't use the erratum sequence in SyntheticSections. + if (isa<SyntheticSection>(isec)) + continue; + // Use sectionMap to make sure we only scan Thumb code and not Arm or inline + // data. We have already sorted mapSyms in ascending order and removed + // consecutive mapping symbols of the same type. Our range of executable + // instructions to scan is therefore [thumbSym->value, nonThumbSym->value) + // or [thumbSym->value, section size). + std::vector<const Defined *> &mapSyms = sectionMap[isec]; + + auto thumbSym = mapSyms.begin(); + while (thumbSym != mapSyms.end()) { + auto nonThumbSym = std::next(thumbSym); + uint64_t off = (*thumbSym)->value; + uint64_t limit = (nonThumbSym == mapSyms.end()) ? isec->data().size() + : (*nonThumbSym)->value; + + while (off < limit) { + ScanResult sr = scanCortexA8Errata657417(isec, off, limit); + if (sr.off) + implementPatch(sr, isec, patches); + } + if (nonThumbSym == mapSyms.end()) + break; + thumbSym = std::next(nonThumbSym); + } + } + return patches; +} + +bool ARMErr657417Patcher::createFixes() { + if (!initialized) + init(); + + bool addressesChanged = false; + for (OutputSection *os : outputSections) { + if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR)) + continue; + for (BaseCommand *bc : os->sectionCommands) + if (auto *isd = dyn_cast<InputSectionDescription>(bc)) { + std::vector<Patch657417Section *> patches = + patchInputSectionDescription(*isd); + if (!patches.empty()) { + insertPatches(*isd, patches); + addressesChanged = true; + } + } + } + return addressesChanged; +} + +} // namespace elf +} // namespace lld diff --git a/ELF/ARMErrataFix.h b/ELF/ARMErrataFix.h new file mode 100644 index 000000000000..5a39bcc75cd3 --- /dev/null +++ b/ELF/ARMErrataFix.h @@ -0,0 +1,51 @@ +//===- ARMErrataFix.h -------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_ELF_ARMA8ERRATAFIX_H +#define LLD_ELF_ARMA8ERRATAFIX_H + +#include "lld/Common/LLVM.h" +#include "llvm/ADT/DenseMap.h" +#include <map> +#include <vector> + +namespace lld { +namespace elf { + +class Defined; +class InputSection; +struct InputSectionDescription; +class OutputSection; +class Patch657417Section; + +class ARMErr657417Patcher { +public: + // Return true if Patches have been added to the OutputSections. + bool createFixes(); + +private: + std::vector<Patch657417Section *> + patchInputSectionDescription(InputSectionDescription &isd); + + void insertPatches(InputSectionDescription &isd, + std::vector<Patch657417Section *> &patches); + + void init(); + + // A cache of the mapping symbols defined by the InputSection sorted in order + // of ascending value with redundant symbols removed. These describe + // the ranges of code and data in an executable InputSection. + llvm::DenseMap<InputSection *, std::vector<const Defined *>> sectionMap; + + bool initialized = false; +}; + +} // namespace elf +} // namespace lld + +#endif diff --git a/ELF/Arch/AArch64.cpp b/ELF/Arch/AArch64.cpp index 4d4789702f03..5cf07029fa1d 100644 --- a/ELF/Arch/AArch64.cpp +++ b/ELF/Arch/AArch64.cpp @@ -17,13 +17,14 @@ using namespace llvm; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { // Page(Expr) is the page address of the expression Expr, defined // as (Expr & ~0xFFF). (This applies even if the machine page size // supported by the platform has a different value.) -uint64_t elf::getAArch64Page(uint64_t expr) { +uint64_t getAArch64Page(uint64_t expr) { return expr & ~static_cast<uint64_t>(0xFFF); } @@ -76,6 +77,26 @@ AArch64::AArch64() { RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const { switch (type) { + case R_AARCH64_ABS16: + case R_AARCH64_ABS32: + case R_AARCH64_ABS64: + case R_AARCH64_ADD_ABS_LO12_NC: + case R_AARCH64_LDST128_ABS_LO12_NC: + case R_AARCH64_LDST16_ABS_LO12_NC: + case R_AARCH64_LDST32_ABS_LO12_NC: + case R_AARCH64_LDST64_ABS_LO12_NC: + case R_AARCH64_LDST8_ABS_LO12_NC: + case R_AARCH64_MOVW_SABS_G0: + case R_AARCH64_MOVW_SABS_G1: + case R_AARCH64_MOVW_SABS_G2: + case R_AARCH64_MOVW_UABS_G0: + case R_AARCH64_MOVW_UABS_G0_NC: + case R_AARCH64_MOVW_UABS_G1: + case R_AARCH64_MOVW_UABS_G1_NC: + case R_AARCH64_MOVW_UABS_G2: + case R_AARCH64_MOVW_UABS_G2_NC: + case R_AARCH64_MOVW_UABS_G3: + return R_ABS; case R_AARCH64_TLSDESC_ADR_PAGE21: return R_AARCH64_TLSDESC_PAGE; case R_AARCH64_TLSDESC_LD64_LO12: @@ -90,6 +111,11 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: + case R_AARCH64_TLSLE_MOVW_TPREL_G0: + case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: + case R_AARCH64_TLSLE_MOVW_TPREL_G1: + case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: + case R_AARCH64_TLSLE_MOVW_TPREL_G2: return R_TLS; case R_AARCH64_CALL26: case R_AARCH64_CONDBR19: @@ -101,6 +127,13 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, case R_AARCH64_PREL64: case R_AARCH64_ADR_PREL_LO21: case R_AARCH64_LD_PREL_LO19: + case R_AARCH64_MOVW_PREL_G0: + case R_AARCH64_MOVW_PREL_G0_NC: + case R_AARCH64_MOVW_PREL_G1: + case R_AARCH64_MOVW_PREL_G1_NC: + case R_AARCH64_MOVW_PREL_G2: + case R_AARCH64_MOVW_PREL_G2_NC: + case R_AARCH64_MOVW_PREL_G3: return R_PC; case R_AARCH64_ADR_PREL_PG_HI21: case R_AARCH64_ADR_PREL_PG_HI21_NC: @@ -114,7 +147,9 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, case R_AARCH64_NONE: return R_NONE; default: - return R_ABS; + error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + + ") against symbol " + toString(s)); + return R_NONE; } } @@ -247,6 +282,26 @@ static void or32AArch64Imm(uint8_t *l, uint64_t imm) { or32le(l, (imm & 0xFFF) << 10); } +// Update the immediate field in an AArch64 movk, movn or movz instruction +// for a signed relocation, and update the opcode of a movn or movz instruction +// to match the sign of the operand. +static void writeSMovWImm(uint8_t *loc, uint32_t imm) { + uint32_t inst = read32le(loc); + // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. + if (!(inst & (1 << 29))) { + // movn or movz. + if (imm & 0x10000) { + // Change opcode to movn, which takes an inverted operand. + imm ^= 0xFFFF; + inst &= ~(1 << 30); + } else { + // Change opcode to movz. + inst |= 1 << 30; + } + } + write32le(loc, inst | ((imm & 0xFFFF) << 5)); +} + void AArch64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { switch (type) { case R_AARCH64_ABS16: @@ -326,18 +381,56 @@ void AArch64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { checkAlignment(loc, val, 16, type); or32AArch64Imm(loc, getBits(val, 4, 11)); break; + case R_AARCH64_MOVW_UABS_G0: + checkUInt(loc, val, 16, type); + LLVM_FALLTHROUGH; case R_AARCH64_MOVW_UABS_G0_NC: or32le(loc, (val & 0xFFFF) << 5); break; + case R_AARCH64_MOVW_UABS_G1: + checkUInt(loc, val, 32, type); + LLVM_FALLTHROUGH; case R_AARCH64_MOVW_UABS_G1_NC: or32le(loc, (val & 0xFFFF0000) >> 11); break; + case R_AARCH64_MOVW_UABS_G2: + checkUInt(loc, val, 48, type); + LLVM_FALLTHROUGH; case R_AARCH64_MOVW_UABS_G2_NC: or32le(loc, (val & 0xFFFF00000000) >> 27); break; case R_AARCH64_MOVW_UABS_G3: or32le(loc, (val & 0xFFFF000000000000) >> 43); break; + case R_AARCH64_MOVW_PREL_G0: + case R_AARCH64_MOVW_SABS_G0: + case R_AARCH64_TLSLE_MOVW_TPREL_G0: + checkInt(loc, val, 17, type); + LLVM_FALLTHROUGH; + case R_AARCH64_MOVW_PREL_G0_NC: + case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: + writeSMovWImm(loc, val); + break; + case R_AARCH64_MOVW_PREL_G1: + case R_AARCH64_MOVW_SABS_G1: + case R_AARCH64_TLSLE_MOVW_TPREL_G1: + checkInt(loc, val, 33, type); + LLVM_FALLTHROUGH; + case R_AARCH64_MOVW_PREL_G1_NC: + case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: + writeSMovWImm(loc, val >> 16); + break; + case R_AARCH64_MOVW_PREL_G2: + case R_AARCH64_MOVW_SABS_G2: + case R_AARCH64_TLSLE_MOVW_TPREL_G2: + checkInt(loc, val, 49, type); + LLVM_FALLTHROUGH; + case R_AARCH64_MOVW_PREL_G2_NC: + writeSMovWImm(loc, val >> 32); + break; + case R_AARCH64_MOVW_PREL_G3: + writeSMovWImm(loc, val >> 48); + break; case R_AARCH64_TSTBR14: checkInt(loc, val, 16, type); or32le(loc, (val & 0xFFFC) << 3); @@ -351,7 +444,7 @@ void AArch64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { or32AArch64Imm(loc, val); break; default: - error(getErrorLocation(loc) + "unrecognized relocation " + toString(type)); + llvm_unreachable("unknown relocation"); } } @@ -587,4 +680,7 @@ static TargetInfo *getTargetInfo() { return &t; } -TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); } +TargetInfo *getAArch64TargetInfo() { return getTargetInfo(); } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/AMDGPU.cpp b/ELF/Arch/AMDGPU.cpp index f2e32ca0996d..b42ca7746742 100644 --- a/ELF/Arch/AMDGPU.cpp +++ b/ELF/Arch/AMDGPU.cpp @@ -17,8 +17,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class AMDGPU final : public TargetInfo { @@ -107,7 +108,10 @@ RelType AMDGPU::getDynRel(RelType type) const { return R_AMDGPU_NONE; } -TargetInfo *elf::getAMDGPUTargetInfo() { +TargetInfo *getAMDGPUTargetInfo() { static AMDGPU target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/ARM.cpp b/ELF/Arch/ARM.cpp index 64adc33c07ae..41baea496d36 100644 --- a/ELF/Arch/ARM.cpp +++ b/ELF/Arch/ARM.cpp @@ -18,8 +18,9 @@ using namespace llvm; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class ARM final : public TargetInfo { @@ -600,7 +601,10 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const { } } -TargetInfo *elf::getARMTargetInfo() { +TargetInfo *getARMTargetInfo() { static ARM target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/AVR.cpp b/ELF/Arch/AVR.cpp index 869f0fe0c525..cb33ff448ba4 100644 --- a/ELF/Arch/AVR.cpp +++ b/ELF/Arch/AVR.cpp @@ -36,8 +36,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class AVR final : public TargetInfo { @@ -70,7 +71,10 @@ void AVR::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { } } -TargetInfo *elf::getAVRTargetInfo() { +TargetInfo *getAVRTargetInfo() { static AVR target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/Hexagon.cpp b/ELF/Arch/Hexagon.cpp index c497a6df7987..67264a2272dd 100644 --- a/ELF/Arch/Hexagon.cpp +++ b/ELF/Arch/Hexagon.cpp @@ -19,8 +19,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class Hexagon final : public TargetInfo { @@ -29,6 +30,7 @@ public: uint32_t calcEFlags() const override; RelExpr getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const override; + RelType getDynRel(RelType type) const override; void relocateOne(uint8_t *loc, RelType type, uint64_t val) const override; void writePltHeader(uint8_t *buf) const override; void writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, uint64_t pltEntryAddr, @@ -86,25 +88,47 @@ static uint32_t applyMask(uint32_t mask, uint32_t data) { RelExpr Hexagon::getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const { switch (type) { + case R_HEX_NONE: + return R_NONE; + case R_HEX_6_X: + case R_HEX_8_X: + case R_HEX_9_X: + case R_HEX_10_X: + case R_HEX_11_X: + case R_HEX_12_X: + case R_HEX_16_X: + case R_HEX_32: + case R_HEX_32_6_X: + case R_HEX_HI16: + case R_HEX_LO16: + return R_ABS; case R_HEX_B9_PCREL: - case R_HEX_B9_PCREL_X: case R_HEX_B13_PCREL: case R_HEX_B15_PCREL: - case R_HEX_B15_PCREL_X: case R_HEX_6_PCREL_X: case R_HEX_32_PCREL: return R_PC; + case R_HEX_B9_PCREL_X: + case R_HEX_B15_PCREL_X: case R_HEX_B22_PCREL: case R_HEX_PLT_B22_PCREL: case R_HEX_B22_PCREL_X: case R_HEX_B32_PCREL_X: return R_PLT_PC; + case R_HEX_GOTREL_11_X: + case R_HEX_GOTREL_16_X: + case R_HEX_GOTREL_32_6_X: + case R_HEX_GOTREL_HI16: + case R_HEX_GOTREL_LO16: + return R_GOTPLTREL; case R_HEX_GOT_11_X: case R_HEX_GOT_16_X: case R_HEX_GOT_32_6_X: - return R_HEXAGON_GOT; + return R_GOTPLT; default: - return R_ABS; + error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + + ") against symbol " + toString(s)); + return R_NONE; } } @@ -197,6 +221,7 @@ void Hexagon::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { break; case R_HEX_11_X: case R_HEX_GOT_11_X: + case R_HEX_GOTREL_11_X: or32le(loc, applyMask(findMaskR11(read32le(loc)), val & 0x3f)); break; case R_HEX_12_X: @@ -204,6 +229,7 @@ void Hexagon::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { break; case R_HEX_16_X: // These relocs only have 6 effective bits. case R_HEX_GOT_16_X: + case R_HEX_GOTREL_16_X: or32le(loc, applyMask(findMaskR16(read32le(loc)), val & 0x3f)); break; case R_HEX_32: @@ -212,18 +238,22 @@ void Hexagon::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { break; case R_HEX_32_6_X: case R_HEX_GOT_32_6_X: + case R_HEX_GOTREL_32_6_X: or32le(loc, applyMask(0x0fff3fff, val >> 6)); break; case R_HEX_B9_PCREL: + checkInt(loc, val, 11, type); or32le(loc, applyMask(0x003000fe, val >> 2)); break; case R_HEX_B9_PCREL_X: or32le(loc, applyMask(0x003000fe, val & 0x3f)); break; case R_HEX_B13_PCREL: + checkInt(loc, val, 15, type); or32le(loc, applyMask(0x00202ffe, val >> 2)); break; case R_HEX_B15_PCREL: + checkInt(loc, val, 17, type); or32le(loc, applyMask(0x00df20fe, val >> 2)); break; case R_HEX_B15_PCREL_X: @@ -231,6 +261,7 @@ void Hexagon::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { break; case R_HEX_B22_PCREL: case R_HEX_PLT_B22_PCREL: + checkInt(loc, val, 22, type); or32le(loc, applyMask(0x1ff3ffe, val >> 2)); break; case R_HEX_B22_PCREL_X: @@ -239,15 +270,16 @@ void Hexagon::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { case R_HEX_B32_PCREL_X: or32le(loc, applyMask(0x0fff3fff, val >> 6)); break; + case R_HEX_GOTREL_HI16: case R_HEX_HI16: or32le(loc, applyMask(0x00c03fff, val >> 16)); break; + case R_HEX_GOTREL_LO16: case R_HEX_LO16: or32le(loc, applyMask(0x00c03fff, val)); break; default: - error(getErrorLocation(loc) + "unrecognized relocation " + toString(type)); - break; + llvm_unreachable("unknown relocation"); } } @@ -285,7 +317,16 @@ void Hexagon::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, relocateOne(buf + 4, R_HEX_6_PCREL_X, gotPltEntryAddr - pltEntryAddr); } -TargetInfo *elf::getHexagonTargetInfo() { +RelType Hexagon::getDynRel(RelType type) const { + if (type == R_HEX_32) + return type; + return R_HEX_NONE; +} + +TargetInfo *getHexagonTargetInfo() { static Hexagon target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/MSP430.cpp b/ELF/Arch/MSP430.cpp index 90664396c85e..f03e8181923b 100644 --- a/ELF/Arch/MSP430.cpp +++ b/ELF/Arch/MSP430.cpp @@ -26,8 +26,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class MSP430 final : public TargetInfo { @@ -87,7 +88,10 @@ void MSP430::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { } } -TargetInfo *elf::getMSP430TargetInfo() { +TargetInfo *getMSP430TargetInfo() { static MSP430 target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/Mips.cpp b/ELF/Arch/Mips.cpp index 24b3957acd99..d8fa306a6205 100644 --- a/ELF/Arch/Mips.cpp +++ b/ELF/Arch/Mips.cpp @@ -14,15 +14,13 @@ #include "Thunks.h" #include "lld/Common/ErrorHandler.h" #include "llvm/Object/ELF.h" -#include "llvm/Support/Endian.h" using namespace llvm; using namespace llvm::object; -using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { namespace { template <class ELFT> class MIPS final : public TargetInfo { public: @@ -85,8 +83,14 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s, switch (type) { case R_MIPS_JALR: + // If the target symbol is not preemptible and is not microMIPS, + // it might be possible to replace jalr/jr instruction by bal/b. + // It depends on the target symbol's offset. + if (!s.isPreemptible && !(s.getVA() & 0x1)) + return R_PC; + return R_NONE; case R_MICROMIPS_JALR: - return R_HINT; + return R_NONE; case R_MIPS_GPREL16: case R_MIPS_GPREL32: case R_MICROMIPS_GPREL16: @@ -120,15 +124,16 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s, case R_MIPS_TLS_DTPREL_LO16: case R_MIPS_TLS_DTPREL32: case R_MIPS_TLS_DTPREL64: + case R_MICROMIPS_TLS_DTPREL_HI16: + case R_MICROMIPS_TLS_DTPREL_LO16: + return R_ABS; case R_MIPS_TLS_TPREL_HI16: case R_MIPS_TLS_TPREL_LO16: case R_MIPS_TLS_TPREL32: case R_MIPS_TLS_TPREL64: - case R_MICROMIPS_TLS_DTPREL_HI16: - case R_MICROMIPS_TLS_DTPREL_LO16: case R_MICROMIPS_TLS_TPREL_HI16: case R_MICROMIPS_TLS_TPREL_LO16: - return R_ABS; + return R_TLS; case R_MIPS_PC32: case R_MIPS_PC16: case R_MIPS_PC19_S2: @@ -192,7 +197,7 @@ void MIPS<ELFT>::writeGotPlt(uint8_t *buf, const Symbol &) const { uint64_t va = in.plt->getVA(); if (isMicroMips()) va |= 1; - write32<ELFT::TargetEndianness>(buf, va); + write32(buf, va); } template <endianness E> static uint32_t readShuffle(const uint8_t *loc) { @@ -202,19 +207,18 @@ template <endianness E> static uint32_t readShuffle(const uint8_t *loc) { // as early as possible. To do so, little-endian binaries keep 16-bit // words in a big-endian order. That is why we have to swap these // words to get a correct value. - uint32_t v = read32<E>(loc); + uint32_t v = read32(loc); if (E == support::little) return (v << 16) | (v >> 16); return v; } -template <endianness E> static void writeValue(uint8_t *loc, uint64_t v, uint8_t bitsSize, uint8_t shift) { - uint32_t instr = read32<E>(loc); + uint32_t instr = read32(loc); uint32_t mask = 0xffffffff >> (32 - bitsSize); uint32_t data = (instr & ~mask) | ((v >> shift) & mask); - write32<E>(loc, data); + write32(loc, data); } template <endianness E> @@ -225,7 +229,7 @@ static void writeShuffleValue(uint8_t *loc, uint64_t v, uint8_t bitsSize, if (E == support::little) std::swap(words[0], words[1]); - writeValue<E>(loc, v, bitsSize, shift); + writeValue(loc, v, bitsSize, shift); if (E == support::little) std::swap(words[0], words[1]); @@ -234,94 +238,92 @@ static void writeShuffleValue(uint8_t *loc, uint64_t v, uint8_t bitsSize, template <endianness E> static void writeMicroRelocation16(uint8_t *loc, uint64_t v, uint8_t bitsSize, uint8_t shift) { - uint16_t instr = read16<E>(loc); + uint16_t instr = read16(loc); uint16_t mask = 0xffff >> (16 - bitsSize); uint16_t data = (instr & ~mask) | ((v >> shift) & mask); - write16<E>(loc, data); + write16(loc, data); } template <class ELFT> void MIPS<ELFT>::writePltHeader(uint8_t *buf) const { - const endianness e = ELFT::TargetEndianness; if (isMicroMips()) { uint64_t gotPlt = in.gotPlt->getVA(); uint64_t plt = in.plt->getVA(); // Overwrite trap instructions written by Writer::writeTrapInstr. memset(buf, 0, pltHeaderSize); - write16<e>(buf, isMipsR6() ? 0x7860 : 0x7980); // addiupc v1, (GOTPLT) - . - write16<e>(buf + 4, 0xff23); // lw $25, 0($3) - write16<e>(buf + 8, 0x0535); // subu16 $2, $2, $3 - write16<e>(buf + 10, 0x2525); // srl16 $2, $2, 2 - write16<e>(buf + 12, 0x3302); // addiu $24, $2, -2 - write16<e>(buf + 14, 0xfffe); - write16<e>(buf + 16, 0x0dff); // move $15, $31 + write16(buf, isMipsR6() ? 0x7860 : 0x7980); // addiupc v1, (GOTPLT) - . + write16(buf + 4, 0xff23); // lw $25, 0($3) + write16(buf + 8, 0x0535); // subu16 $2, $2, $3 + write16(buf + 10, 0x2525); // srl16 $2, $2, 2 + write16(buf + 12, 0x3302); // addiu $24, $2, -2 + write16(buf + 14, 0xfffe); + write16(buf + 16, 0x0dff); // move $15, $31 if (isMipsR6()) { - write16<e>(buf + 18, 0x0f83); // move $28, $3 - write16<e>(buf + 20, 0x472b); // jalrc $25 - write16<e>(buf + 22, 0x0c00); // nop + write16(buf + 18, 0x0f83); // move $28, $3 + write16(buf + 20, 0x472b); // jalrc $25 + write16(buf + 22, 0x0c00); // nop relocateOne(buf, R_MICROMIPS_PC19_S2, gotPlt - plt); } else { - write16<e>(buf + 18, 0x45f9); // jalrc $25 - write16<e>(buf + 20, 0x0f83); // move $28, $3 - write16<e>(buf + 22, 0x0c00); // nop + write16(buf + 18, 0x45f9); // jalrc $25 + write16(buf + 20, 0x0f83); // move $28, $3 + write16(buf + 22, 0x0c00); // nop relocateOne(buf, R_MICROMIPS_PC23_S2, gotPlt - plt); } return; } if (config->mipsN32Abi) { - write32<e>(buf, 0x3c0e0000); // lui $14, %hi(&GOTPLT[0]) - write32<e>(buf + 4, 0x8dd90000); // lw $25, %lo(&GOTPLT[0])($14) - write32<e>(buf + 8, 0x25ce0000); // addiu $14, $14, %lo(&GOTPLT[0]) - write32<e>(buf + 12, 0x030ec023); // subu $24, $24, $14 - write32<e>(buf + 16, 0x03e07825); // move $15, $31 - write32<e>(buf + 20, 0x0018c082); // srl $24, $24, 2 + write32(buf, 0x3c0e0000); // lui $14, %hi(&GOTPLT[0]) + write32(buf + 4, 0x8dd90000); // lw $25, %lo(&GOTPLT[0])($14) + write32(buf + 8, 0x25ce0000); // addiu $14, $14, %lo(&GOTPLT[0]) + write32(buf + 12, 0x030ec023); // subu $24, $24, $14 + write32(buf + 16, 0x03e07825); // move $15, $31 + write32(buf + 20, 0x0018c082); // srl $24, $24, 2 } else if (ELFT::Is64Bits) { - write32<e>(buf, 0x3c0e0000); // lui $14, %hi(&GOTPLT[0]) - write32<e>(buf + 4, 0xddd90000); // ld $25, %lo(&GOTPLT[0])($14) - write32<e>(buf + 8, 0x25ce0000); // addiu $14, $14, %lo(&GOTPLT[0]) - write32<e>(buf + 12, 0x030ec023); // subu $24, $24, $14 - write32<e>(buf + 16, 0x03e07825); // move $15, $31 - write32<e>(buf + 20, 0x0018c0c2); // srl $24, $24, 3 + write32(buf, 0x3c0e0000); // lui $14, %hi(&GOTPLT[0]) + write32(buf + 4, 0xddd90000); // ld $25, %lo(&GOTPLT[0])($14) + write32(buf + 8, 0x25ce0000); // addiu $14, $14, %lo(&GOTPLT[0]) + write32(buf + 12, 0x030ec023); // subu $24, $24, $14 + write32(buf + 16, 0x03e07825); // move $15, $31 + write32(buf + 20, 0x0018c0c2); // srl $24, $24, 3 } else { - write32<e>(buf, 0x3c1c0000); // lui $28, %hi(&GOTPLT[0]) - write32<e>(buf + 4, 0x8f990000); // lw $25, %lo(&GOTPLT[0])($28) - write32<e>(buf + 8, 0x279c0000); // addiu $28, $28, %lo(&GOTPLT[0]) - write32<e>(buf + 12, 0x031cc023); // subu $24, $24, $28 - write32<e>(buf + 16, 0x03e07825); // move $15, $31 - write32<e>(buf + 20, 0x0018c082); // srl $24, $24, 2 + write32(buf, 0x3c1c0000); // lui $28, %hi(&GOTPLT[0]) + write32(buf + 4, 0x8f990000); // lw $25, %lo(&GOTPLT[0])($28) + write32(buf + 8, 0x279c0000); // addiu $28, $28, %lo(&GOTPLT[0]) + write32(buf + 12, 0x031cc023); // subu $24, $24, $28 + write32(buf + 16, 0x03e07825); // move $15, $31 + write32(buf + 20, 0x0018c082); // srl $24, $24, 2 } uint32_t jalrInst = config->zHazardplt ? 0x0320fc09 : 0x0320f809; - write32<e>(buf + 24, jalrInst); // jalr.hb $25 or jalr $25 - write32<e>(buf + 28, 0x2718fffe); // subu $24, $24, 2 + write32(buf + 24, jalrInst); // jalr.hb $25 or jalr $25 + write32(buf + 28, 0x2718fffe); // subu $24, $24, 2 uint64_t gotPlt = in.gotPlt->getVA(); - writeValue<e>(buf, gotPlt + 0x8000, 16, 16); - writeValue<e>(buf + 4, gotPlt, 16, 0); - writeValue<e>(buf + 8, gotPlt, 16, 0); + writeValue(buf, gotPlt + 0x8000, 16, 16); + writeValue(buf + 4, gotPlt, 16, 0); + writeValue(buf + 8, gotPlt, 16, 0); } template <class ELFT> void MIPS<ELFT>::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, uint64_t pltEntryAddr, int32_t index, unsigned relOff) const { - const endianness e = ELFT::TargetEndianness; if (isMicroMips()) { // Overwrite trap instructions written by Writer::writeTrapInstr. memset(buf, 0, pltEntrySize); if (isMipsR6()) { - write16<e>(buf, 0x7840); // addiupc $2, (GOTPLT) - . - write16<e>(buf + 4, 0xff22); // lw $25, 0($2) - write16<e>(buf + 8, 0x0f02); // move $24, $2 - write16<e>(buf + 10, 0x4723); // jrc $25 / jr16 $25 + write16(buf, 0x7840); // addiupc $2, (GOTPLT) - . + write16(buf + 4, 0xff22); // lw $25, 0($2) + write16(buf + 8, 0x0f02); // move $24, $2 + write16(buf + 10, 0x4723); // jrc $25 / jr16 $25 relocateOne(buf, R_MICROMIPS_PC19_S2, gotPltEntryAddr - pltEntryAddr); } else { - write16<e>(buf, 0x7900); // addiupc $2, (GOTPLT) - . - write16<e>(buf + 4, 0xff22); // lw $25, 0($2) - write16<e>(buf + 8, 0x4599); // jrc $25 / jr16 $25 - write16<e>(buf + 10, 0x0f02); // move $24, $2 + write16(buf, 0x7900); // addiupc $2, (GOTPLT) - . + write16(buf + 4, 0xff22); // lw $25, 0($2) + write16(buf + 8, 0x4599); // jrc $25 / jr16 $25 + write16(buf + 10, 0x0f02); // move $24, $2 relocateOne(buf, R_MICROMIPS_PC23_S2, gotPltEntryAddr - pltEntryAddr); } return; @@ -332,13 +334,13 @@ void MIPS<ELFT>::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, : (config->zHazardplt ? 0x03200408 : 0x03200008); uint32_t addInst = ELFT::Is64Bits ? 0x65f80000 : 0x25f80000; - write32<e>(buf, 0x3c0f0000); // lui $15, %hi(.got.plt entry) - write32<e>(buf + 4, loadInst); // l[wd] $25, %lo(.got.plt entry)($15) - write32<e>(buf + 8, jrInst); // jr $25 / jr.hb $25 - write32<e>(buf + 12, addInst); // [d]addiu $24, $15, %lo(.got.plt entry) - writeValue<e>(buf, gotPltEntryAddr + 0x8000, 16, 16); - writeValue<e>(buf + 4, gotPltEntryAddr, 16, 0); - writeValue<e>(buf + 12, gotPltEntryAddr, 16, 0); + write32(buf, 0x3c0f0000); // lui $15, %hi(.got.plt entry) + write32(buf + 4, loadInst); // l[wd] $25, %lo(.got.plt entry)($15) + write32(buf + 8, jrInst); // jr $25 / jr.hb $25 + write32(buf + 12, addInst); // [d]addiu $24, $15, %lo(.got.plt entry) + writeValue(buf, gotPltEntryAddr + 0x8000, 16, 16); + writeValue(buf + 4, gotPltEntryAddr, 16, 0); + writeValue(buf + 12, gotPltEntryAddr, 16, 0); } template <class ELFT> @@ -372,16 +374,16 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const { case R_MIPS_GPREL32: case R_MIPS_TLS_DTPREL32: case R_MIPS_TLS_TPREL32: - return SignExtend64<32>(read32<e>(buf)); + return SignExtend64<32>(read32(buf)); case R_MIPS_26: // FIXME (simon): If the relocation target symbol is not a PLT entry // we should use another expression for calculation: // ((A << 2) | (P & 0xf0000000)) >> 2 - return SignExtend64<28>(read32<e>(buf) << 2); + return SignExtend64<28>(read32(buf) << 2); case R_MIPS_GOT16: case R_MIPS_HI16: case R_MIPS_PCHI16: - return SignExtend64<16>(read32<e>(buf)) << 16; + return SignExtend64<16>(read32(buf)) << 16; case R_MIPS_GPREL16: case R_MIPS_LO16: case R_MIPS_PCLO16: @@ -389,7 +391,7 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const { case R_MIPS_TLS_DTPREL_LO16: case R_MIPS_TLS_TPREL_HI16: case R_MIPS_TLS_TPREL_LO16: - return SignExtend64<16>(read32<e>(buf)); + return SignExtend64<16>(read32(buf)); case R_MICROMIPS_GOT16: case R_MICROMIPS_HI16: return SignExtend64<16>(readShuffle<e>(buf)) << 16; @@ -403,21 +405,21 @@ int64_t MIPS<ELFT>::getImplicitAddend(const uint8_t *buf, RelType type) const { case R_MICROMIPS_GPREL7_S2: return SignExtend64<9>(readShuffle<e>(buf) << 2); case R_MIPS_PC16: - return SignExtend64<18>(read32<e>(buf) << 2); + return SignExtend64<18>(read32(buf) << 2); case R_MIPS_PC19_S2: - return SignExtend64<21>(read32<e>(buf) << 2); + return SignExtend64<21>(read32(buf) << 2); case R_MIPS_PC21_S2: - return SignExtend64<23>(read32<e>(buf) << 2); + return SignExtend64<23>(read32(buf) << 2); case R_MIPS_PC26_S2: - return SignExtend64<28>(read32<e>(buf) << 2); + return SignExtend64<28>(read32(buf) << 2); case R_MIPS_PC32: - return SignExtend64<32>(read32<e>(buf)); + return SignExtend64<32>(read32(buf)); case R_MICROMIPS_26_S1: return SignExtend64<27>(readShuffle<e>(buf) << 1); case R_MICROMIPS_PC7_S1: - return SignExtend64<8>(read16<e>(buf) << 1); + return SignExtend64<8>(read16(buf) << 1); case R_MICROMIPS_PC10_S1: - return SignExtend64<11>(read16<e>(buf) << 1); + return SignExtend64<11>(read16(buf) << 1); case R_MICROMIPS_PC16_S1: return SignExtend64<17>(readShuffle<e>(buf) << 1); case R_MICROMIPS_PC18_S3: @@ -487,9 +489,9 @@ static uint64_t fixupCrossModeJump(uint8_t *loc, RelType type, uint64_t val) { switch (type) { case R_MIPS_26: { - uint32_t inst = read32<e>(loc) >> 26; + uint32_t inst = read32(loc) >> 26; if (inst == 0x3 || inst == 0x1d) { // JAL or JALX - writeValue<e>(loc, 0x1d << 26, 32, 0); + writeValue(loc, 0x1d << 26, 32, 0); return val; } break; @@ -538,11 +540,6 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { type == R_MICROMIPS_TLS_DTPREL_HI16 || type == R_MICROMIPS_TLS_DTPREL_LO16) { val -= 0x8000; - } else if (type == R_MIPS_TLS_TPREL_HI16 || type == R_MIPS_TLS_TPREL_LO16 || - type == R_MIPS_TLS_TPREL32 || type == R_MIPS_TLS_TPREL64 || - type == R_MICROMIPS_TLS_TPREL_HI16 || - type == R_MICROMIPS_TLS_TPREL_LO16) { - val -= 0x7000; } switch (type) { @@ -550,25 +547,25 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { case R_MIPS_GPREL32: case R_MIPS_TLS_DTPREL32: case R_MIPS_TLS_TPREL32: - write32<e>(loc, val); + write32(loc, val); break; case R_MIPS_64: case R_MIPS_TLS_DTPREL64: case R_MIPS_TLS_TPREL64: - write64<e>(loc, val); + write64(loc, val); break; case R_MIPS_26: - writeValue<e>(loc, val, 26, 2); + writeValue(loc, val, 26, 2); break; case R_MIPS_GOT16: // The R_MIPS_GOT16 relocation's value in "relocatable" linking mode // is updated addend (not a GOT index). In that case write high 16 bits // to store a correct addend value. if (config->relocatable) { - writeValue<e>(loc, val + 0x8000, 16, 16); + writeValue(loc, val + 0x8000, 16, 16); } else { checkInt(loc, val, 16, type); - writeValue<e>(loc, val, 16, 0); + writeValue(loc, val, 16, 0); } break; case R_MICROMIPS_GOT16: @@ -595,7 +592,7 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { case R_MIPS_PCLO16: case R_MIPS_TLS_DTPREL_LO16: case R_MIPS_TLS_TPREL_LO16: - writeValue<e>(loc, val, 16, 0); + writeValue(loc, val, 16, 0); break; case R_MICROMIPS_GPREL16: case R_MICROMIPS_TLS_GD: @@ -621,7 +618,7 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { case R_MIPS_PCHI16: case R_MIPS_TLS_DTPREL_HI16: case R_MIPS_TLS_TPREL_HI16: - writeValue<e>(loc, val + 0x8000, 16, 16); + writeValue(loc, val + 0x8000, 16, 16); break; case R_MICROMIPS_CALL_HI16: case R_MICROMIPS_GOT_HI16: @@ -631,37 +628,51 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { writeShuffleValue<e>(loc, val + 0x8000, 16, 16); break; case R_MIPS_HIGHER: - writeValue<e>(loc, val + 0x80008000, 16, 32); + writeValue(loc, val + 0x80008000, 16, 32); break; case R_MIPS_HIGHEST: - writeValue<e>(loc, val + 0x800080008000, 16, 48); + writeValue(loc, val + 0x800080008000, 16, 48); break; case R_MIPS_JALR: + val -= 4; + // Replace jalr/jr instructions by bal/b if the target + // offset fits into the 18-bit range. + if (isInt<18>(val)) { + switch (read32(loc)) { + case 0x0320f809: // jalr $25 => bal sym + write32(loc, 0x04110000 | ((val >> 2) & 0xffff)); + break; + case 0x03200008: // jr $25 => b sym + write32(loc, 0x10000000 | ((val >> 2) & 0xffff)); + break; + } + } + break; case R_MICROMIPS_JALR: // Ignore this optimization relocation for now break; case R_MIPS_PC16: checkAlignment(loc, val, 4, type); checkInt(loc, val, 18, type); - writeValue<e>(loc, val, 16, 2); + writeValue(loc, val, 16, 2); break; case R_MIPS_PC19_S2: checkAlignment(loc, val, 4, type); checkInt(loc, val, 21, type); - writeValue<e>(loc, val, 19, 2); + writeValue(loc, val, 19, 2); break; case R_MIPS_PC21_S2: checkAlignment(loc, val, 4, type); checkInt(loc, val, 23, type); - writeValue<e>(loc, val, 21, 2); + writeValue(loc, val, 21, 2); break; case R_MIPS_PC26_S2: checkAlignment(loc, val, 4, type); checkInt(loc, val, 28, type); - writeValue<e>(loc, val, 26, 2); + writeValue(loc, val, 26, 2); break; case R_MIPS_PC32: - writeValue<e>(loc, val, 32, 0); + writeValue(loc, val, 32, 0); break; case R_MICROMIPS_26_S1: case R_MICROMIPS_PC26_S1: @@ -707,7 +718,7 @@ template <class ELFT> bool MIPS<ELFT>::usesOnlyLowPageBits(RelType type) const { } // Return true if the symbol is a PIC function. -template <class ELFT> bool elf::isMipsPIC(const Defined *sym) { +template <class ELFT> bool isMipsPIC(const Defined *sym) { if (!sym->isFunc()) return false; @@ -725,17 +736,20 @@ template <class ELFT> bool elf::isMipsPIC(const Defined *sym) { return file->getObj().getHeader()->e_flags & EF_MIPS_PIC; } -template <class ELFT> TargetInfo *elf::getMipsTargetInfo() { +template <class ELFT> TargetInfo *getMipsTargetInfo() { static MIPS<ELFT> target; return ⌖ } -template TargetInfo *elf::getMipsTargetInfo<ELF32LE>(); -template TargetInfo *elf::getMipsTargetInfo<ELF32BE>(); -template TargetInfo *elf::getMipsTargetInfo<ELF64LE>(); -template TargetInfo *elf::getMipsTargetInfo<ELF64BE>(); +template TargetInfo *getMipsTargetInfo<ELF32LE>(); +template TargetInfo *getMipsTargetInfo<ELF32BE>(); +template TargetInfo *getMipsTargetInfo<ELF64LE>(); +template TargetInfo *getMipsTargetInfo<ELF64BE>(); + +template bool isMipsPIC<ELF32LE>(const Defined *); +template bool isMipsPIC<ELF32BE>(const Defined *); +template bool isMipsPIC<ELF64LE>(const Defined *); +template bool isMipsPIC<ELF64BE>(const Defined *); -template bool elf::isMipsPIC<ELF32LE>(const Defined *); -template bool elf::isMipsPIC<ELF32BE>(const Defined *); -template bool elf::isMipsPIC<ELF64LE>(const Defined *); -template bool elf::isMipsPIC<ELF64BE>(const Defined *); +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/MipsArchTree.cpp b/ELF/Arch/MipsArchTree.cpp index f64d03756457..923458afae0d 100644 --- a/ELF/Arch/MipsArchTree.cpp +++ b/ELF/Arch/MipsArchTree.cpp @@ -23,8 +23,8 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { namespace { struct ArchTreeEdge { @@ -166,17 +166,17 @@ static ArchTreeEdge archTree[] = { {EF_MIPS_ARCH_2, EF_MIPS_ARCH_1}, }; -static bool isArchMatched(uint32_t New, uint32_t res) { - if (New == res) +static bool isArchMatched(uint32_t newFlags, uint32_t res) { + if (newFlags == res) return true; - if (New == EF_MIPS_ARCH_32 && isArchMatched(EF_MIPS_ARCH_64, res)) + if (newFlags == EF_MIPS_ARCH_32 && isArchMatched(EF_MIPS_ARCH_64, res)) return true; - if (New == EF_MIPS_ARCH_32R2 && isArchMatched(EF_MIPS_ARCH_64R2, res)) + if (newFlags == EF_MIPS_ARCH_32R2 && isArchMatched(EF_MIPS_ARCH_64R2, res)) return true; for (const auto &edge : archTree) { if (res == edge.child) { res = edge.parent; - if (res == New) + if (res == newFlags) return true; } } @@ -278,28 +278,34 @@ static uint32_t getArchFlags(ArrayRef<FileFlags> files) { uint32_t ret = files[0].flags & (EF_MIPS_ARCH | EF_MIPS_MACH); for (const FileFlags &f : files.slice(1)) { - uint32_t New = f.flags & (EF_MIPS_ARCH | EF_MIPS_MACH); + uint32_t newFlags = f.flags & (EF_MIPS_ARCH | EF_MIPS_MACH); // Check ISA compatibility. - if (isArchMatched(New, ret)) + if (isArchMatched(newFlags, ret)) continue; - if (!isArchMatched(ret, New)) { + if (!isArchMatched(ret, newFlags)) { error("incompatible target ISA:\n>>> " + toString(files[0].file) + ": " + getFullArchName(ret) + "\n>>> " + toString(f.file) + ": " + - getFullArchName(New)); + getFullArchName(newFlags)); return 0; } - ret = New; + ret = newFlags; } return ret; } -template <class ELFT> uint32_t elf::calcMipsEFlags() { +template <class ELFT> uint32_t calcMipsEFlags() { std::vector<FileFlags> v; for (InputFile *f : objectFiles) v.push_back({f, cast<ObjFile<ELFT>>(f)->getObj().getHeader()->e_flags}); - if (v.empty()) - return 0; + if (v.empty()) { + // If we don't have any input files, we'll have to rely on the information + // we can derive from emulation information, since this at least gets us + // ABI. + if (config->emulation.empty() || config->is64) + return 0; + return config->mipsN32Abi ? EF_MIPS_ABI2 : EF_MIPS_ABI_O32; + } checkFlags(v); return getMiscFlags(v) | getPicFlags(v) | getArchFlags(v); } @@ -344,8 +350,7 @@ static StringRef getMipsFpAbiName(uint8_t fpAbi) { } } -uint8_t elf::getMipsFpAbiFlag(uint8_t oldFlag, uint8_t newFlag, - StringRef fileName) { +uint8_t getMipsFpAbiFlag(uint8_t oldFlag, uint8_t newFlag, StringRef fileName) { if (compareMipsFpAbi(newFlag, oldFlag) >= 0) return newFlag; if (compareMipsFpAbi(oldFlag, newFlag) < 0) @@ -361,7 +366,7 @@ template <class ELFT> static bool isN32Abi(const InputFile *f) { return false; } -bool elf::isMipsN32Abi(const InputFile *f) { +bool isMipsN32Abi(const InputFile *f) { switch (config->ekind) { case ELF32LEKind: return isN32Abi<ELF32LE>(f); @@ -376,14 +381,17 @@ bool elf::isMipsN32Abi(const InputFile *f) { } } -bool elf::isMicroMips() { return config->eflags & EF_MIPS_MICROMIPS; } +bool isMicroMips() { return config->eflags & EF_MIPS_MICROMIPS; } -bool elf::isMipsR6() { +bool isMipsR6() { uint32_t arch = config->eflags & EF_MIPS_ARCH; return arch == EF_MIPS_ARCH_32R6 || arch == EF_MIPS_ARCH_64R6; } -template uint32_t elf::calcMipsEFlags<ELF32LE>(); -template uint32_t elf::calcMipsEFlags<ELF32BE>(); -template uint32_t elf::calcMipsEFlags<ELF64LE>(); -template uint32_t elf::calcMipsEFlags<ELF64BE>(); +template uint32_t calcMipsEFlags<ELF32LE>(); +template uint32_t calcMipsEFlags<ELF32BE>(); +template uint32_t calcMipsEFlags<ELF64LE>(); +template uint32_t calcMipsEFlags<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/PPC.cpp b/ELF/Arch/PPC.cpp index 46c5891e4f8a..c4eecb9a29c2 100644 --- a/ELF/Arch/PPC.cpp +++ b/ELF/Arch/PPC.cpp @@ -16,8 +16,9 @@ using namespace llvm; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class PPC final : public TargetInfo { @@ -61,7 +62,7 @@ static void writeFromHalf16(uint8_t *loc, uint32_t insn) { write32(config->isLE ? loc : loc - 2, insn); } -void elf::writePPC32GlinkSection(uint8_t *buf, size_t numEntries) { +void writePPC32GlinkSection(uint8_t *buf, size_t numEntries) { // On PPC Secure PLT ABI, bl foo@plt jumps to a call stub, which loads an // absolute address from a specific .plt slot (usually called .got.plt on // other targets) and jumps there. @@ -190,6 +191,13 @@ bool PPC::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { RelExpr PPC::getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const { switch (type) { + case R_PPC_NONE: + return R_NONE; + case R_PPC_ADDR16_HA: + case R_PPC_ADDR16_HI: + case R_PPC_ADDR16_LO: + case R_PPC_ADDR32: + return R_ABS; case R_PPC_DTPREL16: case R_PPC_DTPREL16_HA: case R_PPC_DTPREL16_HI: @@ -227,7 +235,9 @@ RelExpr PPC::getRelExpr(RelType type, const Symbol &s, case R_PPC_TPREL16_HI: return R_TLS; default: - return R_ABS; + error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + + ") against symbol " + toString(s)); + return R_NONE; } } @@ -319,7 +329,7 @@ void PPC::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { break; } default: - error(getErrorLocation(loc) + "unrecognized relocation " + toString(type)); + llvm_unreachable("unknown relocation"); } } @@ -426,7 +436,10 @@ void PPC::relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const { } } -TargetInfo *elf::getPPCTargetInfo() { +TargetInfo *getPPCTargetInfo() { static PPC target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/PPC64.cpp b/ELF/Arch/PPC64.cpp index 70d284cfad71..905903fa4d66 100644 --- a/ELF/Arch/PPC64.cpp +++ b/ELF/Arch/PPC64.cpp @@ -16,8 +16,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { static uint64_t ppc64TocOffset = 0x8000; static uint64_t dynamicThreadPointerOffset = 0x8000; @@ -59,7 +60,7 @@ enum DFormOpcd { ADDI = 14 }; -uint64_t elf::getPPC64TocBase() { +uint64_t getPPC64TocBase() { // The TOC consists of sections .got, .toc, .tocbss, .plt in that order. The // TOC starts where the first of these sections starts. We always create a // .got when we see a relocation that uses it, so for us the start is always @@ -73,7 +74,7 @@ uint64_t elf::getPPC64TocBase() { return tocVA + ppc64TocOffset; } -unsigned elf::getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther) { +unsigned getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther) { // The offset is encoded into the 3 most significant bits of the st_other // field, with some special values described in section 3.4.1 of the ABI: // 0 --> Zero offset between the GEP and LEP, and the function does NOT use @@ -98,7 +99,7 @@ unsigned elf::getPPC64GlobalEntryToLocalEntryOffset(uint8_t stOther) { return 0; } -bool elf::isPPC64SmallCodeModelTocReloc(RelType type) { +bool isPPC64SmallCodeModelTocReloc(RelType type) { // The only small code model relocations that access the .toc section. return type == R_PPC64_TOC16 || type == R_PPC64_TOC16_DS; } @@ -153,8 +154,8 @@ getRelaTocSymAndAddend(InputSectionBase *tocSec, uint64_t offset) { // ld/lwa 3, 0(3) # load the value from the address // // Returns true if the relaxation is performed. -bool elf::tryRelaxPPC64TocIndirection(RelType type, const Relocation &rel, - uint8_t *bufLoc) { +bool tryRelaxPPC64TocIndirection(RelType type, const Relocation &rel, + uint8_t *bufLoc) { assert(config->tocOptimize); if (rel.addend < 0) return false; @@ -175,6 +176,10 @@ bool elf::tryRelaxPPC64TocIndirection(RelType type, const Relocation &rel, if (!d || d->isPreemptible) return false; + // R_PPC64_ADDR64 should have created a canonical PLT for the non-preemptable + // ifunc and changed its type to STT_FUNC. + assert(!d->isGnuIFunc()); + // Two instructions can materialize a 32-bit signed offset from the toc base. uint64_t tocRelative = d->getVA(addend) - getPPC64TocBase(); if (!isInt<32>(tocRelative)) @@ -454,7 +459,7 @@ void PPC64::relaxTlsLdToLe(uint8_t *loc, RelType type, uint64_t val) const { } } -unsigned elf::getPPCDFormOp(unsigned secondaryOp) { +unsigned getPPCDFormOp(unsigned secondaryOp) { switch (secondaryOp) { case LBZX: return LBZ; @@ -532,6 +537,21 @@ void PPC64::relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const { RelExpr PPC64::getRelExpr(RelType type, const Symbol &s, const uint8_t *loc) const { switch (type) { + case R_PPC64_NONE: + return R_NONE; + case R_PPC64_ADDR16: + case R_PPC64_ADDR16_DS: + case R_PPC64_ADDR16_HA: + case R_PPC64_ADDR16_HI: + case R_PPC64_ADDR16_HIGHER: + case R_PPC64_ADDR16_HIGHERA: + case R_PPC64_ADDR16_HIGHEST: + case R_PPC64_ADDR16_HIGHESTA: + case R_PPC64_ADDR16_LO: + case R_PPC64_ADDR16_LO_DS: + case R_PPC64_ADDR32: + case R_PPC64_ADDR64: + return R_ABS; case R_PPC64_GOT16: case R_PPC64_GOT16_DS: case R_PPC64_GOT16_HA: @@ -554,6 +574,7 @@ RelExpr PPC64::getRelExpr(RelType type, const Symbol &s, return R_PPC64_CALL_PLT; case R_PPC64_REL16_LO: case R_PPC64_REL16_HA: + case R_PPC64_REL16_HI: case R_PPC64_REL32: case R_PPC64_REL64: return R_PC; @@ -607,7 +628,9 @@ RelExpr PPC64::getRelExpr(RelType type, const Symbol &s, case R_PPC64_TLS: return R_TLSIE_HINT; default: - return R_ABS; + error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + + ") against symbol " + toString(s)); + return R_NONE; } } @@ -870,7 +893,7 @@ void PPC64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { write64(loc, val - dynamicThreadPointerOffset); break; default: - error(getErrorLocation(loc) + "unrecognized relocation " + toString(type)); + llvm_unreachable("unknown relocation"); } } @@ -1071,7 +1094,10 @@ bool PPC64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, return true; } -TargetInfo *elf::getPPC64TargetInfo() { +TargetInfo *getPPC64TargetInfo() { static PPC64 target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/RISCV.cpp b/ELF/Arch/RISCV.cpp index 6f16ade57177..e7c0e36e0327 100644 --- a/ELF/Arch/RISCV.cpp +++ b/ELF/Arch/RISCV.cpp @@ -14,8 +14,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { @@ -436,7 +437,10 @@ void RISCV::relocateOne(uint8_t *loc, const RelType type, } } -TargetInfo *elf::getRISCVTargetInfo() { +TargetInfo *getRISCVTargetInfo() { static RISCV target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/SPARCV9.cpp b/ELF/Arch/SPARCV9.cpp index 5299206dd919..a0afdff08a63 100644 --- a/ELF/Arch/SPARCV9.cpp +++ b/ELF/Arch/SPARCV9.cpp @@ -16,8 +16,9 @@ using namespace llvm; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class SPARCV9 final : public TargetInfo { @@ -143,7 +144,10 @@ void SPARCV9::writePlt(uint8_t *buf, uint64_t gotEntryAddr, relocateOne(buf + 4, R_SPARC_WDISP19, -(off + 4 - pltEntrySize)); } -TargetInfo *elf::getSPARCV9TargetInfo() { +TargetInfo *getSPARCV9TargetInfo() { static SPARCV9 target; return ⌖ } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/X86.cpp b/ELF/Arch/X86.cpp index e1dd231e8e8d..b27a6e302e78 100644 --- a/ELF/Arch/X86.cpp +++ b/ELF/Arch/X86.cpp @@ -16,8 +16,9 @@ using namespace llvm; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class X86 : public TargetInfo { @@ -539,7 +540,7 @@ void RetpolineNoPic::writePlt(uint8_t *buf, uint64_t gotPltEntryAddr, write32le(buf + 22, -off - 26); } -TargetInfo *elf::getX86TargetInfo() { +TargetInfo *getX86TargetInfo() { if (config->zRetpolineplt) { if (config->isPic) { static RetpolinePic t; @@ -552,3 +553,6 @@ TargetInfo *elf::getX86TargetInfo() { static X86 t; return &t; } + +} // namespace elf +} // namespace lld diff --git a/ELF/Arch/X86_64.cpp b/ELF/Arch/X86_64.cpp index de67aa5c33dc..bb8d92fc61b9 100644 --- a/ELF/Arch/X86_64.cpp +++ b/ELF/Arch/X86_64.cpp @@ -18,8 +18,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { class X86_64 : public TargetInfo { @@ -698,4 +699,7 @@ static TargetInfo *getTargetInfo() { return &t; } -TargetInfo *elf::getX86_64TargetInfo() { return getTargetInfo(); } +TargetInfo *getX86_64TargetInfo() { return getTargetInfo(); } + +} // namespace elf +} // namespace lld diff --git a/ELF/CMakeLists.txt b/ELF/CMakeLists.txt index 70578746483d..1ba79bec73df 100644 --- a/ELF/CMakeLists.txt +++ b/ELF/CMakeLists.txt @@ -22,6 +22,7 @@ add_lld_library(lldELF Arch/SPARCV9.cpp Arch/X86.cpp Arch/X86_64.cpp + ARMErrataFix.cpp CallGraphSort.cpp DWARF.cpp Driver.cpp diff --git a/ELF/CallGraphSort.cpp b/ELF/CallGraphSort.cpp index 9aaadd481833..6f8ef8954af3 100644 --- a/ELF/CallGraphSort.cpp +++ b/ELF/CallGraphSort.cpp @@ -45,9 +45,12 @@ #include "SymbolTable.h" #include "Symbols.h" +#include <numeric> + using namespace llvm; -using namespace lld; -using namespace lld::elf; + +namespace lld { +namespace elf { namespace { struct Edge { @@ -56,7 +59,7 @@ struct Edge { }; struct Cluster { - Cluster(int sec, size_t s) : sections{sec}, size(s) {} + Cluster(int sec, size_t s) : next(sec), prev(sec), size(s) {} double getDensity() const { if (size == 0) @@ -64,7 +67,8 @@ struct Cluster { return double(weight) / double(size); } - std::vector<int> sections; + int next; + int prev; size_t size = 0; uint64_t weight = 0; uint64_t initialWeight = 0; @@ -80,8 +84,6 @@ public: private: std::vector<Cluster> clusters; std::vector<const InputSectionBase *> sections; - - void groupClusters(); }; // Maximum ammount the combined cluster density can be worse than the original @@ -103,7 +105,7 @@ CallGraphSort::CallGraphSort() { DenseMap<const InputSectionBase *, int> secToCluster; auto getOrCreateNode = [&](const InputSectionBase *isec) -> int { - auto res = secToCluster.insert(std::make_pair(isec, clusters.size())); + auto res = secToCluster.try_emplace(isec, clusters.size()); if (res.second) { sections.push_back(isec); clusters.emplace_back(clusters.size(), isec->getSize()); @@ -151,83 +153,88 @@ static bool isNewDensityBad(Cluster &a, Cluster &b) { return newDensity < a.getDensity() / MAX_DENSITY_DEGRADATION; } -static void mergeClusters(Cluster &into, Cluster &from) { - into.sections.insert(into.sections.end(), from.sections.begin(), - from.sections.end()); +// Find the leader of V's belonged cluster (represented as an equivalence +// class). We apply union-find path-halving technique (simple to implement) in +// the meantime as it decreases depths and the time complexity. +static int getLeader(std::vector<int> &leaders, int v) { + while (leaders[v] != v) { + leaders[v] = leaders[leaders[v]]; + v = leaders[v]; + } + return v; +} + +static void mergeClusters(std::vector<Cluster> &cs, Cluster &into, int intoIdx, + Cluster &from, int fromIdx) { + int tail1 = into.prev, tail2 = from.prev; + into.prev = tail2; + cs[tail2].next = intoIdx; + from.prev = tail1; + cs[tail1].next = fromIdx; into.size += from.size; into.weight += from.weight; - from.sections.clear(); from.size = 0; from.weight = 0; } // Group InputSections into clusters using the Call-Chain Clustering heuristic // then sort the clusters by density. -void CallGraphSort::groupClusters() { - std::vector<int> sortedSecs(clusters.size()); - std::vector<Cluster *> secToCluster(clusters.size()); - - for (size_t i = 0; i < clusters.size(); ++i) { - sortedSecs[i] = i; - secToCluster[i] = &clusters[i]; - } +DenseMap<const InputSectionBase *, int> CallGraphSort::run() { + std::vector<int> sorted(clusters.size()); + std::vector<int> leaders(clusters.size()); - llvm::stable_sort(sortedSecs, [&](int a, int b) { + std::iota(leaders.begin(), leaders.end(), 0); + std::iota(sorted.begin(), sorted.end(), 0); + llvm::stable_sort(sorted, [&](int a, int b) { return clusters[a].getDensity() > clusters[b].getDensity(); }); - for (int si : sortedSecs) { - // clusters[si] is the same as secToClusters[si] here because it has not - // been merged into another cluster yet. - Cluster &c = clusters[si]; + for (int l : sorted) { + // The cluster index is the same as the index of its leader here because + // clusters[L] has not been merged into another cluster yet. + Cluster &c = clusters[l]; // Don't consider merging if the edge is unlikely. if (c.bestPred.from == -1 || c.bestPred.weight * 10 <= c.initialWeight) continue; - Cluster *predC = secToCluster[c.bestPred.from]; - if (predC == &c) + int predL = getLeader(leaders, c.bestPred.from); + if (l == predL) continue; + Cluster *predC = &clusters[predL]; if (c.size + predC->size > MAX_CLUSTER_SIZE) continue; if (isNewDensityBad(*predC, c)) continue; - // NOTE: Consider using a disjoint-set to track section -> cluster mapping - // if this is ever slow. - for (int si : c.sections) - secToCluster[si] = predC; - - mergeClusters(*predC, c); + leaders[l] = predL; + mergeClusters(clusters, *predC, predL, c, l); } - // Remove empty or dead nodes. Invalidates all cluster indices. - llvm::erase_if(clusters, [](const Cluster &c) { - return c.size == 0 || c.sections.empty(); + // Sort remaining non-empty clusters by density. + sorted.clear(); + for (int i = 0, e = (int)clusters.size(); i != e; ++i) + if (clusters[i].size > 0) + sorted.push_back(i); + llvm::stable_sort(sorted, [&](int a, int b) { + return clusters[a].getDensity() > clusters[b].getDensity(); }); - // Sort by density. - llvm::stable_sort(clusters, [](const Cluster &a, const Cluster &b) { - return a.getDensity() > b.getDensity(); - }); -} - -DenseMap<const InputSectionBase *, int> CallGraphSort::run() { - groupClusters(); - - // Generate order. DenseMap<const InputSectionBase *, int> orderMap; - ssize_t curOrder = 1; - - for (const Cluster &c : clusters) - for (int secIndex : c.sections) - orderMap[sections[secIndex]] = curOrder++; + int curOrder = 1; + for (int leader : sorted) + for (int i = leader;;) { + orderMap[sections[i]] = curOrder++; + i = clusters[i].next; + if (i == leader) + break; + } if (!config->printSymbolOrder.empty()) { std::error_code ec; - raw_fd_ostream os(config->printSymbolOrder, ec, sys::fs::F_None); + raw_fd_ostream os(config->printSymbolOrder, ec, sys::fs::OF_None); if (ec) { error("cannot open " + config->printSymbolOrder + ": " + ec.message()); return orderMap; @@ -235,15 +242,19 @@ DenseMap<const InputSectionBase *, int> CallGraphSort::run() { // Print the symbols ordered by C3, in the order of increasing curOrder // Instead of sorting all the orderMap, just repeat the loops above. - for (const Cluster &c : clusters) - for (int secIndex : c.sections) + for (int leader : sorted) + for (int i = leader;;) { // Search all the symbols in the file of the section // and find out a Defined symbol with name that is within the section. - for (Symbol *sym: sections[secIndex]->file->getSymbols()) + for (Symbol *sym : sections[i]->file->getSymbols()) if (!sym->isSection()) // Filter out section-type symbols here. if (auto *d = dyn_cast<Defined>(sym)) - if (sections[secIndex] == d->section) + if (sections[i] == d->section) os << sym->getName() << "\n"; + i = clusters[i].next; + if (i == leader) + break; + } } return orderMap; @@ -254,6 +265,9 @@ DenseMap<const InputSectionBase *, int> CallGraphSort::run() { // This first builds a call graph based on the profile data then merges sections // according to the C³ huristic. All clusters are then sorted by a density // metric to further improve locality. -DenseMap<const InputSectionBase *, int> elf::computeCallGraphProfileOrder() { +DenseMap<const InputSectionBase *, int> computeCallGraphProfileOrder() { return CallGraphSort().run(); } + +} // namespace elf +} // namespace lld diff --git a/ELF/Config.h b/ELF/Config.h index ff9d3dc0933c..0c68a8485fa2 100644 --- a/ELF/Config.h +++ b/ELF/Config.h @@ -61,6 +61,9 @@ enum class Target2Policy { Abs, Rel, GotRel }; // For tracking ARM Float Argument PCS enum class ARMVFPArgKind { Default, Base, VFP, ToolChain }; +// For -z noseparate-code, -z separate-code and -z separate-loadable-segments. +enum class SeparateSegmentKind { None, Code, Loadable }; + struct SymbolVersion { llvm::StringRef name; bool isExternCpp; @@ -71,8 +74,8 @@ struct SymbolVersion { // can be found in version script if it is used for link. struct VersionDefinition { llvm::StringRef name; - uint16_t id = 0; - std::vector<SymbolVersion> globals; + uint16_t id; + std::vector<SymbolVersion> patterns; }; // This struct contains the global configuration for the linker. @@ -117,8 +120,6 @@ struct Configuration { std::vector<llvm::StringRef> symbolOrderingFile; std::vector<llvm::StringRef> undefined; std::vector<SymbolVersion> dynamicList; - std::vector<SymbolVersion> versionScriptGlobals; - std::vector<SymbolVersion> versionScriptLocals; std::vector<uint8_t> buildIdVector; llvm::MapVector<std::pair<const InputSectionBase *, const InputSectionBase *>, uint64_t> @@ -147,6 +148,7 @@ struct Configuration { bool executeOnly; bool exportDynamic; bool fixCortexA53Errata843419; + bool fixCortexA8; bool forceBTI; bool formatBinary = false; bool requireCET; @@ -222,8 +224,8 @@ struct Configuration { Target2Policy target2; ARMVFPArgKind armVFPArgs = ARMVFPArgKind::Default; BuildIdKind buildId = BuildIdKind::None; + SeparateSegmentKind zSeparate; ELFKind ekind = ELFNoneKind; - uint16_t defaultSymbolVersion = llvm::ELF::VER_NDX_GLOBAL; uint16_t emachine = llvm::ELF::EM_NONE; llvm::Optional<uint64_t> imageBase; uint64_t commonPageSize; @@ -309,6 +311,12 @@ struct Configuration { // The only instance of Configuration struct. extern Configuration *config; +// The first two elements of versionDefinitions represent VER_NDX_LOCAL and +// VER_NDX_GLOBAL. This helper returns other elements. +static inline ArrayRef<VersionDefinition> namedVersionDefs() { + return llvm::makeArrayRef(config->versionDefinitions).slice(2); +} + static inline void errorOrWarn(const Twine &msg) { if (!config->noinhibitExec) error(msg); diff --git a/ELF/DWARF.cpp b/ELF/DWARF.cpp index 1e4b36f71b54..a00189a0e3a2 100644 --- a/ELF/DWARF.cpp +++ b/ELF/DWARF.cpp @@ -22,9 +22,9 @@ using namespace llvm; using namespace llvm::object; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { template <class ELFT> LLDDwarfObj<ELFT>::LLDDwarfObj(ObjFile<ELFT> *obj) { for (InputSectionBase *sec : obj->getSections()) { if (!sec) @@ -33,11 +33,12 @@ template <class ELFT> LLDDwarfObj<ELFT>::LLDDwarfObj(ObjFile<ELFT> *obj) { if (LLDDWARFSection *m = StringSwitch<LLDDWARFSection *>(sec->name) .Case(".debug_addr", &addrSection) - .Case(".debug_gnu_pubnames", &gnuPubNamesSection) - .Case(".debug_gnu_pubtypes", &gnuPubTypesSection) + .Case(".debug_gnu_pubnames", &gnuPubnamesSection) + .Case(".debug_gnu_pubtypes", &gnuPubtypesSection) .Case(".debug_info", &infoSection) - .Case(".debug_ranges", &rangeSection) - .Case(".debug_rnglists", &rngListsSection) + .Case(".debug_ranges", &rangesSection) + .Case(".debug_rnglists", &rnglistsSection) + .Case(".debug_str_offsets", &strOffsetsSection) .Case(".debug_line", &lineSection) .Default(nullptr)) { m->Data = toStringRef(sec->data()); @@ -50,7 +51,7 @@ template <class ELFT> LLDDwarfObj<ELFT>::LLDDwarfObj(ObjFile<ELFT> *obj) { else if (sec->name == ".debug_str") strSection = toStringRef(sec->data()); else if (sec->name == ".debug_line_str") - lineStringSection = toStringRef(sec->data()); + lineStrSection = toStringRef(sec->data()); } } @@ -123,7 +124,10 @@ Optional<RelocAddrEntry> LLDDwarfObj<ELFT>::find(const llvm::DWARFSection &s, return findAux(*sec.sec, pos, sec.sec->template rels<ELFT>()); } -template class elf::LLDDwarfObj<ELF32LE>; -template class elf::LLDDwarfObj<ELF32BE>; -template class elf::LLDDwarfObj<ELF64LE>; -template class elf::LLDDwarfObj<ELF64BE>; +template class LLDDwarfObj<ELF32LE>; +template class LLDDwarfObj<ELF32BE>; +template class LLDDwarfObj<ELF64LE>; +template class LLDDwarfObj<ELF64BE>; + +} // namespace elf +} // namespace lld diff --git a/ELF/DWARF.h b/ELF/DWARF.h index 426022945007..51ec9092f172 100644 --- a/ELF/DWARF.h +++ b/ELF/DWARF.h @@ -32,12 +32,16 @@ public: f(infoSection); } - const llvm::DWARFSection &getRangeSection() const override { - return rangeSection; + const llvm::DWARFSection &getRangesSection() const override { + return rangesSection; } const llvm::DWARFSection &getRnglistsSection() const override { - return rngListsSection; + return rnglistsSection; + } + + const llvm::DWARFSection &getStrOffsetsSection() const override { + return strOffsetsSection; } const llvm::DWARFSection &getLineSection() const override { @@ -48,18 +52,18 @@ public: return addrSection; } - const llvm::DWARFSection &getGnuPubNamesSection() const override { - return gnuPubNamesSection; + const llvm::DWARFSection &getGnuPubnamesSection() const override { + return gnuPubnamesSection; } - const llvm::DWARFSection &getGnuPubTypesSection() const override { - return gnuPubTypesSection; + const llvm::DWARFSection &getGnuPubtypesSection() const override { + return gnuPubtypesSection; } StringRef getFileName() const override { return ""; } StringRef getAbbrevSection() const override { return abbrevSection; } - StringRef getStringSection() const override { return strSection; } - StringRef getLineStringSection() const override { return lineStringSection; } + StringRef getStrSection() const override { return strSection; } + StringRef getLineStrSection() const override { return lineStrSection; } bool isLittleEndian() const override { return ELFT::TargetEndianness == llvm::support::little; @@ -74,16 +78,17 @@ private: uint64_t pos, ArrayRef<RelTy> rels) const; - LLDDWARFSection gnuPubNamesSection; - LLDDWARFSection gnuPubTypesSection; + LLDDWARFSection gnuPubnamesSection; + LLDDWARFSection gnuPubtypesSection; LLDDWARFSection infoSection; - LLDDWARFSection rangeSection; - LLDDWARFSection rngListsSection; + LLDDWARFSection rangesSection; + LLDDWARFSection rnglistsSection; + LLDDWARFSection strOffsetsSection; LLDDWARFSection lineSection; LLDDWARFSection addrSection; StringRef abbrevSection; StringRef strSection; - StringRef lineStringSection; + StringRef lineStrSection; }; } // namespace elf diff --git a/ELF/Driver.cpp b/ELF/Driver.cpp index fbfc71d22b7e..96257a4c7624 100644 --- a/ELF/Driver.cpp +++ b/ELF/Driver.cpp @@ -48,6 +48,7 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/LTO/LTO.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compression.h" #include "llvm/Support/GlobPattern.h" @@ -65,24 +66,23 @@ using namespace llvm::object; using namespace llvm::sys; using namespace llvm::support; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { -Configuration *elf::config; -LinkerDriver *elf::driver; +Configuration *config; +LinkerDriver *driver; static void setConfigs(opt::InputArgList &args); static void readConfigs(opt::InputArgList &args); -bool elf::link(ArrayRef<const char *> args, bool canExitEarly, - raw_ostream &error) { +bool link(ArrayRef<const char *> args, bool canExitEarly, raw_ostream &error) { errorHandler().logName = args::getFilenameWithoutExe(args[0]); errorHandler().errorLimitExceededMsg = "too many errors emitted, stopping now (use " "-error-limit=0 to see all errors)"; errorHandler().errorOS = &error; errorHandler().exitEarly = canExitEarly; - errorHandler().colorDiagnostics = error.has_colors(); + enableColors(error.has_colors()); inputSections.clear(); outputSections.clear(); @@ -299,6 +299,9 @@ static void checkOptions() { if (config->fixCortexA53Errata843419 && config->emachine != EM_AARCH64) error("--fix-cortex-a53-843419 is only supported on AArch64 targets"); + if (config->fixCortexA8 && config->emachine != EM_ARM) + error("--fix-cortex-a8 is only supported on ARM targets"); + if (config->tocOptimize && config->emachine != EM_PPC64) error("--toc-optimize is only supported on the PowerPC64 target"); @@ -314,6 +317,9 @@ static void checkOptions() { if (!config->relocatable && !config->defineCommon) error("-no-define-common not supported in non relocatable output"); + if (config->strip == StripPolicy::All && config->emitRelocs) + error("--strip-all and --emit-relocs may not be used together"); + if (config->zText && config->zIfuncNoplt) error("-z text and -z ifunc-noplt may not be used together"); @@ -328,6 +334,8 @@ static void checkOptions() { error("-r and --icf may not be used together"); if (config->pie) error("-r and -pie may not be used together"); + if (config->exportDynamic) + error("-r and --export-dynamic may not be used together"); } if (config->executeOnly) { @@ -373,17 +381,32 @@ static bool getZFlag(opt::InputArgList &args, StringRef k1, StringRef k2, return Default; } +static SeparateSegmentKind getZSeparate(opt::InputArgList &args) { + for (auto *arg : args.filtered_reverse(OPT_z)) { + StringRef v = arg->getValue(); + if (v == "noseparate-code") + return SeparateSegmentKind::None; + if (v == "separate-code") + return SeparateSegmentKind::Code; + if (v == "separate-loadable-segments") + return SeparateSegmentKind::Loadable; + } + return SeparateSegmentKind::None; +} + static bool isKnownZFlag(StringRef s) { return s == "combreloc" || s == "copyreloc" || s == "defs" || s == "execstack" || s == "global" || s == "hazardplt" || s == "ifunc-noplt" || s == "initfirst" || s == "interpose" || s == "keep-text-section-prefix" || s == "lazy" || s == "muldefs" || + s == "separate-code" || s == "separate-loadable-segments" || s == "nocombreloc" || s == "nocopyreloc" || s == "nodefaultlib" || s == "nodelete" || s == "nodlopen" || s == "noexecstack" || - s == "nokeep-text-section-prefix" || s == "norelro" || s == "notext" || - s == "now" || s == "origin" || s == "relro" || s == "retpolineplt" || - s == "rodynamic" || s == "text" || s == "wxneeded" || - s.startswith("common-page-size") || s.startswith("max-page-size=") || + s == "nokeep-text-section-prefix" || s == "norelro" || + s == "noseparate-code" || s == "notext" || s == "now" || + s == "origin" || s == "relro" || s == "retpolineplt" || + s == "rodynamic" || s == "text" || s == "undefs" || s == "wxneeded" || + s.startswith("common-page-size=") || s.startswith("max-page-size=") || s.startswith("stack-size="); } @@ -513,6 +536,8 @@ static UnresolvedPolicy getUnresolvedSymbolPolicy(opt::InputArgList &args) { case OPT_z: if (StringRef(arg->getValue()) == "defs") return errorOrWarn; + if (StringRef(arg->getValue()) == "undefs") + return UnresolvedPolicy::Ignore; continue; } } @@ -747,6 +772,12 @@ static bool getCompressDebugSections(opt::InputArgList &args) { return true; } +static StringRef getAliasSpelling(opt::Arg *arg) { + if (const opt::Arg *alias = arg->getAlias()) + return alias->getSpelling(); + return arg->getSpelling(); +} + static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args, unsigned id) { auto *arg = args.getLastArg(id); @@ -756,7 +787,7 @@ static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args, StringRef s = arg->getValue(); std::pair<StringRef, StringRef> ret = s.split(';'); if (ret.second.empty()) - error(arg->getSpelling() + " expects 'old;new' format, but got " + s); + error(getAliasSpelling(arg) + " expects 'old;new' format, but got " + s); return ret; } @@ -829,6 +860,7 @@ static void readConfigs(opt::InputArgList &args) { config->filterList = args::getStrings(args, OPT_filter); config->fini = args.getLastArgValue(OPT_fini, "_fini"); config->fixCortexA53Errata843419 = args.hasArg(OPT_fix_cortex_a53_843419); + config->fixCortexA8 = args.hasArg(OPT_fix_cortex_a8); config->forceBTI = args.hasArg(OPT_force_bti); config->requireCET = args.hasArg(OPT_require_cet); config->gcSections = args.hasFlag(OPT_gc_sections, OPT_no_gc_sections, false); @@ -847,7 +879,7 @@ static void readConfigs(opt::InputArgList &args) { config->ltoNewPassManager = args.hasArg(OPT_lto_new_pass_manager); config->ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes); config->ltoo = args::getInteger(args, OPT_lto_O, 2); - config->ltoObjPath = args.getLastArgValue(OPT_plugin_opt_obj_path_eq); + config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq); config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1); config->ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile); config->mapFile = args.getLastArgValue(OPT_Map); @@ -892,17 +924,15 @@ static void readConfigs(opt::InputArgList &args) { config->thinLTOCachePolicy = CHECK( parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)), "--thinlto-cache-policy: invalid cache policy"); - config->thinLTOEmitImportsFiles = - args.hasArg(OPT_plugin_opt_thinlto_emit_imports_files); - config->thinLTOIndexOnly = args.hasArg(OPT_plugin_opt_thinlto_index_only) || - args.hasArg(OPT_plugin_opt_thinlto_index_only_eq); - config->thinLTOIndexOnlyArg = - args.getLastArgValue(OPT_plugin_opt_thinlto_index_only_eq); + config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files); + config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) || + args.hasArg(OPT_thinlto_index_only_eq); + config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq); config->thinLTOJobs = args::getInteger(args, OPT_thinlto_jobs, -1u); config->thinLTOObjectSuffixReplace = - getOldNewOptions(args, OPT_plugin_opt_thinlto_object_suffix_replace_eq); + getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq); config->thinLTOPrefixReplace = - getOldNewOptions(args, OPT_plugin_opt_thinlto_prefix_replace_eq); + getOldNewOptions(args, OPT_thinlto_prefix_replace_eq); config->trace = args.hasArg(OPT_trace); config->undefined = args::getStrings(args, OPT_undefined); config->undefinedVersion = @@ -935,6 +965,7 @@ static void readConfigs(opt::InputArgList &args) { config->zRelro = getZFlag(args, "relro", "norelro", true); config->zRetpolineplt = hasZOption(args, "retpolineplt"); config->zRodynamic = hasZOption(args, "rodynamic"); + config->zSeparate = getZSeparate(args); config->zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0); config->zText = getZFlag(args, "text", "notext", true); config->zWxneeded = hasZOption(args, "wxneeded"); @@ -966,7 +997,8 @@ static void readConfigs(opt::InputArgList &args) { StringRef s = arg->getValue(); std::tie(config->ekind, config->emachine, config->osabi) = parseEmulation(s); - config->mipsN32Abi = (s == "elf32btsmipn32" || s == "elf32ltsmipn32"); + config->mipsN32Abi = + (s.startswith("elf32btsmipn32") || s.startswith("elf32ltsmipn32")); config->emulation = s; } @@ -1009,30 +1041,33 @@ static void readConfigs(opt::InputArgList &args) { } } + assert(config->versionDefinitions.empty()); + config->versionDefinitions.push_back({"local", (uint16_t)VER_NDX_LOCAL, {}}); + config->versionDefinitions.push_back( + {"global", (uint16_t)VER_NDX_GLOBAL, {}}); + // If --retain-symbol-file is used, we'll keep only the symbols listed in // the file and discard all others. if (auto *arg = args.getLastArg(OPT_retain_symbols_file)) { - config->defaultSymbolVersion = VER_NDX_LOCAL; + config->versionDefinitions[VER_NDX_LOCAL].patterns.push_back( + {"*", /*isExternCpp=*/false, /*hasWildcard=*/true}); if (Optional<MemoryBufferRef> buffer = readFile(arg->getValue())) for (StringRef s : args::getLines(*buffer)) - config->versionScriptGlobals.push_back( - {s, /*IsExternCpp*/ false, /*HasWildcard*/ false}); + config->versionDefinitions[VER_NDX_GLOBAL].patterns.push_back( + {s, /*isExternCpp=*/false, /*hasWildcard=*/false}); } - bool hasExportDynamic = - args.hasFlag(OPT_export_dynamic, OPT_no_export_dynamic, false); - // Parses -dynamic-list and -export-dynamic-symbol. They make some // symbols private. Note that -export-dynamic takes precedence over them // as it says all symbols should be exported. - if (!hasExportDynamic) { + if (!config->exportDynamic) { for (auto *arg : args.filtered(OPT_dynamic_list)) if (Optional<MemoryBufferRef> buffer = readFile(arg->getValue())) readDynamicList(*buffer); for (auto *arg : args.filtered(OPT_export_dynamic_symbol)) config->dynamicList.push_back( - {arg->getValue(), /*IsExternCpp*/ false, /*HasWildcard*/ false}); + {arg->getValue(), /*isExternCpp=*/false, /*hasWildcard=*/false}); } // If --export-dynamic-symbol=foo is given and symbol foo is defined in @@ -1658,12 +1693,6 @@ template <class ELFT> static uint32_t getAndFeatures() { return ret; } -static const char *libcallRoutineNames[] = { -#define HANDLE_LIBCALL(code, name) name, -#include "llvm/IR/RuntimeLibcalls.def" -#undef HANDLE_LIBCALL -}; - // Do actual linking. Note that when this function is called, // all linker scripts have already been parsed. template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { @@ -1754,7 +1783,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { // libcall symbols will be added to the link after LTO when we add the LTO // object file to the link. if (!bitcodeFiles.empty()) - for (const char *s : libcallRoutineNames) + for (auto *s : lto::LTO::getRuntimeLibcallSymbols()) handleLibcall(s); // Return if there were name resolution errors. @@ -1879,20 +1908,54 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { "feature detected"); } - // This adds a .comment section containing a version string. We have to add it - // before mergeSections because the .comment section is a mergeable section. + // This adds a .comment section containing a version string. if (!config->relocatable) inputSections.push_back(createCommentSection()); // Replace common symbols with regular symbols. replaceCommonSymbols(); - // Do size optimizations: garbage collection, merging of SHF_MERGE sections - // and identical code folding. + // Split SHF_MERGE and .eh_frame sections into pieces in preparation for garbage collection. splitSections<ELFT>(); + + // Garbage collection and removal of shared symbols from unused shared objects. markLive<ELFT>(); demoteSharedSymbols(); - mergeSections(); + + // Make copies of any input sections that need to be copied into each + // partition. + copySectionsIntoPartitions(); + + // Create synthesized sections such as .got and .plt. This is called before + // processSectionCommands() so that they can be placed by SECTIONS commands. + createSyntheticSections<ELFT>(); + + // Some input sections that are used for exception handling need to be moved + // into synthetic sections. Do that now so that they aren't assigned to + // output sections in the usual way. + if (!config->relocatable) + combineEhSections(); + + // Create output sections described by SECTIONS commands. + script->processSectionCommands(); + + // Linker scripts control how input sections are assigned to output sections. + // Input sections that were not handled by scripts are called "orphans", and + // they are assigned to output sections by the default rule. Process that. + script->addOrphanSections(); + + // Migrate InputSectionDescription::sectionBases to sections. This includes + // merging MergeInputSections into a single MergeSyntheticSection. From this + // point onwards InputSectionDescription::sections should be used instead of + // sectionBases. + for (BaseCommand *base : script->sectionCommands) + if (auto *sec = dyn_cast<OutputSection>(base)) + sec->finalizeInputSections(); + llvm::erase_if(inputSections, + [](InputSectionBase *s) { return isa<MergeInputSection>(s); }); + + // Two input sections with different output sections should not be folded. + // ICF runs after processSectionCommands() so that we know the output sections. if (config->icf != ICFLevel::None) { findKeepUniqueSections<ELFT>(args); doIcf<ELFT>(); @@ -1909,3 +1972,6 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) { // Write the result to the file. writeResult<ELFT>(); } + +} // namespace elf +} // namespace lld diff --git a/ELF/DriverUtils.cpp b/ELF/DriverUtils.cpp index 87f0aa2a9827..43987cd5d4d4 100644 --- a/ELF/DriverUtils.cpp +++ b/ELF/DriverUtils.cpp @@ -30,8 +30,8 @@ using namespace llvm; using namespace llvm::sys; using namespace llvm::opt; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { // Create OptTable @@ -59,15 +59,15 @@ static void handleColorDiagnostics(opt::InputArgList &args) { if (!arg) return; if (arg->getOption().getID() == OPT_color_diagnostics) { - errorHandler().colorDiagnostics = true; + enableColors(true); } else if (arg->getOption().getID() == OPT_no_color_diagnostics) { - errorHandler().colorDiagnostics = false; + enableColors(false); } else { StringRef s = arg->getValue(); if (s == "always") - errorHandler().colorDiagnostics = true; + enableColors(true); else if (s == "never") - errorHandler().colorDiagnostics = false; + enableColors(false); else if (s != "auto") error("unknown option: --color-diagnostics=" + s); } @@ -143,7 +143,7 @@ opt::InputArgList ELFOptTable::parse(ArrayRef<const char *> argv) { return args; } -void elf::printHelp() { +void printHelp() { ELFOptTable().PrintHelp( outs(), (config->progName + " [options] file...").str().c_str(), "lld", false /*ShowHidden*/, true /*ShowAllAliases*/); @@ -165,7 +165,7 @@ static std::string rewritePath(StringRef s) { // Reconstructs command line arguments so that so that you can re-run // the same command with the same inputs. This is for --reproduce. -std::string elf::createResponseFile(const opt::InputArgList &args) { +std::string createResponseFile(const opt::InputArgList &args) { SmallString<0> data; raw_svector_ostream os(data); os << "--chroot .\n"; @@ -216,7 +216,7 @@ static Optional<std::string> findFile(StringRef path1, const Twine &path2) { return None; } -Optional<std::string> elf::findFromSearchPaths(StringRef path) { +Optional<std::string> findFromSearchPaths(StringRef path) { for (StringRef dir : config->searchPaths) if (Optional<std::string> s = findFile(dir, path)) return s; @@ -225,7 +225,7 @@ Optional<std::string> elf::findFromSearchPaths(StringRef path) { // This is for -l<basename>. We'll look for lib<basename>.so or lib<basename>.a from // search paths. -Optional<std::string> elf::searchLibraryBaseName(StringRef name) { +Optional<std::string> searchLibraryBaseName(StringRef name) { for (StringRef dir : config->searchPaths) { if (!config->isStatic) if (Optional<std::string> s = findFile(dir, "lib" + name + ".so")) @@ -237,17 +237,20 @@ Optional<std::string> elf::searchLibraryBaseName(StringRef name) { } // This is for -l<namespec>. -Optional<std::string> elf::searchLibrary(StringRef name) { - if (name.startswith(":")) - return findFromSearchPaths(name.substr(1)); - return searchLibraryBaseName (name); +Optional<std::string> searchLibrary(StringRef name) { + if (name.startswith(":")) + return findFromSearchPaths(name.substr(1)); + return searchLibraryBaseName(name); } // If a linker/version script doesn't exist in the current directory, we also // look for the script in the '-L' search paths. This matches the behaviour of // '-T', --version-script=, and linker script INPUT() command in ld.bfd. -Optional<std::string> elf::searchScript(StringRef name) { +Optional<std::string> searchScript(StringRef name) { if (fs::exists(name)) return name.str(); return findFromSearchPaths(name); } + +} // namespace elf +} // namespace lld diff --git a/ELF/EhFrame.cpp b/ELF/EhFrame.cpp index b3245dd01669..a9c66f29446c 100644 --- a/ELF/EhFrame.cpp +++ b/ELF/EhFrame.cpp @@ -30,9 +30,8 @@ using namespace llvm::ELF; using namespace llvm::dwarf; using namespace llvm::object; -using namespace lld; -using namespace lld::elf; - +namespace lld { +namespace elf { namespace { class EhReader { public: @@ -57,7 +56,7 @@ private: }; } -size_t elf::readEhRecordSize(InputSectionBase *s, size_t off) { +size_t readEhRecordSize(InputSectionBase *s, size_t off) { return EhReader(s, s->data().slice(off)).readEhRecordSize(); } @@ -149,7 +148,7 @@ void EhReader::skipAugP() { d = d.slice(size); } -uint8_t elf::getFdeEncoding(EhSectionPiece *p) { +uint8_t getFdeEncoding(EhSectionPiece *p) { return EhReader(p->sec, p->data()).getFdeEncoding(); } @@ -195,3 +194,6 @@ uint8_t EhReader::getFdeEncoding() { } return DW_EH_PE_absptr; } + +} // namespace elf +} // namespace lld diff --git a/ELF/ICF.cpp b/ELF/ICF.cpp index 8b01d06b0248..dce76f79c9b3 100644 --- a/ELF/ICF.cpp +++ b/ELF/ICF.cpp @@ -74,6 +74,8 @@ #include "ICF.h" #include "Config.h" +#include "LinkerScript.h" +#include "OutputSections.h" #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" @@ -86,12 +88,12 @@ #include <algorithm> #include <atomic> -using namespace lld; -using namespace lld::elf; using namespace llvm; using namespace llvm::ELF; using namespace llvm::object; +namespace lld { +namespace elf { namespace { template <class ELFT> class ICF { public: @@ -304,10 +306,8 @@ bool ICF<ELFT>::equalsConstant(const InputSection *a, const InputSection *b) { return false; // If two sections have different output sections, we cannot merge them. - // FIXME: This doesn't do the right thing in the case where there is a linker - // script. We probably need to move output section assignment before ICF to - // get the correct behaviour here. - if (getOutputSectionName(a) != getOutputSectionName(b)) + assert(a->getParent() && b->getParent()); + if (a->getParent() != b->getParent()) return false; if (a->areRelocsRela) @@ -446,10 +446,11 @@ static void print(const Twine &s) { // The main function of ICF. template <class ELFT> void ICF<ELFT>::run() { // Collect sections to merge. - for (InputSectionBase *sec : inputSections) - if (auto *s = dyn_cast<InputSection>(sec)) - if (isEligible(s)) - sections.push_back(s); + for (InputSectionBase *sec : inputSections) { + auto *s = cast<InputSection>(sec); + if (isEligible(s)) + sections.push_back(s); + } // Initially, we use hash values to partition sections. parallelForEach(sections, [&](InputSection *s) { @@ -499,12 +500,24 @@ template <class ELFT> void ICF<ELFT>::run() { isec->markDead(); } }); + + // InputSectionDescription::sections is populated by processSectionCommands(). + // ICF may fold some input sections assigned to output sections. Remove them. + for (BaseCommand *base : script->sectionCommands) + if (auto *sec = dyn_cast<OutputSection>(base)) + for (BaseCommand *sub_base : sec->sectionCommands) + if (auto *isd = dyn_cast<InputSectionDescription>(sub_base)) + llvm::erase_if(isd->sections, + [](InputSection *isec) { return !isec->isLive(); }); } // ICF entry point function. -template <class ELFT> void elf::doIcf() { ICF<ELFT>().run(); } +template <class ELFT> void doIcf() { ICF<ELFT>().run(); } + +template void doIcf<ELF32LE>(); +template void doIcf<ELF32BE>(); +template void doIcf<ELF64LE>(); +template void doIcf<ELF64BE>(); -template void elf::doIcf<ELF32LE>(); -template void elf::doIcf<ELF32BE>(); -template void elf::doIcf<ELF64LE>(); -template void elf::doIcf<ELF64BE>(); +} // namespace elf +} // namespace lld diff --git a/ELF/InputFiles.cpp b/ELF/InputFiles.cpp index 98b88283cf09..fdf935a30856 100644 --- a/ELF/InputFiles.cpp +++ b/ELF/InputFiles.cpp @@ -17,7 +17,6 @@ #include "lld/Common/Memory.h" #include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/Analysis.h" -#include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/LTO/LTO.h" @@ -37,18 +36,31 @@ using namespace llvm::sys; using namespace llvm::sys::fs; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; +namespace lld { +// Returns "<internal>", "foo.a(bar.o)" or "baz.o". +std::string toString(const elf::InputFile *f) { + if (!f) + return "<internal>"; + + if (f->toStringCache.empty()) { + if (f->archiveName.empty()) + f->toStringCache = f->getName(); + else + f->toStringCache = (f->archiveName + "(" + f->getName() + ")").str(); + } + return f->toStringCache; +} +namespace elf { bool InputFile::isInGroup; uint32_t InputFile::nextGroupId; -std::vector<BinaryFile *> elf::binaryFiles; -std::vector<BitcodeFile *> elf::bitcodeFiles; -std::vector<LazyObjFile *> elf::lazyObjFiles; -std::vector<InputFile *> elf::objectFiles; -std::vector<SharedFile *> elf::sharedFiles; +std::vector<BinaryFile *> binaryFiles; +std::vector<BitcodeFile *> bitcodeFiles; +std::vector<LazyObjFile *> lazyObjFiles; +std::vector<InputFile *> objectFiles; +std::vector<SharedFile *> sharedFiles; -std::unique_ptr<TarWriter> elf::tar; +std::unique_ptr<TarWriter> tar; static ELFKind getELFKind(MemoryBufferRef mb, StringRef archiveName) { unsigned char size; @@ -88,7 +100,7 @@ InputFile::InputFile(Kind k, MemoryBufferRef m) ++nextGroupId; } -Optional<MemoryBufferRef> elf::readFile(StringRef path) { +Optional<MemoryBufferRef> readFile(StringRef path) { // The --chroot option changes our virtual root directory. // This is useful when you are dealing with files created by --reproduce. if (!config->chroot.empty() && path.startswith("/")) @@ -127,18 +139,18 @@ static bool isCompatible(InputFile *file) { if (!config->emulation.empty()) { error(toString(file) + " is incompatible with " + config->emulation); - } else { - InputFile *existing; - if (!objectFiles.empty()) - existing = objectFiles[0]; - else if (!sharedFiles.empty()) - existing = sharedFiles[0]; - else - existing = bitcodeFiles[0]; - - error(toString(file) + " is incompatible with " + toString(existing)); + return false; } + InputFile *existing; + if (!objectFiles.empty()) + existing = objectFiles[0]; + else if (!sharedFiles.empty()) + existing = sharedFiles[0]; + else + existing = bitcodeFiles[0]; + + error(toString(file) + " is incompatible with " + toString(existing)); return false; } @@ -188,7 +200,7 @@ template <class ELFT> static void doParseFile(InputFile *file) { } // Add symbols in File to the symbol table. -void elf::parseFile(InputFile *file) { +void parseFile(InputFile *file) { switch (config->ekind) { case ELF32LEKind: doParseFile<ELF32LE>(file); @@ -252,57 +264,8 @@ std::string InputFile::getSrcMsg(const Symbol &sym, InputSectionBase &sec, } template <class ELFT> void ObjFile<ELFT>::initializeDwarf() { - dwarf = llvm::make_unique<DWARFContext>(make_unique<LLDDwarfObj<ELFT>>(this)); - for (std::unique_ptr<DWARFUnit> &cu : dwarf->compile_units()) { - auto report = [](Error err) { - handleAllErrors(std::move(err), - [](ErrorInfoBase &info) { warn(info.message()); }); - }; - Expected<const DWARFDebugLine::LineTable *> expectedLT = - dwarf->getLineTableForUnit(cu.get(), report); - const DWARFDebugLine::LineTable *lt = nullptr; - if (expectedLT) - lt = *expectedLT; - else - report(expectedLT.takeError()); - if (!lt) - continue; - lineTables.push_back(lt); - - // Loop over variable records and insert them to variableLoc. - for (const auto &entry : cu->dies()) { - DWARFDie die(cu.get(), &entry); - // Skip all tags that are not variables. - if (die.getTag() != dwarf::DW_TAG_variable) - continue; - - // Skip if a local variable because we don't need them for generating - // error messages. In general, only non-local symbols can fail to be - // linked. - if (!dwarf::toUnsigned(die.find(dwarf::DW_AT_external), 0)) - continue; - - // Get the source filename index for the variable. - unsigned file = dwarf::toUnsigned(die.find(dwarf::DW_AT_decl_file), 0); - if (!lt->hasFileAtIndex(file)) - continue; - - // Get the line number on which the variable is declared. - unsigned line = dwarf::toUnsigned(die.find(dwarf::DW_AT_decl_line), 0); - - // Here we want to take the variable name to add it into variableLoc. - // Variable can have regular and linkage name associated. At first, we try - // to get linkage name as it can be different, for example when we have - // two variables in different namespaces of the same object. Use common - // name otherwise, but handle the case when it also absent in case if the - // input object file lacks some debug info. - StringRef name = - dwarf::toString(die.find(dwarf::DW_AT_linkage_name), - dwarf::toString(die.find(dwarf::DW_AT_name), "")); - if (!name.empty()) - variableLoc.insert({name, {lt, file, line}}); - } - } + dwarf = make<DWARFCache>(std::make_unique<DWARFContext>( + std::make_unique<LLDDwarfObj<ELFT>>(this))); } // Returns the pair of file name and line number describing location of data @@ -312,19 +275,7 @@ Optional<std::pair<std::string, unsigned>> ObjFile<ELFT>::getVariableLoc(StringRef name) { llvm::call_once(initDwarfLine, [this]() { initializeDwarf(); }); - // Return if we have no debug information about data object. - auto it = variableLoc.find(name); - if (it == variableLoc.end()) - return None; - - // Take file name string from line table. - std::string fileName; - if (!it->second.lt->getFileNameByIndex( - it->second.file, {}, - DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath, fileName)) - return None; - - return std::make_pair(fileName, it->second.line); + return dwarf->getVariableLoc(name); } // Returns source line information for a given offset @@ -346,28 +297,7 @@ Optional<DILineInfo> ObjFile<ELFT>::getDILineInfo(InputSectionBase *s, // Use fake address calcuated by adding section file offset and offset in // section. See comments for ObjectInfo class. - DILineInfo info; - for (const llvm::DWARFDebugLine::LineTable *lt : lineTables) { - if (lt->getFileLineInfoForAddress( - {s->getOffsetInFile() + offset, sectionIndex}, nullptr, - DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath, info)) - return info; - } - return None; -} - -// Returns "<internal>", "foo.a(bar.o)" or "baz.o". -std::string lld::toString(const InputFile *f) { - if (!f) - return "<internal>"; - - if (f->toStringCache.empty()) { - if (f->archiveName.empty()) - f->toStringCache = f->getName(); - else - f->toStringCache = (f->archiveName + "(" + f->getName() + ")").str(); - } - return f->toStringCache; + return dwarf->getDILineInfo(s->getOffsetInFile() + offset, sectionIndex); } ELFFileBase::ELFFileBase(Kind k, MemoryBufferRef mb) : InputFile(k, mb) { @@ -484,7 +414,8 @@ StringRef ObjFile<ELFT>::getShtGroupSignature(ArrayRef<Elf_Shdr> sections, return signature; } -template <class ELFT> bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec) { +template <class ELFT> +bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec, StringRef name) { // On a regular link we don't merge sections if -O0 (default is -O1). This // sometimes makes the linker significantly faster, although the output will // be bigger. @@ -516,14 +447,16 @@ template <class ELFT> bool ObjFile<ELFT>::shouldMerge(const Elf_Shdr &sec) { if (entSize == 0) return false; if (sec.sh_size % entSize) - fatal(toString(this) + - ": SHF_MERGE section size must be a multiple of sh_entsize"); + fatal(toString(this) + ":(" + name + "): SHF_MERGE section size (" + + Twine(sec.sh_size) + ") must be a multiple of sh_entsize (" + + Twine(entSize) + ")"); uint64_t flags = sec.sh_flags; if (!(flags & SHF_MERGE)) return false; if (flags & SHF_WRITE) - fatal(toString(this) + ": writable SHF_MERGE section is not supported"); + fatal(toString(this) + ":(" + name + + "): writable SHF_MERGE section is not supported"); return true; } @@ -573,7 +506,7 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) { this->sectionStringTable = CHECK(obj.getSectionStringTable(objSections), this); - for (size_t i = 0, e = objSections.size(); i < e; i++) { + for (size_t i = 0, e = objSections.size(); i < e; ++i) { if (this->sections[i] == &InputSection::discarded) continue; const Elf_Shdr &sec = objSections[i]; @@ -652,25 +585,29 @@ void ObjFile<ELFT>::initializeSections(bool ignoreComdats) { default: this->sections[i] = createInputSection(sec); } + } + + for (size_t i = 0, e = objSections.size(); i < e; ++i) { + if (this->sections[i] == &InputSection::discarded) + continue; + const Elf_Shdr &sec = objSections[i]; + if (!(sec.sh_flags & SHF_LINK_ORDER)) + continue; // .ARM.exidx sections have a reverse dependency on the InputSection they // have a SHF_LINK_ORDER dependency, this is identified by the sh_link. - if (sec.sh_flags & SHF_LINK_ORDER) { - InputSectionBase *linkSec = nullptr; - if (sec.sh_link < this->sections.size()) - linkSec = this->sections[sec.sh_link]; - if (!linkSec) - fatal(toString(this) + - ": invalid sh_link index: " + Twine(sec.sh_link)); - - InputSection *isec = cast<InputSection>(this->sections[i]); - linkSec->dependentSections.push_back(isec); - if (!isa<InputSection>(linkSec)) - error("a section " + isec->name + - " with SHF_LINK_ORDER should not refer a non-regular " - "section: " + - toString(linkSec)); - } + InputSectionBase *linkSec = nullptr; + if (sec.sh_link < this->sections.size()) + linkSec = this->sections[sec.sh_link]; + if (!linkSec) + fatal(toString(this) + ": invalid sh_link index: " + Twine(sec.sh_link)); + + InputSection *isec = cast<InputSection>(this->sections[i]); + linkSec->dependentSections.push_back(isec); + if (!isa<InputSection>(linkSec)) + error("a section " + isec->name + + " with SHF_LINK_ORDER should not refer a non-regular section: " + + toString(linkSec)); } } @@ -1030,7 +967,7 @@ InputSectionBase *ObjFile<ELFT>::createInputSection(const Elf_Shdr &sec) { if (name == ".eh_frame" && !config->relocatable) return make<EhInputSection>(*this, sec, name); - if (shouldMerge(sec)) + if (shouldMerge(sec, name)) return make<MergeInputSection>(*this, sec, name); return make<InputSection>(*this, sec, name); } @@ -1094,6 +1031,7 @@ template <class ELFT> void ObjFile<ELFT>::initializeSymbols() { // Handle global undefined symbols. if (eSym.st_shndx == SHN_UNDEF) { this->symbols[i]->resolve(Undefined{this, name, binding, stOther, type}); + this->symbols[i]->referenced = true; continue; } @@ -1144,7 +1082,7 @@ void ArchiveFile::fetch(const Archive::Symbol &sym) { Archive::Child c = CHECK(sym.getMember(), toString(this) + ": could not get the member for symbol " + - sym.getName()); + toELFString(sym)); if (!seen.insert(c.getChildOffset()).second) return; @@ -1153,7 +1091,7 @@ void ArchiveFile::fetch(const Archive::Symbol &sym) { CHECK(c.getMemoryBufferRef(), toString(this) + ": could not get the buffer for the member defining symbol " + - sym.getName()); + toELFString(sym)); if (tar && c.getParent()->isThin()) tar->append(relativeToRoot(CHECK(c.getFullName(), this)), mb.getBuffer()); @@ -1470,10 +1408,12 @@ static Symbol *createBitcodeSymbol(const std::vector<bool> &keptComdats, int c = objSym.getComdatIndex(); if (objSym.isUndefined() || (c != -1 && !keptComdats[c])) { - Undefined New(&f, name, binding, visibility, type); + Undefined newSym(&f, name, binding, visibility, type); if (canOmitFromDynSym) - New.exportDynamic = false; - return symtab->addSymbol(New); + newSym.exportDynamic = false; + Symbol *ret = symtab->addSymbol(newSym); + ret->referenced = true; + return ret; } if (objSym.isCommon()) @@ -1481,10 +1421,10 @@ static Symbol *createBitcodeSymbol(const std::vector<bool> &keptComdats, CommonSymbol{&f, name, binding, visibility, STT_OBJECT, objSym.getCommonAlignment(), objSym.getCommonSize()}); - Defined New(&f, name, binding, visibility, type, 0, 0, nullptr); + Defined newSym(&f, name, binding, visibility, type, 0, 0, nullptr); if (canOmitFromDynSym) - New.exportDynamic = false; - return symtab->addSymbol(New); + newSym.exportDynamic = false; + return symtab->addSymbol(newSym); } template <class ELFT> void BitcodeFile::parse() { @@ -1523,8 +1463,8 @@ void BinaryFile::parse() { STV_DEFAULT, STT_OBJECT, data.size(), 0, nullptr}); } -InputFile *elf::createObjectFile(MemoryBufferRef mb, StringRef archiveName, - uint64_t offsetInArchive) { +InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName, + uint64_t offsetInArchive) { if (isBitcode(mb)) return make<BitcodeFile>(mb, archiveName, offsetInArchive); @@ -1615,7 +1555,7 @@ template <class ELFT> void LazyObjFile::parse() { } } -std::string elf::replaceThinLTOSuffix(StringRef path) { +std::string replaceThinLTOSuffix(StringRef path) { StringRef suffix = config->thinLTOObjectSuffixReplace.first; StringRef repl = config->thinLTOObjectSuffixReplace.second; @@ -1634,12 +1574,15 @@ template void LazyObjFile::parse<ELF32BE>(); template void LazyObjFile::parse<ELF64LE>(); template void LazyObjFile::parse<ELF64BE>(); -template class elf::ObjFile<ELF32LE>; -template class elf::ObjFile<ELF32BE>; -template class elf::ObjFile<ELF64LE>; -template class elf::ObjFile<ELF64BE>; +template class ObjFile<ELF32LE>; +template class ObjFile<ELF32BE>; +template class ObjFile<ELF64LE>; +template class ObjFile<ELF64BE>; template void SharedFile::parse<ELF32LE>(); template void SharedFile::parse<ELF32BE>(); template void SharedFile::parse<ELF64LE>(); template void SharedFile::parse<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/InputFiles.h b/ELF/InputFiles.h index 5ccc3d402b37..cde6bc617764 100644 --- a/ELF/InputFiles.h +++ b/ELF/InputFiles.h @@ -10,13 +10,13 @@ #define LLD_ELF_INPUT_FILES_H #include "Config.h" +#include "lld/Common/DWARF.h" #include "lld/Common/ErrorHandler.h" #include "lld/Common/LLVM.h" #include "lld/Common/Reproduce.h" #include "llvm/ADT/CachedHashString.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" #include "llvm/IR/Comdat.h" #include "llvm/Object/Archive.h" #include "llvm/Object/ELF.h" @@ -26,22 +26,19 @@ namespace llvm { class TarWriter; -struct DILineInfo; namespace lto { class InputFile; } } // namespace llvm namespace lld { -namespace elf { -class InputFile; -class InputSectionBase; -} // Returns "<internal>", "foo.a(bar.o)" or "baz.o". std::string toString(const elf::InputFile *f); namespace elf { +class InputFile; +class InputSectionBase; using llvm::object::Archive; @@ -261,7 +258,7 @@ private: InputSectionBase *createInputSection(const Elf_Shdr &sec); StringRef getSectionName(const Elf_Shdr &sec); - bool shouldMerge(const Elf_Shdr &sec); + bool shouldMerge(const Elf_Shdr &sec, StringRef name); // Each ELF symbol contains a section index which the symbol belongs to. // However, because the number of bits dedicated for that is limited, a @@ -284,14 +281,7 @@ private: // reporting. Linker may find reasonable number of errors in a // single object file, so we cache debugging information in order to // parse it only once for each object file we link. - std::unique_ptr<llvm::DWARFContext> dwarf; - std::vector<const llvm::DWARFDebugLine::LineTable *> lineTables; - struct VarLoc { - const llvm::DWARFDebugLine::LineTable *lt; - unsigned file; - unsigned line; - }; - llvm::DenseMap<StringRef, VarLoc> variableLoc; + DWARFCache *dwarf; llvm::once_flag initDwarfLine; }; diff --git a/ELF/InputSection.cpp b/ELF/InputSection.cpp index a024ac307b0a..0c93d2e10959 100644 --- a/ELF/InputSection.cpp +++ b/ELF/InputSection.cpp @@ -37,16 +37,15 @@ using namespace llvm::support; using namespace llvm::support::endian; using namespace llvm::sys; -using namespace lld; -using namespace lld::elf; - -std::vector<InputSectionBase *> elf::inputSections; - +namespace lld { // Returns a string to construct an error message. -std::string lld::toString(const InputSectionBase *sec) { +std::string toString(const elf::InputSectionBase *sec) { return (toString(sec->file) + ":(" + sec->name + ")").str(); } +namespace elf { +std::vector<InputSectionBase *> inputSections; + template <class ELFT> static ArrayRef<uint8_t> getSectionContents(ObjFile<ELFT> &file, const typename ELFT::Shdr &hdr) { @@ -608,26 +607,39 @@ static int64_t getTlsTpOffset(const Symbol &s) { if (&s == ElfSym::tlsModuleBase) return 0; + // There are 2 TLS layouts. Among targets we support, x86 uses TLS Variant 2 + // while most others use Variant 1. At run time TP will be aligned to p_align. + + // Variant 1. TP will be followed by an optional gap (which is the size of 2 + // pointers on ARM/AArch64, 0 on other targets), followed by alignment + // padding, then the static TLS blocks. The alignment padding is added so that + // (TP + gap + padding) is congruent to p_vaddr modulo p_align. + // + // Variant 2. Static TLS blocks, followed by alignment padding are placed + // before TP. The alignment padding is added so that (TP - padding - + // p_memsz) is congruent to p_vaddr modulo p_align. + PhdrEntry *tls = Out::tlsPhdr; switch (config->emachine) { + // Variant 1. case EM_ARM: case EM_AARCH64: - // Variant 1. The thread pointer points to a TCB with a fixed 2-word size, - // followed by a variable amount of alignment padding, followed by the TLS - // segment. - return s.getVA(0) + alignTo(config->wordsize * 2, Out::tlsPhdr->p_align); - case EM_386: - case EM_X86_64: - // Variant 2. The TLS segment is located just before the thread pointer. - return s.getVA(0) - alignTo(Out::tlsPhdr->p_memsz, Out::tlsPhdr->p_align); + return s.getVA(0) + config->wordsize * 2 + + ((tls->p_vaddr - config->wordsize * 2) & (tls->p_align - 1)); + case EM_MIPS: case EM_PPC: case EM_PPC64: - // The thread pointer points to a fixed offset from the start of the - // executable's TLS segment. An offset of 0x7000 allows a signed 16-bit - // offset to reach 0x1000 of TCB/thread-library data and 0xf000 of the - // program's TLS segment. - return s.getVA(0) - 0x7000; + // Adjusted Variant 1. TP is placed with a displacement of 0x7000, which is + // to allow a signed 16-bit offset to reach 0x1000 of TCB/thread-library + // data and 0xf000 of the program's TLS segment. + return s.getVA(0) + (tls->p_vaddr & (tls->p_align - 1)) - 0x7000; case EM_RISCV: - return s.getVA(0); + return s.getVA(0) + (tls->p_vaddr & (tls->p_align - 1)); + + // Variant 2. + case EM_386: + case EM_X86_64: + return s.getVA(0) - tls->p_memsz - + ((-tls->p_vaddr - tls->p_memsz) & (tls->p_align - 1)); default: llvm_unreachable("unhandled Config->EMachine"); } @@ -671,8 +683,6 @@ static uint64_t getRelocTargetVA(const InputFile *file, RelType type, int64_t a, case R_GOT_PC: case R_RELAX_TLS_GD_TO_IE: return sym.getGotVA() + a - p; - case R_HEXAGON_GOT: - return sym.getGotVA() - in.gotPlt->getVA(); case R_MIPS_GOTREL: return sym.getVA(a) - in.mipsGot->getGp(file); case R_MIPS_GOT_GP: @@ -1071,7 +1081,7 @@ void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf, end, f->stOther)) continue; if (!getFile<ELFT>()->someNoSplitStack) - error(lld::toString(this) + ": " + f->getName() + + error(toString(this) + ": " + f->getName() + " (with -fsplit-stack) calls " + rel.sym->getName() + " (without -fsplit-stack), but couldn't adjust its prologue"); } @@ -1334,3 +1344,6 @@ template void EhInputSection::split<ELF32LE>(); template void EhInputSection::split<ELF32BE>(); template void EhInputSection::split<ELF64LE>(); template void EhInputSection::split<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/InputSection.h b/ELF/InputSection.h index 3a974074e0e5..d7c953262a41 100644 --- a/ELF/InputSection.h +++ b/ELF/InputSection.h @@ -54,22 +54,9 @@ public: unsigned sectionKind : 3; - // The next three bit fields are only used by InputSectionBase, but we + // The next two bit fields are only used by InputSectionBase, but we // put them here so the struct packs better. - // True if this section has already been placed to a linker script - // output section. This is needed because, in a linker script, you - // can refer to the same section more than once. For example, in - // the following linker script, - // - // .foo : { *(.text) } - // .bar : { *(.text) } - // - // .foo takes all .text sections, and .bar becomes empty. To achieve - // this, we need to memorize whether a section has been placed or - // not for each input section. - unsigned assigned : 1; - unsigned bss : 1; // Set for sections that should not be folded by ICF. @@ -108,9 +95,9 @@ protected: SectionBase(Kind sectionKind, StringRef name, uint64_t flags, uint64_t entsize, uint64_t alignment, uint32_t type, uint32_t info, uint32_t link) - : name(name), repl(this), sectionKind(sectionKind), assigned(false), - bss(false), keepUnique(false), partition(0), alignment(alignment), - flags(flags), entsize(entsize), type(type), link(link), info(info) {} + : name(name), repl(this), sectionKind(sectionKind), bss(false), + keepUnique(false), partition(0), alignment(alignment), flags(flags), + entsize(entsize), type(type), link(link), info(info) {} }; // This corresponds to a section of an input file. diff --git a/ELF/LTO.cpp b/ELF/LTO.cpp index 28d4bfe77c5d..6da409568c8b 100644 --- a/ELF/LTO.cpp +++ b/ELF/LTO.cpp @@ -42,15 +42,15 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { // Creates an empty file to store a list of object files for final // linking of distributed ThinLTO. static std::unique_ptr<raw_fd_ostream> openFile(StringRef file) { std::error_code ec; auto ret = - llvm::make_unique<raw_fd_ostream>(file, ec, sys::fs::OpenFlags::F_None); + std::make_unique<raw_fd_ostream>(file, ec, sys::fs::OpenFlags::OF_None); if (ec) { error("cannot open " + file + ": " + ec.message()); return nullptr; @@ -76,7 +76,9 @@ static lto::Config createConfig() { c.Options.FunctionSections = true; c.Options.DataSections = true; - if (config->relocatable) + if (auto relocModel = getRelocModelFromCMModel()) + c.RelocModel = *relocModel; + else if (config->relocatable) c.RelocModel = None; else if (config->isPic) c.RelocModel = Reloc::PIC_; @@ -139,7 +141,7 @@ BitcodeCompiler::BitcodeCompiler() { backend = lto::createInProcessThinBackend(config->thinLTOJobs); } - ltoObj = llvm::make_unique<lto::LTO>(createConfig(), backend, + ltoObj = std::make_unique<lto::LTO>(createConfig(), backend, config->ltoPartitions); // Initialize usedStartStop. @@ -249,8 +251,8 @@ std::vector<InputFile *> BitcodeCompiler::compile() { if (!bitcodeFiles.empty()) checkError(ltoObj->run( [&](size_t task) { - return llvm::make_unique<lto::NativeObjectStream>( - llvm::make_unique<raw_svector_ostream>(buf[task])); + return std::make_unique<lto::NativeObjectStream>( + std::make_unique<raw_svector_ostream>(buf[task])); }, cache)); @@ -301,3 +303,6 @@ std::vector<InputFile *> BitcodeCompiler::compile() { ret.push_back(createObjectFile(*file)); return ret; } + +} // namespace elf +} // namespace lld diff --git a/ELF/LinkerScript.cpp b/ELF/LinkerScript.cpp index 49e44d780476..cebbd89168be 100644 --- a/ELF/LinkerScript.cpp +++ b/ELF/LinkerScript.cpp @@ -43,29 +43,27 @@ using namespace llvm; using namespace llvm::ELF; using namespace llvm::object; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; -LinkerScript *elf::script; +namespace lld { +namespace elf { +LinkerScript *script; -static uint64_t getOutputSectionVA(SectionBase *inputSec, StringRef loc) { - if (OutputSection *os = inputSec->getOutputSection()) - return os->addr; - error(loc + ": unable to evaluate expression: input section " + - inputSec->name + " has no output section assigned"); - return 0; +static uint64_t getOutputSectionVA(SectionBase *sec) { + OutputSection *os = sec->getOutputSection(); + assert(os && "input section has no output section assigned"); + return os ? os->addr : 0; } uint64_t ExprValue::getValue() const { if (sec) - return alignTo(sec->getOffset(val) + getOutputSectionVA(sec, loc), + return alignTo(sec->getOffset(val) + getOutputSectionVA(sec), alignment); return alignTo(val, alignment); } uint64_t ExprValue::getSecAddr() const { if (sec) - return sec->getOffset(0) + getOutputSectionVA(sec, loc); + return sec->getOffset(0) + getOutputSectionVA(sec); return 0; } @@ -73,7 +71,7 @@ uint64_t ExprValue::getSectionOffset() const { // If the alignment is trivial, we don't have to compute the full // value to know the offset. This allows this function to succeed in // cases where the output section is not yet known. - if (alignment == 1 && (!sec || !sec->getOutputSection())) + if (alignment == 1 && !sec) return val; return getValue() - getSecAddr(); } @@ -157,8 +155,8 @@ static bool shouldDefineSym(SymbolAssignment *cmd) { return false; } -// This function is called from processSectionCommands, -// while we are fixing the output section layout. +// Called by processSymbolAssignments() to assign definitions to +// linker-script-defined symbols. void LinkerScript::addSymbol(SymbolAssignment *cmd) { if (!shouldDefineSym(cmd)) return; @@ -181,12 +179,12 @@ void LinkerScript::addSymbol(SymbolAssignment *cmd) { // write expressions like this: `alignment = 16; . = ALIGN(., alignment)`. uint64_t symValue = value.sec ? 0 : value.getValue(); - Defined New(nullptr, cmd->name, STB_GLOBAL, visibility, STT_NOTYPE, symValue, - 0, sec); + Defined newSym(nullptr, cmd->name, STB_GLOBAL, visibility, STT_NOTYPE, + symValue, 0, sec); Symbol *sym = symtab->insert(cmd->name); - sym->mergeProperties(New); - sym->replace(New); + sym->mergeProperties(newSym); + sym->replace(newSym); cmd->sym = cast<Defined>(sym); } @@ -197,19 +195,57 @@ static void declareSymbol(SymbolAssignment *cmd) { return; uint8_t visibility = cmd->hidden ? STV_HIDDEN : STV_DEFAULT; - Defined New(nullptr, cmd->name, STB_GLOBAL, visibility, STT_NOTYPE, 0, 0, - nullptr); + Defined newSym(nullptr, cmd->name, STB_GLOBAL, visibility, STT_NOTYPE, 0, 0, + nullptr); // We can't calculate final value right now. Symbol *sym = symtab->insert(cmd->name); - sym->mergeProperties(New); - sym->replace(New); + sym->mergeProperties(newSym); + sym->replace(newSym); cmd->sym = cast<Defined>(sym); cmd->provide = false; sym->scriptDefined = true; } +using SymbolAssignmentMap = + DenseMap<const Defined *, std::pair<SectionBase *, uint64_t>>; + +// Collect section/value pairs of linker-script-defined symbols. This is used to +// check whether symbol values converge. +static SymbolAssignmentMap +getSymbolAssignmentValues(const std::vector<BaseCommand *> §ionCommands) { + SymbolAssignmentMap ret; + for (BaseCommand *base : sectionCommands) { + if (auto *cmd = dyn_cast<SymbolAssignment>(base)) { + if (cmd->sym) // sym is nullptr for dot. + ret.try_emplace(cmd->sym, + std::make_pair(cmd->sym->section, cmd->sym->value)); + continue; + } + for (BaseCommand *sub_base : cast<OutputSection>(base)->sectionCommands) + if (auto *cmd = dyn_cast<SymbolAssignment>(sub_base)) + if (cmd->sym) + ret.try_emplace(cmd->sym, + std::make_pair(cmd->sym->section, cmd->sym->value)); + } + return ret; +} + +// Returns the lexicographical smallest (for determinism) Defined whose +// section/value has changed. +static const Defined * +getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) { + const Defined *changed = nullptr; + for (auto &it : oldValues) { + const Defined *sym = it.first; + if (std::make_pair(sym->section, sym->value) != it.second && + (!changed || sym->getName() < changed->getName())) + changed = sym; + } + return changed; +} + // This method is used to handle INSERT AFTER statement. Here we rebuild // the list of script commands to mix sections inserted into. void LinkerScript::processInsertCommands() { @@ -305,46 +341,44 @@ bool LinkerScript::shouldKeep(InputSectionBase *s) { } // A helper function for the SORT() command. -static std::function<bool(InputSectionBase *, InputSectionBase *)> -getComparator(SortSectionPolicy k) { - switch (k) { - case SortSectionPolicy::Alignment: - return [](InputSectionBase *a, InputSectionBase *b) { - // ">" is not a mistake. Sections with larger alignments are placed - // before sections with smaller alignments in order to reduce the - // amount of padding necessary. This is compatible with GNU. - return a->alignment > b->alignment; - }; - case SortSectionPolicy::Name: - return [](InputSectionBase *a, InputSectionBase *b) { - return a->name < b->name; - }; - case SortSectionPolicy::Priority: - return [](InputSectionBase *a, InputSectionBase *b) { - return getPriority(a->name) < getPriority(b->name); - }; - default: - llvm_unreachable("unknown sort policy"); - } -} - -// A helper function for the SORT() command. -static bool matchConstraints(ArrayRef<InputSection *> sections, +static bool matchConstraints(ArrayRef<InputSectionBase *> sections, ConstraintKind kind) { if (kind == ConstraintKind::NoConstraint) return true; bool isRW = llvm::any_of( - sections, [](InputSection *sec) { return sec->flags & SHF_WRITE; }); + sections, [](InputSectionBase *sec) { return sec->flags & SHF_WRITE; }); return (isRW && kind == ConstraintKind::ReadWrite) || (!isRW && kind == ConstraintKind::ReadOnly); } -static void sortSections(MutableArrayRef<InputSection *> vec, +static void sortSections(MutableArrayRef<InputSectionBase *> vec, SortSectionPolicy k) { - if (k != SortSectionPolicy::Default && k != SortSectionPolicy::None) - llvm::stable_sort(vec, getComparator(k)); + auto alignmentComparator = [](InputSectionBase *a, InputSectionBase *b) { + // ">" is not a mistake. Sections with larger alignments are placed + // before sections with smaller alignments in order to reduce the + // amount of padding necessary. This is compatible with GNU. + return a->alignment > b->alignment; + }; + auto nameComparator = [](InputSectionBase *a, InputSectionBase *b) { + return a->name < b->name; + }; + auto priorityComparator = [](InputSectionBase *a, InputSectionBase *b) { + return getPriority(a->name) < getPriority(b->name); + }; + + switch (k) { + case SortSectionPolicy::Default: + case SortSectionPolicy::None: + return; + case SortSectionPolicy::Alignment: + return llvm::stable_sort(vec, alignmentComparator); + case SortSectionPolicy::Name: + return llvm::stable_sort(vec, nameComparator); + case SortSectionPolicy::Priority: + return llvm::stable_sort(vec, priorityComparator); + } } // Sort sections as instructed by SORT-family commands and --sort-section @@ -358,7 +392,7 @@ static void sortSections(MutableArrayRef<InputSection *> vec, // --sort-section is handled as an inner SORT command. // 3. If one SORT command is given, and if it is SORT_NONE, don't sort. // 4. If no SORT command is given, sort according to --sort-section. -static void sortInputSections(MutableArrayRef<InputSection *> vec, +static void sortInputSections(MutableArrayRef<InputSectionBase *> vec, const SectionPattern &pat) { if (pat.sortOuter == SortSectionPolicy::None) return; @@ -371,16 +405,16 @@ static void sortInputSections(MutableArrayRef<InputSection *> vec, } // Compute and remember which sections the InputSectionDescription matches. -std::vector<InputSection *> +std::vector<InputSectionBase *> LinkerScript::computeInputSections(const InputSectionDescription *cmd) { - std::vector<InputSection *> ret; + std::vector<InputSectionBase *> ret; // Collects all sections that satisfy constraints of Cmd. for (const SectionPattern &pat : cmd->sectionPatterns) { size_t sizeBefore = ret.size(); for (InputSectionBase *sec : inputSections) { - if (!sec->isLive() || sec->assigned) + if (!sec->isLive() || sec->parent) continue; // For -emit-relocs we have to ignore entries like @@ -388,9 +422,9 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd) { // which are common because they are in the default bfd script. // We do not ignore SHT_REL[A] linker-synthesized sections here because // want to support scripts that do custom layout for them. - if (auto *isec = dyn_cast<InputSection>(sec)) - if (isec->getRelocatedSection()) - continue; + if (isa<InputSection>(sec) && + cast<InputSection>(sec)->getRelocatedSection()) + continue; std::string filename = getFilename(sec->file); if (!cmd->filePat.match(filename) || @@ -398,88 +432,60 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd) { !pat.sectionPat.match(sec->name)) continue; - // It is safe to assume that Sec is an InputSection - // because mergeable or EH input sections have already been - // handled and eliminated. - ret.push_back(cast<InputSection>(sec)); - sec->assigned = true; + ret.push_back(sec); } - sortInputSections(MutableArrayRef<InputSection *>(ret).slice(sizeBefore), - pat); + sortInputSections( + MutableArrayRef<InputSectionBase *>(ret).slice(sizeBefore), pat); } return ret; } -void LinkerScript::discard(ArrayRef<InputSection *> v) { - for (InputSection *s : v) { - if (s == in.shStrTab || s == mainPart->relaDyn || s == mainPart->relrDyn) - error("discarding " + s->name + " section is not allowed"); - - // You can discard .hash and .gnu.hash sections by linker scripts. Since - // they are synthesized sections, we need to handle them differently than - // other regular sections. - if (s == mainPart->gnuHashTab) - mainPart->gnuHashTab = nullptr; - if (s == mainPart->hashTab) - mainPart->hashTab = nullptr; - - s->assigned = false; - s->markDead(); - discard(s->dependentSections); - } +void LinkerScript::discard(InputSectionBase *s) { + if (s == in.shStrTab || s == mainPart->relaDyn || s == mainPart->relrDyn) + error("discarding " + s->name + " section is not allowed"); + + // You can discard .hash and .gnu.hash sections by linker scripts. Since + // they are synthesized sections, we need to handle them differently than + // other regular sections. + if (s == mainPart->gnuHashTab) + mainPart->gnuHashTab = nullptr; + if (s == mainPart->hashTab) + mainPart->hashTab = nullptr; + + s->markDead(); + s->parent = nullptr; + for (InputSection *ds : s->dependentSections) + discard(ds); } -std::vector<InputSection *> +std::vector<InputSectionBase *> LinkerScript::createInputSectionList(OutputSection &outCmd) { - std::vector<InputSection *> ret; + std::vector<InputSectionBase *> ret; for (BaseCommand *base : outCmd.sectionCommands) { if (auto *cmd = dyn_cast<InputSectionDescription>(base)) { - cmd->sections = computeInputSections(cmd); - ret.insert(ret.end(), cmd->sections.begin(), cmd->sections.end()); + cmd->sectionBases = computeInputSections(cmd); + for (InputSectionBase *s : cmd->sectionBases) + s->parent = &outCmd; + ret.insert(ret.end(), cmd->sectionBases.begin(), cmd->sectionBases.end()); } } return ret; } +// Create output sections described by SECTIONS commands. void LinkerScript::processSectionCommands() { - // A symbol can be assigned before any section is mentioned in the linker - // script. In an DSO, the symbol values are addresses, so the only important - // section values are: - // * SHN_UNDEF - // * SHN_ABS - // * Any value meaning a regular section. - // To handle that, create a dummy aether section that fills the void before - // the linker scripts switches to another section. It has an index of one - // which will map to whatever the first actual section is. - aether = make<OutputSection>("", 0, SHF_ALLOC); - aether->sectionIndex = 1; - - // Ctx captures the local AddressState and makes it accessible deliberately. - // This is needed as there are some cases where we cannot just - // thread the current state through to a lambda function created by the - // script parser. - auto deleter = make_unique<AddressState>(); - ctx = deleter.get(); - ctx->outSec = aether; - size_t i = 0; - // Add input sections to output sections. for (BaseCommand *base : sectionCommands) { - // Handle symbol assignments outside of any output section. - if (auto *cmd = dyn_cast<SymbolAssignment>(base)) { - addSymbol(cmd); - continue; - } - if (auto *sec = dyn_cast<OutputSection>(base)) { - std::vector<InputSection *> v = createInputSectionList(*sec); + std::vector<InputSectionBase *> v = createInputSectionList(*sec); // The output section name `/DISCARD/' is special. // Any input section assigned to it is discarded. if (sec->name == "/DISCARD/") { - discard(v); + for (InputSectionBase *s : v) + discard(s); sec->sectionCommands.clear(); continue; } @@ -493,17 +499,11 @@ void LinkerScript::processSectionCommands() { // way to "make it as if it wasn't present" is to make it empty. if (!matchConstraints(v, sec->constraint)) { for (InputSectionBase *s : v) - s->assigned = false; + s->parent = nullptr; sec->sectionCommands.clear(); continue; } - // A directive may contain symbol definitions like this: - // ".foo : { ...; bar = .; }". Handle them. - for (BaseCommand *base : sec->sectionCommands) - if (auto *outCmd = dyn_cast<SymbolAssignment>(base)) - addSymbol(outCmd); - // Handle subalign (e.g. ".foo : SUBALIGN(32) { ... }"). If subalign // is given, input sections are aligned to that value, whether the // given value is larger or smaller than the original section alignment. @@ -513,17 +513,40 @@ void LinkerScript::processSectionCommands() { s->alignment = subalign; } - // Add input sections to an output section. - for (InputSection *s : v) - sec->addSection(s); + // Set the partition field the same way OutputSection::recordSection() + // does. Partitions cannot be used with the SECTIONS command, so this is + // always 1. + sec->partition = 1; sec->sectionIndex = i++; - if (sec->noload) - sec->type = SHT_NOBITS; - if (sec->nonAlloc) - sec->flags &= ~(uint64_t)SHF_ALLOC; } } +} + +void LinkerScript::processSymbolAssignments() { + // Dot outside an output section still represents a relative address, whose + // sh_shndx should not be SHN_UNDEF or SHN_ABS. Create a dummy aether section + // that fills the void outside a section. It has an index of one, which is + // indistinguishable from any other regular section index. + aether = make<OutputSection>("", 0, SHF_ALLOC); + aether->sectionIndex = 1; + + // ctx captures the local AddressState and makes it accessible deliberately. + // This is needed as there are some cases where we cannot just thread the + // current state through to a lambda function created by the script parser. + AddressState state; + ctx = &state; + ctx->outSec = aether; + + for (BaseCommand *base : sectionCommands) { + if (auto *cmd = dyn_cast<SymbolAssignment>(base)) + addSymbol(cmd); + else + for (BaseCommand *sub_base : cast<OutputSection>(base)->sectionCommands) + if (auto *cmd = dyn_cast<SymbolAssignment>(sub_base)) + addSymbol(cmd); + } + ctx = nullptr; } @@ -539,7 +562,7 @@ static OutputSection *findByName(ArrayRef<BaseCommand *> vec, static OutputSection *createSection(InputSectionBase *isec, StringRef outsecName) { OutputSection *sec = script->createOutputSection(outsecName, "<internal>"); - sec->addSection(cast<InputSection>(isec)); + sec->recordSection(isec); return sec; } @@ -568,7 +591,7 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map, OutputSection *out = sec->getRelocatedSection()->getOutputSection(); if (out->relocationSection) { - out->relocationSection->addSection(sec); + out->relocationSection->recordSection(sec); return nullptr; } @@ -576,12 +599,6 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map, return out->relocationSection; } - // When control reaches here, mergeable sections have already been merged into - // synthetic sections. For relocatable case we want to create one output - // section per syntetic section so that they have a valid sh_entsize. - if (config->relocatable && (isec->flags & SHF_MERGE)) - return createSection(isec, outsecName); - // The ELF spec just says // ---------------------------------------------------------------- // In the first phase, input sections that match in name, type and @@ -628,7 +645,21 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map, for (OutputSection *sec : v) { if (sec->partition != isec->partition) continue; - sec->addSection(cast<InputSection>(isec)); + + if (config->relocatable && (isec->flags & SHF_LINK_ORDER)) { + // Merging two SHF_LINK_ORDER sections with different sh_link fields will + // change their semantics, so we only merge them in -r links if they will + // end up being linked to the same output section. The casts are fine + // because everything in the map was created by the orphan placement code. + auto *firstIsec = cast<InputSectionBase>( + cast<InputSectionDescription>(sec->sectionCommands[0]) + ->sectionBases[0]); + if (firstIsec->getLinkOrderDep()->getOutputSection() != + isec->getLinkOrderDep()->getOutputSection()) + continue; + } + + sec->recordSection(isec); return nullptr; } @@ -642,25 +673,30 @@ void LinkerScript::addOrphanSections() { StringMap<TinyPtrVector<OutputSection *>> map; std::vector<OutputSection *> v; - auto add = [&](InputSectionBase *s) { - if (!s->isLive() || s->parent) - return; - - StringRef name = getOutputSectionName(s); - - if (config->orphanHandling == OrphanHandlingPolicy::Error) - error(toString(s) + " is being placed in '" + name + "'"); - else if (config->orphanHandling == OrphanHandlingPolicy::Warn) - warn(toString(s) + " is being placed in '" + name + "'"); - - if (OutputSection *sec = findByName(sectionCommands, name)) { - sec->addSection(cast<InputSection>(s)); - return; + std::function<void(InputSectionBase *)> add; + add = [&](InputSectionBase *s) { + if (s->isLive() && !s->parent) { + StringRef name = getOutputSectionName(s); + + if (config->orphanHandling == OrphanHandlingPolicy::Error) + error(toString(s) + " is being placed in '" + name + "'"); + else if (config->orphanHandling == OrphanHandlingPolicy::Warn) + warn(toString(s) + " is being placed in '" + name + "'"); + + if (OutputSection *sec = findByName(sectionCommands, name)) { + sec->recordSection(s); + } else { + if (OutputSection *os = addInputSec(map, s, name)) + v.push_back(os); + assert(isa<MergeInputSection>(s) || + s->getOutputSection()->sectionIndex == UINT32_MAX); + } } - if (OutputSection *os = addInputSec(map, s, name)) - v.push_back(os); - assert(s->getOutputSection()->sectionIndex == UINT32_MAX); + if (config->relocatable) + for (InputSectionBase *depSec : s->dependentSections) + if (depSec->flags & SHF_LINK_ORDER) + add(depSec); }; // For futher --emit-reloc handling code we need target output section @@ -668,6 +704,12 @@ void LinkerScript::addOrphanSections() { // to create target sections first. We do not want priority handling // for synthetic sections because them are special. for (InputSectionBase *isec : inputSections) { + // In -r links, SHF_LINK_ORDER sections are added while adding their parent + // sections because we need to know the parent's output section before we + // can select an output section for the SHF_LINK_ORDER section. + if (config->relocatable && (isec->flags & SHF_LINK_ORDER)) + continue; + if (auto *sec = dyn_cast<InputSection>(isec)) if (InputSectionBase *rel = sec->getRelocatedSection()) if (auto *relIS = dyn_cast_or_null<InputSectionBase>(rel->parent)) @@ -772,6 +814,14 @@ void LinkerScript::assignOffsets(OutputSection *sec) { if ((sec->flags & SHF_ALLOC) && sec->addrExpr) setDot(sec->addrExpr, sec->location, false); + // If the address of the section has been moved forward by an explicit + // expression so that it now starts past the current curPos of the enclosing + // region, we need to expand the current region to account for the space + // between the previous section, if any, and the start of this section. + if (ctx->memRegion && ctx->memRegion->curPos < dot) + expandMemoryRegion(ctx->memRegion, dot - ctx->memRegion->curPos, + ctx->memRegion->name, sec->name); + switchTo(sec); if (sec->lmaExpr) @@ -972,17 +1022,13 @@ static uint64_t computeBase(uint64_t min, bool allocateHeaders) { return alignDown(min, config->maxPageSize); } -// Try to find an address for the file and program headers output sections, -// which were unconditionally added to the first PT_LOAD segment earlier. -// -// When using the default layout, we check if the headers fit below the first -// allocated section. When using a linker script, we also check if the headers -// are covered by the output section. This allows omitting the headers by not -// leaving enough space for them in the linker script; this pattern is common -// in embedded systems. +// When the SECTIONS command is used, try to find an address for the file and +// program headers output sections, which can be added to the first PT_LOAD +// segment when program headers are created. // -// If there isn't enough space for these sections, we'll remove them from the -// PT_LOAD segment, and we'll also remove the PT_PHDR segment. +// We check if the headers fit below the first allocated section. If there isn't +// enough space for these sections, we'll remove them from the PT_LOAD segment, +// and we'll also remove the PT_PHDR segment. void LinkerScript::allocateHeaders(std::vector<PhdrEntry *> &phdrs) { uint64_t min = std::numeric_limits<uint64_t>::max(); for (OutputSection *sec : outputSections) @@ -1028,32 +1074,30 @@ LinkerScript::AddressState::AddressState() { } } -static uint64_t getInitialDot() { - // By default linker scripts use an initial value of 0 for '.', - // but prefer -image-base if set. - if (script->hasSectionsCommand) - return config->imageBase ? *config->imageBase : 0; - - uint64_t startAddr = UINT64_MAX; - // The sections with -T<section> have been sorted in order of ascending - // address. We must lower startAddr if the lowest -T<section address> as - // calls to setDot() must be monotonically increasing. - for (auto &kv : config->sectionStartMap) - startAddr = std::min(startAddr, kv.second); - return std::min(startAddr, target->getImageBase() + elf::getHeaderSize()); -} - // Here we assign addresses as instructed by linker script SECTIONS // sub-commands. Doing that allows us to use final VA values, so here // we also handle rest commands like symbol assignments and ASSERTs. -void LinkerScript::assignAddresses() { - dot = getInitialDot(); +// Returns a symbol that has changed its section or value, or nullptr if no +// symbol has changed. +const Defined *LinkerScript::assignAddresses() { + if (script->hasSectionsCommand) { + // With a linker script, assignment of addresses to headers is covered by + // allocateHeaders(). + dot = config->imageBase.getValueOr(0); + } else { + // Assign addresses to headers right now. + dot = target->getImageBase(); + Out::elfHeader->addr = dot; + Out::programHeaders->addr = dot + Out::elfHeader->size; + dot += getHeaderSize(); + } - auto deleter = make_unique<AddressState>(); + auto deleter = std::make_unique<AddressState>(); ctx = deleter.get(); errorOnMissingSection = true; switchTo(aether); + SymbolAssignmentMap oldValues = getSymbolAssignmentValues(sectionCommands); for (BaseCommand *base : sectionCommands) { if (auto *cmd = dyn_cast<SymbolAssignment>(base)) { cmd->addr = dot; @@ -1063,7 +1107,9 @@ void LinkerScript::assignAddresses() { } assignOffsets(cast<OutputSection>(base)); } + ctx = nullptr; + return getChangedSymbolAssignment(oldValues); } // Creates program headers as instructed by PHDRS linker script command. @@ -1156,3 +1202,6 @@ std::vector<size_t> LinkerScript::getPhdrIndices(OutputSection *cmd) { } return ret; } + +} // namespace elf +} // namespace lld diff --git a/ELF/LinkerScript.h b/ELF/LinkerScript.h index 9e9c08ef10ba..621b8baeaae6 100644 --- a/ELF/LinkerScript.h +++ b/ELF/LinkerScript.h @@ -168,6 +168,12 @@ struct InputSectionDescription : BaseCommand { // will be associated with this InputSectionDescription. std::vector<SectionPattern> sectionPatterns; + // Includes InputSections and MergeInputSections. Used temporarily during + // assignment of input sections to output sections. + std::vector<InputSectionBase *> sectionBases; + + // Used after the finalizeInputSections() pass. MergeInputSections have been + // merged into MergeSyntheticSections. std::vector<InputSection *> sections; // Temporary record of synthetic ThunkSection instances and the pass that @@ -226,10 +232,10 @@ class LinkerScript final { void expandOutputSection(uint64_t size); void expandMemoryRegions(uint64_t size); - std::vector<InputSection *> + std::vector<InputSectionBase *> computeInputSections(const InputSectionDescription *); - std::vector<InputSection *> createInputSectionList(OutputSection &cmd); + std::vector<InputSectionBase *> createInputSectionList(OutputSection &cmd); std::vector<size_t> getPhdrIndices(OutputSection *sec); @@ -259,7 +265,7 @@ public: bool hasPhdrsCommands() { return !phdrsCommands.empty(); } uint64_t getDot() { return dot; } - void discard(ArrayRef<InputSection *> v); + void discard(InputSectionBase *s); ExprValue getSymbolValue(StringRef name, const Twine &loc); @@ -271,9 +277,10 @@ public: bool needsInterpSection(); bool shouldKeep(InputSectionBase *s); - void assignAddresses(); + const Defined *assignAddresses(); void allocateHeaders(std::vector<PhdrEntry *> &phdrs); void processSectionCommands(); + void processSymbolAssignments(); void declareSymbols(); // Used to handle INSERT AFTER statements. diff --git a/ELF/MapFile.cpp b/ELF/MapFile.cpp index a4a6238fc655..4d76e22f37f5 100644 --- a/ELF/MapFile.cpp +++ b/ELF/MapFile.cpp @@ -34,13 +34,12 @@ using namespace llvm; using namespace llvm::object; -using namespace lld; -using namespace lld::elf; - +namespace lld { +namespace elf { using SymbolMapTy = DenseMap<const SectionBase *, SmallVector<Defined *, 4>>; -static const std::string indent8 = " "; // 8 spaces -static const std::string indent16 = " "; // 16 spaces +static constexpr char indent8[] = " "; // 8 spaces +static constexpr char indent16[] = " "; // 16 spaces // Print out the first three columns of a line. static void writeHeader(raw_ostream &os, uint64_t vma, uint64_t lma, @@ -139,13 +138,13 @@ static void printEhFrame(raw_ostream &os, const EhFrameSection *sec) { } } -void elf::writeMapFile() { +void writeMapFile() { if (config->mapFile.empty()) return; // Open a map file for writing. std::error_code ec; - raw_fd_ostream os(config->mapFile, ec, sys::fs::F_None); + raw_fd_ostream os(config->mapFile, ec, sys::fs::OF_None); if (ec) { error("cannot open " + config->mapFile + ": " + ec.message()); return; @@ -228,7 +227,7 @@ static void print(StringRef a, StringRef b) { // // In this case, strlen is defined by libc.so.6 and used by other two // files. -void elf::writeCrossReferenceTable() { +void writeCrossReferenceTable() { if (!config->cref) return; @@ -259,3 +258,6 @@ void elf::writeCrossReferenceTable() { print("", toString(file)); } } + +} // namespace elf +} // namespace lld diff --git a/ELF/MarkLive.cpp b/ELF/MarkLive.cpp index 36b847f725b8..02ab6d18e537 100644 --- a/ELF/MarkLive.cpp +++ b/ELF/MarkLive.cpp @@ -37,11 +37,11 @@ using namespace llvm; using namespace llvm::ELF; using namespace llvm::object; -using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; +namespace endian = llvm::support::endian; +namespace lld { +namespace elf { namespace { template <class ELFT> class MarkLive { public: @@ -141,7 +141,7 @@ void MarkLive<ELFT>::scanEhFrameSection(EhInputSection &eh, if (firstRelI == (unsigned)-1) continue; - if (read32<ELFT::TargetEndianness>(piece.data().data() + 4) == 0) { + if (endian::read32<ELFT::TargetEndianness>(piece.data().data() + 4) == 0) { // This is a CIE, we only need to worry about the first relocation. It is // known to point to the personality function. resolveReloc(eh, rels[firstRelI], false); @@ -291,6 +291,10 @@ template <class ELFT> void MarkLive<ELFT>::mark() { // GOT, which means that the ifunc must be available when the main partition is // loaded) and TLS symbols (because we only know how to correctly process TLS // relocations for the main partition). +// +// We also need to move sections whose names are C identifiers that are referred +// to from __start_/__stop_ symbols because there will only be one set of +// symbols for the whole program. template <class ELFT> void MarkLive<ELFT>::moveToMain() { for (InputFile *file : objectFiles) for (Symbol *s : file->getSymbols()) @@ -299,13 +303,21 @@ template <class ELFT> void MarkLive<ELFT>::moveToMain() { d->section->isLive()) markSymbol(s); + for (InputSectionBase *sec : inputSections) { + if (!sec->isLive() || !isValidCIdentifier(sec->name)) + continue; + if (symtab->find(("__start_" + sec->name).str()) || + symtab->find(("__stop_" + sec->name).str())) + enqueue(sec, 0); + } + mark(); } // Before calling this function, Live bits are off for all // input sections. This function make some or all of them on // so that they are emitted to the output file. -template <class ELFT> void elf::markLive() { +template <class ELFT> void markLive() { // If -gc-sections is not given, no sections are removed. if (!config->gcSections) { for (InputSectionBase *sec : inputSections) @@ -367,7 +379,10 @@ template <class ELFT> void elf::markLive() { message("removing unused section " + toString(sec)); } -template void elf::markLive<ELF32LE>(); -template void elf::markLive<ELF32BE>(); -template void elf::markLive<ELF64LE>(); -template void elf::markLive<ELF64BE>(); +template void markLive<ELF32LE>(); +template void markLive<ELF32BE>(); +template void markLive<ELF64LE>(); +template void markLive<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/Options.td b/ELF/Options.td index 3ebb46f2e1b2..c540efb25e90 100644 --- a/ELF/Options.td +++ b/ELF/Options.td @@ -171,6 +171,9 @@ defm fini: Eq<"fini", "Specify a finalizer function">, MetaVarName<"<symbol>">; def fix_cortex_a53_843419: F<"fix-cortex-a53-843419">, HelpText<"Apply fixes for AArch64 Cortex-A53 erratum 843419">; +def fix_cortex_a8: F<"fix-cortex-a8">, + HelpText<"Apply fixes for ARM Cortex-A8 erratum 657417">; + // This option is intentionally hidden from the user as the implementation // is not complete. def require_cet: F<"require-cet">; @@ -306,7 +309,7 @@ def push_state: F<"push-state">, def print_map: F<"print-map">, HelpText<"Print a link map to the standard output">; -defm reproduce: Eq<"reproduce", "Dump linker invocation and input files for debugging">; +defm reproduce: Eq<"reproduce", "Write a tar file containing input files and command line options to reproduce link">; defm rpath: Eq<"rpath", "Add a DT_RUNPATH to the output">; @@ -478,6 +481,7 @@ def lto_cs_profile_generate: F<"lto-cs-profile-generate">, HelpText<"Perform context senstive PGO instrumentation">; def lto_cs_profile_file: J<"lto-cs-profile-file=">, HelpText<"Context sensitive profile file path">; +def lto_obj_path_eq: J<"lto-obj-path=">; def lto_sample_profile: J<"lto-sample-profile=">, HelpText<"Sample profile file path">; def disable_verify: F<"disable-verify">; @@ -495,7 +499,12 @@ def save_temps: F<"save-temps">; def thinlto_cache_dir: J<"thinlto-cache-dir=">, HelpText<"Path to ThinLTO cached object file directory">; defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">; +def thinlto_emit_imports_files: F<"thinlto-emit-imports-files">; +def thinlto_index_only: F<"thinlto-index-only">; +def thinlto_index_only_eq: J<"thinlto-index-only=">; def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">; +def thinlto_object_suffix_replace_eq: J<"thinlto-object-suffix-replace=">; +def thinlto_prefix_replace_eq: J<"thinlto-prefix-replace=">; def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for -lto-O">; def: F<"plugin-opt=debug-pass-manager">, @@ -509,19 +518,31 @@ def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">; def: F<"plugin-opt=new-pass-manager">, Alias<lto_new_pass_manager>, HelpText<"Alias for -lto-new-pass-manager">; -def plugin_opt_obj_path_eq: J<"plugin-opt=obj-path=">; def: F<"plugin-opt=cs-profile-generate">, Alias<lto_cs_profile_generate>, HelpText<"Alias for -lto-cs-profile-generate">; def: J<"plugin-opt=cs-profile-path=">, Alias<lto_cs_profile_file>, HelpText<"Alias for -lto-cs-profile-file">; +def: J<"plugin-opt=obj-path=">, + Alias<lto_obj_path_eq>, + HelpText<"Alias for -lto-obj-path=">; def: J<"plugin-opt=sample-profile=">, Alias<lto_sample_profile>, HelpText<"Alias for -lto-sample-profile">; def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for -save-temps">; -def plugin_opt_thinlto_emit_imports_files: F<"plugin-opt=thinlto-emit-imports-files">; -def plugin_opt_thinlto_index_only: F<"plugin-opt=thinlto-index-only">; -def plugin_opt_thinlto_index_only_eq: J<"plugin-opt=thinlto-index-only=">; -def plugin_opt_thinlto_object_suffix_replace_eq: J<"plugin-opt=thinlto-object-suffix-replace=">; -def plugin_opt_thinlto_prefix_replace_eq: J<"plugin-opt=thinlto-prefix-replace=">; +def: F<"plugin-opt=thinlto-emit-imports-files">, + Alias<thinlto_emit_imports_files>, + HelpText<"Alias for -thinlto-emit-imports-files">; +def: F<"plugin-opt=thinlto-index-only">, + Alias<thinlto_index_only>, + HelpText<"Alias for -thinlto-index-only">; +def: J<"plugin-opt=thinlto-index-only=">, + Alias<thinlto_index_only_eq>, + HelpText<"Alias for -thinlto-index-only=">; +def: J<"plugin-opt=thinlto-object-suffix-replace=">, + Alias<thinlto_object_suffix_replace_eq>, + HelpText<"Alias for -thinlto-object-suffix-replace=">; +def: J<"plugin-opt=thinlto-prefix-replace=">, + Alias<thinlto_prefix_replace_eq>, + HelpText<"Alias for -thinlto-prefix-replace=">; // Ignore LTO plugin-related options. // clang -flto passes -plugin and -plugin-opt to the linker. This is required diff --git a/ELF/OutputSections.cpp b/ELF/OutputSections.cpp index a89bd509bc96..ea7c96eb676a 100644 --- a/ELF/OutputSections.cpp +++ b/ELF/OutputSections.cpp @@ -27,9 +27,8 @@ using namespace llvm::object; using namespace llvm::support::endian; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; - +namespace lld { +namespace elf { uint8_t *Out::bufferStart; uint8_t Out::first; PhdrEntry *Out::tlsPhdr; @@ -39,7 +38,7 @@ OutputSection *Out::preinitArray; OutputSection *Out::initArray; OutputSection *Out::finiArray; -std::vector<OutputSection *> elf::outputSections; +std::vector<OutputSection *> outputSections; uint32_t OutputSection::getPhdrFlags() const { uint32_t ret = 0; @@ -83,12 +82,32 @@ static bool canMergeToProgbits(unsigned type) { type == SHT_NOTE; } -void OutputSection::addSection(InputSection *isec) { +// Record that isec will be placed in the OutputSection. isec does not become +// permanent until finalizeInputSections() is called. The function should not be +// used after finalizeInputSections() is called. If you need to add an +// InputSection post finalizeInputSections(), then you must do the following: +// +// 1. Find or create an InputSectionDescription to hold InputSection. +// 2. Add the InputSection to the InputSectionDesciption::sections. +// 3. Call commitSection(isec). +void OutputSection::recordSection(InputSectionBase *isec) { + partition = isec->partition; + isec->parent = this; + if (sectionCommands.empty() || + !isa<InputSectionDescription>(sectionCommands.back())) + sectionCommands.push_back(make<InputSectionDescription>("")); + auto *isd = cast<InputSectionDescription>(sectionCommands.back()); + isd->sectionBases.push_back(isec); +} + +// Update fields (type, flags, alignment, etc) according to the InputSection +// isec. Also check whether the InputSection flags and type are consistent with +// other InputSections. +void OutputSection::commitSection(InputSection *isec) { if (!hasInputSections) { // If IS is the first section to be added to this section, - // initialize Partition, Type, Entsize and flags from IS. + // initialize type, entsize and flags from isec. hasInputSections = true; - partition = isec->partition; type = isec->type; entsize = isec->entsize; flags = isec->flags; @@ -110,6 +129,8 @@ void OutputSection::addSection(InputSection *isec) { type = SHT_PROGBITS; } } + if (noload) + type = SHT_NOBITS; isec->parent = this; uint64_t andMask = @@ -118,6 +139,8 @@ void OutputSection::addSection(InputSection *isec) { uint64_t andFlags = (flags & isec->flags) & andMask; uint64_t orFlags = (flags | isec->flags) & orMask; flags = andFlags | orFlags; + if (nonAlloc) + flags &= ~(uint64_t)SHF_ALLOC; alignment = std::max(alignment, isec->alignment); @@ -126,15 +149,69 @@ void OutputSection::addSection(InputSection *isec) { // set sh_entsize to 0. if (entsize != isec->entsize) entsize = 0; +} - if (!isec->assigned) { - isec->assigned = true; - if (sectionCommands.empty() || - !isa<InputSectionDescription>(sectionCommands.back())) - sectionCommands.push_back(make<InputSectionDescription>("")); - auto *isd = cast<InputSectionDescription>(sectionCommands.back()); - isd->sections.push_back(isec); +// This function scans over the InputSectionBase list sectionBases to create +// InputSectionDescription::sections. +// +// It removes MergeInputSections from the input section array and adds +// new synthetic sections at the location of the first input section +// that it replaces. It then finalizes each synthetic section in order +// to compute an output offset for each piece of each input section. +void OutputSection::finalizeInputSections() { + std::vector<MergeSyntheticSection *> mergeSections; + for (BaseCommand *base : sectionCommands) { + auto *cmd = dyn_cast<InputSectionDescription>(base); + if (!cmd) + continue; + cmd->sections.reserve(cmd->sectionBases.size()); + for (InputSectionBase *s : cmd->sectionBases) { + MergeInputSection *ms = dyn_cast<MergeInputSection>(s); + if (!ms) { + cmd->sections.push_back(cast<InputSection>(s)); + continue; + } + + // We do not want to handle sections that are not alive, so just remove + // them instead of trying to merge. + if (!ms->isLive()) + continue; + + auto i = llvm::find_if(mergeSections, [=](MergeSyntheticSection *sec) { + // While we could create a single synthetic section for two different + // values of Entsize, it is better to take Entsize into consideration. + // + // With a single synthetic section no two pieces with different Entsize + // could be equal, so we may as well have two sections. + // + // Using Entsize in here also allows us to propagate it to the synthetic + // section. + // + // SHF_STRINGS section with different alignments should not be merged. + return sec->flags == ms->flags && sec->entsize == ms->entsize && + (sec->alignment == ms->alignment || !(sec->flags & SHF_STRINGS)); + }); + if (i == mergeSections.end()) { + MergeSyntheticSection *syn = + createMergeSynthetic(name, ms->type, ms->flags, ms->alignment); + mergeSections.push_back(syn); + i = std::prev(mergeSections.end()); + syn->entsize = ms->entsize; + cmd->sections.push_back(syn); + } + (*i)->addSection(ms); + } + + // sectionBases should not be used from this point onwards. Clear it to + // catch misuses. + cmd->sectionBases.clear(); + + // Some input sections may be removed from the list after ICF. + for (InputSection *s : cmd->sections) + commitSection(s); } + for (auto *ms : mergeSections) + ms->finalizeContents(); } static void sortByOrder(MutableArrayRef<InputSection *> in, @@ -148,7 +225,7 @@ static void sortByOrder(MutableArrayRef<InputSection *> in, in[i] = v[i].second; } -uint64_t elf::getHeaderSize() { +uint64_t getHeaderSize() { if (config->oFormatBinary) return 0; return Out::elfHeader->size + Out::programHeaders->size; @@ -368,7 +445,7 @@ void OutputSection::sortCtorsDtors() { // If an input string is in the form of "foo.N" where N is a number, // return N. Otherwise, returns 65536, which is one greater than the // lowest priority. -int elf::getPriority(StringRef s) { +int getPriority(StringRef s) { size_t pos = s.rfind('.'); if (pos == StringRef::npos) return 65536; @@ -378,7 +455,7 @@ int elf::getPriority(StringRef s) { return v; } -std::vector<InputSection *> elf::getInputSections(OutputSection *os) { +std::vector<InputSection *> getInputSections(OutputSection *os) { std::vector<InputSection *> ret; for (BaseCommand *base : os->sectionCommands) if (auto *isd = dyn_cast<InputSectionDescription>(base)) @@ -419,3 +496,6 @@ template void OutputSection::maybeCompress<ELF32LE>(); template void OutputSection::maybeCompress<ELF32BE>(); template void OutputSection::maybeCompress<ELF64LE>(); template void OutputSection::maybeCompress<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/OutputSections.h b/ELF/OutputSections.h index fff8327ea376..a24294eedf35 100644 --- a/ELF/OutputSections.h +++ b/ELF/OutputSections.h @@ -71,7 +71,9 @@ public: uint64_t addr = 0; uint32_t shName = 0; - void addSection(InputSection *isec); + void recordSection(InputSectionBase *isec); + void commitSection(InputSection *isec); + void finalizeInputSections(); // The following members are normally only used in linker scripts. MemoryRegion *memRegion = nullptr; diff --git a/ELF/Relocations.cpp b/ELF/Relocations.cpp index ee48f4808136..ab3030d91017 100644 --- a/ELF/Relocations.cpp +++ b/ELF/Relocations.cpp @@ -62,9 +62,8 @@ using namespace llvm::ELF; using namespace llvm::object; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; - +namespace lld { +namespace elf { static Optional<std::string> getLinkerScriptLocation(const Symbol &sym) { for (BaseCommand *base : script->sectionCommands) if (auto *cmd = dyn_cast<SymbolAssignment>(base)) @@ -344,9 +343,9 @@ static bool needsPlt(RelExpr expr) { // returns false for TLS variables even though they need GOT, because // TLS variables uses GOT differently than the regular variables. static bool needsGot(RelExpr expr) { - return oneof<R_GOT, R_GOT_OFF, R_HEXAGON_GOT, R_MIPS_GOT_LOCAL_PAGE, - R_MIPS_GOT_OFF, R_MIPS_GOT_OFF32, R_AARCH64_GOT_PAGE_PC, - R_GOT_PC, R_GOTPLT>(expr); + return oneof<R_GOT, R_GOT_OFF, R_MIPS_GOT_LOCAL_PAGE, R_MIPS_GOT_OFF, + R_MIPS_GOT_OFF32, R_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTPLT>( + expr); } // True if this expression is of the form Sym - X, where X is a position in the @@ -369,7 +368,7 @@ static bool isRelExpr(RelExpr expr) { static bool isStaticLinkTimeConstant(RelExpr e, RelType type, const Symbol &sym, InputSectionBase &s, uint64_t relOff) { // These expressions always compute a constant - if (oneof<R_DTPREL, R_GOTPLT, R_GOT_OFF, R_HEXAGON_GOT, R_TLSLD_GOT_OFF, + if (oneof<R_DTPREL, R_GOTPLT, R_GOT_OFF, R_TLSLD_GOT_OFF, R_MIPS_GOT_LOCAL_PAGE, R_MIPS_GOTREL, R_MIPS_GOT_OFF, R_MIPS_GOT_OFF32, R_MIPS_GOT_GP_PC, R_MIPS_TLSGD, R_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTONLY_PC, R_GOTPLTONLY_PC, @@ -510,10 +509,8 @@ static void replaceWithDefined(Symbol &sym, SectionBase *sec, uint64_t value, sym.gotIndex = old.gotIndex; sym.verdefIndex = old.verdefIndex; sym.ppc64BranchltIndex = old.ppc64BranchltIndex; - sym.isPreemptible = true; sym.exportDynamic = true; sym.isUsedInRegularObj = true; - sym.used = true; } // Reserve space in .bss or .bss.rel.ro for copy relocation. @@ -569,10 +566,16 @@ template <class ELFT> static void addCopyRelSymbol(SharedSymbol &ss) { bool isRO = isReadOnly<ELFT>(ss); BssSection *sec = make<BssSection>(isRO ? ".bss.rel.ro" : ".bss", symSize, ss.alignment); - if (isRO) - in.bssRelRo->getParent()->addSection(sec); - else - in.bss->getParent()->addSection(sec); + OutputSection *osec = (isRO ? in.bssRelRo : in.bss)->getParent(); + + // At this point, sectionBases has been migrated to sections. Append sec to + // sections. + if (osec->sectionCommands.empty() || + !isa<InputSectionDescription>(osec->sectionCommands.back())) + osec->sectionCommands.push_back(make<InputSectionDescription>("")); + auto *isd = cast<InputSectionDescription>(osec->sectionCommands.back()); + isd->sections.push_back(sec); + osec->commitSection(sec); // Look through the DSO's dynamic symbol table for aliases and create a // dynamic symbol for each one. This causes the copy relocation to correctly @@ -693,8 +696,75 @@ struct UndefinedDiag { static std::vector<UndefinedDiag> undefs; +// Suggest an alternative spelling of an "undefined symbol" diagnostic. Returns +// the suggested symbol, which is either in the symbol table, or in the same +// file of sym. +static const Symbol *getAlternativeSpelling(const Undefined &sym) { + // Build a map of local defined symbols. + DenseMap<StringRef, const Symbol *> map; + if (sym.file && !isa<SharedFile>(sym.file)) { + for (const Symbol *s : sym.file->getSymbols()) + if (s->isLocal() && s->isDefined()) + map.try_emplace(s->getName(), s); + } + + auto suggest = [&](StringRef newName) -> const Symbol * { + // If defined locally. + if (const Symbol *s = map.lookup(newName)) + return s; + + // If in the symbol table and not undefined. + if (const Symbol *s = symtab->find(newName)) + if (!s->isUndefined()) + return s; + + return nullptr; + }; + + // This loop enumerates all strings of Levenshtein distance 1 as typo + // correction candidates and suggests the one that exists as a non-undefined + // symbol. + StringRef name = sym.getName(); + for (size_t i = 0, e = name.size(); i != e + 1; ++i) { + // Insert a character before name[i]. + std::string newName = (name.substr(0, i) + "0" + name.substr(i)).str(); + for (char c = '0'; c <= 'z'; ++c) { + newName[i] = c; + if (const Symbol *s = suggest(newName)) + return s; + } + if (i == e) + break; + + // Substitute name[i]. + newName = name; + for (char c = '0'; c <= 'z'; ++c) { + newName[i] = c; + if (const Symbol *s = suggest(newName)) + return s; + } + + // Transpose name[i] and name[i+1]. This is of edit distance 2 but it is + // common. + if (i + 1 < e) { + newName[i] = name[i + 1]; + newName[i + 1] = name[i]; + if (const Symbol *s = suggest(newName)) + return s; + } + + // Delete name[i]. + newName = (name.substr(0, i) + name.substr(i + 1)).str(); + if (const Symbol *s = suggest(newName)) + return s; + } + + return nullptr; +} + template <class ELFT> -static void reportUndefinedSymbol(const UndefinedDiag &undef) { +static void reportUndefinedSymbol(const UndefinedDiag &undef, + bool correctSpelling) { Symbol &sym = *undef.sym; auto visibility = [&]() -> std::string { @@ -734,6 +804,14 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef) { msg += ("\n>>> referenced " + Twine(undef.locs.size() - i) + " more times") .str(); + if (correctSpelling) + if (const Symbol *corrected = + getAlternativeSpelling(cast<Undefined>(sym))) { + msg += "\n>>> did you mean: " + toString(*corrected); + if (corrected->file) + msg += "\n>>> defined in: " + toString(corrected->file); + } + if (sym.getName().startswith("_ZTV")) msg += "\nthe vtable symbol may be undefined because the class is missing " "its key function (see https://lld.llvm.org/missingkeyfunction)"; @@ -744,7 +822,7 @@ static void reportUndefinedSymbol(const UndefinedDiag &undef) { error(msg); } -template <class ELFT> void elf::reportUndefinedSymbols() { +template <class ELFT> void reportUndefinedSymbols() { // Find the first "undefined symbol" diagnostic for each diagnostic, and // collect all "referenced from" lines at the first diagnostic. DenseMap<Symbol *, UndefinedDiag *> firstRef; @@ -757,23 +835,21 @@ template <class ELFT> void elf::reportUndefinedSymbols() { firstRef[undef.sym] = &undef; } - for (const UndefinedDiag &undef : undefs) { - if (!undef.locs.empty()) - reportUndefinedSymbol<ELFT>(undef); - } + // Enable spell corrector for the first 2 diagnostics. + for (auto it : enumerate(undefs)) + if (!it.value().locs.empty()) + reportUndefinedSymbol<ELFT>(it.value(), it.index() < 2); undefs.clear(); } // Report an undefined symbol if necessary. // Returns true if the undefined symbol will produce an error message. -template <class ELFT> static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec, uint64_t offset) { if (!sym.isUndefined() || sym.isWeak()) return false; - bool canBeExternal = !sym.isLocal() && sym.computeBinding() != STB_LOCAL && - sym.visibility == STV_DEFAULT; + bool canBeExternal = !sym.isLocal() && sym.visibility == STV_DEFAULT; if (config->unresolvedSymbols == UnresolvedPolicy::Ignore && canBeExternal) return false; @@ -997,56 +1073,29 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, } } - if (!canWrite && (config->isPic && !isRelExpr(expr))) { - error( - "can't create dynamic relocation " + toString(type) + " against " + - (sym.getName().empty() ? "local symbol" : "symbol: " + toString(sym)) + - " in readonly segment; recompile object files with -fPIC " - "or pass '-Wl,-z,notext' to allow text relocations in the output" + - getLocation(sec, sym, offset)); - return; - } - - // Copy relocations (for STT_OBJECT) and canonical PLT (for STT_FUNC) are only - // possible in an executable. - // - // Among R_ABS relocatoin types, symbolicRel has the same size as the word - // size. Others have fewer bits and may cause runtime overflow in -pie/-shared - // mode. Disallow them. - if (config->shared || - (config->pie && expr == R_ABS && type != target->symbolicRel)) { - errorOrWarn( - "relocation " + toString(type) + " cannot be used against " + - (sym.getName().empty() ? "local symbol" : "symbol " + toString(sym)) + - "; recompile with -fPIC" + getLocation(sec, sym, offset)); - return; - } - - // If the symbol is undefined we already reported any relevant errors. - if (sym.isUndefined()) - return; - - if (!canDefineSymbolInExecutable(sym)) { - error("cannot preempt symbol: " + toString(sym) + - getLocation(sec, sym, offset)); - return; - } + // When producing an executable, we can perform copy relocations (for + // STT_OBJECT) and canonical PLT (for STT_FUNC). + if (!config->shared) { + if (!canDefineSymbolInExecutable(sym)) { + errorOrWarn("cannot preempt symbol: " + toString(sym) + + getLocation(sec, sym, offset)); + return; + } - if (sym.isObject()) { - // Produce a copy relocation. - if (auto *ss = dyn_cast<SharedSymbol>(&sym)) { - if (!config->zCopyreloc) - error("unresolvable relocation " + toString(type) + - " against symbol '" + toString(*ss) + - "'; recompile with -fPIC or remove '-z nocopyreloc'" + - getLocation(sec, sym, offset)); - addCopyRelSymbol<ELFT>(*ss); + if (sym.isObject()) { + // Produce a copy relocation. + if (auto *ss = dyn_cast<SharedSymbol>(&sym)) { + if (!config->zCopyreloc) + error("unresolvable relocation " + toString(type) + + " against symbol '" + toString(*ss) + + "'; recompile with -fPIC or remove '-z nocopyreloc'" + + getLocation(sec, sym, offset)); + addCopyRelSymbol<ELFT>(*ss); + } + sec.relocations.push_back({expr, type, offset, addend, &sym}); + return; } - sec.relocations.push_back({expr, type, offset, addend, &sym}); - return; - } - if (sym.isFunc()) { // This handles a non PIC program call to function in a shared library. In // an ideal world, we could just report an error saying the relocation can // overflow at runtime. In the real world with glibc, crt1.o has a @@ -1074,18 +1123,37 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, // compiled without -fPIE/-fPIC and doesn't maintain ebx. // * If a library definition gets preempted to the executable, it will have // the wrong ebx value. - if (config->pie && config->emachine == EM_386) - errorOrWarn("symbol '" + toString(sym) + - "' cannot be preempted; recompile with -fPIE" + - getLocation(sec, sym, offset)); - if (!sym.isInPlt()) - addPltEntry<ELFT>(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym); - if (!sym.isDefined()) - replaceWithDefined( - sym, in.plt, - target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0); - sym.needsPltAddr = true; - sec.relocations.push_back({expr, type, offset, addend, &sym}); + if (sym.isFunc()) { + if (config->pie && config->emachine == EM_386) + errorOrWarn("symbol '" + toString(sym) + + "' cannot be preempted; recompile with -fPIE" + + getLocation(sec, sym, offset)); + if (!sym.isInPlt()) + addPltEntry<ELFT>(in.plt, in.gotPlt, in.relaPlt, target->pltRel, sym); + if (!sym.isDefined()) + replaceWithDefined( + sym, in.plt, + target->pltHeaderSize + target->pltEntrySize * sym.pltIndex, 0); + sym.needsPltAddr = true; + sec.relocations.push_back({expr, type, offset, addend, &sym}); + return; + } + } + + if (config->isPic) { + if (!canWrite && !isRelExpr(expr)) + errorOrWarn( + "can't create dynamic relocation " + toString(type) + " against " + + (sym.getName().empty() ? "local symbol" + : "symbol: " + toString(sym)) + + " in readonly segment; recompile object files with -fPIC " + "or pass '-Wl,-z,notext' to allow text relocations in the output" + + getLocation(sec, sym, offset)); + else + errorOrWarn( + "relocation " + toString(type) + " cannot be used against " + + (sym.getName().empty() ? "local symbol" : "symbol " + toString(sym)) + + "; recompile with -fPIC" + getLocation(sec, sym, offset)); return; } @@ -1093,15 +1161,6 @@ static void processRelocAux(InputSectionBase &sec, RelExpr expr, RelType type, getLocation(sec, sym, offset)); } -struct IRelativeReloc { - RelType type; - InputSectionBase *sec; - uint64_t offset; - Symbol *sym; -}; - -static std::vector<IRelativeReloc> iRelativeRelocs; - template <class ELFT, class RelTy> static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, RelTy *end) { @@ -1125,7 +1184,7 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, // Error if the target symbol is undefined. Symbol index 0 may be used by // marker relocations, e.g. R_*_NONE and R_ARM_V4BX. Don't error on them. - if (symIndex != 0 && maybeReportUndefined<ELFT>(sym, sec, rel.r_offset)) + if (symIndex != 0 && maybeReportUndefined(sym, sec, rel.r_offset)) return; const uint8_t *relocatedAddr = sec.data().begin() + rel.r_offset; @@ -1269,12 +1328,6 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, // correctly, the IRELATIVE relocations are stored in an array which a // statically linked executable's startup code must enumerate using the // linker-defined symbols __rela?_iplt_{start,end}. - // - // - An absolute relocation to a non-preemptible ifunc (such as a global - // variable containing a pointer to the ifunc) needs to be relocated in - // the exact same way as a GOT entry, so we can avoid needing to make the - // PLT entry canonical by translating such relocations into IRELATIVE - // relocations in the relaIplt. if (!sym.isInPlt()) { // Create PLT and GOTPLT slots for the symbol. sym.isInIplt = true; @@ -1291,17 +1344,6 @@ static void scanReloc(InputSectionBase &sec, OffsetGetter &getOffset, RelTy *&i, *directSym); sym.pltIndex = directSym->pltIndex; } - if (expr == R_ABS && addend == 0 && (sec.flags & SHF_WRITE)) { - // We might be able to represent this as an IRELATIVE. But we don't know - // yet whether some later relocation will make the symbol point to a - // canonical PLT, which would make this either a dynamic RELATIVE (PIC) or - // static (non-PIC) relocation. So we keep a record of the information - // required to process the relocation, and after scanRelocs() has been - // called on all relocations, the relocation is resolved by - // addIRelativeRelocs(). - iRelativeRelocs.push_back({type, &sec, offset, &sym}); - return; - } if (needsGot(expr)) { // Redirect GOT accesses to point to the Igot. // @@ -1362,28 +1404,13 @@ static void scanRelocs(InputSectionBase &sec, ArrayRef<RelTy> rels) { }); } -template <class ELFT> void elf::scanRelocations(InputSectionBase &s) { +template <class ELFT> void scanRelocations(InputSectionBase &s) { if (s.areRelocsRela) scanRelocs<ELFT>(s, s.relas<ELFT>()); else scanRelocs<ELFT>(s, s.rels<ELFT>()); } -// Figure out which representation to use for any absolute relocs to -// non-preemptible ifuncs that we visited during scanRelocs(). -void elf::addIRelativeRelocs() { - for (IRelativeReloc &r : iRelativeRelocs) { - if (r.sym->type == STT_GNU_IFUNC) - in.relaIplt->addReloc( - {target->iRelativeRel, r.sec, r.offset, true, r.sym, 0}); - else if (config->isPic) - addRelativeReloc(r.sec, r.offset, r.sym, 0, R_ABS, r.type); - else - r.sec->relocations.push_back({R_ABS, r.type, r.offset, 0, r.sym}); - } - iRelativeRelocs.clear(); -} - static bool mergeCmp(const InputSection *a, const InputSection *b) { // std::merge requires a strict weak ordering. if (a->outSecOff < b->outSecOff) @@ -1745,11 +1772,6 @@ bool ThunkCreator::createThunks(ArrayRef<OutputSection *> outputSections) { if (pass == 0 && target->getThunkSectionSpacing()) createInitialThunkSections(outputSections); - // With Thunk Size much smaller than branch range we expect to - // converge quickly; if we get to 10 something has gone wrong. - if (pass == 10) - fatal("thunk creation not converged"); - // Create all the Thunks and insert them into synthetic ThunkSections. The // ThunkSections are later inserted back into InputSectionDescriptions. // We separate the creation of ThunkSections from the insertion of the @@ -1809,11 +1831,14 @@ bool ThunkCreator::createThunks(ArrayRef<OutputSection *> outputSections) { return addressesChanged; } -template void elf::scanRelocations<ELF32LE>(InputSectionBase &); -template void elf::scanRelocations<ELF32BE>(InputSectionBase &); -template void elf::scanRelocations<ELF64LE>(InputSectionBase &); -template void elf::scanRelocations<ELF64BE>(InputSectionBase &); -template void elf::reportUndefinedSymbols<ELF32LE>(); -template void elf::reportUndefinedSymbols<ELF32BE>(); -template void elf::reportUndefinedSymbols<ELF64LE>(); -template void elf::reportUndefinedSymbols<ELF64BE>(); +template void scanRelocations<ELF32LE>(InputSectionBase &); +template void scanRelocations<ELF32BE>(InputSectionBase &); +template void scanRelocations<ELF64LE>(InputSectionBase &); +template void scanRelocations<ELF64BE>(InputSectionBase &); +template void reportUndefinedSymbols<ELF32LE>(); +template void reportUndefinedSymbols<ELF32BE>(); +template void reportUndefinedSymbols<ELF64LE>(); +template void reportUndefinedSymbols<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/Relocations.h b/ELF/Relocations.h index d74d7b9b458e..befe15b8f3b9 100644 --- a/ELF/Relocations.h +++ b/ELF/Relocations.h @@ -82,7 +82,6 @@ enum RelExpr { R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC, R_AARCH64_TLSDESC_PAGE, R_ARM_SBREL, - R_HEXAGON_GOT, R_MIPS_GOTREL, R_MIPS_GOT_GP, R_MIPS_GOT_GP_PC, diff --git a/ELF/ScriptLexer.cpp b/ELF/ScriptLexer.cpp index 953a3df8a31c..e0ff56fec3f3 100644 --- a/ELF/ScriptLexer.cpp +++ b/ELF/ScriptLexer.cpp @@ -36,9 +36,9 @@ #include "llvm/ADT/Twine.h" using namespace llvm; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { // Returns a whole line containing the current token. StringRef ScriptLexer::getLine() { StringRef s = getCurrentMB().getBuffer(); @@ -298,3 +298,6 @@ MemoryBufferRef ScriptLexer::getCurrentMB() { return mb; llvm_unreachable("getCurrentMB: failed to find a token"); } + +} // namespace elf +} // namespace lld diff --git a/ELF/ScriptParser.cpp b/ELF/ScriptParser.cpp index 8f0aa660145a..fd8de3b54bd7 100644 --- a/ELF/ScriptParser.cpp +++ b/ELF/ScriptParser.cpp @@ -37,9 +37,9 @@ using namespace llvm; using namespace llvm::ELF; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; +namespace lld { +namespace elf { namespace { class ScriptParser final : ScriptLexer { public: @@ -720,7 +720,7 @@ Expr ScriptParser::readAssert() { return [=] { if (!e().getValue()) - error(msg); + errorOrWarn(msg); return script->getDot(); }; } @@ -1268,7 +1268,7 @@ Expr ScriptParser::readPrimary() { return [=] { return cmd->size; }; } if (tok == "SIZEOF_HEADERS") - return [=] { return elf::getHeaderSize(); }; + return [=] { return getHeaderSize(); }; // Tok is the dot. if (tok == ".") @@ -1344,16 +1344,10 @@ void ScriptParser::readAnonymousDeclaration() { std::vector<SymbolVersion> locals; std::vector<SymbolVersion> globals; std::tie(locals, globals) = readSymbols(); - - for (SymbolVersion v : locals) { - if (v.name == "*") - config->defaultSymbolVersion = VER_NDX_LOCAL; - else - config->versionScriptLocals.push_back(v); - } - - for (SymbolVersion v : globals) - config->versionScriptGlobals.push_back(v); + for (const SymbolVersion &pat : locals) + config->versionDefinitions[VER_NDX_LOCAL].patterns.push_back(pat); + for (const SymbolVersion &pat : globals) + config->versionDefinitions[VER_NDX_GLOBAL].patterns.push_back(pat); expect(";"); } @@ -1365,22 +1359,14 @@ void ScriptParser::readVersionDeclaration(StringRef verStr) { std::vector<SymbolVersion> locals; std::vector<SymbolVersion> globals; std::tie(locals, globals) = readSymbols(); - - for (SymbolVersion v : locals) { - if (v.name == "*") - config->defaultSymbolVersion = VER_NDX_LOCAL; - else - config->versionScriptLocals.push_back(v); - } + for (const SymbolVersion &pat : locals) + config->versionDefinitions[VER_NDX_LOCAL].patterns.push_back(pat); // Create a new version definition and add that to the global symbols. VersionDefinition ver; ver.name = verStr; - ver.globals = globals; - - // User-defined version number starts from 2 because 0 and 1 are - // reserved for VER_NDX_LOCAL and VER_NDX_GLOBAL, respectively. - ver.id = config->versionDefinitions.size() + 2; + ver.patterns = globals; + ver.id = config->versionDefinitions.size(); config->versionDefinitions.push_back(ver); // Each version may have a parent version. For example, "Ver2" @@ -1525,18 +1511,19 @@ std::pair<uint32_t, uint32_t> ScriptParser::readMemoryAttributes() { return {flags, negFlags}; } -void elf::readLinkerScript(MemoryBufferRef mb) { +void readLinkerScript(MemoryBufferRef mb) { ScriptParser(mb).readLinkerScript(); } -void elf::readVersionScript(MemoryBufferRef mb) { +void readVersionScript(MemoryBufferRef mb) { ScriptParser(mb).readVersionScript(); } -void elf::readDynamicList(MemoryBufferRef mb) { - ScriptParser(mb).readDynamicList(); -} +void readDynamicList(MemoryBufferRef mb) { ScriptParser(mb).readDynamicList(); } -void elf::readDefsym(StringRef name, MemoryBufferRef mb) { +void readDefsym(StringRef name, MemoryBufferRef mb) { ScriptParser(mb).readDefsym(name); } + +} // namespace elf +} // namespace lld diff --git a/ELF/SymbolTable.cpp b/ELF/SymbolTable.cpp index 3faeed8c2bdc..5f6008ef908b 100644 --- a/ELF/SymbolTable.cpp +++ b/ELF/SymbolTable.cpp @@ -27,10 +27,9 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; - -SymbolTable *elf::symtab; +namespace lld { +namespace elf { +SymbolTable *symtab; void SymbolTable::wrap(Symbol *sym, Symbol *real, Symbol *wrap) { // Swap symbols as instructed by -wrap. @@ -71,21 +70,26 @@ Symbol *SymbolTable::insert(StringRef name) { Symbol *sym = reinterpret_cast<Symbol *>(make<SymbolUnion>()); symVector.push_back(sym); + // *sym was not initialized by a constructor. Fields that may get referenced + // when it is a placeholder must be initialized here. sym->setName(name); sym->symbolKind = Symbol::PlaceholderKind; - sym->versionId = config->defaultSymbolVersion; + sym->versionId = VER_NDX_GLOBAL; sym->visibility = STV_DEFAULT; sym->isUsedInRegularObj = false; sym->exportDynamic = false; + sym->inDynamicList = false; sym->canInline = true; + sym->referenced = false; + sym->traced = false; sym->scriptDefined = false; sym->partition = 1; return sym; } -Symbol *SymbolTable::addSymbol(const Symbol &New) { - Symbol *sym = symtab->insert(New.getName()); - sym->resolve(New); +Symbol *SymbolTable::addSymbol(const Symbol &newSym) { + Symbol *sym = symtab->insert(newSym.getName()); + sym->resolve(newSym); return sym; } @@ -118,10 +122,7 @@ StringMap<std::vector<Symbol *>> &SymbolTable::getDemangledSyms() { for (Symbol *sym : symVector) { if (!sym->isDefined() && !sym->isCommon()) continue; - if (Optional<std::string> s = demangleItanium(sym->getName())) - (*demangledSyms)[*s].push_back(sym); - else - (*demangledSyms)[sym->getName()].push_back(sym); + (*demangledSyms)[demangleItanium(sym->getName())].push_back(sym); } } return *demangledSyms; @@ -162,12 +163,8 @@ void SymbolTable::handleDynamicList() { else syms = findByVersion(ver); - for (Symbol *b : syms) { - if (!config->shared) - b->exportDynamic = true; - else if (b->includeInDynsym()) - b->isPreemptible = true; - } + for (Symbol *sym : syms) + sym->inDynamicList = true; } } @@ -192,7 +189,7 @@ void SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId, return "VER_NDX_LOCAL"; if (ver == VER_NDX_GLOBAL) return "VER_NDX_GLOBAL"; - return ("version '" + config->versionDefinitions[ver - 2].name + "'").str(); + return ("version '" + config->versionDefinitions[ver].name + "'").str(); }; // Assign the version. @@ -203,8 +200,12 @@ void SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId, if (sym->getName().contains('@')) continue; - if (sym->versionId == config->defaultSymbolVersion) + // If the version has not been assigned, verdefIndex is -1. Use an arbitrary + // number (0) to indicate the version has been assigned. + if (sym->verdefIndex == UINT32_C(-1)) { + sym->verdefIndex = 0; sym->versionId = versionId; + } if (sym->versionId == versionId) continue; @@ -214,15 +215,14 @@ void SymbolTable::assignExactVersion(SymbolVersion ver, uint16_t versionId, } void SymbolTable::assignWildcardVersion(SymbolVersion ver, uint16_t versionId) { - if (!ver.hasWildcard) - return; - // Exact matching takes precendence over fuzzy matching, // so we set a version to a symbol only if no version has been assigned // to the symbol. This behavior is compatible with GNU. - for (Symbol *b : findAllByVersion(ver)) - if (b->versionId == config->defaultSymbolVersion) - b->versionId = versionId; + for (Symbol *sym : findAllByVersion(ver)) + if (sym->verdefIndex == UINT32_C(-1)) { + sym->verdefIndex = 0; + sym->versionId = versionId; + } } // This function processes version scripts by updating the versionId @@ -233,26 +233,24 @@ void SymbolTable::assignWildcardVersion(SymbolVersion ver, uint16_t versionId) { void SymbolTable::scanVersionScript() { // First, we assign versions to exact matching symbols, // i.e. version definitions not containing any glob meta-characters. - for (SymbolVersion &ver : config->versionScriptGlobals) - assignExactVersion(ver, VER_NDX_GLOBAL, "global"); - for (SymbolVersion &ver : config->versionScriptLocals) - assignExactVersion(ver, VER_NDX_LOCAL, "local"); for (VersionDefinition &v : config->versionDefinitions) - for (SymbolVersion &ver : v.globals) - assignExactVersion(ver, v.id, v.name); - - // Next, we assign versions to fuzzy matching symbols, - // i.e. version definitions containing glob meta-characters. - for (SymbolVersion &ver : config->versionScriptGlobals) - assignWildcardVersion(ver, VER_NDX_GLOBAL); - for (SymbolVersion &ver : config->versionScriptLocals) - assignWildcardVersion(ver, VER_NDX_LOCAL); - - // Note that because the last match takes precedence over previous matches, - // we iterate over the definitions in the reverse order. + for (SymbolVersion &pat : v.patterns) + assignExactVersion(pat, v.id, v.name); + + // Next, assign versions to wildcards that are not "*". Note that because the + // last match takes precedence over previous matches, we iterate over the + // definitions in the reverse order. for (VersionDefinition &v : llvm::reverse(config->versionDefinitions)) - for (SymbolVersion &ver : v.globals) - assignWildcardVersion(ver, v.id); + for (SymbolVersion &pat : v.patterns) + if (pat.hasWildcard && pat.name != "*") + assignWildcardVersion(pat, v.id); + + // Then, assign versions to "*". In GNU linkers they have lower priority than + // other wildcards. + for (VersionDefinition &v : config->versionDefinitions) + for (SymbolVersion &pat : v.patterns) + if (pat.hasWildcard && pat.name == "*") + assignWildcardVersion(pat, v.id); // Symbol themselves might know their versions because symbols // can contain versions in the form of <name>@<version>. @@ -262,7 +260,10 @@ void SymbolTable::scanVersionScript() { // isPreemptible is false at this point. To correctly compute the binding of a // Defined (which is used by includeInDynsym()), we need to know if it is - // VER_NDX_LOCAL or not. If defaultSymbolVersion is VER_NDX_LOCAL, we should - // compute symbol versions before handling --dynamic-list. + // VER_NDX_LOCAL or not. Compute symbol versions before handling + // --dynamic-list. handleDynamicList(); } + +} // namespace elf +} // namespace lld diff --git a/ELF/SymbolTable.h b/ELF/SymbolTable.h index b64707f4ab02..d3be0cb6450f 100644 --- a/ELF/SymbolTable.h +++ b/ELF/SymbolTable.h @@ -43,7 +43,7 @@ public: Symbol *insert(StringRef name); - Symbol *addSymbol(const Symbol &New); + Symbol *addSymbol(const Symbol &newSym); void scanVersionScript(); diff --git a/ELF/Symbols.cpp b/ELF/Symbols.cpp index 62c552e04828..c0cba21cfe8d 100644 --- a/ELF/Symbols.cpp +++ b/ELF/Symbols.cpp @@ -23,9 +23,20 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; +namespace lld { +// Returns a symbol for an error message. +static std::string demangle(StringRef symName) { + if (elf::config->demangle) + return demangleItanium(symName); + return symName; +} +std::string toString(const elf::Symbol &b) { return demangle(b.getName()); } +std::string toELFString(const Archive::Symbol &b) { + return demangle(b.getName()); +} + +namespace elf { Defined *ElfSym::bss; Defined *ElfSym::etext1; Defined *ElfSym::etext2; @@ -213,7 +224,7 @@ void Symbol::parseSymbolVersion() { if (isDefault) verstr = verstr.substr(1); - for (VersionDefinition &ver : config->versionDefinitions) { + for (const VersionDefinition &ver : namedVersionDefs()) { if (ver.name != verstr) continue; @@ -250,20 +261,20 @@ void Symbol::fetch() const { } MemoryBufferRef LazyArchive::getMemberBuffer() { - Archive::Child c = CHECK( - sym.getMember(), "could not get the member for symbol " + sym.getName()); + Archive::Child c = + CHECK(sym.getMember(), + "could not get the member for symbol " + toELFString(sym)); return CHECK(c.getMemoryBufferRef(), "could not get the buffer for the member defining symbol " + - sym.getName()); + toELFString(sym)); } uint8_t Symbol::computeBinding() const { if (config->relocatable) return binding; - if (visibility != STV_DEFAULT && visibility != STV_PROTECTED) - return STB_LOCAL; - if (versionId == VER_NDX_LOCAL && isDefined() && !isPreemptible) + if ((visibility != STV_DEFAULT && visibility != STV_PROTECTED) || + versionId == VER_NDX_LOCAL) return STB_LOCAL; if (!config->gnuUnique && binding == STB_GNU_UNIQUE) return STB_GLOBAL; @@ -281,11 +292,11 @@ bool Symbol::includeInDynsym() const { if (isUndefWeak() && config->pie && sharedFiles.empty()) return false; - return isUndefined() || isShared() || exportDynamic; + return isUndefined() || isShared() || exportDynamic || inDynamicList; } // Print out a log message for --trace-symbol. -void elf::printTraceSymbol(const Symbol *sym) { +void printTraceSymbol(const Symbol *sym) { std::string s; if (sym->isUndefined()) s = ": reference to "; @@ -301,7 +312,7 @@ void elf::printTraceSymbol(const Symbol *sym) { message(toString(sym->file) + s + sym->getName()); } -void elf::maybeWarnUnorderableSymbol(const Symbol *sym) { +void maybeWarnUnorderableSymbol(const Symbol *sym) { if (!config->warnSymbolOrdering) return; @@ -331,14 +342,6 @@ void elf::maybeWarnUnorderableSymbol(const Symbol *sym) { report(": unable to order discarded symbol: "); } -// Returns a symbol for an error message. -std::string lld::toString(const Symbol &b) { - if (config->demangle) - if (Optional<std::string> s = demangleItanium(b.getName())) - return *s; - return b.getName(); -} - static uint8_t getMinVisibility(uint8_t va, uint8_t vb) { if (va == STV_DEFAULT) return vb; @@ -485,17 +488,13 @@ void Symbol::resolveUndefined(const Undefined &other) { if (dyn_cast_or_null<SharedFile>(other.file)) return; - if (isUndefined()) { - // The binding may "upgrade" from weak to non-weak. - if (other.binding != STB_WEAK) + if (isUndefined() || isShared()) { + // The binding will be weak if there is at least one reference and all are + // weak. The binding has one opportunity to change to weak: if the first + // reference is weak. + if (other.binding != STB_WEAK || !referenced) binding = other.binding; - } else if (auto *s = dyn_cast<SharedSymbol>(this)) { - // The binding of a SharedSymbol will be weak if there is at least one - // reference and all are weak. The binding has one opportunity to change to - // weak: if the first reference is weak. - if (other.binding != STB_WEAK || !s->referenced) - binding = other.binding; - s->referenced = true; + referenced = true; } } @@ -553,7 +552,7 @@ int Symbol::compare(const Symbol *other) const { auto *oldSym = cast<Defined>(this); auto *newSym = cast<Defined>(other); - if (other->file && isa<BitcodeFile>(other->file)) + if (dyn_cast_or_null<BitcodeFile>(other->file)) return 0; if (!oldSym->section && !newSym->section && oldSym->value == newSym->value && @@ -651,6 +650,9 @@ void Symbol::resolveShared(const SharedSymbol &other) { uint8_t bind = binding; replace(other); binding = bind; - cast<SharedSymbol>(this)->referenced = true; + referenced = true; } } + +} // namespace elf +} // namespace lld diff --git a/ELF/Symbols.h b/ELF/Symbols.h index d640495b0e01..d43568fe295c 100644 --- a/ELF/Symbols.h +++ b/ELF/Symbols.h @@ -21,6 +21,13 @@ #include "llvm/Object/ELF.h" namespace lld { +std::string toString(const elf::Symbol &); + +// There are two different ways to convert an Archive::Symbol to a string: +// One for Microsoft name mangling and one for Itanium name mangling. +// Call the functions toCOFFString and toELFString, not just toString. +std::string toELFString(const llvm::object::Archive::Symbol &); + namespace elf { class CommonSymbol; class Defined; @@ -30,12 +37,6 @@ class LazyObject; class SharedSymbol; class Symbol; class Undefined; -} // namespace elf - -std::string toString(const elf::Symbol &); -std::string toString(const elf::InputFile *); - -namespace elf { // This is a StringRef-like container that doesn't run strlen(). // @@ -112,21 +113,35 @@ public: // are unreferenced except by other bitcode objects. unsigned isUsedInRegularObj : 1; - // If this flag is true and the symbol has protected or default visibility, it - // will appear in .dynsym. This flag is set by interposable DSO symbols in - // executables, by most symbols in DSOs and executables built with - // --export-dynamic, and by dynamic lists. + // Used by a Defined symbol with protected or default visibility, to record + // whether it is required to be exported into .dynsym. This is set when any of + // the following conditions hold: + // + // - If there is an interposable symbol from a DSO. + // - If -shared or --export-dynamic is specified, any symbol in an object + // file/bitcode sets this property, unless suppressed by LTO + // canBeOmittedFromSymbolTable(). unsigned exportDynamic : 1; + // True if the symbol is in the --dynamic-list file. A Defined symbol with + // protected or default visibility with this property is required to be + // exported into .dynsym. + unsigned inDynamicList : 1; + // False if LTO shouldn't inline whatever this symbol points to. If a symbol // is overwritten after LTO, LTO shouldn't inline the symbol because it // doesn't know the final contents of the symbol. unsigned canInline : 1; + // Used by Undefined and SharedSymbol to track if there has been at least one + // undefined reference to the symbol. The binding may change to STB_WEAK if + // the first undefined reference from a non-shared object is weak. + unsigned referenced : 1; + // True if this symbol is specified by --trace-symbol option. unsigned traced : 1; - inline void replace(const Symbol &New); + inline void replace(const Symbol &newSym); bool includeInDynsym() const; uint8_t computeBinding() const; @@ -224,9 +239,10 @@ protected: : file(file), nameData(name.data), nameSize(name.size), binding(binding), type(type), stOther(stOther), symbolKind(k), visibility(stOther & 3), isUsedInRegularObj(!file || file->kind() == InputFile::ObjKind), - exportDynamic(isExportDynamic(k, visibility)), canInline(false), - traced(false), needsPltAddr(false), isInIplt(false), gotInIgot(false), - isPreemptible(false), used(!config->gcSections), needsTocRestore(false), + exportDynamic(isExportDynamic(k, visibility)), inDynamicList(false), + canInline(false), referenced(false), traced(false), needsPltAddr(false), + isInIplt(false), gotInIgot(false), isPreemptible(false), + used(!config->gcSections), needsTocRestore(false), scriptDefined(false) {} public: @@ -363,11 +379,6 @@ public: uint64_t value; // st_value uint64_t size; // st_size uint32_t alignment; - - // This is true if there has been at least one undefined reference to the - // symbol. The binding may change to STB_WEAK if the first undefined reference - // is weak. - bool referenced = false; }; // LazyArchive and LazyObject represent a symbols that is not yet in the link, @@ -507,7 +518,7 @@ size_t Symbol::getSymbolSize() const { // replace() replaces "this" object with a given symbol by memcpy'ing // it over to "this". This function is called as a result of name // resolution, e.g. to replace an undefind symbol with a defined symbol. -void Symbol::replace(const Symbol &New) { +void Symbol::replace(const Symbol &newSym) { using llvm::ELF::STT_TLS; // Symbols representing thread-local variables must be referenced by @@ -515,22 +526,23 @@ void Symbol::replace(const Symbol &New) { // non-TLS relocations, so there's a clear distinction between TLS // and non-TLS symbols. It is an error if the same symbol is defined // as a TLS symbol in one file and as a non-TLS symbol in other file. - if (symbolKind != PlaceholderKind && !isLazy() && !New.isLazy()) { - bool tlsMismatch = (type == STT_TLS && New.type != STT_TLS) || - (type != STT_TLS && New.type == STT_TLS); - if (tlsMismatch) - error("TLS attribute mismatch: " + toString(*this) + "\n>>> defined in " + - toString(New.file) + "\n>>> defined in " + toString(file)); - } + if (symbolKind != PlaceholderKind && !isLazy() && !newSym.isLazy() && + (type == STT_TLS) != (newSym.type == STT_TLS)) + error("TLS attribute mismatch: " + toString(*this) + "\n>>> defined in " + + toString(newSym.file) + "\n>>> defined in " + toString(file)); Symbol old = *this; - memcpy(this, &New, New.getSymbolSize()); + memcpy(this, &newSym, newSym.getSymbolSize()); + // old may be a placeholder. The referenced fields must be initialized in + // SymbolTable::insert. versionId = old.versionId; visibility = old.visibility; isUsedInRegularObj = old.isUsedInRegularObj; exportDynamic = old.exportDynamic; + inDynamicList = old.inDynamicList; canInline = old.canInline; + referenced = old.referenced; traced = old.traced; isPreemptible = old.isPreemptible; scriptDefined = old.scriptDefined; diff --git a/ELF/SyntheticSections.cpp b/ELF/SyntheticSections.cpp index f6d0f190d84d..ff35bb7bd10c 100644 --- a/ELF/SyntheticSections.cpp +++ b/ELF/SyntheticSections.cpp @@ -45,13 +45,12 @@ using namespace llvm::ELF; using namespace llvm::object; using namespace llvm::support; -using namespace lld; -using namespace lld::elf; - using llvm::support::endian::read32le; using llvm::support::endian::write32le; using llvm::support::endian::write64le; +namespace lld { +namespace elf { constexpr size_t MergeNoTailSection::numShards; static uint64_t readUint(uint8_t *buf) { @@ -82,7 +81,7 @@ static ArrayRef<uint8_t> getVersion() { // With this feature, you can identify LLD-generated binaries easily // by "readelf --string-dump .comment <file>". // The returned object is a mergeable string section. -MergeInputSection *elf::createCommentSection() { +MergeInputSection *createCommentSection() { return make<MergeInputSection>(SHF_MERGE | SHF_STRINGS, SHT_PROGBITS, 1, getVersion(), ".comment"); } @@ -138,7 +137,7 @@ MipsAbiFlagsSection<ELFT> *MipsAbiFlagsSection<ELFT>::create() { flags.ases |= s->ases; flags.flags1 |= s->flags1; flags.flags2 |= s->flags2; - flags.fp_abi = elf::getMipsFpAbiFlag(flags.fp_abi, s->fp_abi, filename); + flags.fp_abi = getMipsFpAbiFlag(flags.fp_abi, s->fp_abi, filename); }; if (create) @@ -252,19 +251,17 @@ MipsReginfoSection<ELFT> *MipsReginfoSection<ELFT>::create() { return make<MipsReginfoSection<ELFT>>(reginfo); } -InputSection *elf::createInterpSection() { +InputSection *createInterpSection() { // StringSaver guarantees that the returned string ends with '\0'. StringRef s = saver.save(config->dynamicLinker); ArrayRef<uint8_t> contents = {(const uint8_t *)s.data(), s.size() + 1}; - auto *sec = make<InputSection>(nullptr, SHF_ALLOC, SHT_PROGBITS, 1, contents, - ".interp"); - sec->markLive(); - return sec; + return make<InputSection>(nullptr, SHF_ALLOC, SHT_PROGBITS, 1, contents, + ".interp"); } -Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value, - uint64_t size, InputSectionBase §ion) { +Defined *addSyntheticLocal(StringRef name, uint8_t type, uint64_t value, + uint64_t size, InputSectionBase §ion) { auto *s = make<Defined>(section.file, name, STB_LOCAL, STV_DEFAULT, type, value, size, §ion); if (in.symTab) @@ -402,7 +399,7 @@ bool EhFrameSection::isFdeLive(EhSectionPiece &fde, ArrayRef<RelTy> rels) { // a list of FDEs. This function searches an existing CIE or create a new // one and associates FDEs to the CIE. template <class ELFT, class RelTy> -void EhFrameSection::addSectionAux(EhInputSection *sec, ArrayRef<RelTy> rels) { +void EhFrameSection::addRecords(EhInputSection *sec, ArrayRef<RelTy> rels) { offsetToCie.clear(); for (EhSectionPiece &piece : sec->pieces) { // The empty record is the end marker. @@ -428,8 +425,17 @@ void EhFrameSection::addSectionAux(EhInputSection *sec, ArrayRef<RelTy> rels) { } } -template <class ELFT> void EhFrameSection::addSection(InputSectionBase *c) { - auto *sec = cast<EhInputSection>(c); +template <class ELFT> +void EhFrameSection::addSectionAux(EhInputSection *sec) { + if (!sec->isLive()) + return; + if (sec->areRelocsRela) + addRecords<ELFT>(sec, sec->template relas<ELFT>()); + else + addRecords<ELFT>(sec, sec->template rels<ELFT>()); +} + +void EhFrameSection::addSection(EhInputSection *sec) { sec->parent = this; alignment = std::max(alignment, sec->alignment); @@ -437,14 +443,6 @@ template <class ELFT> void EhFrameSection::addSection(InputSectionBase *c) { for (auto *ds : sec->dependentSections) dependentSections.push_back(ds); - - if (sec->pieces.empty()) - return; - - if (sec->areRelocsRela) - addSectionAux<ELFT>(sec, sec->template relas<ELFT>()); - else - addSectionAux<ELFT>(sec, sec->template rels<ELFT>()); } static void writeCieFde(uint8_t *buf, ArrayRef<uint8_t> d) { @@ -461,6 +459,28 @@ static void writeCieFde(uint8_t *buf, ArrayRef<uint8_t> d) { void EhFrameSection::finalizeContents() { assert(!this->size); // Not finalized. + + switch (config->ekind) { + case ELFNoneKind: + llvm_unreachable("invalid ekind"); + case ELF32LEKind: + for (EhInputSection *sec : sections) + addSectionAux<ELF32LE>(sec); + break; + case ELF32BEKind: + for (EhInputSection *sec : sections) + addSectionAux<ELF32BE>(sec); + break; + case ELF64LEKind: + for (EhInputSection *sec : sections) + addSectionAux<ELF64LE>(sec); + break; + case ELF64BEKind: + for (EhInputSection *sec : sections) + addSectionAux<ELF64BE>(sec); + break; + } + size_t off = 0; for (CieRecord *rec : cieRecords) { rec->cie->outputOff = off; @@ -1162,10 +1182,12 @@ void StringTableSection::writeTo(uint8_t *buf) { } } -// Returns the number of version definition entries. Because the first entry -// is for the version definition itself, it is the number of versioned symbols -// plus one. Note that we don't support multiple versions yet. -static unsigned getVerDefNum() { return config->versionDefinitions.size() + 1; } +// Returns the number of entries in .gnu.version_d: the number of +// non-VER_NDX_LOCAL-non-VER_NDX_GLOBAL definitions, plus 1. +// Note that we don't support vd_cnt > 1 yet. +static unsigned getVerDefNum() { + return namedVersionDefs().size() + 1; +} template <class ELFT> DynamicSection<ELFT>::DynamicSection() @@ -1218,6 +1240,25 @@ void DynamicSection<ELFT>::addSym(int32_t tag, Symbol *sym) { entries.push_back({tag, [=] { return sym->getVA(); }}); } +// The output section .rela.dyn may include these synthetic sections: +// +// - part.relaDyn +// - in.relaIplt: this is included if in.relaIplt is named .rela.dyn +// - in.relaPlt: this is included if a linker script places .rela.plt inside +// .rela.dyn +// +// DT_RELASZ is the total size of the included sections. +static std::function<uint64_t()> addRelaSz(RelocationBaseSection *relaDyn) { + return [=]() { + size_t size = relaDyn->getSize(); + if (in.relaIplt->getParent() == relaDyn->getParent()) + size += in.relaIplt->getSize(); + if (in.relaPlt->getParent() == relaDyn->getParent()) + size += in.relaPlt->getSize(); + return size; + }; +} + // A Linker script may assign the RELA relocation sections to the same // output section. When this occurs we cannot just use the OutputSection // Size. Moreover the [DT_JMPREL, DT_JMPREL + DT_PLTRELSZ) is permitted to @@ -1232,7 +1273,7 @@ static uint64_t addPltRelSz() { // Add remaining entries to complete .dynamic contents. template <class ELFT> void DynamicSection<ELFT>::finalizeContents() { - elf::Partition &part = getPartition(); + Partition &part = getPartition(); bool isMain = part.name.empty(); for (StringRef s : config->filterList) @@ -1306,9 +1347,11 @@ template <class ELFT> void DynamicSection<ELFT>::finalizeContents() { if (OutputSection *sec = part.dynStrTab->getParent()) this->link = sec->sectionIndex; - if (part.relaDyn->isNeeded()) { + if (part.relaDyn->isNeeded() || + (in.relaIplt->isNeeded() && + part.relaDyn->getParent() == in.relaIplt->getParent())) { addInSec(part.relaDyn->dynamicTag, part.relaDyn); - addSize(part.relaDyn->sizeDynamicTag, part.relaDyn->getParent()); + entries.push_back({part.relaDyn->sizeDynamicTag, addRelaSz(part.relaDyn)}); bool isRela = config->isRela; addInt(isRela ? DT_RELAENT : DT_RELENT, @@ -1679,6 +1722,56 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() { relativeGroups.emplace_back(std::move(group)); } + // For non-relative relocations, we would like to: + // 1. Have relocations with the same symbol offset to be consecutive, so + // that the runtime linker can speed-up symbol lookup by implementing an + // 1-entry cache. + // 2. Group relocations by r_info to reduce the size of the relocation + // section. + // Since the symbol offset is the high bits in r_info, sorting by r_info + // allows us to do both. + // + // For Rela, we also want to sort by r_addend when r_info is the same. This + // enables us to group by r_addend as well. + llvm::stable_sort(nonRelatives, [](const Elf_Rela &a, const Elf_Rela &b) { + if (a.r_info != b.r_info) + return a.r_info < b.r_info; + if (config->isRela) + return a.r_addend < b.r_addend; + return false; + }); + + // Group relocations with the same r_info. Note that each group emits a group + // header and that may make the relocation section larger. It is hard to + // estimate the size of a group header as the encoded size of that varies + // based on r_info. However, we can approximate this trade-off by the number + // of values encoded. Each group header contains 3 values, and each relocation + // in a group encodes one less value, as compared to when it is not grouped. + // Therefore, we only group relocations if there are 3 or more of them with + // the same r_info. + // + // For Rela, the addend for most non-relative relocations is zero, and thus we + // can usually get a smaller relocation section if we group relocations with 0 + // addend as well. + std::vector<Elf_Rela> ungroupedNonRelatives; + std::vector<std::vector<Elf_Rela>> nonRelativeGroups; + for (auto i = nonRelatives.begin(), e = nonRelatives.end(); i != e;) { + auto j = i + 1; + while (j != e && i->r_info == j->r_info && + (!config->isRela || i->r_addend == j->r_addend)) + ++j; + if (j - i < 3 || (config->isRela && i->r_addend != 0)) + ungroupedNonRelatives.insert(ungroupedNonRelatives.end(), i, j); + else + nonRelativeGroups.emplace_back(i, j); + i = j; + } + + // Sort ungrouped relocations by offset to minimize the encoded length. + llvm::sort(ungroupedNonRelatives, [](const Elf_Rela &a, const Elf_Rela &b) { + return a.r_offset < b.r_offset; + }); + unsigned hasAddendIfRela = config->isRela ? RELOCATION_GROUP_HAS_ADDEND_FLAG : 0; @@ -1733,14 +1826,23 @@ bool AndroidPackedRelocationSection<ELFT>::updateAllocSize() { } } - // Finally the non-relative relocations. - llvm::sort(nonRelatives, [](const Elf_Rela &a, const Elf_Rela &b) { - return a.r_offset < b.r_offset; - }); - if (!nonRelatives.empty()) { - add(nonRelatives.size()); + // Grouped non-relatives. + for (ArrayRef<Elf_Rela> g : nonRelativeGroups) { + add(g.size()); + add(RELOCATION_GROUPED_BY_INFO_FLAG); + add(g[0].r_info); + for (const Elf_Rela &r : g) { + add(r.r_offset - offset); + offset = r.r_offset; + } + addend = 0; + } + + // Finally the ungrouped non-relative relocations. + if (!ungroupedNonRelatives.empty()) { + add(ungroupedNonRelatives.size()); add(hasAddendIfRela); - for (Elf_Rela &r : nonRelatives) { + for (Elf_Rela &r : ungroupedNonRelatives) { add(r.r_offset - offset); offset = r.r_offset; add(r.r_info); @@ -1852,6 +1954,14 @@ template <class ELFT> bool RelrSection<ELFT>::updateAllocSize() { } } + // Don't allow the section to shrink; otherwise the size of the section can + // oscillate infinitely. Trailing 1s do not decode to more relocations. + if (relrRelocs.size() < oldSize) { + log(".relr.dyn needs " + Twine(oldSize - relrRelocs.size()) + + " padding word(s)"); + relrRelocs.resize(oldSize, Elf_Relr(1)); + } + return relrRelocs.size() != oldSize; } @@ -2452,6 +2562,10 @@ readAddressAreas(DWARFContext &dwarf, InputSection *sec) { uint32_t cuIdx = 0; for (std::unique_ptr<DWARFUnit> &cu : dwarf.compile_units()) { + if (Error e = cu->tryExtractDIEsIfNeeded(false)) { + error(toString(sec) + ": " + toString(std::move(e))); + return {}; + } Expected<DWARFAddressRangesVector> ranges = cu->collectAddressRanges(); if (!ranges) { error(toString(sec) + ": " + toString(ranges.takeError())); @@ -2481,9 +2595,9 @@ readAddressAreas(DWARFContext &dwarf, InputSection *sec) { template <class ELFT> static std::vector<GdbIndexSection::NameAttrEntry> readPubNamesAndTypes(const LLDDwarfObj<ELFT> &obj, - const std::vector<GdbIndexSection::CuEntry> &cUs) { - const DWARFSection &pubNames = obj.getGnuPubNamesSection(); - const DWARFSection &pubTypes = obj.getGnuPubTypesSection(); + const std::vector<GdbIndexSection::CuEntry> &cus) { + const DWARFSection &pubNames = obj.getGnuPubnamesSection(); + const DWARFSection &pubTypes = obj.getGnuPubtypesSection(); std::vector<GdbIndexSection::NameAttrEntry> ret; for (const DWARFSection *pub : {&pubNames, &pubTypes}) { @@ -2493,12 +2607,11 @@ readPubNamesAndTypes(const LLDDwarfObj<ELFT> &obj, // don't know how many compilation units precede this object to compute // cuIndex, we compute (kind << 24 | cuIndexInThisObject) instead, and add // the number of preceding compilation units later. - uint32_t i = - lower_bound(cUs, set.Offset, - [](GdbIndexSection::CuEntry cu, uint32_t offset) { - return cu.cuOffset < offset; - }) - - cUs.begin(); + uint32_t i = llvm::partition_point(cus, + [&](GdbIndexSection::CuEntry cu) { + return cu.cuOffset < set.Offset; + }) - + cus.begin(); for (const DWARFDebugPubTable::Entry &ent : set.Entries) ret.push_back({{ent.Name, computeGdbHash(ent.Name)}, (ent.Descriptor.toBits() << 24) | i}); @@ -2603,7 +2716,7 @@ template <class ELFT> GdbIndexSection *GdbIndexSection::create() { parallelForEachN(0, sections.size(), [&](size_t i) { ObjFile<ELFT> *file = sections[i]->getFile<ELFT>(); - DWARFContext dwarf(make_unique<LLDDwarfObj<ELFT>>(file)); + DWARFContext dwarf(std::make_unique<LLDDwarfObj<ELFT>>(file)); chunks[i].sec = sections[i]; chunks[i].compilationUnits = readCuList(dwarf); @@ -2750,7 +2863,7 @@ StringRef VersionDefinitionSection::getFileDefName() { void VersionDefinitionSection::finalizeContents() { fileDefNameOff = getPartition().dynStrTab->addString(getFileDefName()); - for (VersionDefinition &v : config->versionDefinitions) + for (const VersionDefinition &v : namedVersionDefs()) verDefNameOffs.push_back(getPartition().dynStrTab->addString(v.name)); if (OutputSection *sec = getPartition().dynStrTab->getParent()) @@ -2784,7 +2897,7 @@ void VersionDefinitionSection::writeTo(uint8_t *buf) { writeOne(buf, 1, getFileDefName(), fileDefNameOff); auto nameOffIt = verDefNameOffs.begin(); - for (VersionDefinition &v : config->versionDefinitions) { + for (const VersionDefinition &v : namedVersionDefs()) { buf += EntrySize; writeOne(buf, v.id, v.name, *nameOffIt++); } @@ -2826,7 +2939,7 @@ bool VersionTableSection::isNeeded() const { return getPartition().verDef || getPartition().verNeed->isNeeded(); } -void elf::addVerneed(Symbol *ss) { +void addVerneed(Symbol *ss) { auto &file = cast<SharedFile>(*ss->file); if (ss->verdefIndex == VER_NDX_GLOBAL) { ss->versionId = VER_NDX_GLOBAL; @@ -3009,17 +3122,16 @@ void MergeNoTailSection::finalizeContents() { }); } -static MergeSyntheticSection *createMergeSynthetic(StringRef name, - uint32_t type, - uint64_t flags, - uint32_t alignment) { +MergeSyntheticSection *createMergeSynthetic(StringRef name, uint32_t type, + uint64_t flags, + uint32_t alignment) { bool shouldTailMerge = (flags & SHF_STRINGS) && config->optimize >= 2; if (shouldTailMerge) return make<MergeTailSection>(name, type, flags, alignment); return make<MergeNoTailSection>(name, type, flags, alignment); } -template <class ELFT> void elf::splitSections() { +template <class ELFT> void splitSections() { // splitIntoPieces needs to be called on each MergeInputSection // before calling finalizeContents(). parallelForEach(inputSections, [](InputSectionBase *sec) { @@ -3030,63 +3142,6 @@ template <class ELFT> void elf::splitSections() { }); } -// This function scans over the inputsections to create mergeable -// synthetic sections. -// -// It removes MergeInputSections from the input section array and adds -// new synthetic sections at the location of the first input section -// that it replaces. It then finalizes each synthetic section in order -// to compute an output offset for each piece of each input section. -void elf::mergeSections() { - std::vector<MergeSyntheticSection *> mergeSections; - for (InputSectionBase *&s : inputSections) { - MergeInputSection *ms = dyn_cast<MergeInputSection>(s); - if (!ms) - continue; - - // We do not want to handle sections that are not alive, so just remove - // them instead of trying to merge. - if (!ms->isLive()) { - s = nullptr; - continue; - } - - StringRef outsecName = getOutputSectionName(ms); - - auto i = llvm::find_if(mergeSections, [=](MergeSyntheticSection *sec) { - // While we could create a single synthetic section for two different - // values of Entsize, it is better to take Entsize into consideration. - // - // With a single synthetic section no two pieces with different Entsize - // could be equal, so we may as well have two sections. - // - // Using Entsize in here also allows us to propagate it to the synthetic - // section. - // - // SHF_STRINGS section with different alignments should not be merged. - return sec->name == outsecName && sec->flags == ms->flags && - sec->entsize == ms->entsize && - (sec->alignment == ms->alignment || !(sec->flags & SHF_STRINGS)); - }); - if (i == mergeSections.end()) { - MergeSyntheticSection *syn = - createMergeSynthetic(outsecName, ms->type, ms->flags, ms->alignment); - mergeSections.push_back(syn); - i = std::prev(mergeSections.end()); - s = syn; - syn->entsize = ms->entsize; - } else { - s = nullptr; - } - (*i)->addSection(ms); - } - for (auto *ms : mergeSections) - ms->finalizeContents(); - - std::vector<InputSectionBase *> &v = inputSections; - v.erase(std::remove(v.begin(), v.end(), nullptr), v.end()); -} - MipsRldMapSection::MipsRldMapSection() : SyntheticSection(SHF_ALLOC | SHF_WRITE, SHT_PROGBITS, config->wordsize, ".rld_map") {} @@ -3102,17 +3157,23 @@ static InputSection *findExidxSection(InputSection *isec) { return nullptr; } +static bool isValidExidxSectionDep(InputSection *isec) { + return (isec->flags & SHF_ALLOC) && (isec->flags & SHF_EXECINSTR) && + isec->getSize() > 0; +} + bool ARMExidxSyntheticSection::addSection(InputSection *isec) { if (isec->type == SHT_ARM_EXIDX) { - exidxSections.push_back(isec); - return true; + if (InputSection* dep = isec->getLinkOrderDep()) + if (isValidExidxSectionDep(dep)) { + exidxSections.push_back(isec); + return true; + } + return false; } - if ((isec->flags & SHF_ALLOC) && (isec->flags & SHF_EXECINSTR) && - isec->getSize() > 0) { + if (isValidExidxSectionDep(isec)) { executableSections.push_back(isec); - if (empty && findExidxSection(isec)) - empty = false; return false; } @@ -3177,11 +3238,20 @@ static bool isDuplicateArmExidxSec(InputSection *prev, InputSection *cur) { // The .ARM.exidx table must be sorted in ascending order of the address of the // functions the table describes. Optionally duplicate adjacent table entries -// can be removed. At the end of the function the ExecutableSections must be +// can be removed. At the end of the function the executableSections must be // sorted in ascending order of address, Sentinel is set to the InputSection // with the highest address and any InputSections that have mergeable // .ARM.exidx table entries are removed from it. void ARMExidxSyntheticSection::finalizeContents() { + // The executableSections and exidxSections that we use to derive the final + // contents of this SyntheticSection are populated before + // processSectionCommands() and ICF. A /DISCARD/ entry in SECTIONS command or + // ICF may remove executable InputSections and their dependent .ARM.exidx + // section that we recorded earlier. + auto isDiscarded = [](const InputSection *isec) { return !isec->isLive(); }; + llvm::erase_if(executableSections, isDiscarded); + llvm::erase_if(exidxSections, isDiscarded); + // Sort the executable sections that may or may not have associated // .ARM.exidx sections by order of ascending address. This requires the // relative positions of InputSections to be known. @@ -3270,6 +3340,12 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) { assert(size == offset + 8); } +bool ARMExidxSyntheticSection::isNeeded() const { + return llvm::find_if(exidxSections, [](InputSection *isec) { + return isec->isLive(); + }) != exidxSections.end(); +} + bool ARMExidxSyntheticSection::classof(const SectionBase *d) { return d->kind() == InputSectionBase::Synthetic && d->type == SHT_ARM_EXIDX; } @@ -3389,23 +3465,6 @@ bool PPC64LongBranchTargetSection::isNeeded() const { return !finalized || !entries.empty(); } -RISCVSdataSection::RISCVSdataSection() - : SyntheticSection(SHF_ALLOC | SHF_WRITE, SHT_PROGBITS, 1, ".sdata") {} - -bool RISCVSdataSection::isNeeded() const { - if (!ElfSym::riscvGlobalPointer) - return false; - - // __global_pointer$ is defined relative to .sdata . If the section does not - // exist, create a dummy one. - for (BaseCommand *base : getParent()->sectionCommands) - if (auto *isd = dyn_cast<InputSectionDescription>(base)) - for (InputSection *isec : isd->sections) - if (isec != this) - return false; - return true; -} - static uint8_t getAbiVersion() { // MIPS non-PIC executable gets ABI version 1. if (config->emachine == EM_MIPS) { @@ -3426,7 +3485,7 @@ static uint8_t getAbiVersion() { return 0; } -template <typename ELFT> void elf::writeEhdr(uint8_t *buf, Partition &part) { +template <typename ELFT> void writeEhdr(uint8_t *buf, Partition &part) { // For executable segments, the trap instructions are written before writing // the header. Setting Elf header bytes to zero ensures that any unused bytes // in header are zero-cleared, instead of having trap instructions. @@ -3452,7 +3511,7 @@ template <typename ELFT> void elf::writeEhdr(uint8_t *buf, Partition &part) { } } -template <typename ELFT> void elf::writePhdrs(uint8_t *buf, Partition &part) { +template <typename ELFT> void writePhdrs(uint8_t *buf, Partition &part) { // Write the program header table. auto *hBuf = reinterpret_cast<typename ELFT::Phdr *>(buf); for (PhdrEntry *p : part.phdrs) { @@ -3527,92 +3586,90 @@ void PartitionIndexSection::writeTo(uint8_t *buf) { } } -InStruct elf::in; +InStruct in; -std::vector<Partition> elf::partitions; -Partition *elf::mainPart; +std::vector<Partition> partitions; +Partition *mainPart; template GdbIndexSection *GdbIndexSection::create<ELF32LE>(); template GdbIndexSection *GdbIndexSection::create<ELF32BE>(); template GdbIndexSection *GdbIndexSection::create<ELF64LE>(); template GdbIndexSection *GdbIndexSection::create<ELF64BE>(); -template void elf::splitSections<ELF32LE>(); -template void elf::splitSections<ELF32BE>(); -template void elf::splitSections<ELF64LE>(); -template void elf::splitSections<ELF64BE>(); - -template void EhFrameSection::addSection<ELF32LE>(InputSectionBase *); -template void EhFrameSection::addSection<ELF32BE>(InputSectionBase *); -template void EhFrameSection::addSection<ELF64LE>(InputSectionBase *); -template void EhFrameSection::addSection<ELF64BE>(InputSectionBase *); +template void splitSections<ELF32LE>(); +template void splitSections<ELF32BE>(); +template void splitSections<ELF64LE>(); +template void splitSections<ELF64BE>(); template void PltSection::addEntry<ELF32LE>(Symbol &Sym); template void PltSection::addEntry<ELF32BE>(Symbol &Sym); template void PltSection::addEntry<ELF64LE>(Symbol &Sym); template void PltSection::addEntry<ELF64BE>(Symbol &Sym); -template class elf::MipsAbiFlagsSection<ELF32LE>; -template class elf::MipsAbiFlagsSection<ELF32BE>; -template class elf::MipsAbiFlagsSection<ELF64LE>; -template class elf::MipsAbiFlagsSection<ELF64BE>; - -template class elf::MipsOptionsSection<ELF32LE>; -template class elf::MipsOptionsSection<ELF32BE>; -template class elf::MipsOptionsSection<ELF64LE>; -template class elf::MipsOptionsSection<ELF64BE>; - -template class elf::MipsReginfoSection<ELF32LE>; -template class elf::MipsReginfoSection<ELF32BE>; -template class elf::MipsReginfoSection<ELF64LE>; -template class elf::MipsReginfoSection<ELF64BE>; - -template class elf::DynamicSection<ELF32LE>; -template class elf::DynamicSection<ELF32BE>; -template class elf::DynamicSection<ELF64LE>; -template class elf::DynamicSection<ELF64BE>; - -template class elf::RelocationSection<ELF32LE>; -template class elf::RelocationSection<ELF32BE>; -template class elf::RelocationSection<ELF64LE>; -template class elf::RelocationSection<ELF64BE>; - -template class elf::AndroidPackedRelocationSection<ELF32LE>; -template class elf::AndroidPackedRelocationSection<ELF32BE>; -template class elf::AndroidPackedRelocationSection<ELF64LE>; -template class elf::AndroidPackedRelocationSection<ELF64BE>; - -template class elf::RelrSection<ELF32LE>; -template class elf::RelrSection<ELF32BE>; -template class elf::RelrSection<ELF64LE>; -template class elf::RelrSection<ELF64BE>; - -template class elf::SymbolTableSection<ELF32LE>; -template class elf::SymbolTableSection<ELF32BE>; -template class elf::SymbolTableSection<ELF64LE>; -template class elf::SymbolTableSection<ELF64BE>; - -template class elf::VersionNeedSection<ELF32LE>; -template class elf::VersionNeedSection<ELF32BE>; -template class elf::VersionNeedSection<ELF64LE>; -template class elf::VersionNeedSection<ELF64BE>; - -template void elf::writeEhdr<ELF32LE>(uint8_t *Buf, Partition &Part); -template void elf::writeEhdr<ELF32BE>(uint8_t *Buf, Partition &Part); -template void elf::writeEhdr<ELF64LE>(uint8_t *Buf, Partition &Part); -template void elf::writeEhdr<ELF64BE>(uint8_t *Buf, Partition &Part); - -template void elf::writePhdrs<ELF32LE>(uint8_t *Buf, Partition &Part); -template void elf::writePhdrs<ELF32BE>(uint8_t *Buf, Partition &Part); -template void elf::writePhdrs<ELF64LE>(uint8_t *Buf, Partition &Part); -template void elf::writePhdrs<ELF64BE>(uint8_t *Buf, Partition &Part); - -template class elf::PartitionElfHeaderSection<ELF32LE>; -template class elf::PartitionElfHeaderSection<ELF32BE>; -template class elf::PartitionElfHeaderSection<ELF64LE>; -template class elf::PartitionElfHeaderSection<ELF64BE>; - -template class elf::PartitionProgramHeadersSection<ELF32LE>; -template class elf::PartitionProgramHeadersSection<ELF32BE>; -template class elf::PartitionProgramHeadersSection<ELF64LE>; -template class elf::PartitionProgramHeadersSection<ELF64BE>; +template class MipsAbiFlagsSection<ELF32LE>; +template class MipsAbiFlagsSection<ELF32BE>; +template class MipsAbiFlagsSection<ELF64LE>; +template class MipsAbiFlagsSection<ELF64BE>; + +template class MipsOptionsSection<ELF32LE>; +template class MipsOptionsSection<ELF32BE>; +template class MipsOptionsSection<ELF64LE>; +template class MipsOptionsSection<ELF64BE>; + +template class MipsReginfoSection<ELF32LE>; +template class MipsReginfoSection<ELF32BE>; +template class MipsReginfoSection<ELF64LE>; +template class MipsReginfoSection<ELF64BE>; + +template class DynamicSection<ELF32LE>; +template class DynamicSection<ELF32BE>; +template class DynamicSection<ELF64LE>; +template class DynamicSection<ELF64BE>; + +template class RelocationSection<ELF32LE>; +template class RelocationSection<ELF32BE>; +template class RelocationSection<ELF64LE>; +template class RelocationSection<ELF64BE>; + +template class AndroidPackedRelocationSection<ELF32LE>; +template class AndroidPackedRelocationSection<ELF32BE>; +template class AndroidPackedRelocationSection<ELF64LE>; +template class AndroidPackedRelocationSection<ELF64BE>; + +template class RelrSection<ELF32LE>; +template class RelrSection<ELF32BE>; +template class RelrSection<ELF64LE>; +template class RelrSection<ELF64BE>; + +template class SymbolTableSection<ELF32LE>; +template class SymbolTableSection<ELF32BE>; +template class SymbolTableSection<ELF64LE>; +template class SymbolTableSection<ELF64BE>; + +template class VersionNeedSection<ELF32LE>; +template class VersionNeedSection<ELF32BE>; +template class VersionNeedSection<ELF64LE>; +template class VersionNeedSection<ELF64BE>; + +template void writeEhdr<ELF32LE>(uint8_t *Buf, Partition &Part); +template void writeEhdr<ELF32BE>(uint8_t *Buf, Partition &Part); +template void writeEhdr<ELF64LE>(uint8_t *Buf, Partition &Part); +template void writeEhdr<ELF64BE>(uint8_t *Buf, Partition &Part); + +template void writePhdrs<ELF32LE>(uint8_t *Buf, Partition &Part); +template void writePhdrs<ELF32BE>(uint8_t *Buf, Partition &Part); +template void writePhdrs<ELF64LE>(uint8_t *Buf, Partition &Part); +template void writePhdrs<ELF64BE>(uint8_t *Buf, Partition &Part); + +template class PartitionElfHeaderSection<ELF32LE>; +template class PartitionElfHeaderSection<ELF32BE>; +template class PartitionElfHeaderSection<ELF64LE>; +template class PartitionElfHeaderSection<ELF64BE>; + +template class PartitionProgramHeadersSection<ELF32LE>; +template class PartitionProgramHeadersSection<ELF32BE>; +template class PartitionProgramHeadersSection<ELF64LE>; +template class PartitionProgramHeadersSection<ELF64BE>; + +} // namespace elf +} // namespace lld diff --git a/ELF/SyntheticSections.h b/ELF/SyntheticSections.h index 1c4dd06e0277..d592dcb84e12 100644 --- a/ELF/SyntheticSections.h +++ b/ELF/SyntheticSections.h @@ -76,7 +76,7 @@ public: return SyntheticSection::classof(d) && d->name == ".eh_frame"; } - template <class ELFT> void addSection(InputSectionBase *s); + void addSection(EhInputSection *sec); std::vector<EhInputSection *> sections; size_t numFdes = 0; @@ -97,7 +97,9 @@ private: uint64_t size = 0; template <class ELFT, class RelTy> - void addSectionAux(EhInputSection *s, llvm::ArrayRef<RelTy> rels); + void addRecords(EhInputSection *s, llvm::ArrayRef<RelTy> rels); + template <class ELFT> + void addSectionAux(EhInputSection *s); template <class ELFT, class RelTy> CieRecord *addCie(EhSectionPiece &piece, ArrayRef<RelTy> rels); @@ -992,7 +994,7 @@ public: size_t getSize() const override { return size; } void writeTo(uint8_t *buf) override; - bool isNeeded() const override { return !empty; } + bool isNeeded() const override; // Sort and remove duplicate entries. void finalizeContents() override; InputSection *getLinkOrderDep() const; @@ -1006,9 +1008,6 @@ public: private: size_t size; - // Empty if ExecutableSections contains no dependent .ARM.exidx sections. - bool empty = true; - // Instead of storing pointers to the .ARM.exidx InputSections from // InputObjects, we store pointers to the executable sections that need // .ARM.exidx sections. We can then use the dependentSections of these to @@ -1098,19 +1097,11 @@ public: void writeTo(uint8_t *buf) override; }; -// Create a dummy .sdata for __global_pointer$ if .sdata does not exist. -class RISCVSdataSection final : public SyntheticSection { -public: - RISCVSdataSection(); - size_t getSize() const override { return 0; } - bool isNeeded() const override; - void writeTo(uint8_t *buf) override {} -}; - InputSection *createInterpSection(); MergeInputSection *createCommentSection(); +MergeSyntheticSection *createMergeSynthetic(StringRef name, uint32_t type, + uint64_t flags, uint32_t alignment); template <class ELFT> void splitSections(); -void mergeSections(); template <typename ELFT> void writeEhdr(uint8_t *buf, Partition &part); template <typename ELFT> void writePhdrs(uint8_t *buf, Partition &part); @@ -1171,7 +1162,6 @@ struct InStruct { PltSection *plt; PltSection *iplt; PPC32Got2Section *ppc32Got2; - RISCVSdataSection *riscvSdata; RelocationBaseSection *relaPlt; RelocationBaseSection *relaIplt; StringTableSection *shStrTab; diff --git a/ELF/Target.cpp b/ELF/Target.cpp index d07478a5178c..024e0cfec27b 100644 --- a/ELF/Target.cpp +++ b/ELF/Target.cpp @@ -34,19 +34,19 @@ using namespace llvm; using namespace llvm::object; using namespace llvm::ELF; -using namespace lld; -using namespace lld::elf; -const TargetInfo *elf::target; - -std::string lld::toString(RelType type) { +namespace lld { +std::string toString(elf::RelType type) { StringRef s = getELFRelocationTypeName(elf::config->emachine, type); if (s == "Unknown") return ("Unknown (" + Twine(type) + ")").str(); return s; } -TargetInfo *elf::getTarget() { +namespace elf { +const TargetInfo *target; + +TargetInfo *getTarget() { switch (config->emachine) { case EM_386: case EM_IAMCU: @@ -91,6 +91,9 @@ TargetInfo *elf::getTarget() { } template <class ELFT> static ErrorPlace getErrPlace(const uint8_t *loc) { + if (!Out::bufferStart) + return {}; + for (InputSectionBase *d : inputSections) { auto *isec = cast<InputSection>(d); if (!isec->getParent()) @@ -103,7 +106,7 @@ template <class ELFT> static ErrorPlace getErrPlace(const uint8_t *loc) { return {}; } -ErrorPlace elf::getErrorPlace(const uint8_t *loc) { +ErrorPlace getErrorPlace(const uint8_t *loc) { switch (config->ekind) { case ELF32LEKind: return getErrPlace<ELF32LE>(loc); @@ -179,3 +182,6 @@ uint64_t TargetInfo::getImageBase() const { return *config->imageBase; return config->isPic ? 0 : defaultImageBase; } + +} // namespace elf +} // namespace lld diff --git a/ELF/Writer.cpp b/ELF/Writer.cpp index b8c8891648a4..dc0f9254596a 100644 --- a/ELF/Writer.cpp +++ b/ELF/Writer.cpp @@ -8,6 +8,7 @@ #include "Writer.h" #include "AArch64ErrataFix.h" +#include "ARMErrataFix.h" #include "CallGraphSort.h" #include "Config.h" #include "LinkerScript.h" @@ -35,9 +36,8 @@ using namespace llvm::object; using namespace llvm::support; using namespace llvm::support::endian; -using namespace lld; -using namespace lld::elf; - +namespace lld { +namespace elf { namespace { // The writer writes a SymbolTable result to a file. template <class ELFT> class Writer { @@ -62,7 +62,6 @@ private: void setReservedSymbolSections(); std::vector<PhdrEntry *> createPhdrs(Partition &part); - void removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrEntry); void addPhdrForSection(Partition &part, unsigned shType, unsigned pType, unsigned pFlags); void assignFileOffsets(); @@ -92,7 +91,7 @@ static bool isSectionPrefix(StringRef prefix, StringRef name) { return name.startswith(prefix) || name == prefix.drop_back(); } -StringRef elf::getOutputSectionName(const InputSectionBase *s) { +StringRef getOutputSectionName(const InputSectionBase *s) { if (config->relocatable) return s->name; @@ -140,10 +139,9 @@ static bool needsInterpSection() { script->needsInterpSection(); } -template <class ELFT> void elf::writeResult() { Writer<ELFT>().run(); } +template <class ELFT> void writeResult() { Writer<ELFT>().run(); } -template <class ELFT> -void Writer<ELFT>::removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrs) { +static void removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrs) { llvm::erase_if(phdrs, [&](const PhdrEntry *p) { if (p->p_type != PT_LOAD) return false; @@ -154,7 +152,7 @@ void Writer<ELFT>::removeEmptyPTLoad(std::vector<PhdrEntry *> &phdrs) { }); } -template <class ELFT> static void copySectionsIntoPartitions() { +void copySectionsIntoPartitions() { std::vector<InputSectionBase *> newSections; for (unsigned part = 2; part != partitions.size() + 1; ++part) { for (InputSectionBase *s : inputSections) { @@ -176,7 +174,7 @@ template <class ELFT> static void copySectionsIntoPartitions() { newSections.end()); } -template <class ELFT> static void combineEhSections() { +void combineEhSections() { for (InputSectionBase *&s : inputSections) { // Ignore dead sections and the partition end marker (.part.end), // whose partition number is out of bounds. @@ -185,7 +183,7 @@ template <class ELFT> static void combineEhSections() { Partition &part = s->getPartition(); if (auto *es = dyn_cast<EhInputSection>(s)) { - part.ehFrame->addSection<ELFT>(es); + part.ehFrame->addSection(es); s = nullptr; } else if (s->kind() == SectionBase::Regular && part.armExidx && part.armExidx->addSection(cast<InputSection>(s))) { @@ -217,7 +215,7 @@ static Defined *addAbsolute(StringRef name) { // The linker is expected to define some symbols depending on // the linking result. This function defines such symbols. -void elf::addReservedSymbols() { +void addReservedSymbols() { if (config->emachine == EM_MIPS) { // Define _gp for MIPS. st_value of _gp symbol will be updated by Writer // so that it points to an absolute address which by default is relative @@ -310,13 +308,23 @@ static OutputSection *findSection(StringRef name, unsigned partition = 1) { return nullptr; } -// Initialize Out members. -template <class ELFT> static void createSyntheticSections() { +template <class ELFT> void createSyntheticSections() { // Initialize all pointers with NULL. This is needed because // you can call lld::elf::main more than once as a library. memset(&Out::first, 0, sizeof(Out)); - auto add = [](InputSectionBase *sec) { inputSections.push_back(sec); }; + // Add the .interp section first because it is not a SyntheticSection. + // The removeUnusedSyntheticSections() function relies on the + // SyntheticSections coming last. + if (needsInterpSection()) { + for (size_t i = 1; i <= partitions.size(); ++i) { + InputSection *sec = createInterpSection(); + sec->partition = i; + inputSections.push_back(sec); + } + } + + auto add = [](SyntheticSection *sec) { inputSections.push_back(sec); }; in.shStrTab = make<StringTableSection>(".shstrtab", false); @@ -355,8 +363,10 @@ template <class ELFT> static void createSyntheticSections() { add(sec); } + StringRef relaDynName = config->isRela ? ".rela.dyn" : ".rel.dyn"; + for (Partition &part : partitions) { - auto add = [&](InputSectionBase *sec) { + auto add = [&](SyntheticSection *sec) { sec->partition = part.getNumber(); inputSections.push_back(sec); }; @@ -378,16 +388,11 @@ template <class ELFT> static void createSyntheticSections() { part.dynStrTab = make<StringTableSection>(".dynstr", true); part.dynSymTab = make<SymbolTableSection<ELFT>>(*part.dynStrTab); part.dynamic = make<DynamicSection<ELFT>>(); - if (config->androidPackDynRelocs) { - part.relaDyn = make<AndroidPackedRelocationSection<ELFT>>( - config->isRela ? ".rela.dyn" : ".rel.dyn"); - } else { - part.relaDyn = make<RelocationSection<ELFT>>( - config->isRela ? ".rela.dyn" : ".rel.dyn", config->zCombreloc); - } - - if (needsInterpSection()) - add(createInterpSection()); + if (config->androidPackDynRelocs) + part.relaDyn = make<AndroidPackedRelocationSection<ELFT>>(relaDynName); + else + part.relaDyn = + make<RelocationSection<ELFT>>(relaDynName, config->zCombreloc); if (config->hasDynSymTab) { part.dynSymTab = make<SymbolTableSection<ELFT>>(*part.dynStrTab); @@ -396,7 +401,7 @@ template <class ELFT> static void createSyntheticSections() { part.verSym = make<VersionTableSection>(); add(part.verSym); - if (!config->versionDefinitions.empty()) { + if (!namedVersionDefs().empty()) { part.verDef = make<VersionDefinitionSection>(); add(part.verDef); } @@ -476,11 +481,6 @@ template <class ELFT> static void createSyntheticSections() { add(in.ppc64LongBranchTarget); } - if (config->emachine == EM_RISCV) { - in.riscvSdata = make<RISCVSdataSection>(); - add(in.riscvSdata); - } - in.gotPlt = make<GotPltSection>(); add(in.gotPlt); in.igotPlt = make<IgotPltSection>(); @@ -504,16 +504,14 @@ template <class ELFT> static void createSyntheticSections() { config->isRela ? ".rela.plt" : ".rel.plt", /*sort=*/false); add(in.relaPlt); - // The relaIplt immediately follows .rel.plt (.rel.dyn for ARM) to ensure - // that the IRelative relocations are processed last by the dynamic loader. - // We cannot place the iplt section in .rel.dyn when Android relocation - // packing is enabled because that would cause a section type mismatch. - // However, because the Android dynamic loader reads .rel.plt after .rel.dyn, - // we can get the desired behaviour by placing the iplt section in .rel.plt. + // The relaIplt immediately follows .rel[a].dyn to ensure that the IRelative + // relocations are processed last by the dynamic loader. We cannot place the + // iplt section in .rel.dyn when Android relocation packing is enabled because + // that would cause a section type mismatch. However, because the Android + // dynamic loader reads .rel.plt after .rel.dyn, we can get the desired + // behaviour by placing the iplt section in .rel.plt. in.relaIplt = make<RelocationSection<ELFT>>( - (config->emachine == EM_ARM && !config->androidPackDynRelocs) - ? ".rel.dyn" - : in.relaPlt->name, + config->androidPackDynRelocs ? in.relaPlt->name : relaDynName, /*sort=*/false); add(in.relaIplt); @@ -544,29 +542,6 @@ template <class ELFT> static void createSyntheticSections() { // The main function of the writer. template <class ELFT> void Writer<ELFT>::run() { - // Make copies of any input sections that need to be copied into each - // partition. - copySectionsIntoPartitions<ELFT>(); - - // Create linker-synthesized sections such as .got or .plt. - // Such sections are of type input section. - createSyntheticSections<ELFT>(); - - // Some input sections that are used for exception handling need to be moved - // into synthetic sections. Do that now so that they aren't assigned to - // output sections in the usual way. - if (!config->relocatable) - combineEhSections<ELFT>(); - - // We want to process linker script commands. When SECTIONS command - // is given we let it create sections. - script->processSectionCommands(); - - // Linker scripts controls how input sections are assigned to output sections. - // Input sections that were not handled by scripts are called "orphans", and - // they are assigned to output sections by the default rule. Process that. - script->addOrphanSections(); - if (config->discard != DiscardPolicy::All) copyLocalSymbols(); @@ -582,15 +557,14 @@ template <class ELFT> void Writer<ELFT>::run() { if (errorCount()) return; - script->assignAddresses(); - // If -compressed-debug-sections is specified, we need to compress // .debug_* sections. Do it right now because it changes the size of // output sections. for (OutputSection *sec : outputSections) sec->maybeCompress<ELFT>(); - script->allocateHeaders(mainPart->phdrs); + if (script->hasSectionsCommand) + script->allocateHeaders(mainPart->phdrs); // Remove empty PT_LOAD to avoid causing the dynamic linker to try to mmap a // 0 sized region. This has to be done late since only after assignAddresses @@ -622,7 +596,8 @@ template <class ELFT> void Writer<ELFT>::run() { return; if (!config->oFormatBinary) { - writeTrapInstr(); + if (config->zSeparate != SeparateSegmentKind::None) + writeTrapInstr(); writeHeader(); writeSections(); } else { @@ -738,7 +713,7 @@ template <class ELFT> void Writer<ELFT>::addSectionSymbols() { }); if (i == sec->sectionCommands.end()) continue; - InputSection *isec = cast<InputSectionDescription>(*i)->sections[0]; + InputSectionBase *isec = cast<InputSectionDescription>(*i)->sections[0]; // Relocations are not using REL[A] section symbols. if (isec->type == SHT_REL || isec->type == SHT_RELA) @@ -1070,7 +1045,7 @@ template <class ELFT> void Writer<ELFT>::setReservedSymbolSections() { ElfSym::globalOffsetTable->section = gotSection; } - // .rela_iplt_{start,end} mark the start and the end of .rela.plt section. + // .rela_iplt_{start,end} mark the start and the end of in.relaIplt. if (ElfSym::relaIpltStart && in.relaIplt->isNeeded()) { ElfSym::relaIpltStart->section = in.relaIplt; ElfSym::relaIpltEnd->section = in.relaIplt; @@ -1298,10 +1273,7 @@ sortISDBySectionOrder(InputSectionDescription *isd, } orderedSections.push_back({isec, i->second}); } - llvm::sort(orderedSections, [&](std::pair<InputSection *, int> a, - std::pair<InputSection *, int> b) { - return a.second < b.second; - }); + llvm::sort(orderedSections, llvm::less_second()); // Find an insertion point for the ordered section list in the unordered // section list. On targets with limited-range branches, this is the mid-point @@ -1536,6 +1508,12 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() { if (!(sec->flags & SHF_LINK_ORDER)) continue; + // The ARM.exidx section use SHF_LINK_ORDER, but we have consolidated + // this processing inside the ARMExidxsyntheticsection::finalizeContents(). + if (!config->relocatable && config->emachine == EM_ARM && + sec->type == SHT_ARM_EXIDX) + continue; + // Link order may be distributed across several InputSectionDescriptions // but sort must consider them all at once. std::vector<InputSection **> scriptSections; @@ -1545,14 +1523,16 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() { for (InputSection *&isec : isd->sections) { scriptSections.push_back(&isec); sections.push_back(isec); + + InputSection *link = isec->getLinkOrderDep(); + if (!link->getParent()) + error(toString(isec) + ": sh_link points to discarded section " + + toString(link)); } } } - // The ARM.exidx section use SHF_LINK_ORDER, but we have consolidated - // this processing inside the ARMExidxsyntheticsection::finalizeContents(). - if (!config->relocatable && config->emachine == EM_ARM && - sec->type == SHT_ARM_EXIDX) + if (errorCount()) continue; llvm::stable_sort(sections, compareByFilePosition); @@ -1569,21 +1549,30 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() { template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() { ThunkCreator tc; AArch64Err843419Patcher a64p; + ARMErr657417Patcher a32p; + script->assignAddresses(); - // For some targets, like x86, this loop iterates only once. + int assignPasses = 0; for (;;) { - bool changed = false; - - script->assignAddresses(); + bool changed = target->needsThunks && tc.createThunks(outputSections); - if (target->needsThunks) - changed |= tc.createThunks(outputSections); + // With Thunk Size much smaller than branch range we expect to + // converge quickly; if we get to 10 something has gone wrong. + if (changed && tc.pass >= 10) { + error("thunk creation not converged"); + break; + } if (config->fixCortexA53Errata843419) { if (changed) script->assignAddresses(); changed |= a64p.createFixes(); } + if (config->fixCortexA8) { + if (changed) + script->assignAddresses(); + changed |= a32p.createFixes(); + } if (in.mipsGot) in.mipsGot->updateAllocSize(); @@ -1594,8 +1583,19 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() { changed |= part.relrDyn->updateAllocSize(); } - if (!changed) - return; + const Defined *changedSym = script->assignAddresses(); + if (!changed) { + // Some symbols may be dependent on section addresses. When we break the + // loop, the symbol values are finalized because a previous + // assignAddresses() finalized section addresses. + if (!changedSym) + break; + if (++assignPasses == 5) { + errorOrWarn("assignment to symbol " + toString(*changedSym) + + " does not converge"); + break; + } + } } } @@ -1655,13 +1655,13 @@ static bool computeIsPreemptible(const Symbol &b) { if (!b.isDefined()) return true; - // If we have a dynamic list it specifies which local symbols are preemptible. - if (config->hasDynamicList) - return false; - if (!config->shared) return false; + // If the dynamic list is present, it specifies preemptable symbols in a DSO. + if (config->hasDynamicList) + return b.inDynamicList; + // -Bsymbolic means that definitions are not preempted. if (config->bsymbolic || (config->bsymbolicFunctions && b.isFunc())) return false; @@ -1696,12 +1696,16 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // Define __rel[a]_iplt_{start,end} symbols if needed. addRelIpltSymbols(); - // RISC-V's gp can address +/- 2 KiB, set it to .sdata + 0x800 if not defined. - // This symbol should only be defined in an executable. - if (config->emachine == EM_RISCV && !config->shared) + // RISC-V's gp can address +/- 2 KiB, set it to .sdata + 0x800. This symbol + // should only be defined in an executable. If .sdata does not exist, its + // value/section does not matter but it has to be relative, so set its + // st_shndx arbitrarily to 1 (Out::elfHeader). + if (config->emachine == EM_RISCV && !config->shared) { + OutputSection *sec = findSection(".sdata"); ElfSym::riscvGlobalPointer = - addOptionalRegular("__global_pointer$", findSection(".sdata"), 0x800, - STV_DEFAULT, STB_GLOBAL); + addOptionalRegular("__global_pointer$", sec ? sec : Out::elfHeader, + 0x800, STV_DEFAULT, STB_GLOBAL); + } if (config->emachine == EM_X86_64) { // On targets that support TLSDESC, _TLS_MODULE_BASE_ is defined in such a @@ -1730,20 +1734,22 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { for (Partition &part : partitions) finalizeSynthetic(part.ehFrame); - symtab->forEachSymbol([](Symbol *s) { - if (!s->isPreemptible) - s->isPreemptible = computeIsPreemptible(*s); - }); + symtab->forEachSymbol( + [](Symbol *s) { s->isPreemptible = computeIsPreemptible(*s); }); + + // Change values of linker-script-defined symbols from placeholders (assigned + // by declareSymbols) to actual definitions. + script->processSymbolAssignments(); // Scan relocations. This must be done after every symbol is declared so that - // we can correctly decide if a dynamic relocation is needed. + // we can correctly decide if a dynamic relocation is needed. This is called + // after processSymbolAssignments() because it needs to know whether a + // linker-script-defined symbol is absolute. if (!config->relocatable) { forEachRelSec(scanRelocations<ELFT>); reportUndefinedSymbols<ELFT>(); } - addIRelativeRelocs(); - if (in.plt && in.plt->isNeeded()) in.plt->addSymbols(); if (in.iplt && in.iplt->isNeeded()) @@ -1880,7 +1886,6 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { finalizeSynthetic(in.plt); finalizeSynthetic(in.iplt); finalizeSynthetic(in.ppc32Got2); - finalizeSynthetic(in.riscvSdata); finalizeSynthetic(in.partIndex); // Dynamic section must be the last one in this list and dynamic @@ -1905,6 +1910,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { // SHFLinkOrder processing must be processed after relative section placements are // known but before addresses are allocated. resolveShfLinkOrder(); + if (errorCount()) + return; // This is used to: // 1) Create "thunks": @@ -2049,27 +2056,32 @@ std::vector<PhdrEntry *> Writer<ELFT>::createPhdrs(Partition &part) { unsigned partNo = part.getNumber(); bool isMain = partNo == 1; - // The first phdr entry is PT_PHDR which describes the program header itself. - if (isMain) - addHdr(PT_PHDR, PF_R)->add(Out::programHeaders); - else - addHdr(PT_PHDR, PF_R)->add(part.programHeaders->getParent()); - - // PT_INTERP must be the second entry if exists. - if (OutputSection *cmd = findSection(".interp", partNo)) - addHdr(PT_INTERP, cmd->getPhdrFlags())->add(cmd); - // Add the first PT_LOAD segment for regular output sections. uint64_t flags = computeFlags(PF_R); PhdrEntry *load = nullptr; - // Add the headers. We will remove them if they don't fit. - // In the other partitions the headers are ordinary sections, so they don't - // need to be added here. - if (isMain) { - load = addHdr(PT_LOAD, flags); - load->add(Out::elfHeader); - load->add(Out::programHeaders); + // nmagic or omagic output does not have PT_PHDR, PT_INTERP, or the readonly + // PT_LOAD. + if (!config->nmagic && !config->omagic) { + // The first phdr entry is PT_PHDR which describes the program header + // itself. + if (isMain) + addHdr(PT_PHDR, PF_R)->add(Out::programHeaders); + else + addHdr(PT_PHDR, PF_R)->add(part.programHeaders->getParent()); + + // PT_INTERP must be the second entry if exists. + if (OutputSection *cmd = findSection(".interp", partNo)) + addHdr(PT_INTERP, cmd->getPhdrFlags())->add(cmd); + + // Add the headers. We will remove them if they don't fit. + // In the other partitions the headers are ordinary sections, so they don't + // need to be added here. + if (isMain) { + load = addHdr(PT_LOAD, flags); + load->add(Out::elfHeader); + load->add(Out::programHeaders); + } } // PT_GNU_RELRO includes all sections that should be marked as @@ -2208,21 +2220,68 @@ void Writer<ELFT>::addPhdrForSection(Partition &part, unsigned shType, part.phdrs.push_back(entry); } -// The first section of each PT_LOAD, the first section in PT_GNU_RELRO and the -// first section after PT_GNU_RELRO have to be page aligned so that the dynamic -// linker can set the permissions. +// Place the first section of each PT_LOAD to a different page (of maxPageSize). +// This is achieved by assigning an alignment expression to addrExpr of each +// such section. template <class ELFT> void Writer<ELFT>::fixSectionAlignments() { - auto pageAlign = [](OutputSection *cmd) { - if (cmd && !cmd->addrExpr) - cmd->addrExpr = [=] { - return alignTo(script->getDot(), config->maxPageSize); - }; + const PhdrEntry *prev; + auto pageAlign = [&](const PhdrEntry *p) { + OutputSection *cmd = p->firstSec; + if (cmd && !cmd->addrExpr) { + // Prefer advancing to align(dot, maxPageSize) + dot%maxPageSize to avoid + // padding in the file contents. + // + // When -z separate-code is used we must not have any overlap in pages + // between an executable segment and a non-executable segment. We align to + // the next maximum page size boundary on transitions between executable + // and non-executable segments. + // + // SHT_LLVM_PART_EHDR marks the start of a partition. The partition + // sections will be extracted to a separate file. Align to the next + // maximum page size boundary so that we can find the ELF header at the + // start. We cannot benefit from overlapping p_offset ranges with the + // previous segment anyway. + if (config->zSeparate == SeparateSegmentKind::Loadable || + (config->zSeparate == SeparateSegmentKind::Code && prev && + (prev->p_flags & PF_X) != (p->p_flags & PF_X)) || + cmd->type == SHT_LLVM_PART_EHDR) + cmd->addrExpr = [] { + return alignTo(script->getDot(), config->maxPageSize); + }; + // PT_TLS is at the start of the first RW PT_LOAD. If `p` includes PT_TLS, + // it must be the RW. Align to p_align(PT_TLS) to make sure + // p_vaddr(PT_LOAD)%p_align(PT_LOAD) = 0. Otherwise, if + // sh_addralign(.tdata) < sh_addralign(.tbss), we will set p_align(PT_TLS) + // to sh_addralign(.tbss), while p_vaddr(PT_TLS)=p_vaddr(PT_LOAD) may not + // be congruent to 0 modulo p_align(PT_TLS). + // + // Technically this is not required, but as of 2019, some dynamic loaders + // don't handle p_vaddr%p_align != 0 correctly, e.g. glibc (i386 and + // x86-64) doesn't make runtime address congruent to p_vaddr modulo + // p_align for dynamic TLS blocks (PR/24606), FreeBSD rtld has the same + // bug, musl (TLS Variant 1 architectures) before 1.1.23 handled TLS + // blocks correctly. We need to keep the workaround for a while. + else if (Out::tlsPhdr && Out::tlsPhdr->firstSec == p->firstSec) + cmd->addrExpr = [] { + return alignTo(script->getDot(), config->maxPageSize) + + alignTo(script->getDot() % config->maxPageSize, + Out::tlsPhdr->p_align); + }; + else + cmd->addrExpr = [] { + return alignTo(script->getDot(), config->maxPageSize) + + script->getDot() % config->maxPageSize; + }; + } }; for (Partition &part : partitions) { + prev = nullptr; for (const PhdrEntry *p : part.phdrs) - if (p->p_type == PT_LOAD && p->firstSec) - pageAlign(p->firstSec); + if (p->p_type == PT_LOAD && p->firstSec) { + pageAlign(p); + prev = p; + } } } @@ -2230,25 +2289,24 @@ template <class ELFT> void Writer<ELFT>::fixSectionAlignments() { // same with its virtual address modulo the page size, so that the loader can // load executables without any address adjustment. static uint64_t computeFileOffset(OutputSection *os, uint64_t off) { - // File offsets are not significant for .bss sections. By convention, we keep - // section offsets monotonically increasing rather than setting to zero. - if (os->type == SHT_NOBITS) - return off; + // The first section in a PT_LOAD has to have congruent offset and address + // modulo the maximum page size. + if (os->ptLoad && os->ptLoad->firstSec == os) + return alignTo(off, os->ptLoad->p_align, os->addr); + + // File offsets are not significant for .bss sections other than the first one + // in a PT_LOAD. By convention, we keep section offsets monotonically + // increasing rather than setting to zero. + if (os->type == SHT_NOBITS) + return off; // If the section is not in a PT_LOAD, we just have to align it. if (!os->ptLoad) return alignTo(off, os->alignment); - // The first section in a PT_LOAD has to have congruent offset and address - // module the page size. - OutputSection *first = os->ptLoad->firstSec; - if (os == first) { - uint64_t alignment = std::max<uint64_t>(os->alignment, config->maxPageSize); - return alignTo(off, alignment, os->addr); - } - // If two sections share the same PT_LOAD the file offset is calculated // using this formula: Off2 = Off1 + (VA2 - VA1). + OutputSection *first = os->ptLoad->firstSec; return first->offset + os->addr - first->addr; } @@ -2289,13 +2347,12 @@ template <class ELFT> void Writer<ELFT>::assignFileOffsets() { for (OutputSection *sec : outputSections) { off = setFileOffset(sec, off); - if (script->hasSectionsCommand) - continue; // If this is a last section of the last executable segment and that // segment is the last loadable segment, align the offset of the // following section to avoid loading non-segments parts of the file. - if (lastRX && lastRX->lastSec == sec) + if (config->zSeparate != SeparateSegmentKind::None && lastRX && + lastRX->lastSec == sec) off = alignTo(off, config->commonPageSize); } @@ -2346,14 +2403,13 @@ template <class ELFT> void Writer<ELFT>::setPhdrs(Partition &part) { p->p_paddr = first->getLMA(); } - if (p->p_type == PT_LOAD) { - p->p_align = std::max<uint64_t>(p->p_align, config->maxPageSize); - } else if (p->p_type == PT_GNU_RELRO) { + if (p->p_type == PT_GNU_RELRO) { p->p_align = 1; - // The glibc dynamic loader rounds the size down, so we need to round up + // musl/glibc ld.so rounds the size down, so we need to round up // to protect the last page. This is a no-op on FreeBSD which always // rounds up. - p->p_memsz = alignTo(p->p_memsz, config->commonPageSize); + p->p_memsz = alignTo(p->p_offset + p->p_memsz, config->commonPageSize) - + p->p_offset; } } } @@ -2568,9 +2624,6 @@ static void fillTrap(uint8_t *i, uint8_t *end) { // We'll leave other pages in segments as-is because the rest will be // overwritten by output sections. template <class ELFT> void Writer<ELFT>::writeTrapInstr() { - if (script->hasSectionsCommand) - return; - for (Partition &part : partitions) { // Fill the last page. for (PhdrEntry *p : part.phdrs) @@ -2683,7 +2736,15 @@ template <class ELFT> void Writer<ELFT>::writeBuildId() { part.buildId->writeBuildId(buildId); } -template void elf::writeResult<ELF32LE>(); -template void elf::writeResult<ELF32BE>(); -template void elf::writeResult<ELF64LE>(); -template void elf::writeResult<ELF64BE>(); +template void createSyntheticSections<ELF32LE>(); +template void createSyntheticSections<ELF32BE>(); +template void createSyntheticSections<ELF64LE>(); +template void createSyntheticSections<ELF64BE>(); + +template void writeResult<ELF32LE>(); +template void writeResult<ELF32BE>(); +template void writeResult<ELF64LE>(); +template void writeResult<ELF64BE>(); + +} // namespace elf +} // namespace lld diff --git a/ELF/Writer.h b/ELF/Writer.h index 784fba9c75a6..3698544d977b 100644 --- a/ELF/Writer.h +++ b/ELF/Writer.h @@ -9,6 +9,7 @@ #ifndef LLD_ELF_WRITER_H #define LLD_ELF_WRITER_H +#include "Config.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" #include <cstdint> @@ -19,13 +20,18 @@ namespace elf { class InputFile; class OutputSection; class InputSectionBase; +void copySectionsIntoPartitions(); +template <class ELFT> void createSyntheticSections(); +void combineEhSections(); template <class ELFT> void writeResult(); // This describes a program header entry. // Each contains type, access flags and range of output sections that will be // placed in it. struct PhdrEntry { - PhdrEntry(unsigned type, unsigned flags) : p_type(type), p_flags(flags) {} + PhdrEntry(unsigned type, unsigned flags) + : p_align(type == llvm::ELF::PT_LOAD ? config->maxPageSize : 0), + p_type(type), p_flags(flags) {} void add(OutputSection *sec); uint64_t p_paddr = 0; |