diff options
Diffstat (limited to 'llvm/lib/Bitcode')
-rw-r--r-- | llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 15 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 404 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/MetadataLoader.cpp | 90 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/MetadataLoader.h | 2 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Reader/ValueList.cpp | 2 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 170 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 49 | ||||
-rw-r--r-- | llvm/lib/Bitcode/Writer/ValueEnumerator.h | 2 |
8 files changed, 481 insertions, 253 deletions
diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp index e70caa83c8c1..2ce064c7685a 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -130,7 +130,7 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(MODULE_CODE, DATALAYOUT) STRINGIFY_CODE(MODULE_CODE, ASM) STRINGIFY_CODE(MODULE_CODE, SECTIONNAME) - STRINGIFY_CODE(MODULE_CODE, DEPLIB) // FIXME: Remove in 4.0 + STRINGIFY_CODE(MODULE_CODE, DEPLIB) // Deprecated, present in old bitcode STRINGIFY_CODE(MODULE_CODE, GLOBALVAR) STRINGIFY_CODE(MODULE_CODE, FUNCTION) STRINGIFY_CODE(MODULE_CODE, ALIAS) @@ -305,6 +305,8 @@ static Optional<const char *> GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(FS, CFI_FUNCTION_DECLS) STRINGIFY_CODE(FS, TYPE_ID) STRINGIFY_CODE(FS, TYPE_ID_METADATA) + STRINGIFY_CODE(FS, BLOCK_COUNT) + STRINGIFY_CODE(FS, PARAM_ACCESS) } case bitc::METADATA_ATTACHMENT_ID: switch (CodeID) { @@ -910,17 +912,14 @@ Error BitcodeAnalyzer::parseBlock(unsigned BlockID, unsigned IndentLevel, Hasher.update(ArrayRef<uint8_t>(Ptr, BlockSize)); Hash = Hasher.result(); } - SmallString<20> RecordedHash; - RecordedHash.resize(20); + std::array<char, 20> RecordedHash; int Pos = 0; for (auto &Val : Record) { assert(!(Val >> 32) && "Unexpected high bits set"); - RecordedHash[Pos++] = (Val >> 24) & 0xFF; - RecordedHash[Pos++] = (Val >> 16) & 0xFF; - RecordedHash[Pos++] = (Val >> 8) & 0xFF; - RecordedHash[Pos++] = (Val >> 0) & 0xFF; + support::endian::write32be(&RecordedHash[Pos], Val); + Pos += 4; } - if (Hash == RecordedHash) + if (Hash == StringRef(RecordedHash.data(), RecordedHash.size())) O->OS << " (match)"; else O->OS << " (!mismatch!)"; diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 33464412edc5..659e26c2bd25 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -27,7 +27,6 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" @@ -153,8 +152,7 @@ static bool convertToString(ArrayRef<uint64_t> Record, unsigned Idx, if (Idx > Record.size()) return true; - for (unsigned i = Idx, e = Record.size(); i != e; ++i) - Result += (char)Record[i]; + Result.append(Record.begin() + Idx, Record.end()); return false; } @@ -578,8 +576,11 @@ public: /// Main interface to parsing a bitcode buffer. /// \returns true if an error occurred. - Error parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata = false, - bool IsImporting = false); + Error parseBitcodeInto( + Module *M, bool ShouldLazyLoadMetadata = false, bool IsImporting = false, + DataLayoutCallbackTy DataLayoutCallback = [](std::string) { + return None; + }); static uint64_t decodeSignRotatedValue(uint64_t V); @@ -724,7 +725,9 @@ private: /// a corresponding error code. Error parseAlignmentValue(uint64_t Exponent, MaybeAlign &Alignment); Error parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind); - Error parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false); + Error parseModule( + uint64_t ResumeBit, bool ShouldLazyLoadMetadata = false, + DataLayoutCallbackTy DataLayoutCallback = [](StringRef) { return None; }); Error parseComdatRecord(ArrayRef<uint64_t> Record); Error parseGlobalVarRecord(ArrayRef<uint64_t> Record); @@ -859,7 +862,7 @@ BitcodeReader::BitcodeReader(BitstreamCursor Stream, StringRef Strtab, LLVMContext &Context) : BitcodeReaderBase(std::move(Stream), Strtab), Context(Context), ValueList(Context, Stream.SizeInBytes()) { - this->ProducerIdentification = ProducerIdentification; + this->ProducerIdentification = std::string(ProducerIdentification); } Error BitcodeReader::materializeForwardReferencedFunctions() { @@ -985,8 +988,10 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, // Decode the flags for GlobalVariable in the summary static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) { - return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false, - (RawFlags & 0x2) ? true : false); + return GlobalVarSummary::GVarFlags( + (RawFlags & 0x1) ? true : false, (RawFlags & 0x2) ? true : false, + (RawFlags & 0x4) ? true : false, + (GlobalObject::VCallVisibility)(RawFlags >> 3)); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { @@ -1216,6 +1221,8 @@ StructType *BitcodeReader::createIdentifiedStructType(LLVMContext &Context) { static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { switch (Val) { case Attribute::EndAttrKinds: + case Attribute::EmptyKey: + case Attribute::TombstoneKey: llvm_unreachable("Synthetic enumerators which should never get here"); case Attribute::None: return 0; @@ -1281,25 +1288,10 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { return 1ULL << 62; case Attribute::NoFree: return 1ULL << 63; - case Attribute::NoSync: - llvm_unreachable("nosync attribute not supported in raw format"); - break; - case Attribute::Dereferenceable: - llvm_unreachable("dereferenceable attribute not supported in raw format"); - break; - case Attribute::DereferenceableOrNull: - llvm_unreachable("dereferenceable_or_null attribute not supported in raw " - "format"); - break; - case Attribute::ArgMemOnly: - llvm_unreachable("argmemonly attribute not supported in raw format"); - break; - case Attribute::AllocSize: - llvm_unreachable("allocsize not supported in raw format"); - break; - case Attribute::SanitizeMemTag: - llvm_unreachable("sanitize_memtag attribute not supported in raw format"); - break; + default: + // Other attributes are not supported in the raw format, + // as we ran out of space. + return 0; } llvm_unreachable("Unsupported attribute type"); } @@ -1309,13 +1301,6 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) { for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; I = Attribute::AttrKind(I + 1)) { - if (I == Attribute::SanitizeMemTag || - I == Attribute::Dereferenceable || - I == Attribute::DereferenceableOrNull || - I == Attribute::ArgMemOnly || - I == Attribute::AllocSize || - I == Attribute::NoSync) - continue; if (uint64_t A = (Val & getRawAttributeMask(I))) { if (I == Attribute::Alignment) B.addAlignmentAttr(1ULL << ((A >> 16) - 1)); @@ -1332,8 +1317,6 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) { /// 'encodeLLVMAttributesForBitcode'. static void decodeLLVMAttributesForBitcode(AttrBuilder &B, uint64_t EncodedAttrs) { - // FIXME: Remove in 4.0. - // The alignment is stored as a 16-bit raw value from bits 31--16. We shift // the bits above 31 down by 11 bits. unsigned Alignment = (EncodedAttrs & (0xffffULL << 16)) >> 16; @@ -1384,7 +1367,7 @@ Error BitcodeReader::parseAttributeBlock() { default: // Default behavior: ignore. break; case bitc::PARAMATTR_CODE_ENTRY_OLD: // ENTRY: [paramidx0, attr0, ...] - // FIXME: Remove in 4.0. + // Deprecated, but still needed to read old bitcode files. if (Record.size() & 1) return error("Invalid record"); @@ -1461,6 +1444,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoInline; case bitc::ATTR_KIND_NO_RECURSE: return Attribute::NoRecurse; + case bitc::ATTR_KIND_NO_MERGE: + return Attribute::NoMerge; case bitc::ATTR_KIND_NON_LAZY_BIND: return Attribute::NonLazyBind; case bitc::ATTR_KIND_NON_NULL: @@ -1481,6 +1466,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoCfCheck; case bitc::ATTR_KIND_NO_UNWIND: return Attribute::NoUnwind; + case bitc::ATTR_KIND_NULL_POINTER_IS_VALID: + return Attribute::NullPointerIsValid; case bitc::ATTR_KIND_OPT_FOR_FUZZING: return Attribute::OptForFuzzing; case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE: @@ -1541,6 +1528,10 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::ImmArg; case bitc::ATTR_KIND_SANITIZE_MEMTAG: return Attribute::SanitizeMemTag; + case bitc::ATTR_KIND_PREALLOCATED: + return Attribute::Preallocated; + case bitc::ATTR_KIND_NOUNDEF: + return Attribute::NoUndef; } } @@ -1656,12 +1647,15 @@ Error BitcodeReader::parseAttributeGroupBlock() { Attribute::AttrKind Kind; if (Error Err = parseAttrKind(Record[++i], &Kind)) return Err; - if (Kind == Attribute::ByVal) + if (Kind == Attribute::ByVal) { B.addByValAttr(HasType ? getTypeByID(Record[++i]) : nullptr); + } else if (Kind == Attribute::Preallocated) { + B.addPreallocatedAttr(getTypeByID(Record[++i])); + } } } - UpgradeFramePointerAttributes(B); + UpgradeAttributes(B); MAttributeGroups[GrpID] = AttributeList::get(Context, Idx, B); break; } @@ -1727,6 +1721,9 @@ Error BitcodeReader::parseTypeTableBody() { case bitc::TYPE_CODE_HALF: // HALF ResultTy = Type::getHalfTy(Context); break; + case bitc::TYPE_CODE_BFLOAT: // BFLOAT + ResultTy = Type::getBFloatTy(Context); + break; case bitc::TYPE_CODE_FLOAT: // FLOAT ResultTy = Type::getFloatTy(Context); break; @@ -1780,7 +1777,7 @@ Error BitcodeReader::parseTypeTableBody() { break; } case bitc::TYPE_CODE_FUNCTION_OLD: { - // FIXME: attrid is dead, remove it in LLVM 4.0 + // Deprecated, but still needed to read old bitcode files. // FUNCTION: [vararg, attrid, retty, paramty x N] if (Record.size() < 3) return error("Invalid record"); @@ -2314,7 +2311,7 @@ Error BitcodeReader::resolveGlobalAndIndirectSymbolInits() { return Error::success(); } -static APInt readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) { +APInt llvm::readWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) { SmallVector<uint64_t, 8> Words(Vals.size()); transform(Vals, Words.begin(), BitcodeReader::decodeSignRotatedValue); @@ -2333,6 +2330,16 @@ Error BitcodeReader::parseConstants() { Type *CurFullTy = Type::getInt32Ty(Context); unsigned NextCstNo = ValueList.size(); + struct DelayedShufTy { + VectorType *OpTy; + VectorType *RTy; + Type *CurFullTy; + uint64_t Op0Idx; + uint64_t Op1Idx; + uint64_t Op2Idx; + unsigned CstNo; + }; + std::vector<DelayedShufTy> DelayedShuffles; while (true) { Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks(); if (!MaybeEntry) @@ -2344,11 +2351,35 @@ Error BitcodeReader::parseConstants() { case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: + // Once all the constants have been read, go through and resolve forward + // references. + // + // We have to treat shuffles specially because they don't have three + // operands anymore. We need to convert the shuffle mask into an array, + // and we can't convert a forward reference. + for (auto &DelayedShuffle : DelayedShuffles) { + VectorType *OpTy = DelayedShuffle.OpTy; + VectorType *RTy = DelayedShuffle.RTy; + uint64_t Op0Idx = DelayedShuffle.Op0Idx; + uint64_t Op1Idx = DelayedShuffle.Op1Idx; + uint64_t Op2Idx = DelayedShuffle.Op2Idx; + uint64_t CstNo = DelayedShuffle.CstNo; + Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy); + Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy); + Type *ShufTy = + VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount()); + Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy); + if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2)) + return error("Invalid shufflevector operands"); + SmallVector<int, 16> Mask; + ShuffleVectorInst::getShuffleMask(Op2, Mask); + Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask); + ValueList.assignValue(V, CstNo, DelayedShuffle.CurFullTy); + } + if (NextCstNo != ValueList.size()) return error("Invalid constant reference"); - // Once all the constants have been read, go through and resolve forward - // references. ValueList.resolveConstantForwardRefs(); return Error::success(); case BitstreamEntry::Record: @@ -2404,6 +2435,9 @@ Error BitcodeReader::parseConstants() { if (CurTy->isHalfTy()) V = ConstantFP::get(Context, APFloat(APFloat::IEEEhalf(), APInt(16, (uint16_t)Record[0]))); + else if (CurTy->isBFloatTy()) + V = ConstantFP::get(Context, APFloat(APFloat::BFloat(), + APInt(16, (uint32_t)Record[0]))); else if (CurTy->isFloatTy()) V = ConstantFP::get(Context, APFloat(APFloat::IEEEsingle(), APInt(32, (uint32_t)Record[0]))); @@ -2469,7 +2503,11 @@ Error BitcodeReader::parseConstants() { if (Record.empty()) return error("Invalid record"); - Type *EltTy = cast<SequentialType>(CurTy)->getElementType(); + Type *EltTy; + if (auto *Array = dyn_cast<ArrayType>(CurTy)) + EltTy = Array->getElementType(); + else + EltTy = cast<VectorType>(CurTy)->getElementType(); if (EltTy->isIntegerTy(8)) { SmallVector<uint8_t, 16> Elts(Record.begin(), Record.end()); if (isa<VectorType>(CurTy)) @@ -2497,21 +2535,27 @@ Error BitcodeReader::parseConstants() { } else if (EltTy->isHalfTy()) { SmallVector<uint16_t, 16> Elts(Record.begin(), Record.end()); if (isa<VectorType>(CurTy)) - V = ConstantDataVector::getFP(Context, Elts); + V = ConstantDataVector::getFP(EltTy, Elts); + else + V = ConstantDataArray::getFP(EltTy, Elts); + } else if (EltTy->isBFloatTy()) { + SmallVector<uint16_t, 16> Elts(Record.begin(), Record.end()); + if (isa<VectorType>(CurTy)) + V = ConstantDataVector::getFP(EltTy, Elts); else - V = ConstantDataArray::getFP(Context, Elts); + V = ConstantDataArray::getFP(EltTy, Elts); } else if (EltTy->isFloatTy()) { SmallVector<uint32_t, 16> Elts(Record.begin(), Record.end()); if (isa<VectorType>(CurTy)) - V = ConstantDataVector::getFP(Context, Elts); + V = ConstantDataVector::getFP(EltTy, Elts); else - V = ConstantDataArray::getFP(Context, Elts); + V = ConstantDataArray::getFP(EltTy, Elts); } else if (EltTy->isDoubleTy()) { SmallVector<uint64_t, 16> Elts(Record.begin(), Record.end()); if (isa<VectorType>(CurTy)) - V = ConstantDataVector::getFP(Context, Elts); + V = ConstantDataVector::getFP(EltTy, Elts); else - V = ConstantDataArray::getFP(Context, Elts); + V = ConstantDataArray::getFP(EltTy, Elts); } else { return error("Invalid type for value"); } @@ -2629,12 +2673,13 @@ Error BitcodeReader::parseConstants() { Type *SelectorTy = Type::getInt1Ty(Context); - // The selector might be an i1 or an <n x i1> + // The selector might be an i1, an <n x i1>, or a <vscale x n x i1> // Get the type from the ValueList before getting a forward ref. if (VectorType *VTy = dyn_cast<VectorType>(CurTy)) if (Value *V = ValueList[Record[0]]) if (SelectorTy != V->getType()) - SelectorTy = VectorType::get(SelectorTy, VTy->getNumElements()); + SelectorTy = VectorType::get(SelectorTy, + VTy->getElementCount()); V = ConstantExpr::getSelect(ValueList.getConstantFwdRef(Record[0], SelectorTy), @@ -2657,8 +2702,10 @@ Error BitcodeReader::parseConstants() { if (!IdxTy) return error("Invalid record"); Op1 = ValueList.getConstantFwdRef(Record[3], IdxTy); - } else // TODO: Remove with llvm 4.0 + } else { + // Deprecated, but still needed to read old bitcode files. Op1 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context)); + } if (!Op1) return error("Invalid record"); V = ConstantExpr::getExtractElement(Op0, Op1); @@ -2678,8 +2725,10 @@ Error BitcodeReader::parseConstants() { if (!IdxTy) return error("Invalid record"); Op2 = ValueList.getConstantFwdRef(Record[3], IdxTy); - } else // TODO: Remove with llvm 4.0 + } else { + // Deprecated, but still needed to read old bitcode files. Op2 = ValueList.getConstantFwdRef(Record[2], Type::getInt32Ty(Context)); + } if (!Op2) return error("Invalid record"); V = ConstantExpr::getInsertElement(Op0, Op1, Op2); @@ -2689,13 +2738,10 @@ Error BitcodeReader::parseConstants() { VectorType *OpTy = dyn_cast<VectorType>(CurTy); if (Record.size() < 3 || !OpTy) return error("Invalid record"); - Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy); - Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy); - Type *ShufTy = VectorType::get(Type::getInt32Ty(Context), - OpTy->getNumElements()); - Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy); - V = ConstantExpr::getShuffleVector(Op0, Op1, Op2); - break; + DelayedShuffles.push_back( + {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2], NextCstNo}); + ++NextCstNo; + continue; } case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval] VectorType *RTy = dyn_cast<VectorType>(CurTy); @@ -2703,13 +2749,10 @@ Error BitcodeReader::parseConstants() { dyn_cast_or_null<VectorType>(getTypeByID(Record[0])); if (Record.size() < 4 || !RTy || !OpTy) return error("Invalid record"); - Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy); - Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy); - Type *ShufTy = VectorType::get(Type::getInt32Ty(Context), - RTy->getNumElements()); - Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy); - V = ConstantExpr::getShuffleVector(Op0, Op1, Op2); - break; + DelayedShuffles.push_back( + {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3], NextCstNo}); + ++NextCstNo; + continue; } case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred] if (Record.size() < 4) @@ -2727,7 +2770,7 @@ Error BitcodeReader::parseConstants() { break; } // This maintains backward compatibility, pre-asm dialect keywords. - // FIXME: Remove with the 4.0 release. + // Deprecated, but still needed to read old bitcode files. case bitc::CST_CODE_INLINEASM_OLD: { if (Record.size() < 2) return error("Invalid record"); @@ -2967,6 +3010,7 @@ Error BitcodeReader::globalCleanup() { return error("Malformed global initializer set"); // Look for intrinsic functions which need to be upgraded at some point + // and functions that need to have their function attributes upgraded. for (Function &F : *TheModule) { MDLoader->upgradeDebugIntrinsics(F); Function *NewFn; @@ -2977,6 +3021,8 @@ Error BitcodeReader::globalCleanup() { // loaded in the same LLVMContext (LTO scenario). In this case we should // remangle intrinsics names as well. RemangledIntrinsics[&F] = Remangled.getValue(); + // Look for functions that rely on old function attribute behavior. + UpgradeFunctionAttributes(F); } // Look for global variables which need to be renamed. @@ -3125,8 +3171,8 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef<uint64_t> Record) { } GlobalValue::VisibilityTypes Visibility = GlobalValue::DefaultVisibility; // Local linkage must have default visibility. + // auto-upgrade `hidden` and `protected` for old bitcode. if (Record.size() > 6 && !GlobalValue::isLocalLinkage(Linkage)) - // FIXME: Change to an error if non-default in 4.0. Visibility = getDecodedVisibility(Record[6]); GlobalVariable::ThreadLocalMode TLM = GlobalVariable::NotThreadLocal; @@ -3255,8 +3301,8 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef<uint64_t> Record) { Func->setSection(SectionTable[Record[6] - 1]); } // Local linkage must have default visibility. + // auto-upgrade `hidden` and `protected` for old bitcode. if (!Func->hasLocalLinkage()) - // FIXME: Change to an error if non-default in 4.0. Func->setVisibility(getDecodedVisibility(Record[7])); if (Record.size() > 8 && Record[8]) { if (Record[8] - 1 >= GCTable.size()) @@ -3363,12 +3409,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( assert(NewGA->getValueType() == flattenPointerTypes(FullTy) && "Incorrect fully structured type provided for GlobalIndirectSymbol"); - // Old bitcode files didn't have visibility field. // Local linkage must have default visibility. + // auto-upgrade `hidden` and `protected` for old bitcode. if (OpNum != Record.size()) { auto VisInd = OpNum++; if (!NewGA->hasLocalLinkage()) - // FIXME: Change to an error if non-default in 4.0. NewGA->setVisibility(getDecodedVisibility(Record[VisInd])); } if (BitCode == bitc::MODULE_CODE_ALIAS || @@ -3402,7 +3447,8 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( } Error BitcodeReader::parseModule(uint64_t ResumeBit, - bool ShouldLazyLoadMetadata) { + bool ShouldLazyLoadMetadata, + DataLayoutCallbackTy DataLayoutCallback) { if (ResumeBit) { if (Error JumpFailed = Stream.JumpToBit(ResumeBit)) return JumpFailed; @@ -3411,6 +3457,26 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, SmallVector<uint64_t, 64> Record; + // Parts of bitcode parsing depend on the datalayout. Make sure we + // finalize the datalayout before we run any of that code. + bool ResolvedDataLayout = false; + auto ResolveDataLayout = [&] { + if (ResolvedDataLayout) + return; + + // datalayout and triple can't be parsed after this point. + ResolvedDataLayout = true; + + // Upgrade data layout string. + std::string DL = llvm::UpgradeDataLayoutString( + TheModule->getDataLayoutStr(), TheModule->getTargetTriple()); + TheModule->setDataLayout(DL); + + if (auto LayoutOverride = + DataLayoutCallback(TheModule->getTargetTriple())) + TheModule->setDataLayout(*LayoutOverride); + }; + // Read all the records for this module. while (true) { Expected<llvm::BitstreamEntry> MaybeEntry = Stream.advance(); @@ -3422,6 +3488,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, case BitstreamEntry::Error: return error("Malformed block"); case BitstreamEntry::EndBlock: + ResolveDataLayout(); return globalCleanup(); case BitstreamEntry::SubBlock: @@ -3486,6 +3553,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, return Err; break; case bitc::FUNCTION_BLOCK_ID: + ResolveDataLayout(); + // If this is the first function body we've seen, reverse the // FunctionsWithBodies list. if (!SeenFirstFunctionBody) { @@ -3572,6 +3641,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, break; } case bitc::MODULE_CODE_TRIPLE: { // TRIPLE: [strchr x N] + if (ResolvedDataLayout) + return error("target triple too late in module"); std::string S; if (convertToString(Record, 0, S)) return error("Invalid record"); @@ -3579,6 +3650,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, break; } case bitc::MODULE_CODE_DATALAYOUT: { // DATALAYOUT: [strchr x N] + if (ResolvedDataLayout) + return error("datalayout too late in module"); std::string S; if (convertToString(Record, 0, S)) return error("Invalid record"); @@ -3593,7 +3666,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, break; } case bitc::MODULE_CODE_DEPLIB: { // DEPLIB: [strchr x N] - // FIXME: Remove in 4.0. + // Deprecated, but still needed to read old bitcode files. std::string S; if (convertToString(Record, 0, S)) return error("Invalid record"); @@ -3623,6 +3696,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, return Err; break; case bitc::MODULE_CODE_FUNCTION: + ResolveDataLayout(); if (Error Err = parseFunctionRecord(Record)) return Err; break; @@ -3650,20 +3724,16 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, break; } Record.clear(); - - // Upgrade data layout string. - std::string DL = llvm::UpgradeDataLayoutString( - TheModule->getDataLayoutStr(), TheModule->getTargetTriple()); - TheModule->setDataLayout(DL); } } Error BitcodeReader::parseBitcodeInto(Module *M, bool ShouldLazyLoadMetadata, - bool IsImporting) { + bool IsImporting, + DataLayoutCallbackTy DataLayoutCallback) { TheModule = M; MDLoader = MetadataLoader(Stream, *M, ValueList, IsImporting, [&](unsigned ID) { return getTypeByID(ID); }); - return parseModule(0, ShouldLazyLoadMetadata); + return parseModule(0, ShouldLazyLoadMetadata, DataLayoutCallback); } Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { @@ -4135,7 +4205,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (!Vec->getType()->isVectorTy()) return error("Invalid type for value"); I = ExtractElementInst::Create(Vec, Idx); - FullTy = FullTy->getVectorElementType(); + FullTy = cast<VectorType>(FullTy)->getElementType(); InstructionList.push_back(I); break; } @@ -4167,9 +4237,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); if (!Vec1->getType()->isVectorTy() || !Vec2->getType()->isVectorTy()) return error("Invalid type for value"); + I = new ShuffleVectorInst(Vec1, Vec2, Mask); - FullTy = VectorType::get(FullTy->getVectorElementType(), - Mask->getType()->getVectorNumElements()); + FullTy = + VectorType::get(cast<VectorType>(FullTy)->getElementType(), + cast<VectorType>(Mask->getType())->getElementCount()); InstructionList.push_back(I); break; } @@ -4763,7 +4835,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { const DataLayout &DL = TheModule->getDataLayout(); unsigned AS = DL.getAllocaAddrSpace(); - AllocaInst *AI = new AllocaInst(Ty, AS, Size, Align); + SmallPtrSet<Type *, 4> Visited; + if (!Align && !Ty->isSized(&Visited)) + return error("alloca of unsized type"); + if (!Align) + Align = DL.getPrefTypeAlign(Ty); + + AllocaInst *AI = new AllocaInst(Ty, AS, Size, *Align); AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); I = AI; @@ -4794,7 +4872,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { MaybeAlign Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align); + SmallPtrSet<Type *, 4> Visited; + if (!Align && !Ty->isSized(&Visited)) + return error("load of unsized type"); + if (!Align) + Align = TheModule->getDataLayout().getABITypeAlign(Ty); + I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align); InstructionList.push_back(I); break; } @@ -4831,7 +4914,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) { MaybeAlign Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align, Ordering, SSID); + if (!Align) + return error("Alignment missing from atomic load"); + I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID); InstructionList.push_back(I); break; } @@ -4853,7 +4938,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { MaybeAlign Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new StoreInst(Val, Ptr, Record[OpNum + 1], Align); + SmallPtrSet<Type *, 4> Visited; + if (!Align && !Val->getType()->isSized(&Visited)) + return error("store of unsized type"); + if (!Align) + Align = TheModule->getDataLayout().getABITypeAlign(Val->getType()); + I = new StoreInst(Val, Ptr, Record[OpNum + 1], *Align); InstructionList.push_back(I); break; } @@ -4886,7 +4976,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) { MaybeAlign Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new StoreInst(Val, Ptr, Record[OpNum + 1], Align, Ordering, SSID); + if (!Align) + return error("Alignment missing from atomic store"); + I = new StoreInst(Val, Ptr, Record[OpNum + 1], *Align, Ordering, SSID); InstructionList.push_back(I); break; } @@ -4930,8 +5022,10 @@ Error BitcodeReader::parseFunctionBody(Function *F) { else FailureOrdering = getDecodedOrdering(Record[OpNum + 3]); - I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering, - SSID); + Align Alignment( + TheModule->getDataLayout().getTypeStoreSize(Cmp->getType())); + I = new AtomicCmpXchgInst(Ptr, Cmp, New, Alignment, SuccessOrdering, + FailureOrdering, SSID); FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); cast<AtomicCmpXchgInst>(I)->setVolatile(Record[OpNum]); @@ -4968,7 +5062,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Ordering == AtomicOrdering::Unordered) return error("Invalid record"); SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); - I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID); + Align Alignment( + TheModule->getDataLayout().getTypeStoreSize(Val->getType())); + I = new AtomicRMWInst(Operation, Ptr, Val, Alignment, Ordering, SSID); FullTy = getPointerElementFlatType(FullTy); cast<AtomicRMWInst>(I)->setVolatile(Record[OpNum+1]); InstructionList.push_back(I); @@ -5161,8 +5257,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { !FullTy->isPointerTy() && !isa<StructType>(FullTy) && !isa<ArrayType>(FullTy) && (!isa<VectorType>(FullTy) || - FullTy->getVectorElementType()->isFloatingPointTy() || - FullTy->getVectorElementType()->isIntegerTy()) && + cast<VectorType>(FullTy)->getElementType()->isFloatingPointTy() || + cast<VectorType>(FullTy)->getElementType()->isIntegerTy()) && "Structured types must be assigned with corresponding non-opaque " "pointer type"); } @@ -5277,7 +5373,7 @@ Error BitcodeReader::materialize(GlobalValue *GV) { for (auto UI = I.first->materialized_user_begin(), UE = I.first->user_end(); UI != UE;) // Don't expect any other users than call sites - CallSite(*UI++).setCalledFunction(I.second); + cast<CallBase>(*UI++)->setCalledFunction(I.second); // Finish fn->subprogram upgrade for materialized functions. if (DISubprogram *SP = MDLoader->lookupSubprogramForFunction(F)) @@ -5294,6 +5390,9 @@ Error BitcodeReader::materialize(GlobalValue *GV) { } } + // Look for functions that rely on old function attribute behavior. + UpgradeFunctionAttributes(*F); + // Bring in any functions that this function forward-referenced via // blockaddresses. return materializeForwardReferencedFunctions(); @@ -5722,6 +5821,41 @@ static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record, parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId); } +static std::vector<FunctionSummary::ParamAccess> +parseParamAccesses(ArrayRef<uint64_t> Record) { + auto ReadRange = [&]() { + APInt Lower(FunctionSummary::ParamAccess::RangeWidth, + BitcodeReader::decodeSignRotatedValue(Record.front())); + Record = Record.drop_front(); + APInt Upper(FunctionSummary::ParamAccess::RangeWidth, + BitcodeReader::decodeSignRotatedValue(Record.front())); + Record = Record.drop_front(); + ConstantRange Range{Lower, Upper}; + assert(!Range.isFullSet()); + assert(!Range.isUpperSignWrapped()); + return Range; + }; + + std::vector<FunctionSummary::ParamAccess> PendingParamAccesses; + while (!Record.empty()) { + PendingParamAccesses.emplace_back(); + FunctionSummary::ParamAccess &ParamAccess = PendingParamAccesses.back(); + ParamAccess.ParamNo = Record.front(); + Record = Record.drop_front(); + ParamAccess.Use = ReadRange(); + ParamAccess.Calls.resize(Record.front()); + Record = Record.drop_front(); + for (auto &Call : ParamAccess.Calls) { + Call.ParamNo = Record.front(); + Record = Record.drop_front(); + Call.Callee = Record.front(); + Record = Record.drop_front(); + Call.Offsets = ReadRange(); + } + } + return PendingParamAccesses; +} + void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableInfo( ArrayRef<uint64_t> Record, size_t &Slot, TypeIdCompatibleVtableInfo &TypeId) { @@ -5799,6 +5933,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { PendingTypeCheckedLoadVCalls; std::vector<FunctionSummary::ConstVCall> PendingTypeTestAssumeConstVCalls, PendingTypeCheckedLoadConstVCalls; + std::vector<FunctionSummary::ParamAccess> PendingParamAccesses; while (true) { Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks(); @@ -5832,35 +5967,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { default: // Default behavior: ignore. break; case bitc::FS_FLAGS: { // [flags] - uint64_t Flags = Record[0]; - // Scan flags. - assert(Flags <= 0x3f && "Unexpected bits in flag"); - - // 1 bit: WithGlobalValueDeadStripping flag. - // Set on combined index only. - if (Flags & 0x1) - TheIndex.setWithGlobalValueDeadStripping(); - // 1 bit: SkipModuleByDistributedBackend flag. - // Set on combined index only. - if (Flags & 0x2) - TheIndex.setSkipModuleByDistributedBackend(); - // 1 bit: HasSyntheticEntryCounts flag. - // Set on combined index only. - if (Flags & 0x4) - TheIndex.setHasSyntheticEntryCounts(); - // 1 bit: DisableSplitLTOUnit flag. - // Set on per module indexes. It is up to the client to validate - // the consistency of this flag across modules being linked. - if (Flags & 0x8) - TheIndex.setEnableSplitLTOUnit(); - // 1 bit: PartiallySplitLTOUnits flag. - // Set on combined index only. - if (Flags & 0x10) - TheIndex.setPartiallySplitLTOUnits(); - // 1 bit: WithAttributePropagation flag. - // Set on combined index only. - if (Flags & 0x20) - TheIndex.setWithAttributePropagation(); + TheIndex.setFlags(Record[0]); break; } case bitc::FS_VALUE_GUID: { // [valueid, refguid] @@ -5925,7 +6032,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), - std::move(PendingTypeCheckedLoadConstVCalls)); + std::move(PendingTypeCheckedLoadConstVCalls), + std::move(PendingParamAccesses)); auto VIAndOriginalGUID = getValueInfoFromValueId(ValueID); FS->setModulePath(getThisModule()->first()); FS->setOriginalName(VIAndOriginalGUID.second); @@ -5965,7 +6073,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t RawFlags = Record[1]; unsigned RefArrayStart = 2; GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false, - /* WriteOnly */ false); + /* WriteOnly */ false, + /* Constant */ false, + GlobalObject::VCallVisibilityPublic); auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); if (Version >= 5) { GVF = getDecodedGVarFlags(Record[2]); @@ -6065,7 +6175,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::move(PendingTypeTestAssumeVCalls), std::move(PendingTypeCheckedLoadVCalls), std::move(PendingTypeTestAssumeConstVCalls), - std::move(PendingTypeCheckedLoadConstVCalls)); + std::move(PendingTypeCheckedLoadConstVCalls), + std::move(PendingParamAccesses)); LastSeenSummary = FS.get(); LastSeenGUID = VI.getGUID(); FS->setModulePath(ModuleIdMap[ModuleId]); @@ -6101,7 +6212,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t RawFlags = Record[2]; unsigned RefArrayStart = 3; GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false, - /* WriteOnly */ false); + /* WriteOnly */ false, + /* Constant */ false, + GlobalObject::VCallVisibilityPublic); auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); if (Version >= 5) { GVF = getDecodedGVarFlags(Record[3]); @@ -6181,6 +6294,15 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { case bitc::FS_TYPE_ID_METADATA: parseTypeIdCompatibleVtableSummaryRecord(Record); break; + + case bitc::FS_BLOCK_COUNT: + TheIndex.addBlockCount(Record[0]); + break; + + case bitc::FS_PARAM_ACCESS: { + PendingParamAccesses = parseParamAccesses(Record); + break; + } } } llvm_unreachable("Exit infinite loop"); @@ -6452,7 +6574,8 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { /// everything. Expected<std::unique_ptr<Module>> BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, - bool ShouldLazyLoadMetadata, bool IsImporting) { + bool ShouldLazyLoadMetadata, bool IsImporting, + DataLayoutCallbackTy DataLayoutCallback) { BitstreamCursor Stream(Buffer); std::string ProducerIdentification; @@ -6477,8 +6600,8 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, M->setMaterializer(R); // Delay parsing Metadata if ShouldLazyLoadMetadata is true. - if (Error Err = - R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata, IsImporting)) + if (Error Err = R->parseBitcodeInto(M.get(), ShouldLazyLoadMetadata, + IsImporting, DataLayoutCallback)) return std::move(Err); if (MaterializeAll) { @@ -6496,7 +6619,8 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, Expected<std::unique_ptr<Module>> BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, bool IsImporting) { - return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting); + return getModuleImpl(Context, false, ShouldLazyLoadMetadata, IsImporting, + [](StringRef) { return None; }); } // Parse the specified bitcode buffer and merge the index into CombinedIndex. @@ -6662,19 +6786,21 @@ Expected<std::unique_ptr<Module>> llvm::getOwningLazyBitcodeModule( } Expected<std::unique_ptr<Module>> -BitcodeModule::parseModule(LLVMContext &Context) { - return getModuleImpl(Context, true, false, false); +BitcodeModule::parseModule(LLVMContext &Context, + DataLayoutCallbackTy DataLayoutCallback) { + return getModuleImpl(Context, true, false, false, DataLayoutCallback); // TODO: Restore the use-lists to the in-memory state when the bitcode was // written. We must defer until the Module has been fully materialized. } -Expected<std::unique_ptr<Module>> llvm::parseBitcodeFile(MemoryBufferRef Buffer, - LLVMContext &Context) { +Expected<std::unique_ptr<Module>> +llvm::parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context, + DataLayoutCallbackTy DataLayoutCallback) { Expected<BitcodeModule> BM = getSingleModule(Buffer); if (!BM) return BM.takeError(); - return BM->parseModule(Context); + return BM->parseModule(Context, DataLayoutCallback); } Expected<std::string> llvm::getBitcodeTargetTriple(MemoryBufferRef Buffer) { diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp index d16c3b0ff59d..a8bf579bd180 100644 --- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp @@ -1132,7 +1132,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_OLD_FN_NODE: { - // FIXME: Remove in 4.0. + // Deprecated, but still needed to read old bitcode files. // This is a LocalAsMetadata record, the only type of function-local // metadata. if (Record.size() % 2 == 1) @@ -1162,7 +1162,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_OLD_NODE: { - // FIXME: Remove in 4.0. + // Deprecated, but still needed to read old bitcode files. if (Record.size() % 2 == 1) return error("Invalid record"); @@ -1258,14 +1258,24 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( // Operand 'count' is interpreted as: // - Signed integer (version 0) // - Metadata node (version 1) + // Operand 'lowerBound' is interpreted as: + // - Signed integer (version 0 and 1) + // - Metadata node (version 2) + // Operands 'upperBound' and 'stride' are interpreted as: + // - Metadata node (version 2) switch (Record[0] >> 1) { case 0: Val = GET_OR_DISTINCT(DISubrange, - (Context, Record[1], unrotateSign(Record.back()))); + (Context, Record[1], unrotateSign(Record[2]))); break; case 1: Val = GET_OR_DISTINCT(DISubrange, (Context, getMDOrNull(Record[1]), - unrotateSign(Record.back()))); + unrotateSign(Record[2]))); + break; + case 2: + Val = GET_OR_DISTINCT( + DISubrange, (Context, getMDOrNull(Record[1]), getMDOrNull(Record[2]), + getMDOrNull(Record[3]), getMDOrNull(Record[4]))); break; default: return error("Invalid record: Unsupported version of DISubrange"); @@ -1277,14 +1287,24 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_ENUMERATOR: { - if (Record.size() != 3) + if (Record.size() < 3) return error("Invalid record"); IsDistinct = Record[0] & 1; bool IsUnsigned = Record[0] & 2; + bool IsBigInt = Record[0] & 4; + APInt Value; + + if (IsBigInt) { + const uint64_t BitWidth = Record[1]; + const size_t NumWords = Record.size() - 3; + Value = readWideAPInt(makeArrayRef(&Record[3], NumWords), BitWidth); + } else + Value = APInt(64, unrotateSign(Record[1]), !IsUnsigned); + MetadataList.assignValue( - GET_OR_DISTINCT(DIEnumerator, (Context, unrotateSign(Record[1]), - IsUnsigned, getMDString(Record[2]))), + GET_OR_DISTINCT(DIEnumerator, + (Context, Value, IsUnsigned, getMDString(Record[2]))), NextMetadataNo); NextMetadataNo++; break; @@ -1330,7 +1350,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPOSITE_TYPE: { - if (Record.size() < 16 || Record.size() > 17) + if (Record.size() < 16 || Record.size() > 18) return error("Invalid record"); // If we have a UUID and this is not a forward declaration, lookup the @@ -1354,6 +1374,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Metadata *VTableHolder = nullptr; Metadata *TemplateParams = nullptr; Metadata *Discriminator = nullptr; + Metadata *DataLocation = nullptr; auto *Identifier = getMDString(Record[15]); // If this module is being parsed so that it can be ThinLTO imported // into another module, composite types only need to be imported @@ -1376,13 +1397,15 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( TemplateParams = getMDOrNull(Record[14]); if (Record.size() > 16) Discriminator = getMDOrNull(Record[16]); + if (Record.size() > 17) + DataLocation = getMDOrNull(Record[17]); } DICompositeType *CT = nullptr; if (Identifier) CT = DICompositeType::buildODRType( Context, *Identifier, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams, Discriminator); + VTableHolder, TemplateParams, Discriminator, DataLocation); // Create a node if we didn't get a lazy ODR type. if (!CT) @@ -1390,7 +1413,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( (Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams, - Identifier, Discriminator)); + Identifier, Discriminator, DataLocation)); if (!IsNotUsedInTypeRef && Identifier) MetadataList.addTypeRef(*Identifier, *cast<DICompositeType>(CT)); @@ -1418,15 +1441,19 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( } case bitc::METADATA_MODULE: { - if (Record.size() != 6) + if (Record.size() < 5 || Record.size() > 8) return error("Invalid record"); + unsigned Offset = Record.size() >= 7 ? 2 : 1; IsDistinct = Record[0]; MetadataList.assignValue( - GET_OR_DISTINCT(DIModule, - (Context, getMDOrNull(Record[1]), - getMDString(Record[2]), getMDString(Record[3]), - getMDString(Record[4]), getMDString(Record[5]))), + GET_OR_DISTINCT( + DIModule, + (Context, Record.size() >= 7 ? getMDOrNull(Record[1]) : nullptr, + getMDOrNull(Record[0 + Offset]), getMDString(Record[1 + Offset]), + getMDString(Record[2 + Offset]), getMDString(Record[3 + Offset]), + getMDString(Record[4 + Offset]), + Record.size() <= 7 ? 0 : Record[7])), NextMetadataNo); NextMetadataNo++; break; @@ -1457,7 +1484,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_COMPILE_UNIT: { - if (Record.size() < 14 || Record.size() > 19) + if (Record.size() < 14 || Record.size() > 22) return error("Invalid record"); // Ignore Record[0], which indicates whether this compile unit is @@ -1473,7 +1500,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( Record.size() <= 16 ? true : Record[16], Record.size() <= 17 ? false : Record[17], Record.size() <= 18 ? 0 : Record[18], - Record.size() <= 19 ? 0 : Record[19]); + Record.size() <= 19 ? 0 : Record[19], + Record.size() <= 20 ? nullptr : getMDString(Record[20]), + Record.size() <= 21 ? nullptr : getMDString(Record[21])); MetadataList.assignValue(CU, NextMetadataNo); NextMetadataNo++; @@ -1667,27 +1696,34 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( break; } case bitc::METADATA_TEMPLATE_TYPE: { - if (Record.size() != 3) + if (Record.size() < 3 || Record.size() > 4) return error("Invalid record"); IsDistinct = Record[0]; - MetadataList.assignValue(GET_OR_DISTINCT(DITemplateTypeParameter, - (Context, getMDString(Record[1]), - getDITypeRefOrNull(Record[2]))), - NextMetadataNo); + MetadataList.assignValue( + GET_OR_DISTINCT(DITemplateTypeParameter, + (Context, getMDString(Record[1]), + getDITypeRefOrNull(Record[2]), + (Record.size() == 4) ? getMDOrNull(Record[3]) + : getMDOrNull(false))), + NextMetadataNo); NextMetadataNo++; break; } case bitc::METADATA_TEMPLATE_VALUE: { - if (Record.size() != 5) + if (Record.size() < 5 || Record.size() > 6) return error("Invalid record"); IsDistinct = Record[0]; + MetadataList.assignValue( - GET_OR_DISTINCT(DITemplateValueParameter, - (Context, Record[1], getMDString(Record[2]), - getDITypeRefOrNull(Record[3]), - getMDOrNull(Record[4]))), + GET_OR_DISTINCT( + DITemplateValueParameter, + (Context, Record[1], getMDString(Record[2]), + getDITypeRefOrNull(Record[3]), + (Record.size() == 6) ? getMDOrNull(Record[4]) : getMDOrNull(false), + (Record.size() == 6) ? getMDOrNull(Record[5]) + : getMDOrNull(Record[4]))), NextMetadataNo); NextMetadataNo++; break; diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.h b/llvm/lib/Bitcode/Reader/MetadataLoader.h index fe2b20273249..709800850f0d 100644 --- a/llvm/lib/Bitcode/Reader/MetadataLoader.h +++ b/llvm/lib/Bitcode/Reader/MetadataLoader.h @@ -23,11 +23,9 @@ namespace llvm { class BitcodeReaderValueList; class BitstreamCursor; class DISubprogram; -class Error; class Function; class Instruction; class Metadata; -class MDNode; class Module; class Type; diff --git a/llvm/lib/Bitcode/Reader/ValueList.cpp b/llvm/lib/Bitcode/Reader/ValueList.cpp index 431995fd40ac..63a206eeb022 100644 --- a/llvm/lib/Bitcode/Reader/ValueList.cpp +++ b/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -220,6 +220,6 @@ void BitcodeReaderValueList::resolveConstantForwardRefs() { // Update all ValueHandles, they should be the only users at this point. Placeholder->replaceAllUsesWith(RealVal); - Placeholder->deleteValue(); + delete cast<ConstantPlaceHolder>(Placeholder); } } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index dcff7c421fc4..9c15a5f9f193 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -31,7 +31,6 @@ #include "llvm/Config/llvm-config.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" @@ -374,7 +373,7 @@ private: void writeModuleConstants(); bool pushValueAndType(const Value *V, unsigned InstID, SmallVectorImpl<unsigned> &Vals); - void writeOperandBundles(ImmutableCallSite CS, unsigned InstID); + void writeOperandBundles(const CallBase &CB, unsigned InstID); void pushValue(const Value *V, unsigned InstID, SmallVectorImpl<unsigned> &Vals); void pushValueSigned(const Value *V, unsigned InstID, @@ -648,6 +647,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NO_INLINE; case Attribute::NoRecurse: return bitc::ATTR_KIND_NO_RECURSE; + case Attribute::NoMerge: + return bitc::ATTR_KIND_NO_MERGE; case Attribute::NonLazyBind: return bitc::ATTR_KIND_NON_LAZY_BIND; case Attribute::NonNull: @@ -666,6 +667,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoUnwind: return bitc::ATTR_KIND_NO_UNWIND; + case Attribute::NullPointerIsValid: + return bitc::ATTR_KIND_NULL_POINTER_IS_VALID; case Attribute::OptForFuzzing: return bitc::ATTR_KIND_OPT_FOR_FUZZING; case Attribute::OptimizeForSize: @@ -726,10 +729,17 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_IMMARG; case Attribute::SanitizeMemTag: return bitc::ATTR_KIND_SANITIZE_MEMTAG; + case Attribute::Preallocated: + return bitc::ATTR_KIND_PREALLOCATED; + case Attribute::NoUndef: + return bitc::ATTR_KIND_NOUNDEF; case Attribute::EndAttrKinds: llvm_unreachable("Can not encode end-attribute kinds marker."); case Attribute::None: llvm_unreachable("Can not encode none-attribute."); + case Attribute::EmptyKey: + case Attribute::TombstoneKey: + llvm_unreachable("Trying to encode EmptyKey/TombstoneKey"); } llvm_unreachable("Trying to encode unknown attribute"); @@ -875,6 +885,7 @@ void ModuleBitcodeWriter::writeTypeTable() { switch (T->getTypeID()) { case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break; case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break; + case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break; case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break; case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break; case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break; @@ -946,15 +957,16 @@ void ModuleBitcodeWriter::writeTypeTable() { AbbrevToUse = ArrayAbbrev; break; } - case Type::VectorTyID: { + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: { VectorType *VT = cast<VectorType>(T); // VECTOR [numelts, eltty] or // [numelts, eltty, scalable] Code = bitc::TYPE_CODE_VECTOR; - TypeVals.push_back(VT->getNumElements()); + TypeVals.push_back(VT->getElementCount().Min); TypeVals.push_back(VE.getTypeID(VT->getElementType())); - if (VT->isScalable()) - TypeVals.push_back(VT->isScalable()); + if (isa<ScalableVectorType>(VT)) + TypeVals.push_back(true); break; } } @@ -1028,7 +1040,8 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { } static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) { - uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1); + uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) | + (Flags.Constant << 2) | Flags.VCallVisibility << 3; return RawFlags; } @@ -1168,12 +1181,12 @@ void ModuleBitcodeWriter::writeModuleInfo() { std::map<std::string, unsigned> GCMap; unsigned MaxAlignment = 0; unsigned MaxGlobalType = 0; - for (const GlobalValue &GV : M.globals()) { + for (const GlobalVariable &GV : M.globals()) { MaxAlignment = std::max(MaxAlignment, GV.getAlignment()); MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType())); if (GV.hasSection()) { // Give section names unique ID's. - unsigned &Entry = SectionMap[GV.getSection()]; + unsigned &Entry = SectionMap[std::string(GV.getSection())]; if (!Entry) { writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(), 0 /*TODO*/); @@ -1185,7 +1198,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { MaxAlignment = std::max(MaxAlignment, F.getAlignment()); if (F.hasSection()) { // Give section names unique ID's. - unsigned &Entry = SectionMap[F.getSection()]; + unsigned &Entry = SectionMap[std::string(F.getSection())]; if (!Entry) { writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, F.getSection(), 0 /*TODO*/); @@ -1275,7 +1288,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { (VE.getValueID(GV.getInitializer()) + 1)); Vals.push_back(getEncodedLinkage(GV)); Vals.push_back(Log2_32(GV.getAlignment())+1); - Vals.push_back(GV.hasSection() ? SectionMap[GV.getSection()] : 0); + Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())] + : 0); if (GV.isThreadLocal() || GV.getVisibility() != GlobalValue::DefaultVisibility || GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None || @@ -1320,7 +1334,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(getEncodedLinkage(F)); Vals.push_back(VE.getAttributeListID(F.getAttributes())); Vals.push_back(Log2_32(F.getAlignment())+1); - Vals.push_back(F.hasSection() ? SectionMap[F.getSection()] : 0); + Vals.push_back(F.hasSection() ? SectionMap[std::string(F.getSection())] + : 0); Vals.push_back(getEncodedVisibility(F)); Vals.push_back(F.hasGC() ? GCMap[F.getGC()] : 0); Vals.push_back(getEncodedUnnamedAddr(F)); @@ -1506,29 +1521,46 @@ void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N, Record.clear(); } -static uint64_t rotateSign(int64_t I) { - uint64_t U = I; - return I < 0 ? ~(U << 1) : U << 1; -} - void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - const uint64_t Version = 1 << 1; + const uint64_t Version = 2 << 1; Record.push_back((uint64_t)N->isDistinct() | Version); Record.push_back(VE.getMetadataOrNullID(N->getRawCountNode())); - Record.push_back(rotateSign(N->getLowerBound())); + Record.push_back(VE.getMetadataOrNullID(N->getRawLowerBound())); + Record.push_back(VE.getMetadataOrNullID(N->getRawUpperBound())); + Record.push_back(VE.getMetadataOrNullID(N->getRawStride())); Stream.EmitRecord(bitc::METADATA_SUBRANGE, Record, Abbrev); Record.clear(); } +static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) { + if ((int64_t)V >= 0) + Vals.push_back(V << 1); + else + Vals.push_back((-V << 1) | 1); +} + +static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) { + // We have an arbitrary precision integer value to write whose + // bit width is > 64. However, in canonical unsigned integer + // format it is likely that the high bits are going to be zero. + // So, we only write the number of active words. + unsigned NumWords = A.getActiveWords(); + const uint64_t *RawData = A.getRawData(); + for (unsigned i = 0; i < NumWords; i++) + emitSignedInt64(Vals, RawData[i]); +} + void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N, SmallVectorImpl<uint64_t> &Record, unsigned Abbrev) { - Record.push_back((N->isUnsigned() << 1) | N->isDistinct()); - Record.push_back(rotateSign(N->getValue())); + const uint64_t IsBigInt = 1 << 2; + Record.push_back(IsBigInt | (N->isUnsigned() << 1) | N->isDistinct()); + Record.push_back(N->getValue().getBitWidth()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); + emitWideAPInt(Record, N->getValue()); Stream.EmitRecord(bitc::METADATA_ENUMERATOR, Record, Abbrev); Record.clear(); @@ -1597,6 +1629,7 @@ void ModuleBitcodeWriter::writeDICompositeType( Record.push_back(VE.getMetadataOrNullID(N->getTemplateParams().get())); Record.push_back(VE.getMetadataOrNullID(N->getRawIdentifier())); Record.push_back(VE.getMetadataOrNullID(N->getDiscriminator())); + Record.push_back(VE.getMetadataOrNullID(N->getRawDataLocation())); Stream.EmitRecord(bitc::METADATA_COMPOSITE_TYPE, Record, Abbrev); Record.clear(); @@ -1661,6 +1694,9 @@ void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N, Record.push_back(N->getSplitDebugInlining()); Record.push_back(N->getDebugInfoForProfiling()); Record.push_back((unsigned)N->getNameTableKind()); + Record.push_back(N->getRangesBaseAddress()); + Record.push_back(VE.getMetadataOrNullID(N->getRawSysRoot())); + Record.push_back(VE.getMetadataOrNullID(N->getRawSDK())); Stream.EmitRecord(bitc::METADATA_COMPILE_UNIT, Record, Abbrev); Record.clear(); @@ -1776,6 +1812,7 @@ void ModuleBitcodeWriter::writeDIModule(const DIModule *N, Record.push_back(N->isDistinct()); for (auto &I : N->operands()) Record.push_back(VE.getMetadataOrNullID(I)); + Record.push_back(N->getLineNo()); Stream.EmitRecord(bitc::METADATA_MODULE, Record, Abbrev); Record.clear(); @@ -1787,6 +1824,7 @@ void ModuleBitcodeWriter::writeDITemplateTypeParameter( Record.push_back(N->isDistinct()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getType())); + Record.push_back(N->isDefault()); Stream.EmitRecord(bitc::METADATA_TEMPLATE_TYPE, Record, Abbrev); Record.clear(); @@ -1799,6 +1837,7 @@ void ModuleBitcodeWriter::writeDITemplateValueParameter( Record.push_back(N->getTag()); Record.push_back(VE.getMetadataOrNullID(N->getRawName())); Record.push_back(VE.getMetadataOrNullID(N->getType())); + Record.push_back(N->isDefault()); Record.push_back(VE.getMetadataOrNullID(N->getValue())); Stream.EmitRecord(bitc::METADATA_TEMPLATE_VALUE, Record, Abbrev); @@ -2258,13 +2297,6 @@ void ModuleBitcodeWriter::writeSyncScopeNames() { Stream.ExitBlock(); } -static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) { - if ((int64_t)V >= 0) - Vals.push_back(V << 1); - else - Vals.push_back((-V << 1) | 1); -} - void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal) { if (FirstVal == LastVal) return; @@ -2351,21 +2383,14 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, Code = bitc::CST_CODE_INTEGER; AbbrevToUse = CONSTANTS_INTEGER_ABBREV; } else { // Wide integers, > 64 bits in size. - // We have an arbitrary precision integer value to write whose - // bit width is > 64. However, in canonical unsigned integer - // format it is likely that the high bits are going to be zero. - // So, we only write the number of active words. - unsigned NWords = IV->getValue().getActiveWords(); - const uint64_t *RawWords = IV->getValue().getRawData(); - for (unsigned i = 0; i != NWords; ++i) { - emitSignedInt64(Record, RawWords[i]); - } + emitWideAPInt(Record, IV->getValue()); Code = bitc::CST_CODE_WIDE_INTEGER; } } else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { Code = bitc::CST_CODE_FLOAT; Type *Ty = CFP->getType(); - if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy()) { + if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() || + Ty->isDoubleTy()) { Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue()); } else if (Ty->isX86_FP80Ty()) { // api needed to prevent premature destruction @@ -2412,7 +2437,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, } else if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(C)) { Code = bitc::CST_CODE_DATA; - Type *EltTy = CDS->getType()->getElementType(); + Type *EltTy = CDS->getElementType(); if (isa<IntegerType>(EltTy)) { for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) Record.push_back(CDS->getElementAsInteger(i)); @@ -2504,7 +2529,7 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal, } Record.push_back(VE.getValueID(C->getOperand(0))); Record.push_back(VE.getValueID(C->getOperand(1))); - Record.push_back(VE.getValueID(C->getOperand(2))); + Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode())); break; case Instruction::ICmp: case Instruction::FCmp: @@ -2566,10 +2591,10 @@ bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID, return false; } -void ModuleBitcodeWriter::writeOperandBundles(ImmutableCallSite CS, +void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS, unsigned InstID) { SmallVector<unsigned, 64> Record; - LLVMContext &C = CS.getInstruction()->getContext(); + LLVMContext &C = CS.getContext(); for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) { const auto &Bundle = CS.getOperandBundleAt(i); @@ -2691,7 +2716,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Code = bitc::FUNC_CODE_INST_SHUFFLEVEC; pushValueAndType(I.getOperand(0), InstID, Vals); pushValue(I.getOperand(1), InstID, Vals); - pushValue(I.getOperand(2), InstID, Vals); + pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID, + Vals); break; case Instruction::ICmp: case Instruction::FCmp: { @@ -2756,11 +2782,11 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, case Instruction::Invoke: { const InvokeInst *II = cast<InvokeInst>(&I); - const Value *Callee = II->getCalledValue(); + const Value *Callee = II->getCalledOperand(); FunctionType *FTy = II->getFunctionType(); if (II->hasOperandBundles()) - writeOperandBundles(II, InstID); + writeOperandBundles(*II, InstID); Code = bitc::FUNC_CODE_INST_INVOKE; @@ -2832,11 +2858,11 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, } case Instruction::CallBr: { const CallBrInst *CBI = cast<CallBrInst>(&I); - const Value *Callee = CBI->getCalledValue(); + const Value *Callee = CBI->getCalledOperand(); FunctionType *FTy = CBI->getFunctionType(); if (CBI->hasOperandBundles()) - writeOperandBundles(CBI, InstID); + writeOperandBundles(*CBI, InstID); Code = bitc::FUNC_CODE_INST_CALLBR; @@ -2993,7 +3019,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, FunctionType *FTy = CI.getFunctionType(); if (CI.hasOperandBundles()) - writeOperandBundles(&CI, InstID); + writeOperandBundles(CI, InstID); Code = bitc::FUNC_CODE_INST_CALL; @@ -3010,7 +3036,7 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(Flags); Vals.push_back(VE.getTypeID(FTy)); - pushValueAndType(CI.getCalledValue(), InstID, Vals); // Callee + pushValueAndType(CI.getCalledOperand(), InstID, Vals); // Callee // Emit value #'s for the fixed parameters. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) { @@ -3552,6 +3578,29 @@ static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream, FS->type_test_assume_const_vcalls()); WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL, FS->type_checked_load_const_vcalls()); + + auto WriteRange = [&](ConstantRange Range) { + Range = Range.sextOrTrunc(FunctionSummary::ParamAccess::RangeWidth); + assert(Range.getLower().getNumWords() == 1); + assert(Range.getUpper().getNumWords() == 1); + emitSignedInt64(Record, *Range.getLower().getRawData()); + emitSignedInt64(Record, *Range.getUpper().getRawData()); + }; + + if (!FS->paramAccesses().empty()) { + Record.clear(); + for (auto &Arg : FS->paramAccesses()) { + Record.push_back(Arg.ParamNo); + WriteRange(Arg.Use); + Record.push_back(Arg.Calls.size()); + for (auto &Call : Arg.Calls) { + Record.push_back(Call.ParamNo); + Record.push_back(Call.Callee); + WriteRange(Call.Offsets); + } + } + Stream.EmitRecord(bitc::FS_PARAM_ACCESS, Record); + } } /// Collect type IDs from type tests used by function. @@ -3882,6 +3931,9 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { NameVals.clear(); } + Stream.EmitRecord(bitc::FS_BLOCK_COUNT, + ArrayRef<uint64_t>{Index->getBlockCount()}); + Stream.ExitBlock(); } @@ -3893,20 +3945,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion}); // Write the index flags. - uint64_t Flags = 0; - if (Index.withGlobalValueDeadStripping()) - Flags |= 0x1; - if (Index.skipModuleByDistributedBackend()) - Flags |= 0x2; - if (Index.hasSyntheticEntryCounts()) - Flags |= 0x4; - if (Index.enableSplitLTOUnit()) - Flags |= 0x8; - if (Index.partiallySplitLTOUnits()) - Flags |= 0x10; - if (Index.withAttributePropagation()) - Flags |= 0x20; - Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags}); + Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Index.getFlags()}); for (const auto &GVI : valueIds()) { Stream.EmitRecord(bitc::FS_VALUE_GUID, @@ -4178,6 +4217,9 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { } } + Stream.EmitRecord(bitc::FS_BLOCK_COUNT, + ArrayRef<uint64_t>{Index.getBlockCount()}); + Stream.ExitBlock(); } @@ -4200,7 +4242,7 @@ static void writeIdentificationBlock(BitstreamWriter &Stream) { Abbv->Add(BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); auto EpochAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - SmallVector<unsigned, 1> Vals = {bitc::BITCODE_CURRENT_EPOCH}; + constexpr std::array<unsigned, 1> Vals = {{bitc::BITCODE_CURRENT_EPOCH}}; Stream.EmitRecord(bitc::IDENTIFICATION_CODE_EPOCH, Vals, EpochAbbrev); Stream.ExitBlock(); } diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index f59c906c7b75..8bdddc27e95a 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -88,11 +88,16 @@ static void orderValue(const Value *V, OrderMap &OM) { if (OM.lookup(V).first) return; - if (const Constant *C = dyn_cast<Constant>(V)) - if (C->getNumOperands() && !isa<GlobalValue>(C)) + if (const Constant *C = dyn_cast<Constant>(V)) { + if (C->getNumOperands() && !isa<GlobalValue>(C)) { for (const Value *Op : C->operands()) if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op)) orderValue(Op, OM); + if (auto *CE = dyn_cast<ConstantExpr>(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + orderValue(CE->getShuffleMaskForBitcode(), OM); + } + } // Note: we cannot cache this lookup above, since inserting into the map // changes the map's size, and thus affects the other IDs. @@ -155,11 +160,14 @@ static OrderMap orderModule(const Module &M) { for (const Argument &A : F.args()) orderValue(&A, OM); for (const BasicBlock &BB : F) - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Value *Op : I.operands()) if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) || isa<InlineAsm>(*Op)) orderValue(Op, OM); + if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) + orderValue(SVI->getShuffleMaskForBitcode(), OM); + } for (const BasicBlock &BB : F) for (const Instruction &I : BB) orderValue(&I, OM); @@ -223,9 +231,9 @@ static void predictValueUseListOrderImpl(const Value *V, const Function *F, return LU->getOperandNo() > RU->getOperandNo(); }); - if (std::is_sorted( - List.begin(), List.end(), - [](const Entry &L, const Entry &R) { return L.second < R.second; })) + if (llvm::is_sorted(List, [](const Entry &L, const Entry &R) { + return L.second < R.second; + })) // Order is already correct. return; @@ -250,11 +258,17 @@ static void predictValueUseListOrder(const Value *V, const Function *F, predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack); // Recursive descent into constants. - if (const Constant *C = dyn_cast<Constant>(V)) - if (C->getNumOperands()) // Visit GlobalValues. + if (const Constant *C = dyn_cast<Constant>(V)) { + if (C->getNumOperands()) { // Visit GlobalValues. for (const Value *Op : C->operands()) if (isa<Constant>(Op)) // Visit GlobalValues. predictValueUseListOrder(Op, F, OM, Stack); + if (auto *CE = dyn_cast<ConstantExpr>(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM, + Stack); + } + } } static UseListOrderStack predictUseListOrder(const Module &M) { @@ -279,10 +293,14 @@ static UseListOrderStack predictUseListOrder(const Module &M) { for (const Argument &A : F.args()) predictValueUseListOrder(&A, &F, OM, Stack); for (const BasicBlock &BB : F) - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Value *Op : I.operands()) if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues. predictValueUseListOrder(Op, &F, OM, Stack); + if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) + predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM, + Stack); + } for (const BasicBlock &BB : F) for (const Instruction &I : BB) predictValueUseListOrder(&I, &F, OM, Stack); @@ -413,6 +431,8 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateMetadata(&F, MD->getMetadata()); } + if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) + EnumerateType(SVI->getShuffleMaskForBitcode()->getType()); EnumerateType(I.getType()); if (const auto *Call = dyn_cast<CallBase>(&I)) EnumerateAttributes(Call->getAttributes()); @@ -836,6 +856,9 @@ void ValueEnumerator::EnumerateValue(const Value *V) { I != E; ++I) if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress. EnumerateValue(*I); + if (auto *CE = dyn_cast<ConstantExpr>(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + EnumerateValue(CE->getShuffleMaskForBitcode()); // Finally, add the value. Doing this could make the ValueID reference be // dangling, don't reuse it. @@ -913,6 +936,9 @@ void ValueEnumerator::EnumerateOperandType(const Value *V) { EnumerateOperandType(Op); } + if (auto *CE = dyn_cast<ConstantExpr>(C)) + if (CE->getOpcode() == Instruction::ShuffleVector) + EnumerateOperandType(CE->getShuffleMaskForBitcode()); } void ValueEnumerator::EnumerateAttributes(AttributeList PAL) { @@ -958,11 +984,14 @@ void ValueEnumerator::incorporateFunction(const Function &F) { // Add all function-level constants to the value table. for (const BasicBlock &BB : F) { - for (const Instruction &I : BB) + for (const Instruction &I : BB) { for (const Use &OI : I.operands()) { if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI)) EnumerateValue(OI); } + if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) + EnumerateValue(SVI->getShuffleMaskForBitcode()); + } BasicBlocks.push_back(&BB); ValueMap[&BB] = BasicBlocks.size(); } diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.h b/llvm/lib/Bitcode/Writer/ValueEnumerator.h index 112f0b4a1dc4..3c3bd0d9fdc7 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.h +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.h @@ -17,8 +17,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/UniqueVector.h" #include "llvm/IR/Attributes.h" -#include "llvm/IR/Metadata.h" -#include "llvm/IR/Type.h" #include "llvm/IR/UseListOrder.h" #include <cassert> #include <cstdint> |