diff options
Diffstat (limited to 'llvm/lib/IR')
53 files changed, 4819 insertions, 2290 deletions
diff --git a/llvm/lib/IR/AbstractCallSite.cpp b/llvm/lib/IR/AbstractCallSite.cpp index 19b35665c3fa..6504e566ba4b 100644 --- a/llvm/lib/IR/AbstractCallSite.cpp +++ b/llvm/lib/IR/AbstractCallSite.cpp @@ -14,9 +14,8 @@ // //===----------------------------------------------------------------------===// +#include "llvm/IR/AbstractCallSite.h" #include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/IR/CallSite.h" #include "llvm/Support/Debug.h" using namespace llvm; @@ -33,9 +32,9 @@ STATISTIC(NumInvalidAbstractCallSitesUnknownCallee, STATISTIC(NumInvalidAbstractCallSitesNoCallback, "Number of invalid abstract call sites created (no callback)"); -void AbstractCallSite::getCallbackUses(ImmutableCallSite ICS, - SmallVectorImpl<const Use *> &CBUses) { - const Function *Callee = ICS.getCalledFunction(); +void AbstractCallSite::getCallbackUses( + const CallBase &CB, SmallVectorImpl<const Use *> &CallbackUses) { + const Function *Callee = CB.getCalledFunction(); if (!Callee) return; @@ -48,56 +47,58 @@ void AbstractCallSite::getCallbackUses(ImmutableCallSite ICS, auto *CBCalleeIdxAsCM = cast<ConstantAsMetadata>(OpMD->getOperand(0)); uint64_t CBCalleeIdx = cast<ConstantInt>(CBCalleeIdxAsCM->getValue())->getZExtValue(); - CBUses.push_back(ICS.arg_begin() + CBCalleeIdx); + if (CBCalleeIdx < CB.arg_size()) + CallbackUses.push_back(CB.arg_begin() + CBCalleeIdx); } } /// Create an abstract call site from a use. -AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) { +AbstractCallSite::AbstractCallSite(const Use *U) + : CB(dyn_cast<CallBase>(U->getUser())) { // First handle unknown users. - if (!CS) { + if (!CB) { // If the use is actually in a constant cast expression which itself // has only one use, we look through the constant cast expression. // This happens by updating the use @p U to the use of the constant - // cast expression and afterwards re-initializing CS accordingly. + // cast expression and afterwards re-initializing CB accordingly. if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U->getUser())) - if (CE->getNumUses() == 1 && CE->isCast()) { + if (CE->hasOneUse() && CE->isCast()) { U = &*CE->use_begin(); - CS = CallSite(U->getUser()); + CB = dyn_cast<CallBase>(U->getUser()); } - if (!CS) { + if (!CB) { NumInvalidAbstractCallSitesUnknownUse++; return; } } // Then handle direct or indirect calls. Thus, if U is the callee of the - // call site CS it is not a callback and we are done. - if (CS.isCallee(U)) { + // call site CB it is not a callback and we are done. + if (CB->isCallee(U)) { NumDirectAbstractCallSites++; return; } // If we cannot identify the broker function we cannot create a callback and // invalidate the abstract call site. - Function *Callee = CS.getCalledFunction(); + Function *Callee = CB->getCalledFunction(); if (!Callee) { NumInvalidAbstractCallSitesUnknownCallee++; - CS = CallSite(); + CB = nullptr; return; } MDNode *CallbackMD = Callee->getMetadata(LLVMContext::MD_callback); if (!CallbackMD) { NumInvalidAbstractCallSitesNoCallback++; - CS = CallSite(); + CB = nullptr; return; } - unsigned UseIdx = CS.getArgumentNo(U); + unsigned UseIdx = CB->getArgOperandNo(U); MDNode *CallbackEncMD = nullptr; for (const MDOperand &Op : CallbackMD->operands()) { MDNode *OpMD = cast<MDNode>(Op.get()); @@ -112,7 +113,7 @@ AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) { if (!CallbackEncMD) { NumInvalidAbstractCallSitesNoCallback++; - CS = CallSite(); + CB = nullptr; return; } @@ -120,7 +121,7 @@ AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) { assert(CallbackEncMD->getNumOperands() >= 2 && "Incomplete !callback metadata"); - unsigned NumCallOperands = CS.getNumArgOperands(); + unsigned NumCallOperands = CB->getNumArgOperands(); // Skip the var-arg flag at the end when reading the metadata. for (unsigned u = 0, e = CallbackEncMD->getNumOperands() - 1; u < e; u++) { Metadata *OpAsM = CallbackEncMD->getOperand(u).get(); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index acf0e4afef27..fd08310316b3 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -228,9 +228,9 @@ static void predictValueUseListOrderImpl(const Value *V, const Function *F, return LU->getOperandNo() > RU->getOperandNo(); }); - if (std::is_sorted( - List.begin(), List.end(), - [](const Entry &L, const Entry &R) { return L.second < R.second; })) + if (llvm::is_sorted(List, [](const Entry &L, const Entry &R) { + return L.second < R.second; + })) // Order is already correct. return; @@ -462,6 +462,33 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) { isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix); } +static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) { + Out << ", <"; + if (isa<ScalableVectorType>(Ty)) + Out << "vscale x "; + Out << Mask.size() << " x i32> "; + bool FirstElt = true; + if (all_of(Mask, [](int Elt) { return Elt == 0; })) { + Out << "zeroinitializer"; + } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + Out << "undef"; + } else { + Out << "<"; + for (int Elt : Mask) { + if (FirstElt) + FirstElt = false; + else + Out << ", "; + Out << "i32 "; + if (Elt == UndefMaskElem) + Out << "undef"; + else + Out << Elt; + } + Out << ">"; + } +} + namespace { class TypePrinting { @@ -561,6 +588,7 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) { switch (Ty->getTypeID()) { case Type::VoidTyID: OS << "void"; return; case Type::HalfTyID: OS << "half"; return; + case Type::BFloatTyID: OS << "bfloat"; return; case Type::FloatTyID: OS << "float"; return; case Type::DoubleTyID: OS << "double"; return; case Type::X86_FP80TyID: OS << "x86_fp80"; return; @@ -623,12 +651,14 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) { OS << ']'; return; } - case Type::VectorTyID: { + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: { VectorType *PTy = cast<VectorType>(Ty); + ElementCount EC = PTy->getElementCount(); OS << "<"; - if (PTy->isScalable()) + if (EC.Scalable) OS << "vscale x "; - OS << PTy->getNumElements() << " x "; + OS << EC.Min << " x "; print(PTy->getElementType(), OS); OS << '>'; return; @@ -783,7 +813,7 @@ public: /// These functions do the actual initialization. inline void initializeIfNeeded(); - void initializeIndexIfNeeded(); + int initializeIndexIfNeeded(); // Implementation Details private: @@ -806,7 +836,8 @@ private: /// Add all of the module level global variables (and their initializers) /// and function declarations, but not the contents of those functions. void processModule(); - void processIndex(); + // Returns number of allocated slots + int processIndex(); /// Add all of the functions arguments, basic blocks, and instructions. void processFunction(); @@ -920,11 +951,12 @@ inline void SlotTracker::initializeIfNeeded() { processFunction(); } -void SlotTracker::initializeIndexIfNeeded() { +int SlotTracker::initializeIndexIfNeeded() { if (!TheIndex) - return; - processIndex(); + return 0; + int NumSlots = processIndex(); TheIndex = nullptr; ///< Prevent re-processing next time we're called. + return NumSlots; } // Iterate through all the global variables, functions, and global @@ -1019,7 +1051,7 @@ void SlotTracker::processFunction() { } // Iterate through all the GUID in the index and create slots for them. -void SlotTracker::processIndex() { +int SlotTracker::processIndex() { ST_DEBUG("begin processIndex!\n"); assert(TheIndex); @@ -1038,17 +1070,17 @@ void SlotTracker::processIndex() { for (auto &GlobalList : *TheIndex) CreateGUIDSlot(GlobalList.first); + for (auto &TId : TheIndex->typeIdCompatibleVtableMap()) + CreateGUIDSlot(GlobalValue::getGUID(TId.first)); + // Start numbering the TypeIds after the GUIDs. TypeIdNext = GUIDNext; - for (auto TidIter = TheIndex->typeIds().begin(); TidIter != TheIndex->typeIds().end(); TidIter++) CreateTypeIdSlot(TidIter->second.first); - for (auto &TId : TheIndex->typeIdCompatibleVtableMap()) - CreateGUIDSlot(GlobalValue::getGUID(TId.first)); - ST_DEBUG("end processIndex!\n"); + return TypeIdNext; } void SlotTracker::processGlobalObjectMetadata(const GlobalObject &GO) { @@ -1348,7 +1380,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, return; } - // Either half, or some form of long double. + // Either half, bfloat or some form of long double. // These appear as a magic letter identifying the type, then a // fixed number of hex digits. Out << "0x"; @@ -1376,6 +1408,10 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << 'H'; Out << format_hex_no_prefix(API.getZExtValue(), 4, /*Upper=*/true); + } else if (&APF.getSemantics() == &APFloat::BFloat()) { + Out << 'R'; + Out << format_hex_no_prefix(API.getZExtValue(), 4, + /*Upper=*/true); } else llvm_unreachable("Unsupported floating point type"); return; @@ -1475,13 +1511,14 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, } if (isa<ConstantVector>(CV) || isa<ConstantDataVector>(CV)) { - Type *ETy = CV->getType()->getVectorElementType(); + auto *CVVTy = cast<VectorType>(CV->getType()); + Type *ETy = CVVTy->getElementType(); Out << '<'; TypePrinter.print(ETy, Out); Out << ' '; WriteAsOperandInternal(Out, CV->getAggregateElement(0U), &TypePrinter, Machine, Context); - for (unsigned i = 1, e = CV->getType()->getVectorNumElements(); i != e;++i){ + for (unsigned i = 1, e = CVVTy->getNumElements(); i != e; ++i) { Out << ", "; TypePrinter.print(ETy, Out); Out << ' '; @@ -1545,6 +1582,9 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, TypePrinter.print(CE->getType(), Out); } + if (CE->getOpcode() == Instruction::ShuffleVector) + PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask()); + Out << ')'; return; } @@ -1614,6 +1654,8 @@ struct MDFieldPrinter { bool ShouldSkipNull = true); template <class IntTy> void printInt(StringRef Name, IntTy Int, bool ShouldSkipZero = true); + void printAPInt(StringRef Name, const APInt &Int, bool IsUnsigned, + bool ShouldSkipZero); void printBool(StringRef Name, bool Value, Optional<bool> Default = None); void printDIFlags(StringRef Name, DINode::DIFlags Flags); void printDISPFlags(StringRef Name, DISubprogram::DISPFlags Flags); @@ -1689,6 +1731,15 @@ void MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) { Out << FS << Name << ": " << Int; } +void MDFieldPrinter::printAPInt(StringRef Name, const APInt &Int, + bool IsUnsigned, bool ShouldSkipZero) { + if (ShouldSkipZero && Int.isNullValue()) + return; + + Out << FS << Name << ": "; + Int.print(Out, !IsUnsigned); +} + void MDFieldPrinter::printBool(StringRef Name, bool Value, Optional<bool> Default) { if (Default && Value == *Default) @@ -1807,9 +1858,34 @@ static void writeDISubrange(raw_ostream &Out, const DISubrange *N, if (auto *CE = N->getCount().dyn_cast<ConstantInt*>()) Printer.printInt("count", CE->getSExtValue(), /* ShouldSkipZero */ false); else - Printer.printMetadata("count", N->getCount().dyn_cast<DIVariable*>(), - /*ShouldSkipNull */ false); - Printer.printInt("lowerBound", N->getLowerBound()); + Printer.printMetadata("count", N->getCount().dyn_cast<DIVariable *>(), + /*ShouldSkipNull */ true); + + // A lowerBound of constant 0 should not be skipped, since it is different + // from an unspecified lower bound (= nullptr). + auto *LBound = N->getRawLowerBound(); + if (auto *LE = dyn_cast_or_null<ConstantAsMetadata>(LBound)) { + auto *LV = cast<ConstantInt>(LE->getValue()); + Printer.printInt("lowerBound", LV->getSExtValue(), + /* ShouldSkipZero */ false); + } else + Printer.printMetadata("lowerBound", LBound, /*ShouldSkipNull */ true); + + auto *UBound = N->getRawUpperBound(); + if (auto *UE = dyn_cast_or_null<ConstantAsMetadata>(UBound)) { + auto *UV = cast<ConstantInt>(UE->getValue()); + Printer.printInt("upperBound", UV->getSExtValue(), + /* ShouldSkipZero */ false); + } else + Printer.printMetadata("upperBound", UBound, /*ShouldSkipNull */ true); + + auto *Stride = N->getRawStride(); + if (auto *SE = dyn_cast_or_null<ConstantAsMetadata>(Stride)) { + auto *SV = cast<ConstantInt>(SE->getValue()); + Printer.printInt("stride", SV->getSExtValue(), /* ShouldSkipZero */ false); + } else + Printer.printMetadata("stride", Stride, /*ShouldSkipNull */ true); + Out << ")"; } @@ -1818,13 +1894,10 @@ static void writeDIEnumerator(raw_ostream &Out, const DIEnumerator *N, Out << "!DIEnumerator("; MDFieldPrinter Printer(Out); Printer.printString("name", N->getName(), /* ShouldSkipEmpty */ false); - if (N->isUnsigned()) { - auto Value = static_cast<uint64_t>(N->getValue()); - Printer.printInt("value", Value, /* ShouldSkipZero */ false); + Printer.printAPInt("value", N->getValue(), N->isUnsigned(), + /*ShouldSkipZero=*/false); + if (N->isUnsigned()) Printer.printBool("isUnsigned", true); - } else { - Printer.printInt("value", N->getValue(), /* ShouldSkipZero */ false); - } Out << ")"; } @@ -1888,6 +1961,7 @@ static void writeDICompositeType(raw_ostream &Out, const DICompositeType *N, Printer.printMetadata("templateParams", N->getRawTemplateParams()); Printer.printString("identifier", N->getIdentifier()); Printer.printMetadata("discriminator", N->getRawDiscriminator()); + Printer.printMetadata("dataLocation", N->getRawDataLocation()); Out << ")"; } @@ -1945,6 +2019,8 @@ static void writeDICompileUnit(raw_ostream &Out, const DICompileUnit *N, false); Printer.printNameTableKind("nameTableKind", N->getNameTableKind()); Printer.printBool("rangesBaseAddress", N->getRangesBaseAddress(), false); + Printer.printString("sysroot", N->getSysRoot()); + Printer.printString("sdk", N->getSDK()); Out << ")"; } @@ -2057,7 +2133,9 @@ static void writeDIModule(raw_ostream &Out, const DIModule *N, Printer.printString("name", N->getName()); Printer.printString("configMacros", N->getConfigurationMacros()); Printer.printString("includePath", N->getIncludePath()); - Printer.printString("sysroot", N->getSysRoot()); + Printer.printString("apinotes", N->getAPINotesFile()); + Printer.printMetadata("file", N->getRawFile()); + Printer.printInt("line", N->getLineNo()); Out << ")"; } @@ -2071,6 +2149,7 @@ static void writeDITemplateTypeParameter(raw_ostream &Out, MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printString("name", N->getName()); Printer.printMetadata("type", N->getRawType(), /* ShouldSkipNull */ false); + Printer.printBool("defaulted", N->isDefault(), /* Default= */ false); Out << ")"; } @@ -2085,6 +2164,7 @@ static void writeDITemplateValueParameter(raw_ostream &Out, Printer.printTag(N); Printer.printString("name", N->getName()); Printer.printMetadata("type", N->getRawType()); + Printer.printBool("defaulted", N->isDefault(), /* Default= */ false); Printer.printMetadata("value", N->getValue(), /* ShouldSkipNull */ false); Out << ")"; } @@ -2430,10 +2510,10 @@ public: void printTypeIdInfo(const FunctionSummary::TypeIdInfo &TIDInfo); void printVFuncId(const FunctionSummary::VFuncId VFId); void - printNonConstVCalls(const std::vector<FunctionSummary::VFuncId> VCallList, + printNonConstVCalls(const std::vector<FunctionSummary::VFuncId> &VCallList, const char *Tag); void - printConstVCalls(const std::vector<FunctionSummary::ConstVCall> VCallList, + printConstVCalls(const std::vector<FunctionSummary::ConstVCall> &VCallList, const char *Tag); private: @@ -2651,8 +2731,10 @@ void AssemblyWriter::printModule(const Module *M) { printUseLists(nullptr); // Output all of the functions. - for (const Function &F : *M) + for (const Function &F : *M) { + Out << '\n'; printFunction(&F); + } assert(UseListOrders.empty() && "All use-lists should have been consumed"); // Output all attribute groups. @@ -2676,21 +2758,22 @@ void AssemblyWriter::printModule(const Module *M) { void AssemblyWriter::printModuleSummaryIndex() { assert(TheIndex); - Machine.initializeIndexIfNeeded(); + int NumSlots = Machine.initializeIndexIfNeeded(); Out << "\n"; // Print module path entries. To print in order, add paths to a vector // indexed by module slot. std::vector<std::pair<std::string, ModuleHash>> moduleVec; - std::string RegularLTOModuleName = "[Regular LTO]"; + std::string RegularLTOModuleName = + ModuleSummaryIndex::getRegularLTOModuleName(); moduleVec.resize(TheIndex->modulePaths().size()); for (auto &ModPath : TheIndex->modulePaths()) moduleVec[Machine.getModulePathSlot(ModPath.first())] = std::make_pair( // A module id of -1 is a special entry for a regular LTO module created // during the thin link. ModPath.second.first == -1u ? RegularLTOModuleName - : (std::string)ModPath.first(), + : (std::string)std::string(ModPath.first()), ModPath.second.second); unsigned i = 0; @@ -2737,6 +2820,15 @@ void AssemblyWriter::printModuleSummaryIndex() { printTypeIdCompatibleVtableSummary(TId.second); Out << ") ; guid = " << GUID << "\n"; } + + // Don't emit flags when it's not really needed (value is zero by default). + if (TheIndex->getFlags()) { + Out << "^" << NumSlots << " = flags: " << TheIndex->getFlags() << "\n"; + ++NumSlots; + } + + Out << "^" << NumSlots << " = blockcount: " << TheIndex->getBlockCount() + << "\n"; } static const char * @@ -2769,6 +2861,8 @@ static const char *getWholeProgDevirtResByArgKindName( static const char *getTTResKindName(TypeTestResolution::Kind K) { switch (K) { + case TypeTestResolution::Unknown: + return "unknown"; case TypeTestResolution::Unsat: return "unsat"; case TypeTestResolution::ByteArray: @@ -2900,10 +2994,15 @@ void AssemblyWriter::printAliasSummary(const AliasSummary *AS) { } void AssemblyWriter::printGlobalVarSummary(const GlobalVarSummary *GS) { + auto VTableFuncs = GS->vTableFuncs(); Out << ", varFlags: (readonly: " << GS->VarFlags.MaybeReadOnly << ", " - << "writeonly: " << GS->VarFlags.MaybeWriteOnly << ")"; + << "writeonly: " << GS->VarFlags.MaybeWriteOnly << ", " + << "constant: " << GS->VarFlags.Constant; + if (!VTableFuncs.empty()) + Out << ", " + << "vcall_visibility: " << GS->VarFlags.VCallVisibility; + Out << ")"; - auto VTableFuncs = GS->vTableFuncs(); if (!VTableFuncs.empty()) { Out << ", vTableFuncs: ("; FieldSeparator FS; @@ -2986,6 +3085,36 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) { if (const auto *TIdInfo = FS->getTypeIdInfo()) printTypeIdInfo(*TIdInfo); + + auto PrintRange = [&](const ConstantRange &Range) { + Out << "[" << Range.getLower() << ", " << Range.getSignedMax() << "]"; + }; + + if (!FS->paramAccesses().empty()) { + Out << ", params: ("; + FieldSeparator IFS; + for (auto &PS : FS->paramAccesses()) { + Out << IFS; + Out << "(param: " << PS.ParamNo; + Out << ", offset: "; + PrintRange(PS.Use); + if (!PS.Calls.empty()) { + Out << ", calls: ("; + FieldSeparator IFS; + for (auto &Call : PS.Calls) { + Out << IFS; + Out << "(callee: ^" << Machine.getGUIDSlot(Call.Callee); + Out << ", param: " << Call.ParamNo; + Out << ", offset: "; + PrintRange(Call.Offsets); + Out << ")"; + } + Out << ")"; + } + Out << ")"; + } + Out << ")"; + } } void AssemblyWriter::printTypeIdInfo( @@ -3057,7 +3186,7 @@ void AssemblyWriter::printVFuncId(const FunctionSummary::VFuncId VFId) { } void AssemblyWriter::printNonConstVCalls( - const std::vector<FunctionSummary::VFuncId> VCallList, const char *Tag) { + const std::vector<FunctionSummary::VFuncId> &VCallList, const char *Tag) { Out << Tag << ": ("; FieldSeparator FS; for (auto &VFuncId : VCallList) { @@ -3068,7 +3197,8 @@ void AssemblyWriter::printNonConstVCalls( } void AssemblyWriter::printConstVCalls( - const std::vector<FunctionSummary::ConstVCall> VCallList, const char *Tag) { + const std::vector<FunctionSummary::ConstVCall> &VCallList, + const char *Tag) { Out << Tag << ": ("; FieldSeparator FS; for (auto &ConstVCall : VCallList) { @@ -3200,11 +3330,7 @@ static void PrintVisibility(GlobalValue::VisibilityTypes Vis, static void PrintDSOLocation(const GlobalValue &GV, formatted_raw_ostream &Out) { - // GVs with local linkage or non default visibility are implicitly dso_local, - // so we don't print it. - bool Implicit = GV.hasLocalLinkage() || - (!GV.hasExternalWeakLinkage() && !GV.hasDefaultVisibility()); - if (GV.isDSOLocal() && !Implicit) + if (GV.isDSOLocal() && !GV.isImplicitDSOLocal()) Out << "dso_local "; } @@ -3850,7 +3976,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { PrintCallingConv(CI->getCallingConv(), Out); } - Operand = CI->getCalledValue(); + Operand = CI->getCalledOperand(); FunctionType *FTy = CI->getFunctionType(); Type *RetTy = FTy->getReturnType(); const AttributeList &PAL = CI->getAttributes(); @@ -3889,7 +4015,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { writeOperandBundles(CI); } else if (const InvokeInst *II = dyn_cast<InvokeInst>(&I)) { - Operand = II->getCalledValue(); + Operand = II->getCalledOperand(); FunctionType *FTy = II->getFunctionType(); Type *RetTy = FTy->getReturnType(); const AttributeList &PAL = II->getAttributes(); @@ -3932,7 +4058,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { Out << " unwind "; writeOperand(II->getUnwindDest(), true); } else if (const CallBrInst *CBI = dyn_cast<CallBrInst>(&I)) { - Operand = CBI->getCalledValue(); + Operand = CBI->getCalledOperand(); FunctionType *FTy = CBI->getFunctionType(); Type *RetTy = FTy->getReturnType(); const AttributeList &PAL = CBI->getAttributes(); @@ -4079,6 +4205,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) { RMWI->getSyncScopeID()); } else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) { writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID()); + } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) { + PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask()); } // Print Metadata info. @@ -4140,9 +4268,16 @@ void AssemblyWriter::writeAttribute(const Attribute &Attr, bool InAttrGroup) { return; } - assert(Attr.hasAttribute(Attribute::ByVal) && "unexpected type attr"); + assert((Attr.hasAttribute(Attribute::ByVal) || + Attr.hasAttribute(Attribute::Preallocated)) && + "unexpected type attr"); + + if (Attr.hasAttribute(Attribute::ByVal)) { + Out << "byval"; + } else { + Out << "preallocated"; + } - Out << "byval"; if (Type *Ty = Attr.getValueAsType()) { Out << '('; TypePrinter.print(Ty, Out); @@ -4228,6 +4363,17 @@ void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, W.printFunction(this); } +void BasicBlock::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder, + bool IsForDebug) const { + SlotTracker SlotTable(this->getModule()); + formatted_raw_ostream OS(ROS); + AssemblyWriter W(OS, SlotTable, this->getModule(), AAW, + IsForDebug, + ShouldPreserveUseListOrder); + W.printBasicBlock(this); +} + void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, bool ShouldPreserveUseListOrder, bool IsForDebug) const { SlotTracker SlotTable(this); diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h index 15e488bbb13b..5c334348cde3 100644 --- a/llvm/lib/IR/AttributeImpl.h +++ b/llvm/lib/IR/AttributeImpl.h @@ -16,6 +16,7 @@ #define LLVM_LIB_IR_ATTRIBUTEIMPL_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/Attributes.h" @@ -53,8 +54,6 @@ public: AttributeImpl(const AttributeImpl &) = delete; AttributeImpl &operator=(const AttributeImpl &) = delete; - virtual ~AttributeImpl(); - bool isEnumAttribute() const { return KindID == EnumAttrEntry; } bool isIntAttribute() const { return KindID == IntAttrEntry; } bool isStringAttribute() const { return KindID == StringAttrEntry; } @@ -103,6 +102,9 @@ public: } }; +static_assert(std::is_trivially_destructible<AttributeImpl>::value, + "AttributeImpl should be trivially destructible"); + //===----------------------------------------------------------------------===// /// \class /// A set of classes that contain the value of the @@ -111,8 +113,6 @@ public: /// attribute enties, which are for target-dependent attributes. class EnumAttributeImpl : public AttributeImpl { - virtual void anchor(); - Attribute::AttrKind Kind; protected: @@ -129,38 +129,53 @@ public: class IntAttributeImpl : public EnumAttributeImpl { uint64_t Val; - void anchor() override; - public: IntAttributeImpl(Attribute::AttrKind Kind, uint64_t Val) : EnumAttributeImpl(IntAttrEntry, Kind), Val(Val) { - assert((Kind == Attribute::Alignment || Kind == Attribute::StackAlignment || - Kind == Attribute::Dereferenceable || - Kind == Attribute::DereferenceableOrNull || - Kind == Attribute::AllocSize) && + assert(Attribute::doesAttrKindHaveArgument(Kind) && "Wrong kind for int attribute!"); } uint64_t getValue() const { return Val; } }; -class StringAttributeImpl : public AttributeImpl { - virtual void anchor(); +class StringAttributeImpl final + : public AttributeImpl, + private TrailingObjects<StringAttributeImpl, char> { + friend TrailingObjects; - std::string Kind; - std::string Val; + unsigned KindSize; + unsigned ValSize; + size_t numTrailingObjects(OverloadToken<char>) const { + return KindSize + 1 + ValSize + 1; + } public: StringAttributeImpl(StringRef Kind, StringRef Val = StringRef()) - : AttributeImpl(StringAttrEntry), Kind(Kind), Val(Val) {} + : AttributeImpl(StringAttrEntry), KindSize(Kind.size()), + ValSize(Val.size()) { + char *TrailingString = getTrailingObjects<char>(); + // Some users rely on zero-termination. + llvm::copy(Kind, TrailingString); + TrailingString[KindSize] = '\0'; + llvm::copy(Val, &TrailingString[KindSize + 1]); + TrailingString[KindSize + 1 + ValSize] = '\0'; + } - StringRef getStringKind() const { return Kind; } - StringRef getStringValue() const { return Val; } + StringRef getStringKind() const { + return StringRef(getTrailingObjects<char>(), KindSize); + } + StringRef getStringValue() const { + return StringRef(getTrailingObjects<char>() + KindSize + 1, ValSize); + } + + static size_t totalSizeToAlloc(StringRef Kind, StringRef Val) { + return TrailingObjects::totalSizeToAlloc<char>(Kind.size() + 1 + + Val.size() + 1); + } }; class TypeAttributeImpl : public EnumAttributeImpl { - void anchor() override; - Type *Ty; public: @@ -170,6 +185,22 @@ public: Type *getTypeValue() const { return Ty; } }; +class AttributeBitSet { + /// Bitset with a bit for each available attribute Attribute::AttrKind. + uint8_t AvailableAttrs[12] = {}; + static_assert(Attribute::EndAttrKinds <= sizeof(AvailableAttrs) * CHAR_BIT, + "Too many attributes"); + +public: + bool hasAttribute(Attribute::AttrKind Kind) const { + return AvailableAttrs[Kind / 8] & (1 << (Kind % 8)); + } + + void addAttribute(Attribute::AttrKind Kind) { + AvailableAttrs[Kind / 8] |= 1 << (Kind % 8); + } +}; + //===----------------------------------------------------------------------===// /// \class /// This class represents a group of attributes that apply to one @@ -180,11 +211,16 @@ class AttributeSetNode final friend TrailingObjects; unsigned NumAttrs; ///< Number of attributes in this node. - /// Bitset with a bit for each available attribute Attribute::AttrKind. - uint8_t AvailableAttrs[12] = {}; + AttributeBitSet AvailableAttrs; ///< Available enum attributes. + + DenseMap<StringRef, Attribute> StringAttrs; AttributeSetNode(ArrayRef<Attribute> Attrs); + static AttributeSetNode *getSorted(LLVMContext &C, + ArrayRef<Attribute> SortedAttrs); + Optional<Attribute> findEnumAttribute(Attribute::AttrKind Kind) const; + public: // AttributesSetNode is uniqued, these should not be available. AttributeSetNode(const AttributeSetNode &) = delete; @@ -200,7 +236,7 @@ public: unsigned getNumAttributes() const { return NumAttrs; } bool hasAttribute(Attribute::AttrKind Kind) const { - return AvailableAttrs[Kind / 8] & ((uint64_t)1) << (Kind % 8); + return AvailableAttrs.hasAttribute(Kind); } bool hasAttribute(StringRef Kind) const; bool hasAttributes() const { return NumAttrs != 0; } @@ -215,6 +251,7 @@ public: std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; std::string getAsString(bool InAttrGrp) const; Type *getByValType() const; + Type *getPreallocatedType() const; using iterator = const Attribute *; @@ -231,8 +268,6 @@ public: } }; -using IndexAttrPair = std::pair<unsigned, AttributeSet>; - //===----------------------------------------------------------------------===// /// \class /// This class represents a set of attributes that apply to the function, @@ -244,32 +279,34 @@ class AttributeListImpl final friend TrailingObjects; private: - LLVMContext &Context; unsigned NumAttrSets; ///< Number of entries in this set. - /// Bitset with a bit for each available attribute Attribute::AttrKind. - uint8_t AvailableFunctionAttrs[12] = {}; + /// Available enum function attributes. + AttributeBitSet AvailableFunctionAttrs; + /// Union of enum attributes available at any index. + AttributeBitSet AvailableSomewhereAttrs; // Helper fn for TrailingObjects class. size_t numTrailingObjects(OverloadToken<AttributeSet>) { return NumAttrSets; } public: - AttributeListImpl(LLVMContext &C, ArrayRef<AttributeSet> Sets); + AttributeListImpl(ArrayRef<AttributeSet> Sets); // AttributesSetImpt is uniqued, these should not be available. AttributeListImpl(const AttributeListImpl &) = delete; AttributeListImpl &operator=(const AttributeListImpl &) = delete; - void operator delete(void *p) { ::operator delete(p); } - - /// Get the context that created this AttributeListImpl. - LLVMContext &getContext() { return Context; } - /// Return true if the AttributeSet or the FunctionIndex has an /// enum attribute of the given kind. bool hasFnAttribute(Attribute::AttrKind Kind) const { - return AvailableFunctionAttrs[Kind / 8] & ((uint64_t)1) << (Kind % 8); + return AvailableFunctionAttrs.hasAttribute(Kind); } + /// Return true if the specified attribute is set for at least one + /// parameter or for the return value. If Index is not nullptr, the index + /// of a parameter with the specified attribute is provided. + bool hasAttrSomewhere(Attribute::AttrKind Kind, + unsigned *Index = nullptr) const; + using iterator = const AttributeSet *; iterator begin() const { return getTrailingObjects<AttributeSet>(); } @@ -281,6 +318,9 @@ public: void dump() const; }; +static_assert(std::is_trivially_destructible<AttributeListImpl>::value, + "AttributeListImpl should be trivially destructible"); + } // end namespace llvm #endif // LLVM_LIB_IR_ATTRIBUTEIMPL_H diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 5ca99c981739..f67d96a854f4 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -22,6 +22,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Twine.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Function.h" @@ -91,9 +92,9 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind, // If we didn't find any existing attributes of the same shape then create a // new one and insert it. if (!Val) - PA = new EnumAttributeImpl(Kind); + PA = new (pImpl->Alloc) EnumAttributeImpl(Kind); else - PA = new IntAttributeImpl(Kind, Val); + PA = new (pImpl->Alloc) IntAttributeImpl(Kind, Val); pImpl->AttrsSet.InsertNode(PA, InsertPoint); } @@ -113,7 +114,10 @@ Attribute Attribute::get(LLVMContext &Context, StringRef Kind, StringRef Val) { if (!PA) { // If we didn't find any existing attributes of the same shape then create a // new one and insert it. - PA = new StringAttributeImpl(Kind, Val); + void *Mem = + pImpl->Alloc.Allocate(StringAttributeImpl::totalSizeToAlloc(Kind, Val), + alignof(StringAttributeImpl)); + PA = new (Mem) StringAttributeImpl(Kind, Val); pImpl->AttrsSet.InsertNode(PA, InsertPoint); } @@ -134,7 +138,7 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind, if (!PA) { // If we didn't find any existing attributes of the same shape then create a // new one and insert it. - PA = new TypeAttributeImpl(Kind, Ty); + PA = new (pImpl->Alloc) TypeAttributeImpl(Kind, Ty); pImpl->AttrsSet.InsertNode(PA, InsertPoint); } @@ -143,7 +147,7 @@ Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind, } Attribute Attribute::getWithAlignment(LLVMContext &Context, Align A) { - assert(A <= 0x40000000 && "Alignment too large."); + assert(A <= llvm::Value::MaximumAlignment && "Alignment too large."); return get(Context, Alignment, A.value()); } @@ -168,6 +172,10 @@ Attribute Attribute::getWithByValType(LLVMContext &Context, Type *Ty) { return get(Context, ByVal, Ty); } +Attribute Attribute::getWithPreallocatedType(LLVMContext &Context, Type *Ty) { + return get(Context, Preallocated, Ty); +} + Attribute Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const Optional<unsigned> &NumElemsArg) { @@ -176,6 +184,45 @@ Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, return get(Context, AllocSize, packAllocSizeArgs(ElemSizeArg, NumElemsArg)); } +Attribute::AttrKind Attribute::getAttrKindFromName(StringRef AttrName) { + return StringSwitch<Attribute::AttrKind>(AttrName) +#define GET_ATTR_NAMES +#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \ + .Case(#DISPLAY_NAME, Attribute::ENUM_NAME) +#include "llvm/IR/Attributes.inc" + .Default(Attribute::None); +} + +StringRef Attribute::getNameFromAttrKind(Attribute::AttrKind AttrKind) { + switch (AttrKind) { +#define GET_ATTR_NAMES +#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \ + case Attribute::ENUM_NAME: \ + return #DISPLAY_NAME; +#include "llvm/IR/Attributes.inc" + case Attribute::None: + return "none"; + default: + llvm_unreachable("invalid Kind"); + } +} + +bool Attribute::doesAttrKindHaveArgument(Attribute::AttrKind AttrKind) { + return AttrKind == Attribute::Alignment || + AttrKind == Attribute::StackAlignment || + AttrKind == Attribute::Dereferenceable || + AttrKind == Attribute::AllocSize || + AttrKind == Attribute::DereferenceableOrNull; +} + +bool Attribute::isExistingAttribute(StringRef Name) { + return StringSwitch<bool>(Name) +#define GET_ATTR_NAMES +#define ATTRIBUTE_ALL(ENUM_NAME, DISPLAY_NAME) .Case(#DISPLAY_NAME, true) +#include "llvm/IR/Attributes.inc" + .Default(false); +} + //===----------------------------------------------------------------------===// // Attribute Accessor Methods //===----------------------------------------------------------------------===// @@ -328,6 +375,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "noinline"; if (hasAttribute(Attribute::NonLazyBind)) return "nonlazybind"; + if (hasAttribute(Attribute::NoMerge)) + return "nomerge"; if (hasAttribute(Attribute::NonNull)) return "nonnull"; if (hasAttribute(Attribute::NoRedZone)) @@ -336,6 +385,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "noreturn"; if (hasAttribute(Attribute::NoSync)) return "nosync"; + if (hasAttribute(Attribute::NullPointerIsValid)) + return "null_pointer_is_valid"; if (hasAttribute(Attribute::WillReturn)) return "willreturn"; if (hasAttribute(Attribute::NoCfCheck)) @@ -392,6 +443,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "cold"; if (hasAttribute(Attribute::ImmArg)) return "immarg"; + if (hasAttribute(Attribute::NoUndef)) + return "noundef"; if (hasAttribute(Attribute::ByVal)) { std::string Result; @@ -406,6 +459,17 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return Result; } + if (hasAttribute(Attribute::Preallocated)) { + std::string Result; + Result += "preallocated"; + raw_string_ostream OS(Result); + Result += '('; + getValueAsType()->print(OS, false, true); + OS.flush(); + Result += ')'; + return Result; + } + // FIXME: These should be output like this: // // align=4 @@ -464,19 +528,19 @@ std::string Attribute::getAsString(bool InAttrGrp) const { // if (isStringAttribute()) { std::string Result; - Result += (Twine('"') + getKindAsString() + Twine('"')).str(); - - std::string AttrVal = pImpl->getValueAsString(); - if (AttrVal.empty()) return Result; - - // Since some attribute strings contain special characters that cannot be - // printable, those have to be escaped to make the attribute value printable - // as is. e.g. "\01__gnu_mcount_nc" { raw_string_ostream OS(Result); - OS << "=\""; - printEscapedString(AttrVal, OS); - OS << "\""; + OS << '"' << getKindAsString() << '"'; + + // Since some attribute strings contain special characters that cannot be + // printable, those have to be escaped to make the attribute value + // printable as is. e.g. "\01__gnu_mcount_nc" + const auto &AttrVal = pImpl->getValueAsString(); + if (!AttrVal.empty()) { + OS << "=\""; + printEscapedString(AttrVal, OS); + OS << "\""; + } } return Result; } @@ -491,21 +555,14 @@ bool Attribute::operator<(Attribute A) const { return *pImpl < *A.pImpl; } +void Attribute::Profile(FoldingSetNodeID &ID) const { + ID.AddPointer(pImpl); +} + //===----------------------------------------------------------------------===// // AttributeImpl Definition //===----------------------------------------------------------------------===// -// Pin the vtables to this file. -AttributeImpl::~AttributeImpl() = default; - -void EnumAttributeImpl::anchor() {} - -void IntAttributeImpl::anchor() {} - -void StringAttributeImpl::anchor() {} - -void TypeAttributeImpl::anchor() {} - bool AttributeImpl::hasAttribute(Attribute::AttrKind A) const { if (isStringAttribute()) return false; return getKindAsEnum() == A; @@ -542,6 +599,8 @@ Type *AttributeImpl::getValueAsType() const { } bool AttributeImpl::operator<(const AttributeImpl &AI) const { + if (this == &AI) + return false; // This sorts the attributes with Attribute::AttrKinds coming first (sorted // relative to their enum value) and then strings. if (isEnumAttribute()) { @@ -687,6 +746,10 @@ Type *AttributeSet::getByValType() const { return SetNode ? SetNode->getByValType() : nullptr; } +Type *AttributeSet::getPreallocatedType() const { + return SetNode ? SetNode->getPreallocatedType() : nullptr; +} + std::pair<unsigned, Optional<unsigned>> AttributeSet::getAllocSizeArgs() const { return SetNode ? SetNode->getAllocSizeArgs() : std::pair<unsigned, Optional<unsigned>>(0, 0); @@ -721,30 +784,31 @@ AttributeSetNode::AttributeSetNode(ArrayRef<Attribute> Attrs) // There's memory after the node where we can store the entries in. llvm::copy(Attrs, getTrailingObjects<Attribute>()); - static_assert(Attribute::EndAttrKinds <= - sizeof(AvailableAttrs) * CHAR_BIT, - "Too many attributes"); - for (const auto &I : *this) { - if (!I.isStringAttribute()) { - Attribute::AttrKind Kind = I.getKindAsEnum(); - AvailableAttrs[Kind / 8] |= 1ULL << (Kind % 8); - } + if (I.isStringAttribute()) + StringAttrs.insert({ I.getKindAsString(), I }); + else + AvailableAttrs.addAttribute(I.getKindAsEnum()); } } AttributeSetNode *AttributeSetNode::get(LLVMContext &C, ArrayRef<Attribute> Attrs) { - if (Attrs.empty()) + SmallVector<Attribute, 8> SortedAttrs(Attrs.begin(), Attrs.end()); + llvm::sort(SortedAttrs); + return getSorted(C, SortedAttrs); +} + +AttributeSetNode *AttributeSetNode::getSorted(LLVMContext &C, + ArrayRef<Attribute> SortedAttrs) { + if (SortedAttrs.empty()) return nullptr; - // Otherwise, build a key to look up the existing attributes. + // Build a key to look up the existing attributes. LLVMContextImpl *pImpl = C.pImpl; FoldingSetNodeID ID; - SmallVector<Attribute, 8> SortedAttrs(Attrs.begin(), Attrs.end()); - llvm::sort(SortedAttrs); - + assert(llvm::is_sorted(SortedAttrs) && "Expected sorted attributes!"); for (const auto &Attr : SortedAttrs) Attr.Profile(ID); @@ -778,6 +842,9 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) { case Attribute::ByVal: Attr = Attribute::getWithByValType(C, B.getByValType()); break; + case Attribute::Preallocated: + Attr = Attribute::getWithPreallocatedType(C, B.getPreallocatedType()); + break; case Attribute::Alignment: assert(B.getAlignment() && "Alignment must be set"); Attr = Attribute::getWithAlignment(C, *B.getAlignment()); @@ -809,72 +876,81 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, const AttrBuilder &B) { for (const auto &TDA : B.td_attrs()) Attrs.emplace_back(Attribute::get(C, TDA.first, TDA.second)); - return get(C, Attrs); + return getSorted(C, Attrs); } bool AttributeSetNode::hasAttribute(StringRef Kind) const { - for (const auto &I : *this) - if (I.hasAttribute(Kind)) - return true; - return false; + return StringAttrs.count(Kind); +} + +Optional<Attribute> +AttributeSetNode::findEnumAttribute(Attribute::AttrKind Kind) const { + // Do a quick presence check. + if (!hasAttribute(Kind)) + return None; + + // Attributes in a set are sorted by enum value, followed by string + // attributes. Binary search the one we want. + const Attribute *I = + std::lower_bound(begin(), end() - StringAttrs.size(), Kind, + [](Attribute A, Attribute::AttrKind Kind) { + return A.getKindAsEnum() < Kind; + }); + assert(I != end() && I->hasAttribute(Kind) && "Presence check failed?"); + return *I; } Attribute AttributeSetNode::getAttribute(Attribute::AttrKind Kind) const { - if (hasAttribute(Kind)) { - for (const auto &I : *this) - if (I.hasAttribute(Kind)) - return I; - } + if (auto A = findEnumAttribute(Kind)) + return *A; return {}; } Attribute AttributeSetNode::getAttribute(StringRef Kind) const { - for (const auto &I : *this) - if (I.hasAttribute(Kind)) - return I; - return {}; + return StringAttrs.lookup(Kind); } MaybeAlign AttributeSetNode::getAlignment() const { - for (const auto &I : *this) - if (I.hasAttribute(Attribute::Alignment)) - return I.getAlignment(); + if (auto A = findEnumAttribute(Attribute::Alignment)) + return A->getAlignment(); return None; } MaybeAlign AttributeSetNode::getStackAlignment() const { - for (const auto &I : *this) - if (I.hasAttribute(Attribute::StackAlignment)) - return I.getStackAlignment(); + if (auto A = findEnumAttribute(Attribute::StackAlignment)) + return A->getStackAlignment(); return None; } Type *AttributeSetNode::getByValType() const { + if (auto A = findEnumAttribute(Attribute::ByVal)) + return A->getValueAsType(); + return 0; +} + +Type *AttributeSetNode::getPreallocatedType() const { for (const auto &I : *this) - if (I.hasAttribute(Attribute::ByVal)) + if (I.hasAttribute(Attribute::Preallocated)) return I.getValueAsType(); return 0; } uint64_t AttributeSetNode::getDereferenceableBytes() const { - for (const auto &I : *this) - if (I.hasAttribute(Attribute::Dereferenceable)) - return I.getDereferenceableBytes(); + if (auto A = findEnumAttribute(Attribute::Dereferenceable)) + return A->getDereferenceableBytes(); return 0; } uint64_t AttributeSetNode::getDereferenceableOrNullBytes() const { - for (const auto &I : *this) - if (I.hasAttribute(Attribute::DereferenceableOrNull)) - return I.getDereferenceableOrNullBytes(); + if (auto A = findEnumAttribute(Attribute::DereferenceableOrNull)) + return A->getDereferenceableOrNullBytes(); return 0; } std::pair<unsigned, Optional<unsigned>> AttributeSetNode::getAllocSizeArgs() const { - for (const auto &I : *this) - if (I.hasAttribute(Attribute::AllocSize)) - return I.getAllocSizeArgs(); + if (auto A = findEnumAttribute(Attribute::AllocSize)) + return A->getAllocSizeArgs(); return std::make_pair(0, 0); } @@ -893,33 +969,30 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const { //===----------------------------------------------------------------------===// /// Map from AttributeList index to the internal array index. Adding one happens -/// to work, but it relies on unsigned integer wrapping. MSVC warns about -/// unsigned wrapping in constexpr functions, so write out the conditional. LLVM -/// folds it to add anyway. +/// to work, because -1 wraps around to 0. static constexpr unsigned attrIdxToArrayIdx(unsigned Index) { - return Index == AttributeList::FunctionIndex ? 0 : Index + 1; + return Index + 1; } -AttributeListImpl::AttributeListImpl(LLVMContext &C, - ArrayRef<AttributeSet> Sets) - : Context(C), NumAttrSets(Sets.size()) { +AttributeListImpl::AttributeListImpl(ArrayRef<AttributeSet> Sets) + : NumAttrSets(Sets.size()) { assert(!Sets.empty() && "pointless AttributeListImpl"); // There's memory after the node where we can store the entries in. llvm::copy(Sets, getTrailingObjects<AttributeSet>()); - // Initialize AvailableFunctionAttrs summary bitset. - static_assert(Attribute::EndAttrKinds <= - sizeof(AvailableFunctionAttrs) * CHAR_BIT, - "Too many attributes"); + // Initialize AvailableFunctionAttrs and AvailableSomewhereAttrs + // summary bitsets. static_assert(attrIdxToArrayIdx(AttributeList::FunctionIndex) == 0U, "function should be stored in slot 0"); - for (const auto &I : Sets[0]) { - if (!I.isStringAttribute()) { - Attribute::AttrKind Kind = I.getKindAsEnum(); - AvailableFunctionAttrs[Kind / 8] |= 1ULL << (Kind % 8); - } - } + for (const auto &I : Sets[0]) + if (!I.isStringAttribute()) + AvailableFunctionAttrs.addAttribute(I.getKindAsEnum()); + + for (const auto &Set : Sets) + for (const auto &I : Set) + if (!I.isStringAttribute()) + AvailableSomewhereAttrs.addAttribute(I.getKindAsEnum()); } void AttributeListImpl::Profile(FoldingSetNodeID &ID) const { @@ -932,6 +1005,24 @@ void AttributeListImpl::Profile(FoldingSetNodeID &ID, ID.AddPointer(Set.SetNode); } +bool AttributeListImpl::hasAttrSomewhere(Attribute::AttrKind Kind, + unsigned *Index) const { + if (!AvailableSomewhereAttrs.hasAttribute(Kind)) + return false; + + if (Index) { + for (unsigned I = 0, E = NumAttrSets; I != E; ++I) { + if (begin()[I].hasAttribute(Kind)) { + *Index = I - 1; + break; + } + } + } + + return true; +} + + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void AttributeListImpl::dump() const { AttributeList(const_cast<AttributeListImpl *>(this)).dump(); @@ -958,9 +1049,10 @@ AttributeList AttributeList::getImpl(LLVMContext &C, // create a new one and insert it. if (!PA) { // Coallocate entries after the AttributeListImpl itself. - void *Mem = ::operator new( - AttributeListImpl::totalSizeToAlloc<AttributeSet>(AttrSets.size())); - PA = new (Mem) AttributeListImpl(C, AttrSets); + void *Mem = pImpl->Alloc.Allocate( + AttributeListImpl::totalSizeToAlloc<AttributeSet>(AttrSets.size()), + alignof(AttributeListImpl)); + PA = new (Mem) AttributeListImpl(AttrSets); pImpl->AttrsLists.InsertNode(PA, InsertPoint); } @@ -975,11 +1067,12 @@ AttributeList::get(LLVMContext &C, if (Attrs.empty()) return {}; - assert(std::is_sorted(Attrs.begin(), Attrs.end(), - [](const std::pair<unsigned, Attribute> &LHS, - const std::pair<unsigned, Attribute> &RHS) { - return LHS.first < RHS.first; - }) && "Misordered Attributes list!"); + assert(llvm::is_sorted(Attrs, + [](const std::pair<unsigned, Attribute> &LHS, + const std::pair<unsigned, Attribute> &RHS) { + return LHS.first < RHS.first; + }) && + "Misordered Attributes list!"); assert(llvm::none_of(Attrs, [](const std::pair<unsigned, Attribute> &Pair) { return Pair.second.hasAttribute(Attribute::None); @@ -1011,11 +1104,11 @@ AttributeList::get(LLVMContext &C, if (Attrs.empty()) return {}; - assert(std::is_sorted(Attrs.begin(), Attrs.end(), - [](const std::pair<unsigned, AttributeSet> &LHS, - const std::pair<unsigned, AttributeSet> &RHS) { - return LHS.first < RHS.first; - }) && + assert(llvm::is_sorted(Attrs, + [](const std::pair<unsigned, AttributeSet> &LHS, + const std::pair<unsigned, AttributeSet> &RHS) { + return LHS.first < RHS.first; + }) && "Misordered Attributes list!"); assert(llvm::none_of(Attrs, [](const std::pair<unsigned, AttributeSet> &Pair) { @@ -1096,6 +1189,17 @@ AttributeList AttributeList::get(LLVMContext &C, unsigned Index, } AttributeList AttributeList::get(LLVMContext &C, unsigned Index, + ArrayRef<Attribute::AttrKind> Kinds, + ArrayRef<uint64_t> Values) { + assert(Kinds.size() == Values.size() && "Mismatched attribute values."); + SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; + auto VI = Values.begin(); + for (const auto K : Kinds) + Attrs.emplace_back(Index, Attribute::get(C, K, *VI++)); + return get(C, Attrs); +} + +AttributeList AttributeList::get(LLVMContext &C, unsigned Index, ArrayRef<StringRef> Kinds) { SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; for (const auto &K : Kinds) @@ -1184,7 +1288,7 @@ AttributeList AttributeList::addAttributes(LLVMContext &C, unsigned Index, AttributeList AttributeList::addParamAttribute(LLVMContext &C, ArrayRef<unsigned> ArgNos, Attribute A) const { - assert(std::is_sorted(ArgNos.begin(), ArgNos.end())); + assert(llvm::is_sorted(ArgNos)); SmallVector<AttributeSet, 4> AttrSets(this->begin(), this->end()); unsigned MaxIndex = attrIdxToArrayIdx(ArgNos.back() + FirstArgIndex); @@ -1284,8 +1388,6 @@ AttributeList::addAllocSizeAttr(LLVMContext &C, unsigned Index, // AttributeList Accessor Methods //===----------------------------------------------------------------------===// -LLVMContext &AttributeList::getContext() const { return pImpl->getContext(); } - AttributeSet AttributeList::getParamAttributes(unsigned ArgNo) const { return getAttributes(ArgNo + FirstArgIndex); } @@ -1326,17 +1428,7 @@ bool AttributeList::hasParamAttribute(unsigned ArgNo, bool AttributeList::hasAttrSomewhere(Attribute::AttrKind Attr, unsigned *Index) const { - if (!pImpl) return false; - - for (unsigned I = index_begin(), E = index_end(); I != E; ++I) { - if (hasAttribute(I, Attr)) { - if (Index) - *Index = I; - return true; - } - } - - return false; + return pImpl && pImpl->hasAttrSomewhere(Attr, Index); } Attribute AttributeList::getAttribute(unsigned Index, @@ -1360,6 +1452,10 @@ Type *AttributeList::getParamByValType(unsigned Index) const { return getAttributes(Index+FirstArgIndex).getByValType(); } +Type *AttributeList::getParamPreallocatedType(unsigned Index) const { + return getAttributes(Index + FirstArgIndex).getPreallocatedType(); +} + MaybeAlign AttributeList::getStackAlignment(unsigned Index) const { return getAttributes(Index).getStackAlignment(); } @@ -1441,12 +1537,12 @@ void AttrBuilder::clear() { DerefBytes = DerefOrNullBytes = 0; AllocSizeArgs = 0; ByValType = nullptr; + PreallocatedType = nullptr; } AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Val) { assert((unsigned)Val < Attribute::EndAttrKinds && "Attribute out of range!"); - assert(Val != Attribute::Alignment && Val != Attribute::StackAlignment && - Val != Attribute::Dereferenceable && Val != Attribute::AllocSize && + assert(!Attribute::doesAttrKindHaveArgument(Val) && "Adding integer attribute without adding a value!"); Attrs[Val] = true; return *this; @@ -1467,6 +1563,8 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) { StackAlignment = Attr.getStackAlignment(); else if (Kind == Attribute::ByVal) ByValType = Attr.getValueAsType(); + else if (Kind == Attribute::Preallocated) + PreallocatedType = Attr.getValueAsType(); else if (Kind == Attribute::Dereferenceable) DerefBytes = Attr.getDereferenceableBytes(); else if (Kind == Attribute::DereferenceableOrNull) @@ -1477,7 +1575,7 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) { } AttrBuilder &AttrBuilder::addAttribute(StringRef A, StringRef V) { - TargetDepAttrs[A] = V; + TargetDepAttrs[std::string(A)] = std::string(V); return *this; } @@ -1491,6 +1589,8 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) { StackAlignment.reset(); else if (Val == Attribute::ByVal) ByValType = nullptr; + else if (Val == Attribute::Preallocated) + PreallocatedType = nullptr; else if (Val == Attribute::Dereferenceable) DerefBytes = 0; else if (Val == Attribute::DereferenceableOrNull) @@ -1521,7 +1621,7 @@ AttrBuilder &AttrBuilder::addAlignmentAttr(MaybeAlign Align) { if (!Align) return *this; - assert(*Align <= 0x40000000 && "Alignment too large."); + assert(*Align <= llvm::Value::MaximumAlignment && "Alignment too large."); Attrs[Attribute::Alignment] = true; Alignment = Align; @@ -1579,6 +1679,12 @@ AttrBuilder &AttrBuilder::addByValAttr(Type *Ty) { return *this; } +AttrBuilder &AttrBuilder::addPreallocatedAttr(Type *Ty) { + Attrs[Attribute::Preallocated] = true; + PreallocatedType = Ty; + return *this; +} + AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { // FIXME: What if both have alignments, but they don't match?! if (!Alignment) @@ -1599,9 +1705,12 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { if (!ByValType) ByValType = B.ByValType; + if (!PreallocatedType) + PreallocatedType = B.PreallocatedType; + Attrs |= B.Attrs; - for (auto I : B.td_attrs()) + for (const auto &I : B.td_attrs()) TargetDepAttrs[I.first] = I.second; return *this; @@ -1627,9 +1736,12 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) { if (B.ByValType) ByValType = nullptr; + if (B.PreallocatedType) + PreallocatedType = nullptr; + Attrs &= ~B.Attrs; - for (auto I : B.td_attrs()) + for (const auto &I : B.td_attrs()) TargetDepAttrs.erase(I.first); return *this; @@ -1686,7 +1798,8 @@ bool AttrBuilder::operator==(const AttrBuilder &B) { return false; return Alignment == B.Alignment && StackAlignment == B.StackAlignment && - DerefBytes == B.DerefBytes && ByValType == B.ByValType; + DerefBytes == B.DerefBytes && ByValType == B.ByValType && + PreallocatedType == B.PreallocatedType; } //===----------------------------------------------------------------------===// @@ -1704,17 +1817,18 @@ AttrBuilder AttributeFuncs::typeIncompatible(Type *Ty) { if (!Ty->isPointerTy()) // Attribute that only apply to pointers. - Incompatible.addAttribute(Attribute::ByVal) - .addAttribute(Attribute::Nest) - .addAttribute(Attribute::NoAlias) - .addAttribute(Attribute::NoCapture) - .addAttribute(Attribute::NonNull) - .addDereferenceableAttr(1) // the int here is ignored - .addDereferenceableOrNullAttr(1) // the int here is ignored - .addAttribute(Attribute::ReadNone) - .addAttribute(Attribute::ReadOnly) - .addAttribute(Attribute::StructRet) - .addAttribute(Attribute::InAlloca); + Incompatible.addAttribute(Attribute::Nest) + .addAttribute(Attribute::NoAlias) + .addAttribute(Attribute::NoCapture) + .addAttribute(Attribute::NonNull) + .addDereferenceableAttr(1) // the int here is ignored + .addDereferenceableOrNullAttr(1) // the int here is ignored + .addAttribute(Attribute::ReadNone) + .addAttribute(Attribute::ReadOnly) + .addAttribute(Attribute::StructRet) + .addAttribute(Attribute::InAlloca) + .addPreallocatedAttr(Ty) + .addByValAttr(Ty); return Incompatible; } @@ -1837,17 +1951,58 @@ adjustMinLegalVectorWidth(Function &Caller, const Function &Callee) { } } -/// If the inlined function has "null-pointer-is-valid=true" attribute, +/// If the inlined function has null_pointer_is_valid attribute, /// set this attribute in the caller post inlining. static void adjustNullPointerValidAttr(Function &Caller, const Function &Callee) { if (Callee.nullPointerIsDefined() && !Caller.nullPointerIsDefined()) { - Caller.addFnAttr(Callee.getFnAttribute("null-pointer-is-valid")); + Caller.addFnAttr(Attribute::NullPointerIsValid); } } +struct EnumAttr { + static bool isSet(const Function &Fn, + Attribute::AttrKind Kind) { + return Fn.hasFnAttribute(Kind); + } + + static void set(Function &Fn, + Attribute::AttrKind Kind, bool Val) { + if (Val) + Fn.addFnAttr(Kind); + else + Fn.removeFnAttr(Kind); + } +}; + +struct StrBoolAttr { + static bool isSet(const Function &Fn, + StringRef Kind) { + auto A = Fn.getFnAttribute(Kind); + return A.getValueAsString().equals("true"); + } + + static void set(Function &Fn, + StringRef Kind, bool Val) { + Fn.addFnAttr(Kind, Val ? "true" : "false"); + } +}; + +#define GET_ATTR_NAMES +#define ATTRIBUTE_ENUM(ENUM_NAME, DISPLAY_NAME) \ + struct ENUM_NAME##Attr : EnumAttr { \ + static enum Attribute::AttrKind getKind() { \ + return llvm::Attribute::ENUM_NAME; \ + } \ + }; +#define ATTRIBUTE_STRBOOL(ENUM_NAME, DISPLAY_NAME) \ + struct ENUM_NAME##Attr : StrBoolAttr { \ + static StringRef getKind() { return #DISPLAY_NAME; } \ + }; +#include "llvm/IR/Attributes.inc" + #define GET_ATTR_COMPAT_FUNC -#include "AttributesCompatFunc.inc" +#include "llvm/IR/Attributes.inc" bool AttributeFuncs::areInlineCompatible(const Function &Caller, const Function &Callee) { diff --git a/llvm/lib/IR/AttributesCompatFunc.td b/llvm/lib/IR/AttributesCompatFunc.td deleted file mode 100644 index 7c85b3da9ab6..000000000000 --- a/llvm/lib/IR/AttributesCompatFunc.td +++ /dev/null @@ -1 +0,0 @@ -include "llvm/IR/Attributes.td" diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 6e2beeb839b6..1e8fdb506619 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/IntrinsicsAArch64.h" #include "llvm/IR/IntrinsicsARM.h" @@ -42,7 +43,7 @@ static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID, // Check whether this is an old version of the function, which received // v4f32 arguments. Type *Arg0Type = F->getFunctionType()->getParamType(0); - if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) + if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4)) return false; // Yes, it's old, replace it with new version. @@ -99,7 +100,6 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("fma4.vfmadd.s") || // Added in 7.0 Name.startswith("fma.vfmadd.") || // Added in 7.0 Name.startswith("fma.vfmsub.") || // Added in 7.0 - Name.startswith("fma.vfmaddsub.") || // Added in 7.0 Name.startswith("fma.vfmsubadd.") || // Added in 7.0 Name.startswith("fma.vfnmadd.") || // Added in 7.0 Name.startswith("fma.vfnmsub.") || // Added in 7.0 @@ -205,6 +205,8 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0 Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0 Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0 + Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0 + Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0 Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0 Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0 Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0 @@ -317,6 +319,7 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name == "avx.cvtdq2.pd.256" || // Added in 3.9 Name == "avx.cvtdq2.ps.256" || // Added in 7.0 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9 + Name.startswith("vcvtph2ps.") || // Added in 11.0 Name.startswith("avx.vinsertf128.") || // Added in 3.7 Name == "avx2.vinserti128" || // Added in 3.7 Name.startswith("avx512.mask.insert") || // Added in 4.0 @@ -372,8 +375,14 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { Name.startswith("avx2.pblendd.") || // Added in 3.7 Name.startswith("avx.vbroadcastf128") || // Added in 4.0 Name == "avx2.vbroadcasti128" || // Added in 3.7 - Name.startswith("avx512.mask.broadcastf") || // Added in 6.0 - Name.startswith("avx512.mask.broadcasti") || // Added in 6.0 + Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0 + Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0 Name == "xop.vpcmov" || // Added in 3.8 Name == "xop.vpcmov.256" || // Added in 5.0 Name.startswith("avx512.mask.move.s") || // Added in 4.0 @@ -891,11 +900,11 @@ GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) { // to byte shuffles. static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift) { - Type *ResultTy = Op->getType(); - unsigned NumElts = ResultTy->getVectorNumElements() * 8; + auto *ResultTy = cast<VectorType>(Op->getType()); + unsigned NumElts = ResultTy->getNumElements() * 8; // Bitcast from a 64-bit element type to a byte element type. - Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); + Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts); Op = Builder.CreateBitCast(Op, VecTy, "cast"); // We'll be shuffling in zeroes. @@ -904,7 +913,7 @@ static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, // we'll just return the zero vector. if (Shift < 16) { - uint32_t Idxs[64]; + int Idxs[64]; // 256/512-bit version is split into 2/4 16-byte lanes. for (unsigned l = 0; l != NumElts; l += 16) for (unsigned i = 0; i != 16; ++i) { @@ -925,11 +934,11 @@ static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, // to byte shuffles. static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift) { - Type *ResultTy = Op->getType(); - unsigned NumElts = ResultTy->getVectorNumElements() * 8; + auto *ResultTy = cast<VectorType>(Op->getType()); + unsigned NumElts = ResultTy->getNumElements() * 8; // Bitcast from a 64-bit element type to a byte element type. - Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); + Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts); Op = Builder.CreateBitCast(Op, VecTy, "cast"); // We'll be shuffling in zeroes. @@ -938,7 +947,7 @@ static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, // we'll just return the zero vector. if (Shift < 16) { - uint32_t Idxs[64]; + int Idxs[64]; // 256/512-bit version is split into 2/4 16-byte lanes. for (unsigned l = 0; l != NumElts; l += 16) for (unsigned i = 0; i != 16; ++i) { @@ -957,14 +966,14 @@ static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts) { - llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), - cast<IntegerType>(Mask->getType())->getBitWidth()); + llvm::VectorType *MaskTy = FixedVectorType::get( + Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth()); Mask = Builder.CreateBitCast(Mask, MaskTy); // If we have less than 8 elements, then the starting mask was an i8 and // we need to extract down to the right number of elements. if (NumElts < 8) { - uint32_t Indices[4]; + int Indices[4]; for (unsigned i = 0; i != NumElts; ++i) Indices[i] = i; Mask = Builder.CreateShuffleVector(Mask, Mask, @@ -982,7 +991,8 @@ static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, if (C->isAllOnesValue()) return Op0; - Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements()); + Mask = getX86MaskVec(Builder, Mask, + cast<VectorType>(Op0->getType())->getNumElements()); return Builder.CreateSelect(Mask, Op0, Op1); } @@ -993,9 +1003,8 @@ static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, if (C->isAllOnesValue()) return Op0; - llvm::VectorType *MaskTy = - llvm::VectorType::get(Builder.getInt1Ty(), - Mask->getType()->getIntegerBitWidth()); + auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), + Mask->getType()->getIntegerBitWidth()); Mask = Builder.CreateBitCast(Mask, MaskTy); Mask = Builder.CreateExtractElement(Mask, (uint64_t)0); return Builder.CreateSelect(Mask, Op0, Op1); @@ -1010,7 +1019,7 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, bool IsVALIGN) { unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); - unsigned NumElts = Op0->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Op0->getType())->getNumElements(); assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"); assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"); assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!"); @@ -1032,7 +1041,7 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Op0 = llvm::Constant::getNullValue(Op0->getType()); } - uint32_t Indices[64]; + int Indices[64]; // 256-bit palignr operates on 128-bit lanes so we need to handle that for (unsigned l = 0; l < NumElts; l += 16) { for (unsigned i = 0; i != 16; ++i) { @@ -1141,7 +1150,7 @@ static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI, // Funnel shifts amounts are treated as modulo and types are all power-of-2 so // we only care about the lowest log2 bits anyway. if (Amt->getType() != Ty) { - unsigned NumElts = Ty->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Ty)->getNumElements(); Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); Amt = Builder.CreateVectorSplat(NumElts, Amt); } @@ -1211,7 +1220,7 @@ static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI, // Funnel shifts amounts are treated as modulo and types are all power-of-2 so // we only care about the lowest log2 bits anyway. if (Amt->getType() != Ty) { - unsigned NumElts = Ty->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Ty)->getNumElements(); Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); Amt = Builder.CreateVectorSplat(NumElts, Amt); } @@ -1237,18 +1246,20 @@ static Value *UpgradeMaskedStore(IRBuilder<> &Builder, // Cast the pointer to the right type. Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(Data->getType())); - unsigned Align = - Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; + const Align Alignment = + Aligned + ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8) + : Align(1); // If the mask is all ones just emit a regular store. if (const auto *C = dyn_cast<Constant>(Mask)) if (C->isAllOnesValue()) - return Builder.CreateAlignedStore(Data, Ptr, Align); + return Builder.CreateAlignedStore(Data, Ptr, Alignment); // Convert the mask from an integer type to a vector of i1. - unsigned NumElts = Data->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Data->getType())->getNumElements(); Mask = getX86MaskVec(Builder, Mask, NumElts); - return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); + return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask); } static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, @@ -1257,18 +1268,21 @@ static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, Type *ValTy = Passthru->getType(); // Cast the pointer to the right type. Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy)); - unsigned Align = - Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1; + const Align Alignment = + Aligned + ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() / + 8) + : Align(1); // If the mask is all ones just emit a regular store. if (const auto *C = dyn_cast<Constant>(Mask)) if (C->isAllOnesValue()) - return Builder.CreateAlignedLoad(ValTy, Ptr, Align); + return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment); // Convert the mask from an integer type to a vector of i1. - unsigned NumElts = Passthru->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Passthru->getType())->getNumElements(); Mask = getX86MaskVec(Builder, Mask, NumElts); - return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru); + return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru); } static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { @@ -1330,7 +1344,7 @@ static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) { // Applying mask on vector of i1's and make sure result is at least 8 bits wide. static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask) { - unsigned NumElts = Vec->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Vec->getType())->getNumElements(); if (Mask) { const auto *C = dyn_cast<Constant>(Mask); if (!C || !C->isAllOnesValue()) @@ -1338,7 +1352,7 @@ static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, } if (NumElts < 8) { - uint32_t Indices[8]; + int Indices[8]; for (unsigned i = 0; i != NumElts; ++i) Indices[i] = i; for (unsigned i = NumElts; i != 8; ++i) @@ -1353,13 +1367,15 @@ static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, unsigned CC, bool Signed) { Value *Op0 = CI.getArgOperand(0); - unsigned NumElts = Op0->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(Op0->getType())->getNumElements(); Value *Cmp; if (CC == 3) { - Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); + Cmp = Constant::getNullValue( + FixedVectorType::get(Builder.getInt1Ty(), NumElts)); } else if (CC == 7) { - Cmp = Constant::getAllOnesValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); + Cmp = Constant::getAllOnesValue( + FixedVectorType::get(Builder.getInt1Ty(), NumElts)); } else { ICmpInst::Predicate Pred; switch (CC) { @@ -1406,7 +1422,7 @@ static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) { Value* Op = CI.getArgOperand(0); Type* ReturnOp = CI.getType(); - unsigned NumElts = CI.getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI.getType())->getNumElements(); Value *Mask = getX86MaskVec(Builder, Op, NumElts); return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2"); } @@ -1705,7 +1721,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Extract = Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); - StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1); + StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1)); SI->setMetadata(M->getMDKindID("nontemporal"), Node); // Remove intrinsic. @@ -1728,9 +1744,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *BC = Builder.CreateBitCast(Arg0, PointerType::getUnqual(Arg1->getType()), "cast"); - VectorType *VTy = cast<VectorType>(Arg1->getType()); - StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC, - VTy->getBitWidth() / 8); + StoreInst *SI = Builder.CreateAlignedStore( + Arg1, BC, + Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)); SI->setMetadata(M->getMDKindID("nontemporal"), Node); // Remove intrinsic. @@ -1742,13 +1758,13 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Arg0 = CI->getArgOperand(0); Value *Arg1 = CI->getArgOperand(1); - Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); + auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2); Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0); Value *BC = Builder.CreateBitCast(Arg0, PointerType::getUnqual(Elt->getType()), "cast"); - Builder.CreateAlignedStore(Elt, BC, 1); + Builder.CreateAlignedStore(Elt, BC, Align(1)); // Remove intrinsic. CI->eraseFromParent(); @@ -1764,7 +1780,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Arg0 = Builder.CreateBitCast(Arg0, PointerType::getUnqual(Arg1->getType()), "cast"); - Builder.CreateAlignedStore(Arg1, Arg0, 1); + Builder.CreateAlignedStore(Arg1, Arg0, Align(1)); // Remove intrinsic. CI->eraseFromParent(); @@ -1856,7 +1872,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask); } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){ unsigned NumElts = - CI->getArgOperand(1)->getType()->getVectorNumElements(); + cast<VectorType>(CI->getArgOperand(1)->getType())->getNumElements(); Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); @@ -1864,7 +1880,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { unsigned NumElts = CI->getType()->getScalarSizeInBits(); Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); - uint32_t Indices[64]; + int Indices[64]; for (unsigned i = 0; i != NumElts; ++i) Indices[i] = i; @@ -2074,16 +2090,19 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name == "sse2.cvtsi2sd" || Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd")) { - Rep = Builder.CreateSIToFP(CI->getArgOperand(1), - CI->getType()->getVectorElementType()); + Rep = Builder.CreateSIToFP( + CI->getArgOperand(1), + cast<VectorType>(CI->getType())->getElementType()); Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); } else if (IsX86 && Name == "avx512.cvtusi2sd") { - Rep = Builder.CreateUIToFP(CI->getArgOperand(1), - CI->getType()->getVectorElementType()); + Rep = Builder.CreateUIToFP( + CI->getArgOperand(1), + cast<VectorType>(CI->getType())->getElementType()); Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); } else if (IsX86 && Name == "sse2.cvtss2sd") { Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); - Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType()); + Rep = Builder.CreateFPExt( + Rep, cast<VectorType>(CI->getType())->getElementType()); Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); } else if (IsX86 && (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" || @@ -2103,18 +2122,17 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name == "avx.cvt.ps2.pd.256" || Name == "avx512.mask.cvtps2pd.128" || Name == "avx512.mask.cvtps2pd.256")) { - Type *DstTy = CI->getType(); + auto *DstTy = cast<VectorType>(CI->getType()); Rep = CI->getArgOperand(0); - Type *SrcTy = Rep->getType(); + auto *SrcTy = cast<VectorType>(Rep->getType()); - unsigned NumDstElts = DstTy->getVectorNumElements(); - if (NumDstElts < SrcTy->getVectorNumElements()) { + unsigned NumDstElts = DstTy->getNumElements(); + if (NumDstElts < SrcTy->getNumElements()) { assert(NumDstElts == 2 && "Unexpected vector size"); - uint32_t ShuffleMask[2] = { 0, 1 }; - Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask); + Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1}); } - bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy(); + bool IsPS2PD = SrcTy->getElementType()->isFloatTy(); bool IsUnsigned = (StringRef::npos != Name.find("cvtu")); if (IsPS2PD) Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); @@ -2134,6 +2152,22 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (CI->getNumArgOperands() >= 3) Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); + } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") || + Name.startswith("vcvtph2ps."))) { + auto *DstTy = cast<VectorType>(CI->getType()); + Rep = CI->getArgOperand(0); + auto *SrcTy = cast<VectorType>(Rep->getType()); + unsigned NumDstElts = DstTy->getNumElements(); + if (NumDstElts != SrcTy->getNumElements()) { + assert(NumDstElts == 4 && "Unexpected vector size"); + Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3}); + } + Rep = Builder.CreateBitCast( + Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts)); + Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps"); + if (CI->getNumArgOperands() >= 3) + Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, + CI->getArgOperand(1)); } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) { Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), @@ -2143,30 +2177,30 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->getArgOperand(1),CI->getArgOperand(2), /*Aligned*/true); } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) { - Type *ResultTy = CI->getType(); - Type *PtrTy = ResultTy->getVectorElementType(); + auto *ResultTy = cast<VectorType>(CI->getType()); + Type *PtrTy = ResultTy->getElementType(); // Cast the pointer to element type. Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), llvm::PointerType::getUnqual(PtrTy)); Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), - ResultTy->getVectorNumElements()); + ResultTy->getNumElements()); Function *ELd = Intrinsic::getDeclaration(F->getParent(), Intrinsic::masked_expandload, ResultTy); Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) { - Type *ResultTy = CI->getArgOperand(1)->getType(); - Type *PtrTy = ResultTy->getVectorElementType(); + auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType()); + Type *PtrTy = ResultTy->getElementType(); // Cast the pointer to element type. Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), llvm::PointerType::getUnqual(PtrTy)); Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), - ResultTy->getVectorNumElements()); + ResultTy->getNumElements()); Function *CSt = Intrinsic::getDeclaration(F->getParent(), Intrinsic::masked_compressstore, @@ -2174,10 +2208,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); } else if (IsX86 && (Name.startswith("avx512.mask.compress.") || Name.startswith("avx512.mask.expand."))) { - Type *ResultTy = CI->getType(); + auto *ResultTy = cast<VectorType>(CI->getType()); Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), - ResultTy->getVectorNumElements()); + ResultTy->getNumElements()); bool IsCompress = Name[12] == 'c'; Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress @@ -2254,9 +2288,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") || Name.startswith("avx512.vbroadcast.s"))) { // Replace broadcasts with a series of insertelements. - Type *VecTy = CI->getType(); - Type *EltTy = VecTy->getVectorElementType(); - unsigned EltNum = VecTy->getVectorNumElements(); + auto *VecTy = cast<VectorType>(CI->getType()); + Type *EltTy = VecTy->getElementType(); + unsigned EltNum = VecTy->getNumElements(); Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), EltTy->getPointerTo()); Value *Load = Builder.CreateLoad(EltTy, Cast); @@ -2276,7 +2310,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { unsigned NumDstElts = DstTy->getNumElements(); // Extract a subvector of the first NumDstElts lanes and sign/zero extend. - SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); + SmallVector<int, 8> ShuffleMask(NumDstElts); for (unsigned i = 0; i != NumDstElts; ++i) ShuffleMask[i] = i; @@ -2301,18 +2335,19 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") || Name == "avx2.vbroadcasti128")) { // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle. - Type *EltTy = CI->getType()->getVectorElementType(); + Type *EltTy = cast<VectorType>(CI->getType())->getElementType(); unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits(); - Type *VT = VectorType::get(EltTy, NumSrcElts); + auto *VT = FixedVectorType::get(EltTy, NumSrcElts); Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), PointerType::getUnqual(VT)); - Value *Load = Builder.CreateAlignedLoad(VT, Op, 1); + Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1)); if (NumSrcElts == 2) - Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), - { 0, 1, 0, 1 }); + Rep = Builder.CreateShuffleVector( + Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1}); else - Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), - { 0, 1, 2, 3, 0, 1, 2, 3 }); + Rep = + Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), + ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3}); } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") || Name.startswith("avx512.mask.shuf.f"))) { unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); @@ -2321,7 +2356,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits(); unsigned ControlBitsMask = NumLanes - 1; unsigned NumControlBits = NumLanes / 2; - SmallVector<uint32_t, 8> ShuffleMask(0); + SmallVector<int, 8> ShuffleMask(0); for (unsigned l = 0; l != NumLanes; ++l) { unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask; @@ -2338,10 +2373,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") || Name.startswith("avx512.mask.broadcasti"))) { unsigned NumSrcElts = - CI->getArgOperand(0)->getType()->getVectorNumElements(); - unsigned NumDstElts = CI->getType()->getVectorNumElements(); + cast<VectorType>(CI->getArgOperand(0)->getType())->getNumElements(); + unsigned NumDstElts = cast<VectorType>(CI->getType())->getNumElements(); - SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); + SmallVector<int, 8> ShuffleMask(NumDstElts); for (unsigned i = 0; i != NumDstElts; ++i) ShuffleMask[i] = i % NumSrcElts; @@ -2356,8 +2391,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.broadcast.s"))) { // Replace vp?broadcasts with a vector shuffle. Value *Op = CI->getArgOperand(0); - unsigned NumElts = CI->getType()->getVectorNumElements(); - Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts); + ElementCount EC = cast<VectorType>(CI->getType())->getElementCount(); + Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC); Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()), Constant::getNullValue(MaskTy)); @@ -2431,7 +2466,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { VectorType *VecTy = cast<VectorType>(CI->getType()); unsigned NumElts = VecTy->getNumElements(); - SmallVector<uint32_t, 16> Idxs(NumElts); + SmallVector<int, 16> Idxs(NumElts); for (unsigned i = 0; i != NumElts; ++i) Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i; @@ -2442,8 +2477,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); - unsigned DstNumElts = CI->getType()->getVectorNumElements(); - unsigned SrcNumElts = Op1->getType()->getVectorNumElements(); + unsigned DstNumElts = cast<VectorType>(CI->getType())->getNumElements(); + unsigned SrcNumElts = cast<VectorType>(Op1->getType())->getNumElements(); unsigned Scale = DstNumElts / SrcNumElts; // Mask off the high bits of the immediate value; hardware ignores those. @@ -2451,7 +2486,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Extend the second operand into a vector the size of the destination. Value *UndefV = UndefValue::get(Op1->getType()); - SmallVector<uint32_t, 8> Idxs(DstNumElts); + SmallVector<int, 8> Idxs(DstNumElts); for (unsigned i = 0; i != SrcNumElts; ++i) Idxs[i] = i; for (unsigned i = SrcNumElts; i != DstNumElts; ++i) @@ -2486,15 +2521,15 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.vextract"))) { Value *Op0 = CI->getArgOperand(0); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - unsigned DstNumElts = CI->getType()->getVectorNumElements(); - unsigned SrcNumElts = Op0->getType()->getVectorNumElements(); + unsigned DstNumElts = cast<VectorType>(CI->getType())->getNumElements(); + unsigned SrcNumElts = cast<VectorType>(Op0->getType())->getNumElements(); unsigned Scale = SrcNumElts / DstNumElts; // Mask off the high bits of the immediate value; hardware ignores those. Imm = Imm % Scale; // Get indexes for the subvector of the input vector. - SmallVector<uint32_t, 8> Idxs(DstNumElts); + SmallVector<int, 8> Idxs(DstNumElts); for (unsigned i = 0; i != DstNumElts; ++i) { Idxs[i] = i + (Imm * DstNumElts); } @@ -2513,7 +2548,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { VectorType *VecTy = cast<VectorType>(CI->getType()); unsigned NumElts = VecTy->getNumElements(); - SmallVector<uint32_t, 8> Idxs(NumElts); + SmallVector<int, 8> Idxs(NumElts); for (unsigned i = 0; i != NumElts; ++i) Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3); @@ -2534,9 +2569,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); - unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements(); unsigned HalfSize = NumElts / 2; - SmallVector<uint32_t, 8> ShuffleMask(NumElts); + SmallVector<int, 8> ShuffleMask(NumElts); // Determine which operand(s) are actually in use for this instruction. Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); @@ -2570,7 +2605,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { unsigned IdxSize = 64 / VecTy->getScalarSizeInBits(); unsigned IdxMask = ((1 << IdxSize) - 1); - SmallVector<uint32_t, 8> Idxs(NumElts); + SmallVector<int, 8> Idxs(NumElts); // Lookup the bits for this element, wrapping around the immediate every // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need // to offset by the first index of each group. @@ -2586,9 +2621,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.pshufl.w."))) { Value *Op0 = CI->getArgOperand(0); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements(); - SmallVector<uint32_t, 16> Idxs(NumElts); + SmallVector<int, 16> Idxs(NumElts); for (unsigned l = 0; l != NumElts; l += 8) { for (unsigned i = 0; i != 4; ++i) Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l; @@ -2605,9 +2640,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.pshufh.w."))) { Value *Op0 = CI->getArgOperand(0); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements(); - SmallVector<uint32_t, 16> Idxs(NumElts); + SmallVector<int, 16> Idxs(NumElts); for (unsigned l = 0; l != NumElts; l += 8) { for (unsigned i = 0; i != 4; ++i) Idxs[i + l] = i + l; @@ -2624,12 +2659,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); - unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements(); unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); unsigned HalfLaneElts = NumLaneElts / 2; - SmallVector<uint32_t, 16> Idxs(NumElts); + SmallVector<int, 16> Idxs(NumElts); for (unsigned i = 0; i != NumElts; ++i) { // Base index is the starting element of the lane. Idxs[i] = i - (i % NumLaneElts); @@ -2649,14 +2684,14 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.movshdup") || Name.startswith("avx512.mask.movsldup"))) { Value *Op0 = CI->getArgOperand(0); - unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumElts = cast<VectorType>(CI->getType())->getNumElements(); unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); unsigned Offset = 0; if (Name.startswith("avx512.mask.movshdup.")) Offset = 1; - SmallVector<uint32_t, 16> Idxs(NumElts); + SmallVector<int, 16> Idxs(NumElts); for (unsigned l = 0; l != NumElts; l += NumLaneElts) for (unsigned i = 0; i != NumLaneElts; i += 2) { Idxs[i + l + 0] = i + l + Offset; @@ -2671,10 +2706,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.unpckl."))) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); - int NumElts = CI->getType()->getVectorNumElements(); + int NumElts = cast<VectorType>(CI->getType())->getNumElements(); int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); - SmallVector<uint32_t, 64> Idxs(NumElts); + SmallVector<int, 64> Idxs(NumElts); for (int l = 0; l != NumElts; l += NumLaneElts) for (int i = 0; i != NumLaneElts; ++i) Idxs[i + l] = l + (i / 2) + NumElts * (i % 2); @@ -2687,10 +2722,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Name.startswith("avx512.mask.unpckh."))) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); - int NumElts = CI->getType()->getVectorNumElements(); + int NumElts = cast<VectorType>(CI->getType())->getNumElements(); int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); - SmallVector<uint32_t, 64> Idxs(NumElts); + SmallVector<int, 64> Idxs(NumElts); for (int l = 0; l != NumElts; l += NumLaneElts) for (int i = 0; i != NumLaneElts; ++i) Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2); @@ -3047,12 +3082,13 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); Value *Ptr = CI->getArgOperand(0); - VectorType *VTy = cast<VectorType>(CI->getType()); // Convert the type of the pointer to a pointer to the stored type. - Value *BC = - Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast"); - LoadInst *LI = Builder.CreateAlignedLoad(VTy, BC, VTy->getBitWidth() / 8); + Value *BC = Builder.CreateBitCast( + Ptr, PointerType::getUnqual(CI->getType()), "cast"); + LoadInst *LI = Builder.CreateAlignedLoad( + CI->getType(), BC, + Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)); LI->setMetadata(M->getMDKindID("nontemporal"), Node); Rep = LI; } else if (IsX86 && (Name.startswith("fma.vfmadd.") || @@ -3209,28 +3245,26 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->getArgOperand(0); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); - } else if (IsX86 && (Name.startswith("fma.vfmaddsub.p") || - Name.startswith("fma.vfmsubadd.p"))) { - bool IsSubAdd = Name[7] == 's'; - int NumElts = CI->getType()->getVectorNumElements(); + } else if (IsX86 && Name.startswith("fma.vfmsubadd.p")) { + unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); + unsigned EltWidth = CI->getType()->getScalarSizeInBits(); + Intrinsic::ID IID; + if (VecWidth == 128 && EltWidth == 32) + IID = Intrinsic::x86_fma_vfmaddsub_ps; + else if (VecWidth == 256 && EltWidth == 32) + IID = Intrinsic::x86_fma_vfmaddsub_ps_256; + else if (VecWidth == 128 && EltWidth == 64) + IID = Intrinsic::x86_fma_vfmaddsub_pd; + else if (VecWidth == 256 && EltWidth == 64) + IID = Intrinsic::x86_fma_vfmaddsub_pd_256; + else + llvm_unreachable("Unexpected intrinsic"); Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2) }; - - Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, - Ops[0]->getType()); - Value *Odd = Builder.CreateCall(FMA, Ops); Ops[2] = Builder.CreateFNeg(Ops[2]); - Value *Even = Builder.CreateCall(FMA, Ops); - - if (IsSubAdd) - std::swap(Even, Odd); - - SmallVector<uint32_t, 32> Idxs(NumElts); - for (int i = 0; i != NumElts; ++i) - Idxs[i] = i + (i % 2) * NumElts; - - Rep = Builder.CreateShuffleVector(Even, Odd, Idxs); + Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), + Ops); } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") || Name.startswith("avx512.mask3.vfmaddsub.p") || Name.startswith("avx512.maskz.vfmaddsub.p") || @@ -3240,9 +3274,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Drop the "avx512.mask." to make it easier. Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12); bool IsSubAdd = Name[3] == 's'; - if (CI->getNumArgOperands() == 5 && - (!isa<ConstantInt>(CI->getArgOperand(4)) || - cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { + if (CI->getNumArgOperands() == 5) { Intrinsic::ID IID; // Check the character before ".512" in string. if (Name[Name.size()-5] == 's') @@ -3256,10 +3288,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Ops[2] = Builder.CreateFNeg(Ops[2]); Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), - {CI->getArgOperand(0), CI->getArgOperand(1), - CI->getArgOperand(2), CI->getArgOperand(4)}); + Ops); } else { - int NumElts = CI->getType()->getVectorNumElements(); + int NumElts = cast<VectorType>(CI->getType())->getNumElements(); Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2) }; @@ -3273,7 +3304,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (IsSubAdd) std::swap(Even, Odd); - SmallVector<uint32_t, 32> Idxs(NumElts); + SmallVector<int, 32> Idxs(NumElts); for (int i = 0; i != NumElts; ++i) Idxs[i] = i + (i % 2) * NumElts; @@ -3434,7 +3465,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Cast the pointer to the right type. Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), llvm::PointerType::getUnqual(Data->getType())); - Builder.CreateAlignedStore(Data, Ptr, 1); + Builder.CreateAlignedStore(Data, Ptr, Align(1)); // Replace the original call result with the first result of the new call. Value *CF = Builder.CreateExtractValue(NewCall, 0); @@ -3629,13 +3660,13 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // So, the only thing required is a bitcast for both arguments. // First, check the arguments have the old type. Value *Arg0 = CI->getArgOperand(0); - if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) + if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4)) return; // Old intrinsic, add bitcasts Value *Arg1 = CI->getArgOperand(1); - Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); + auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2); Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast"); Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); @@ -3656,11 +3687,11 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Cast the pointer to the right type. Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), llvm::PointerType::getUnqual(Data->getType())); - Builder.CreateAlignedStore(Data, Ptr, 1); + Builder.CreateAlignedStore(Data, Ptr, Align(1)); // Replace the original call result with the first result of the new call. Value *TSC = Builder.CreateExtractValue(NewCall, 0); - std::string Name = CI->getName(); + std::string Name = std::string(CI->getName()); if (!Name.empty()) { CI->setName(Name + ".old"); NewCall->setName(Name); @@ -3726,16 +3757,16 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { auto *MemCI = cast<MemIntrinsic>(NewCall); // All mem intrinsics support dest alignment. const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); - MemCI->setDestAlignment(Align->getZExtValue()); + MemCI->setDestAlignment(Align->getMaybeAlignValue()); // Memcpy/Memmove also support source alignment. if (auto *MTI = dyn_cast<MemTransferInst>(MemCI)) - MTI->setSourceAlignment(Align->getZExtValue()); + MTI->setSourceAlignment(Align->getMaybeAlignValue()); break; } } assert(NewCall && "Should have either set this variable or returned through " "the default case"); - std::string Name = CI->getName(); + std::string Name = std::string(CI->getName()); if (!Name.empty()) { CI->setName(Name + ".old"); NewCall->setName(Name); @@ -4005,6 +4036,12 @@ bool llvm::UpgradeModuleFlags(Module &M) { return false; bool HasObjCFlag = false, HasClassProperties = false, Changed = false; + bool HasSwiftVersionFlag = false; + uint8_t SwiftMajorVersion, SwiftMinorVersion; + uint32_t SwiftABIVersion; + auto Int8Ty = Type::getInt8Ty(M.getContext()); + auto Int32Ty = Type::getInt32Ty(M.getContext()); + for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { MDNode *Op = ModFlags->getOperand(I); if (Op->getNumOperands() != 3) @@ -4050,6 +4087,31 @@ bool llvm::UpgradeModuleFlags(Module &M) { } } } + + // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value. + // If the higher bits are set, it adds new module flag for swift info. + if (ID->getString() == "Objective-C Garbage Collection") { + auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2)); + if (Md) { + assert(Md->getValue() && "Expected non-empty metadata"); + auto Type = Md->getValue()->getType(); + if (Type == Int8Ty) + continue; + unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue(); + if ((Val & 0xff) != Val) { + HasSwiftVersionFlag = true; + SwiftABIVersion = (Val & 0xff00) >> 8; + SwiftMajorVersion = (Val & 0xff000000) >> 24; + SwiftMinorVersion = (Val & 0xff0000) >> 16; + } + Metadata *Ops[3] = { + ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)), + Op->getOperand(1), + ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))}; + ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); + Changed = true; + } + } } // "Objective-C Class Properties" is recently added for Objective-C. We @@ -4063,6 +4125,16 @@ bool llvm::UpgradeModuleFlags(Module &M) { Changed = true; } + if (HasSwiftVersionFlag) { + M.addModuleFlag(Module::Error, "Swift ABI Version", + SwiftABIVersion); + M.addModuleFlag(Module::Error, "Swift Major Version", + ConstantInt::get(Int8Ty, SwiftMajorVersion)); + M.addModuleFlag(Module::Error, "Swift Minor Version", + ConstantInt::get(Int8Ty, SwiftMinorVersion)); + Changed = true; + } + return Changed; } @@ -4077,7 +4149,7 @@ void llvm::UpgradeSectionAttributes(Module &M) { for (auto Component : Components) OS << ',' << Component.trim(); - return OS.str().substr(1); + return std::string(OS.str().substr(1)); }; for (auto &GV : M.globals()) { @@ -4095,6 +4167,43 @@ void llvm::UpgradeSectionAttributes(Module &M) { } } +namespace { +// Prior to LLVM 10.0, the strictfp attribute could be used on individual +// callsites within a function that did not also have the strictfp attribute. +// Since 10.0, if strict FP semantics are needed within a function, the +// function must have the strictfp attribute and all calls within the function +// must also have the strictfp attribute. This latter restriction is +// necessary to prevent unwanted libcall simplification when a function is +// being cloned (such as for inlining). +// +// The "dangling" strictfp attribute usage was only used to prevent constant +// folding and other libcall simplification. The nobuiltin attribute on the +// callsite has the same effect. +struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> { + StrictFPUpgradeVisitor() {} + + void visitCallBase(CallBase &Call) { + if (!Call.isStrictFP()) + return; + if (isa<ConstrainedFPIntrinsic>(&Call)) + return; + // If we get here, the caller doesn't have the strictfp attribute + // but this callsite does. Replace the strictfp attribute with nobuiltin. + Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP); + Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin); + } +}; +} // namespace + +void llvm::UpgradeFunctionAttributes(Function &F) { + // If a function definition doesn't have the strictfp attribute, + // convert any callsite strictfp attributes to nobuiltin. + if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) { + StrictFPUpgradeVisitor SFPV; + SFPV.visit(F); + } +} + static bool isOldLoopArgument(Metadata *MD) { auto *T = dyn_cast_or_null<MDTuple>(MD); if (!T) @@ -4163,19 +4272,19 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { // If X86, and the datalayout matches the expected format, add pointer size // address spaces to the datalayout. if (!Triple(TT).isX86() || DL.contains(AddrSpaces)) - return DL; + return std::string(DL); SmallVector<StringRef, 4> Groups; Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)"); if (!R.match(DL, &Groups)) - return DL; + return std::string(DL); SmallString<1024> Buf; std::string Res = (Groups[1] + AddrSpaces + Groups[3]).toStringRef(Buf).str(); return Res; } -void llvm::UpgradeFramePointerAttributes(AttrBuilder &B) { +void llvm::UpgradeAttributes(AttrBuilder &B) { StringRef FramePointer; if (B.contains("no-frame-pointer-elim")) { // The value can be "true" or "false". @@ -4190,7 +4299,17 @@ void llvm::UpgradeFramePointerAttributes(AttrBuilder &B) { FramePointer = "non-leaf"; B.removeAttribute("no-frame-pointer-elim-non-leaf"); } - if (!FramePointer.empty()) B.addAttribute("frame-pointer", FramePointer); + + if (B.contains("null-pointer-is-valid")) { + // The value can be "true" or "false". + bool NullPointerIsValid = false; + for (const auto &I : B.td_attrs()) + if (I.first == "null-pointer-is-valid") + NullPointerIsValid = I.second == "true"; + B.removeAttribute("null-pointer-is-valid"); + if (NullPointerIsValid) + B.addAttribute(Attribute::NullPointerIsValid); + } } diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index bdee6990f932..64f1d3f3100c 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -33,6 +33,10 @@ LLVMContext &BasicBlock::getContext() const { return getType()->getContext(); } +template <> void llvm::invalidateParentIListOrdering(BasicBlock *BB) { + BB->invalidateOrders(); +} + // Explicit instantiation of SymbolTableListTraits since some of the methods // are not in the public header file... template class llvm::SymbolTableListTraits<Instruction>; @@ -61,6 +65,8 @@ void BasicBlock::insertInto(Function *NewParent, BasicBlock *InsertBefore) { } BasicBlock::~BasicBlock() { + validateInstrOrdering(); + // If the address of the block is taken and it is being deleted (e.g. because // it is dead), this means that there is either a dangling constant expr // hanging off the block, or an undefined use of the block (source code @@ -193,6 +199,18 @@ const CallInst *BasicBlock::getTerminatingDeoptimizeCall() const { return nullptr; } +const CallInst *BasicBlock::getPostdominatingDeoptimizeCall() const { + const BasicBlock* BB = this; + SmallPtrSet<const BasicBlock *, 8> Visited; + Visited.insert(BB); + while (auto *Succ = BB->getUniqueSuccessor()) { + if (!Visited.insert(Succ).second) + return nullptr; + BB = Succ; + } + return BB->getTerminatingDeoptimizeCall(); +} + const Instruction* BasicBlock::getFirstNonPHI() const { for (const Instruction &I : *this) if (!isa<PHINode>(I)) @@ -273,7 +291,7 @@ bool BasicBlock::hasNPredecessorsOrMore(unsigned N) const { } const BasicBlock *BasicBlock::getSingleSuccessor() const { - succ_const_iterator SI = succ_begin(this), E = succ_end(this); + const_succ_iterator SI = succ_begin(this), E = succ_end(this); if (SI == E) return nullptr; // no successors const BasicBlock *TheSucc = *SI; ++SI; @@ -281,7 +299,7 @@ const BasicBlock *BasicBlock::getSingleSuccessor() const { } const BasicBlock *BasicBlock::getUniqueSuccessor() const { - succ_const_iterator SI = succ_begin(this), E = succ_end(this); + const_succ_iterator SI = succ_begin(this), E = succ_end(this); if (SI == E) return nullptr; // No successors const BasicBlock *SuccBB = *SI; ++SI; @@ -299,78 +317,38 @@ iterator_range<BasicBlock::phi_iterator> BasicBlock::phis() { return make_range<phi_iterator>(P, nullptr); } -/// This method is used to notify a BasicBlock that the -/// specified Predecessor of the block is no longer able to reach it. This is -/// actually not used to update the Predecessor list, but is actually used to -/// update the PHI nodes that reside in the block. Note that this should be -/// called while the predecessor still refers to this block. +/// Update PHI nodes in this BasicBlock before removal of predecessor \p Pred. +/// Note that this function does not actually remove the predecessor. /// +/// If \p KeepOneInputPHIs is true then don't remove PHIs that are left with +/// zero or one incoming values, and don't simplify PHIs with all incoming +/// values the same. void BasicBlock::removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs) { - assert((hasNUsesOrMore(16)||// Reduce cost of this assertion for complex CFGs. + // Use hasNUsesOrMore to bound the cost of this assertion for complex CFGs. + assert((hasNUsesOrMore(16) || find(pred_begin(this), pred_end(this), Pred) != pred_end(this)) && - "removePredecessor: BB is not a predecessor!"); - - if (InstList.empty()) return; - PHINode *APN = dyn_cast<PHINode>(&front()); - if (!APN) return; // Quick exit. - - // If there are exactly two predecessors, then we want to nuke the PHI nodes - // altogether. However, we cannot do this, if this in this case: - // - // Loop: - // %x = phi [X, Loop] - // %x2 = add %x, 1 ;; This would become %x2 = add %x2, 1 - // br Loop ;; %x2 does not dominate all uses - // - // This is because the PHI node input is actually taken from the predecessor - // basic block. The only case this can happen is with a self loop, so we - // check for this case explicitly now. - // - unsigned max_idx = APN->getNumIncomingValues(); - assert(max_idx != 0 && "PHI Node in block with 0 predecessors!?!?!"); - if (max_idx == 2) { - BasicBlock *Other = APN->getIncomingBlock(APN->getIncomingBlock(0) == Pred); - - // Disable PHI elimination! - if (this == Other) max_idx = 3; - } + "Pred is not a predecessor!"); - // <= Two predecessors BEFORE I remove one? - if (max_idx <= 2 && !KeepOneInputPHIs) { - // Yup, loop through and nuke the PHI nodes - while (PHINode *PN = dyn_cast<PHINode>(&front())) { - // Remove the predecessor first. - PN->removeIncomingValue(Pred, !KeepOneInputPHIs); - - // If the PHI _HAD_ two uses, replace PHI node with its now *single* value - if (max_idx == 2) { - if (PN->getIncomingValue(0) != PN) - PN->replaceAllUsesWith(PN->getIncomingValue(0)); - else - // We are left with an infinite loop with no entries: kill the PHI. - PN->replaceAllUsesWith(UndefValue::get(PN->getType())); - getInstList().pop_front(); // Remove the PHI node - } - - // If the PHI node already only had one entry, it got deleted by - // removeIncomingValue. - } - } else { - // Okay, now we know that we need to remove predecessor #pred_idx from all - // PHI nodes. Iterate over each PHI node fixing them up - PHINode *PN; - for (iterator II = begin(); (PN = dyn_cast<PHINode>(II)); ) { - ++II; - PN->removeIncomingValue(Pred, false); - // If all incoming values to the Phi are the same, we can replace the Phi - // with that value. - Value* PNV = nullptr; - if (!KeepOneInputPHIs && (PNV = PN->hasConstantValue())) - if (PNV != PN) { + // Return early if there are no PHI nodes to update. + if (!isa<PHINode>(begin())) + return; + unsigned NumPreds = cast<PHINode>(front()).getNumIncomingValues(); + + // Update all PHI nodes. + for (iterator II = begin(); isa<PHINode>(II);) { + PHINode *PN = cast<PHINode>(II++); + PN->removeIncomingValue(Pred, !KeepOneInputPHIs); + if (!KeepOneInputPHIs) { + // If we have a single predecessor, removeIncomingValue erased the PHI + // node itself. + if (NumPreds > 1) { + if (Value *PNV = PN->hasConstantValue()) { + // Replace the PHI node with its constant value. PN->replaceAllUsesWith(PNV); PN->eraseFromParent(); } + } } } } @@ -494,3 +472,29 @@ BasicBlock::iterator llvm::skipDebugIntrinsics(BasicBlock::iterator It) { ++It; return It; } + +void BasicBlock::renumberInstructions() { + unsigned Order = 0; + for (Instruction &I : *this) + I.Order = Order++; + + // Set the bit to indicate that the instruction order valid and cached. + BasicBlockBits Bits = getBasicBlockBits(); + Bits.InstrOrderValid = true; + setBasicBlockBits(Bits); +} + +#ifndef NDEBUG +/// In asserts builds, this checks the numbering. In non-asserts builds, it +/// is defined as a no-op inline function in BasicBlock.h. +void BasicBlock::validateInstrOrdering() const { + if (!isInstrOrderValid()) + return; + const Instruction *Prev = nullptr; + for (const Instruction &I : *this) { + assert((!Prev || Prev->comesBefore(&I)) && + "cached instruction ordering is incorrect"); + Prev = &I; + } +} +#endif diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp index 6e24f03c4cfd..f02246cda7fc 100644 --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -47,14 +47,24 @@ static Constant *BitCastConstantVector(Constant *CV, VectorType *DstTy) { if (CV->isAllOnesValue()) return Constant::getAllOnesValue(DstTy); if (CV->isNullValue()) return Constant::getNullValue(DstTy); + // Do not iterate on scalable vector. The num of elements is unknown at + // compile-time. + if (isa<ScalableVectorType>(DstTy)) + return nullptr; + // If this cast changes element count then we can't handle it here: // doing so requires endianness information. This should be handled by // Analysis/ConstantFolding.cpp - unsigned NumElts = DstTy->getNumElements(); - if (NumElts != CV->getType()->getVectorNumElements()) + unsigned NumElts = cast<FixedVectorType>(DstTy)->getNumElements(); + if (NumElts != cast<FixedVectorType>(CV->getType())->getNumElements()) return nullptr; Type *DstEltTy = DstTy->getElementType(); + // Fast path for splatted constants. + if (Constant *Splat = CV->getSplatValue()) { + return ConstantVector::getSplat(DstTy->getElementCount(), + ConstantExpr::getBitCast(Splat, DstEltTy)); + } SmallVector<Constant*, 16> Result; Type *Ty = IntegerType::get(CV->getContext(), 32); @@ -114,18 +124,9 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { Constant::getNullValue(Type::getInt32Ty(DPTy->getContext())); IdxList.push_back(Zero); Type *ElTy = PTy->getElementType(); - while (ElTy != DPTy->getElementType()) { - if (StructType *STy = dyn_cast<StructType>(ElTy)) { - if (STy->getNumElements() == 0) break; - ElTy = STy->getElementType(0); - IdxList.push_back(Zero); - } else if (SequentialType *STy = - dyn_cast<SequentialType>(ElTy)) { - ElTy = STy->getElementType(); - IdxList.push_back(Zero); - } else { - break; - } + while (ElTy && ElTy != DPTy->getElementType()) { + ElTy = GetElementPtrInst::getTypeAtIndex(ElTy, (uint64_t)0); + IdxList.push_back(Zero); } if (ElTy == DPTy->getElementType()) @@ -138,7 +139,8 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { // and dest type have the same size (otherwise its an illegal cast). if (VectorType *DestPTy = dyn_cast<VectorType>(DestTy)) { if (VectorType *SrcTy = dyn_cast<VectorType>(V->getType())) { - assert(DestPTy->getBitWidth() == SrcTy->getBitWidth() && + assert(DestPTy->getPrimitiveSizeInBits() == + SrcTy->getPrimitiveSizeInBits() && "Not cast between same sized vectors!"); SrcTy = nullptr; // First, check for null. Undef is already handled. @@ -571,12 +573,21 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, // count may be mismatched; don't attempt to handle that here. if ((isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) && DestTy->isVectorTy() && - DestTy->getVectorNumElements() == V->getType()->getVectorNumElements()) { - SmallVector<Constant*, 16> res; + cast<FixedVectorType>(DestTy)->getNumElements() == + cast<FixedVectorType>(V->getType())->getNumElements()) { VectorType *DestVecTy = cast<VectorType>(DestTy); Type *DstEltTy = DestVecTy->getElementType(); + // Fast path for splatted constants. + if (Constant *Splat = V->getSplatValue()) { + return ConstantVector::getSplat( + cast<VectorType>(DestTy)->getElementCount(), + ConstantExpr::getCast(opc, Splat, DstEltTy)); + } + SmallVector<Constant *, 16> res; Type *Ty = IntegerType::get(V->getContext(), 32); - for (unsigned i = 0, e = V->getType()->getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, + e = cast<FixedVectorType>(V->getType())->getNumElements(); + i != e; ++i) { Constant *C = ConstantExpr::getExtractElement(V, ConstantInt::get(Ty, i)); res.push_back(ConstantExpr::getCast(opc, C, DstEltTy)); @@ -738,9 +749,10 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, // If the condition is a vector constant, fold the result elementwise. if (ConstantVector *CondV = dyn_cast<ConstantVector>(Cond)) { + auto *V1VTy = CondV->getType(); SmallVector<Constant*, 16> Result; Type *Ty = IntegerType::get(CondV->getContext(), 32); - for (unsigned i = 0, e = V1->getType()->getVectorNumElements(); i != e;++i){ + for (unsigned i = 0, e = V1VTy->getNumElements(); i != e; ++i) { Constant *V; Constant *V1Element = ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, i)); @@ -759,7 +771,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, } // If we were able to build the vector, return it. - if (Result.size() == V1->getType()->getVectorNumElements()) + if (Result.size() == V1VTy->getNumElements()) return ConstantVector::get(Result); } @@ -767,10 +779,30 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, if (isa<UndefValue>(V1)) return V1; return V2; } - if (isa<UndefValue>(V1)) return V2; - if (isa<UndefValue>(V2)) return V1; + if (V1 == V2) return V1; + // If the true or false value is undef, we can fold to the other value as + // long as the other value isn't poison. + auto NotPoison = [](Constant *C) { + // TODO: We can analyze ConstExpr by opcode to determine if there is any + // possibility of poison. + if (isa<ConstantExpr>(C)) + return false; + + if (isa<ConstantInt>(C) || isa<GlobalVariable>(C) || isa<ConstantFP>(C) || + isa<ConstantPointerNull>(C) || isa<Function>(C)) + return true; + + if (C->getType()->isVectorTy()) + return !C->containsUndefElement() && !C->containsConstantExpression(); + + // TODO: Recursively analyze aggregates or other constants. + return false; + }; + if (isa<UndefValue>(V1) && NotPoison(V2)) return V2; + if (isa<UndefValue>(V2) && NotPoison(V1)) return V1; + if (ConstantExpr *TrueVal = dyn_cast<ConstantExpr>(V1)) { if (TrueVal->getOpcode() == Instruction::Select) if (TrueVal->getOperand(0) == Cond) @@ -787,18 +819,22 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx) { + auto *ValVTy = cast<VectorType>(Val->getType()); + // extractelt undef, C -> undef // extractelt C, undef -> undef if (isa<UndefValue>(Val) || isa<UndefValue>(Idx)) - return UndefValue::get(Val->getType()->getVectorElementType()); + return UndefValue::get(ValVTy->getElementType()); auto *CIdx = dyn_cast<ConstantInt>(Idx); if (!CIdx) return nullptr; - // ee({w,x,y,z}, wrong_value) -> undef - if (CIdx->uge(Val->getType()->getVectorNumElements())) - return UndefValue::get(Val->getType()->getVectorElementType()); + if (auto *ValFVTy = dyn_cast<FixedVectorType>(Val->getType())) { + // ee({w,x,y,z}, wrong_value) -> undef + if (CIdx->uge(ValFVTy->getNumElements())) + return UndefValue::get(ValFVTy->getElementType()); + } // ee (gep (ptr, idx0, ...), idx) -> gep (ee (ptr, idx), ee (idx0, idx), ...) if (auto *CE = dyn_cast<ConstantExpr>(Val)) { @@ -810,17 +846,26 @@ Constant *llvm::ConstantFoldExtractElementInstruction(Constant *Val, if (Op->getType()->isVectorTy()) { Constant *ScalarOp = ConstantExpr::getExtractElement(Op, Idx); if (!ScalarOp) - return nullptr; + return nullptr; Ops.push_back(ScalarOp); } else Ops.push_back(Op); } - return CE->getWithOperands(Ops, CE->getType()->getVectorElementType(), - false, + return CE->getWithOperands(Ops, ValVTy->getElementType(), false, Ops[0]->getType()->getPointerElementType()); } } + // CAZ of type ScalableVectorType and n < CAZ->getMinNumElements() => + // extractelt CAZ, n -> 0 + if (auto *ValSVTy = dyn_cast<ScalableVectorType>(Val->getType())) { + if (!CIdx->uge(ValSVTy->getMinNumElements())) { + if (auto *CAZ = dyn_cast<ConstantAggregateZero>(Val)) + return CAZ->getElementValue(CIdx->getZExtValue()); + } + return nullptr; + } + return Val->getAggregateElement(CIdx); } @@ -835,11 +880,12 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val, // Do not iterate on scalable vector. The num of elements is unknown at // compile-time. - VectorType *ValTy = cast<VectorType>(Val->getType()); - if (ValTy->isScalable()) + if (isa<ScalableVectorType>(Val->getType())) return nullptr; - unsigned NumElts = Val->getType()->getVectorNumElements(); + auto *ValTy = cast<FixedVectorType>(Val->getType()); + + unsigned NumElts = ValTy->getNumElements(); if (CIdx->uge(NumElts)) return UndefValue::get(Val->getType()); @@ -860,31 +906,38 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val, return ConstantVector::get(Result); } -Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, - Constant *V2, - Constant *Mask) { - unsigned MaskNumElts = Mask->getType()->getVectorNumElements(); - Type *EltTy = V1->getType()->getVectorElementType(); +Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, + ArrayRef<int> Mask) { + auto *V1VTy = cast<VectorType>(V1->getType()); + unsigned MaskNumElts = Mask.size(); + ElementCount MaskEltCount = {MaskNumElts, isa<ScalableVectorType>(V1VTy)}; + Type *EltTy = V1VTy->getElementType(); // Undefined shuffle mask -> undefined value. - if (isa<UndefValue>(Mask)) - return UndefValue::get(VectorType::get(EltTy, MaskNumElts)); - - // Don't break the bitcode reader hack. - if (isa<ConstantExpr>(Mask)) return nullptr; + if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { + return UndefValue::get(FixedVectorType::get(EltTy, MaskNumElts)); + } + // If the mask is all zeros this is a splat, no need to go through all + // elements. + if (all_of(Mask, [](int Elt) { return Elt == 0; }) && + !MaskEltCount.Scalable) { + Type *Ty = IntegerType::get(V1->getContext(), 32); + Constant *Elt = + ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0)); + return ConstantVector::getSplat(MaskEltCount, Elt); + } // Do not iterate on scalable vector. The num of elements is unknown at // compile-time. - VectorType *ValTy = cast<VectorType>(V1->getType()); - if (ValTy->isScalable()) + if (isa<ScalableVectorType>(V1VTy)) return nullptr; - unsigned SrcNumElts = V1->getType()->getVectorNumElements(); + unsigned SrcNumElts = V1VTy->getElementCount().Min; // Loop over the shuffle mask, evaluating each element. SmallVector<Constant*, 32> Result; for (unsigned i = 0; i != MaskNumElts; ++i) { - int Elt = ShuffleVectorInst::getMaskValue(Mask, i); + int Elt = Mask[i]; if (Elt == -1) { Result.push_back(UndefValue::get(EltTy)); continue; @@ -930,7 +983,7 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg, if (StructType *ST = dyn_cast<StructType>(Agg->getType())) NumElts = ST->getNumElements(); else - NumElts = cast<SequentialType>(Agg->getType())->getNumElements(); + NumElts = cast<ArrayType>(Agg->getType())->getNumElements(); SmallVector<Constant*, 32> Result; for (unsigned i = 0; i != NumElts; ++i) { @@ -945,18 +998,19 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg, if (StructType *ST = dyn_cast<StructType>(Agg->getType())) return ConstantStruct::get(ST, Result); - if (ArrayType *AT = dyn_cast<ArrayType>(Agg->getType())) - return ConstantArray::get(AT, Result); - return ConstantVector::get(Result); + return ConstantArray::get(cast<ArrayType>(Agg->getType()), Result); } Constant *llvm::ConstantFoldUnaryInstruction(unsigned Opcode, Constant *C) { assert(Instruction::isUnaryOp(Opcode) && "Non-unary instruction detected"); - // Handle scalar UndefValue. Vectors are always evaluated per element. - bool HasScalarUndef = !C->getType()->isVectorTy() && isa<UndefValue>(C); + // Handle scalar UndefValue and scalable vector UndefValue. Fixed-length + // vectors are always evaluated per element. + bool IsScalableVector = isa<ScalableVectorType>(C->getType()); + bool HasScalarUndefOrScalableVectorUndef = + (!C->getType()->isVectorTy() || IsScalableVector) && isa<UndefValue>(C); - if (HasScalarUndef) { + if (HasScalarUndefOrScalableVectorUndef) { switch (static_cast<Instruction::UnaryOps>(Opcode)) { case Instruction::FNeg: return C; // -undef -> undef @@ -966,7 +1020,7 @@ Constant *llvm::ConstantFoldUnaryInstruction(unsigned Opcode, Constant *C) { } // Constant should not be UndefValue, unless these are vector constants. - assert(!HasScalarUndef && "Unexpected UndefValue"); + assert(!HasScalarUndefOrScalableVectorUndef && "Unexpected UndefValue"); // We only have FP UnaryOps right now. assert(!isa<ConstantInt>(C) && "Unexpected Integer UnaryOp"); @@ -978,10 +1032,17 @@ Constant *llvm::ConstantFoldUnaryInstruction(unsigned Opcode, Constant *C) { case Instruction::FNeg: return ConstantFP::get(C->getContext(), neg(CV)); } - } else if (VectorType *VTy = dyn_cast<VectorType>(C->getType())) { - // Fold each element and create a vector constant from those constants. - SmallVector<Constant*, 16> Result; + } else if (auto *VTy = dyn_cast<FixedVectorType>(C->getType())) { + Type *Ty = IntegerType::get(VTy->getContext(), 32); + // Fast path for splatted constants. + if (Constant *Splat = C->getSplatValue()) { + Constant *Elt = ConstantExpr::get(Opcode, Splat); + return ConstantVector::getSplat(VTy->getElementCount(), Elt); + } + + // Fold each element and create a vector constant from those constants. + SmallVector<Constant *, 16> Result; for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { Constant *ExtractIdx = ConstantInt::get(Ty, i); Constant *Elt = ConstantExpr::getExtractElement(C, ExtractIdx); @@ -1013,10 +1074,13 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, return C1; } - // Handle scalar UndefValue. Vectors are always evaluated per element. - bool HasScalarUndef = !C1->getType()->isVectorTy() && - (isa<UndefValue>(C1) || isa<UndefValue>(C2)); - if (HasScalarUndef) { + // Handle scalar UndefValue and scalable vector UndefValue. Fixed-length + // vectors are always evaluated per element. + bool IsScalableVector = isa<ScalableVectorType>(C1->getType()); + bool HasScalarUndefOrScalableVectorUndef = + (!C1->getType()->isVectorTy() || IsScalableVector) && + (isa<UndefValue>(C1) || isa<UndefValue>(C2)); + if (HasScalarUndefOrScalableVectorUndef) { switch (static_cast<Instruction::BinaryOps>(Opcode)) { case Instruction::Xor: if (isa<UndefValue>(C1) && isa<UndefValue>(C2)) @@ -1097,8 +1161,12 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, return C1; // undef << X -> 0 return Constant::getNullValue(C1->getType()); - case Instruction::FAdd: case Instruction::FSub: + // -0.0 - undef --> undef (consistent with "fneg undef") + if (match(C1, m_NegZeroFP()) && isa<UndefValue>(C2)) + return C2; + LLVM_FALLTHROUGH; + case Instruction::FAdd: case Instruction::FMul: case Instruction::FDiv: case Instruction::FRem: @@ -1119,7 +1187,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, } // Neither constant should be UndefValue, unless these are vector constants. - assert(!HasScalarUndef && "Unexpected UndefValue"); + assert((!HasScalarUndefOrScalableVectorUndef) && "Unexpected UndefValue"); // Handle simplifications when the RHS is a constant int. if (ConstantInt *CI2 = dyn_cast<ConstantInt>(C2)) { @@ -1173,7 +1241,8 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, MaybeAlign GVAlign; if (Module *TheModule = GV->getParent()) { - GVAlign = GV->getPointerAlignment(TheModule->getDataLayout()); + const DataLayout &DL = TheModule->getDataLayout(); + GVAlign = GV->getPointerAlignment(DL); // If the function alignment is not specified then assume that it // is 4. @@ -1184,14 +1253,14 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, // increased code size (see https://reviews.llvm.org/D55115) // FIXME: This code should be deleted once existing targets have // appropriate defaults - if (!GVAlign && isa<Function>(GV)) + if (isa<Function>(GV) && !DL.getFunctionPtrAlign()) GVAlign = Align(4); } else if (isa<Function>(GV)) { // Without a datalayout we have to assume the worst case: that the // function pointer isn't aligned at all. GVAlign = llvm::None; - } else { - GVAlign = MaybeAlign(GV->getAlignment()); + } else if (isa<GlobalVariable>(GV)) { + GVAlign = cast<GlobalVariable>(GV)->getAlign(); } if (GVAlign && *GVAlign > 1) { @@ -1329,7 +1398,23 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, return ConstantFP::get(C1->getContext(), C3V); } } - } else if (VectorType *VTy = dyn_cast<VectorType>(C1->getType())) { + } else if (IsScalableVector) { + // Do not iterate on scalable vector. The number of elements is unknown at + // compile-time. + // FIXME: this branch can potentially be removed + return nullptr; + } else if (auto *VTy = dyn_cast<FixedVectorType>(C1->getType())) { + // Fast path for splatted constants. + if (Constant *C2Splat = C2->getSplatValue()) { + if (Instruction::isIntDivRem(Opcode) && C2Splat->isNullValue()) + return UndefValue::get(VTy); + if (Constant *C1Splat = C1->getSplatValue()) { + return ConstantVector::getSplat( + VTy->getElementCount(), + ConstantExpr::get(Opcode, C1Splat, C2Splat)); + } + } + // Fold each element and create a vector constant from those constants. SmallVector<Constant*, 16> Result; Type *Ty = IntegerType::get(VTy->getContext(), 32); @@ -1812,7 +1897,7 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, Type *ResultTy; if (VectorType *VT = dyn_cast<VectorType>(C1->getType())) ResultTy = VectorType::get(Type::getInt1Ty(C1->getContext()), - VT->getNumElements()); + VT->getElementCount()); else ResultTy = Type::getInt1Ty(C1->getContext()); @@ -1942,13 +2027,26 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, return ConstantInt::get(ResultTy, R==APFloat::cmpGreaterThan || R==APFloat::cmpEqual); } - } else if (C1->getType()->isVectorTy()) { + } else if (auto *C1VTy = dyn_cast<VectorType>(C1->getType())) { + + // Do not iterate on scalable vector. The number of elements is unknown at + // compile-time. + if (isa<ScalableVectorType>(C1VTy)) + return nullptr; + + // Fast path for splatted constants. + if (Constant *C1Splat = C1->getSplatValue()) + if (Constant *C2Splat = C2->getSplatValue()) + return ConstantVector::getSplat( + C1VTy->getElementCount(), + ConstantExpr::getCompare(pred, C1Splat, C2Splat)); + // If we can constant fold the comparison of each element, constant fold // the whole vector comparison. SmallVector<Constant*, 4> ResElts; Type *Ty = IntegerType::get(C1->getContext(), 32); // Compare the elements, producing an i1 result or constant expr. - for (unsigned i = 0, e = C1->getType()->getVectorNumElements(); i != e;++i){ + for (unsigned i = 0, e = C1VTy->getElementCount().Min; i != e; ++i) { Constant *C1E = ConstantExpr::getExtractElement(C1, ConstantInt::get(Ty, i)); Constant *C2E = @@ -2202,7 +2300,7 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, if (Idxs.size() == 1 && (Idx0->isNullValue() || isa<UndefValue>(Idx0))) return GEPTy->isVectorTy() && !C->getType()->isVectorTy() ? ConstantVector::getSplat( - cast<VectorType>(GEPTy)->getNumElements(), C) + cast<VectorType>(GEPTy)->getElementCount(), C) : C; if (C->isNullValue()) { @@ -2221,13 +2319,16 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, Type *OrigGEPTy = PointerType::get(Ty, PtrTy->getAddressSpace()); Type *GEPTy = PointerType::get(Ty, PtrTy->getAddressSpace()); if (VectorType *VT = dyn_cast<VectorType>(C->getType())) - GEPTy = VectorType::get(OrigGEPTy, VT->getNumElements()); + GEPTy = VectorType::get(OrigGEPTy, VT->getElementCount()); // The GEP returns a vector of pointers when one of more of // its arguments is a vector. for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { if (auto *VT = dyn_cast<VectorType>(Idxs[i]->getType())) { - GEPTy = VectorType::get(OrigGEPTy, VT->getNumElements()); + assert((!isa<VectorType>(GEPTy) || isa<ScalableVectorType>(GEPTy) == + isa<ScalableVectorType>(VT)) && + "Mismatched GEPTy vector types"); + GEPTy = VectorType::get(OrigGEPTy, VT->getElementCount()); break; } } @@ -2357,10 +2458,11 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, SmallVector<Constant *, 8> NewIdxs; Type *Ty = PointeeTy; Type *Prev = C->getType(); + auto GEPIter = gep_type_begin(PointeeTy, Idxs); bool Unknown = !isa<ConstantInt>(Idxs[0]) && !isa<ConstantDataVector>(Idxs[0]); for (unsigned i = 1, e = Idxs.size(); i != e; - Prev = Ty, Ty = cast<CompositeType>(Ty)->getTypeAtIndex(Idxs[i]), ++i) { + Prev = Ty, Ty = (++GEPIter).getIndexedType(), ++i) { if (!isa<ConstantInt>(Idxs[i]) && !isa<ConstantDataVector>(Idxs[i])) { // We don't know if it's in range or not. Unknown = true; @@ -2379,12 +2481,12 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, // The verify makes sure that GEPs into a struct are in range. continue; } - auto *STy = cast<SequentialType>(Ty); - if (isa<VectorType>(STy)) { + if (isa<VectorType>(Ty)) { // There can be awkward padding in after a non-power of two vector. Unknown = true; continue; } + auto *STy = cast<ArrayType>(Ty); if (ConstantInt *CI = dyn_cast<ConstantInt>(Idxs[i])) { if (isIndexInRangeOfArrayType(STy->getNumElements(), CI)) // It's in range, skip to the next index. @@ -2433,18 +2535,19 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, if (!IsCurrIdxVector && IsPrevIdxVector) CurrIdx = ConstantDataVector::getSplat( - PrevIdx->getType()->getVectorNumElements(), CurrIdx); + cast<FixedVectorType>(PrevIdx->getType())->getNumElements(), CurrIdx); if (!IsPrevIdxVector && IsCurrIdxVector) PrevIdx = ConstantDataVector::getSplat( - CurrIdx->getType()->getVectorNumElements(), PrevIdx); + cast<FixedVectorType>(CurrIdx->getType())->getNumElements(), PrevIdx); Constant *Factor = ConstantInt::get(CurrIdx->getType()->getScalarType(), NumElements); if (UseVector) Factor = ConstantDataVector::getSplat( - IsPrevIdxVector ? PrevIdx->getType()->getVectorNumElements() - : CurrIdx->getType()->getVectorNumElements(), + IsPrevIdxVector + ? cast<FixedVectorType>(PrevIdx->getType())->getNumElements() + : cast<FixedVectorType>(CurrIdx->getType())->getNumElements(), Factor); NewIdxs[i] = ConstantExpr::getSRem(CurrIdx, Factor); @@ -2460,10 +2563,11 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, // overflow trouble. Type *ExtendedTy = Type::getIntNTy(Div->getContext(), CommonExtendedWidth); if (UseVector) - ExtendedTy = VectorType::get( - ExtendedTy, IsPrevIdxVector - ? PrevIdx->getType()->getVectorNumElements() - : CurrIdx->getType()->getVectorNumElements()); + ExtendedTy = FixedVectorType::get( + ExtendedTy, + IsPrevIdxVector + ? cast<FixedVectorType>(PrevIdx->getType())->getNumElements() + : cast<FixedVectorType>(CurrIdx->getType())->getNumElements()); if (!PrevIdx->getType()->isIntOrIntVectorTy(CommonExtendedWidth)) PrevIdx = ConstantExpr::getSExt(PrevIdx, ExtendedTy); diff --git a/llvm/lib/IR/ConstantFold.h b/llvm/lib/IR/ConstantFold.h index 9ad6e14e9e40..0cdd5cf3cbce 100644 --- a/llvm/lib/IR/ConstantFold.h +++ b/llvm/lib/IR/ConstantFold.h @@ -38,7 +38,7 @@ template <typename T> class ArrayRef; Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt, Constant *Idx); Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, - Constant *Mask); + ArrayRef<int> Mask); Constant *ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef<unsigned> Idxs); Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 3d25cb5bfbdf..eabaaa203927 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -802,6 +802,8 @@ ConstantRange ConstantRange::binaryOp(Instruction::BinaryOps BinOp, return binaryAnd(Other); case Instruction::Or: return binaryOr(Other); + case Instruction::Xor: + return binaryXor(Other); // Note: floating point operations applied to abstract ranges are just // ideal integer operations with a lossy representation case Instruction::FAdd: @@ -1194,6 +1196,10 @@ ConstantRange::binaryAnd(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) return getEmpty(); + // Use APInt's implementation of AND for single element ranges. + if (isSingleElement() && Other.isSingleElement()) + return {*getSingleElement() & *Other.getSingleElement()}; + // TODO: replace this with something less conservative APInt umin = APIntOps::umin(Other.getUnsignedMax(), getUnsignedMax()); @@ -1205,12 +1211,28 @@ ConstantRange::binaryOr(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) return getEmpty(); + // Use APInt's implementation of OR for single element ranges. + if (isSingleElement() && Other.isSingleElement()) + return {*getSingleElement() | *Other.getSingleElement()}; + // TODO: replace this with something less conservative APInt umax = APIntOps::umax(getUnsignedMin(), Other.getUnsignedMin()); return getNonEmpty(std::move(umax), APInt::getNullValue(getBitWidth())); } +ConstantRange ConstantRange::binaryXor(const ConstantRange &Other) const { + if (isEmptySet() || Other.isEmptySet()) + return getEmpty(); + + // Use APInt's implementation of XOR for single element ranges. + if (isSingleElement() && Other.isSingleElement()) + return {*getSingleElement() ^ *Other.getSingleElement()}; + + // TODO: replace this with something less conservative + return getFull(); +} + ConstantRange ConstantRange::shl(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index 054375aab6c3..cbbcca20ea51 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -160,8 +160,8 @@ bool Constant::isNotOneValue() const { return !CFP->getValueAPF().bitcastToAPInt().isOneValue(); // Check that vectors don't contain 1 - if (this->getType()->isVectorTy()) { - unsigned NumElts = this->getType()->getVectorNumElements(); + if (auto *VTy = dyn_cast<VectorType>(this->getType())) { + unsigned NumElts = VTy->getNumElements(); for (unsigned i = 0; i != NumElts; ++i) { Constant *Elt = this->getAggregateElement(i); if (!Elt || !Elt->isNotOneValue()) @@ -210,8 +210,8 @@ bool Constant::isNotMinSignedValue() const { return !CFP->getValueAPF().bitcastToAPInt().isMinSignedValue(); // Check that vectors don't contain INT_MIN - if (this->getType()->isVectorTy()) { - unsigned NumElts = this->getType()->getVectorNumElements(); + if (auto *VTy = dyn_cast<VectorType>(this->getType())) { + unsigned NumElts = VTy->getNumElements(); for (unsigned i = 0; i != NumElts; ++i) { Constant *Elt = this->getAggregateElement(i); if (!Elt || !Elt->isNotMinSignedValue()) @@ -227,9 +227,10 @@ bool Constant::isNotMinSignedValue() const { bool Constant::isFiniteNonZeroFP() const { if (auto *CFP = dyn_cast<ConstantFP>(this)) return CFP->getValueAPF().isFiniteNonZero(); - if (!getType()->isVectorTy()) + auto *VTy = dyn_cast<VectorType>(getType()); + if (!VTy) return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { auto *CFP = dyn_cast_or_null<ConstantFP>(this->getAggregateElement(i)); if (!CFP || !CFP->getValueAPF().isFiniteNonZero()) return false; @@ -240,9 +241,10 @@ bool Constant::isFiniteNonZeroFP() const { bool Constant::isNormalFP() const { if (auto *CFP = dyn_cast<ConstantFP>(this)) return CFP->getValueAPF().isNormal(); - if (!getType()->isVectorTy()) + auto *VTy = dyn_cast<FixedVectorType>(getType()); + if (!VTy) return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { auto *CFP = dyn_cast_or_null<ConstantFP>(this->getAggregateElement(i)); if (!CFP || !CFP->getValueAPF().isNormal()) return false; @@ -253,9 +255,10 @@ bool Constant::isNormalFP() const { bool Constant::hasExactInverseFP() const { if (auto *CFP = dyn_cast<ConstantFP>(this)) return CFP->getValueAPF().getExactInverse(nullptr); - if (!getType()->isVectorTy()) + auto *VTy = dyn_cast<FixedVectorType>(getType()); + if (!VTy) return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { auto *CFP = dyn_cast_or_null<ConstantFP>(this->getAggregateElement(i)); if (!CFP || !CFP->getValueAPF().getExactInverse(nullptr)) return false; @@ -266,9 +269,10 @@ bool Constant::hasExactInverseFP() const { bool Constant::isNaN() const { if (auto *CFP = dyn_cast<ConstantFP>(this)) return CFP->isNaN(); - if (!getType()->isVectorTy()) + auto *VTy = dyn_cast<FixedVectorType>(getType()); + if (!VTy) return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) { auto *CFP = dyn_cast_or_null<ConstantFP>(this->getAggregateElement(i)); if (!CFP || !CFP->isNaN()) return false; @@ -282,34 +286,40 @@ bool Constant::isElementWiseEqual(Value *Y) const { return true; // The input value must be a vector constant with the same type. - Type *Ty = getType(); - if (!isa<Constant>(Y) || !Ty->isVectorTy() || Ty != Y->getType()) + auto *VTy = dyn_cast<VectorType>(getType()); + if (!isa<Constant>(Y) || !VTy || VTy != Y->getType()) + return false; + + // TODO: Compare pointer constants? + if (!(VTy->getElementType()->isIntegerTy() || + VTy->getElementType()->isFloatingPointTy())) return false; // They may still be identical element-wise (if they have `undef`s). - // FIXME: This crashes on FP vector constants. - return match(ConstantExpr::getICmp(ICmpInst::Predicate::ICMP_EQ, - const_cast<Constant *>(this), - cast<Constant>(Y)), - m_One()); + // Bitcast to integer to allow exact bitwise comparison for all types. + Type *IntTy = VectorType::getInteger(VTy); + Constant *C0 = ConstantExpr::getBitCast(const_cast<Constant *>(this), IntTy); + Constant *C1 = ConstantExpr::getBitCast(cast<Constant>(Y), IntTy); + Constant *CmpEq = ConstantExpr::getICmp(ICmpInst::ICMP_EQ, C0, C1); + return isa<UndefValue>(CmpEq) || match(CmpEq, m_One()); } bool Constant::containsUndefElement() const { - if (!getType()->isVectorTy()) - return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) - if (isa<UndefValue>(getAggregateElement(i))) - return true; + if (auto *VTy = dyn_cast<VectorType>(getType())) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) + if (isa<UndefValue>(getAggregateElement(i))) + return true; + } return false; } bool Constant::containsConstantExpression() const { - if (!getType()->isVectorTy()) - return false; - for (unsigned i = 0, e = getType()->getVectorNumElements(); i != e; ++i) - if (isa<ConstantExpr>(getAggregateElement(i))) - return true; + if (auto *VTy = dyn_cast<VectorType>(getType())) { + for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i) + if (isa<ConstantExpr>(getAggregateElement(i))) + return true; + } return false; } @@ -322,6 +332,9 @@ Constant *Constant::getNullValue(Type *Ty) { case Type::HalfTyID: return ConstantFP::get(Ty->getContext(), APFloat::getZero(APFloat::IEEEhalf())); + case Type::BFloatTyID: + return ConstantFP::get(Ty->getContext(), + APFloat::getZero(APFloat::BFloat())); case Type::FloatTyID: return ConstantFP::get(Ty->getContext(), APFloat::getZero(APFloat::IEEEsingle())); @@ -342,7 +355,8 @@ Constant *Constant::getNullValue(Type *Ty) { return ConstantPointerNull::get(cast<PointerType>(Ty)); case Type::StructTyID: case Type::ArrayTyID: - case Type::VectorTyID: + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: return ConstantAggregateZero::get(Ty); case Type::TokenTyID: return ConstantTokenNone::get(Ty->getContext()); @@ -364,7 +378,7 @@ Constant *Constant::getIntegerValue(Type *Ty, const APInt &V) { // Broadcast a scalar to a vector, if necessary. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - C = ConstantVector::getSplat(VTy->getNumElements(), C); + C = ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -375,13 +389,13 @@ Constant *Constant::getAllOnesValue(Type *Ty) { APInt::getAllOnesValue(ITy->getBitWidth())); if (Ty->isFloatingPointTy()) { - APFloat FL = APFloat::getAllOnesValue(Ty->getPrimitiveSizeInBits(), - !Ty->isPPC_FP128Ty()); + APFloat FL = APFloat::getAllOnesValue(Ty->getFltSemantics(), + Ty->getPrimitiveSizeInBits()); return ConstantFP::get(Ty->getContext(), FL); } VectorType *VTy = cast<VectorType>(Ty); - return ConstantVector::getSplat(VTy->getNumElements(), + return ConstantVector::getSplat(VTy->getElementCount(), getAllOnesValue(VTy->getElementType())); } @@ -449,7 +463,74 @@ void Constant::destroyConstant() { } // Value has no outstanding references it is safe to delete it now... - delete this; + deleteConstant(this); +} + +void llvm::deleteConstant(Constant *C) { + switch (C->getValueID()) { + case Constant::ConstantIntVal: + delete static_cast<ConstantInt *>(C); + break; + case Constant::ConstantFPVal: + delete static_cast<ConstantFP *>(C); + break; + case Constant::ConstantAggregateZeroVal: + delete static_cast<ConstantAggregateZero *>(C); + break; + case Constant::ConstantArrayVal: + delete static_cast<ConstantArray *>(C); + break; + case Constant::ConstantStructVal: + delete static_cast<ConstantStruct *>(C); + break; + case Constant::ConstantVectorVal: + delete static_cast<ConstantVector *>(C); + break; + case Constant::ConstantPointerNullVal: + delete static_cast<ConstantPointerNull *>(C); + break; + case Constant::ConstantDataArrayVal: + delete static_cast<ConstantDataArray *>(C); + break; + case Constant::ConstantDataVectorVal: + delete static_cast<ConstantDataVector *>(C); + break; + case Constant::ConstantTokenNoneVal: + delete static_cast<ConstantTokenNone *>(C); + break; + case Constant::BlockAddressVal: + delete static_cast<BlockAddress *>(C); + break; + case Constant::UndefValueVal: + delete static_cast<UndefValue *>(C); + break; + case Constant::ConstantExprVal: + if (isa<UnaryConstantExpr>(C)) + delete static_cast<UnaryConstantExpr *>(C); + else if (isa<BinaryConstantExpr>(C)) + delete static_cast<BinaryConstantExpr *>(C); + else if (isa<SelectConstantExpr>(C)) + delete static_cast<SelectConstantExpr *>(C); + else if (isa<ExtractElementConstantExpr>(C)) + delete static_cast<ExtractElementConstantExpr *>(C); + else if (isa<InsertElementConstantExpr>(C)) + delete static_cast<InsertElementConstantExpr *>(C); + else if (isa<ShuffleVectorConstantExpr>(C)) + delete static_cast<ShuffleVectorConstantExpr *>(C); + else if (isa<ExtractValueConstantExpr>(C)) + delete static_cast<ExtractValueConstantExpr *>(C); + else if (isa<InsertValueConstantExpr>(C)) + delete static_cast<InsertValueConstantExpr *>(C); + else if (isa<GetElementPtrConstantExpr>(C)) + delete static_cast<GetElementPtrConstantExpr *>(C); + else if (isa<CompareConstantExpr>(C)) + delete static_cast<CompareConstantExpr *>(C); + else + llvm_unreachable("Unexpected constant expr"); + break; + default: + llvm_unreachable("Unexpected constant"); + } } static bool canTrapImpl(const Constant *C, @@ -633,10 +714,11 @@ Constant *Constant::replaceUndefsWith(Constant *C, Constant *Replacement) { } // Don't know how to deal with this constant. - if (!Ty->isVectorTy()) + auto *VTy = dyn_cast<FixedVectorType>(Ty); + if (!VTy) return C; - unsigned NumElts = Ty->getVectorNumElements(); + unsigned NumElts = VTy->getNumElements(); SmallVector<Constant *, 32> NewC(NumElts); for (unsigned i = 0; i != NumElts; ++i) { Constant *EltC = C->getAggregateElement(i); @@ -675,7 +757,7 @@ Constant *ConstantInt::getTrue(Type *Ty) { assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1."); ConstantInt *TrueC = ConstantInt::getTrue(Ty->getContext()); if (auto *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), TrueC); + return ConstantVector::getSplat(VTy->getElementCount(), TrueC); return TrueC; } @@ -683,7 +765,7 @@ Constant *ConstantInt::getFalse(Type *Ty) { assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1."); ConstantInt *FalseC = ConstantInt::getFalse(Ty->getContext()); if (auto *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), FalseC); + return ConstantVector::getSplat(VTy->getElementCount(), FalseC); return FalseC; } @@ -706,7 +788,7 @@ Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) { // For vectors, broadcast the value. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -730,7 +812,7 @@ Constant *ConstantInt::get(Type *Ty, const APInt& V) { // For vectors, broadcast the value. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -751,6 +833,8 @@ void ConstantInt::destroyConstantImpl() { static const fltSemantics *TypeToFloatSemantics(Type *Ty) { if (Ty->isHalfTy()) return &APFloat::IEEEhalf(); + if (Ty->isBFloatTy()) + return &APFloat::BFloat(); if (Ty->isFloatTy()) return &APFloat::IEEEsingle(); if (Ty->isDoubleTy()) @@ -775,7 +859,7 @@ Constant *ConstantFP::get(Type *Ty, double V) { // For vectors, broadcast the value. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -787,7 +871,7 @@ Constant *ConstantFP::get(Type *Ty, const APFloat &V) { // For vectors, broadcast the value. if (auto *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -800,7 +884,7 @@ Constant *ConstantFP::get(Type *Ty, StringRef Str) { // For vectors, broadcast the value. if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -811,7 +895,7 @@ Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) { Constant *C = get(Ty->getContext(), NaN); if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -820,10 +904,10 @@ Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) { const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType()); APFloat NaN = APFloat::getQNaN(Semantics, Negative, Payload); Constant *C = get(Ty->getContext(), NaN); - + if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); - + return ConstantVector::getSplat(VTy->getElementCount(), C); + return C; } @@ -831,10 +915,10 @@ Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) { const fltSemantics &Semantics = *TypeToFloatSemantics(Ty->getScalarType()); APFloat NaN = APFloat::getSNaN(Semantics, Negative, Payload); Constant *C = get(Ty->getContext(), NaN); - + if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); - + return ConstantVector::getSplat(VTy->getElementCount(), C); + return C; } @@ -844,7 +928,7 @@ Constant *ConstantFP::getNegativeZero(Type *Ty) { Constant *C = get(Ty->getContext(), NegZero); if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -868,6 +952,8 @@ ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) { Type *Ty; if (&V.getSemantics() == &APFloat::IEEEhalf()) Ty = Type::getHalfTy(Context); + else if (&V.getSemantics() == &APFloat::BFloat()) + Ty = Type::getBFloatTy(Context); else if (&V.getSemantics() == &APFloat::IEEEsingle()) Ty = Type::getFloatTy(Context); else if (&V.getSemantics() == &APFloat::IEEEdouble()) @@ -892,7 +978,7 @@ Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) { Constant *C = get(Ty->getContext(), APFloat::getInf(Semantics, Negative)); if (VectorType *VTy = dyn_cast<VectorType>(Ty)) - return ConstantVector::getSplat(VTy->getNumElements(), C); + return ConstantVector::getSplat(VTy->getElementCount(), C); return C; } @@ -917,7 +1003,9 @@ void ConstantFP::destroyConstantImpl() { //===----------------------------------------------------------------------===// Constant *ConstantAggregateZero::getSequentialElement() const { - return Constant::getNullValue(getType()->getSequentialElementType()); + if (auto *AT = dyn_cast<ArrayType>(getType())) + return Constant::getNullValue(AT->getElementType()); + return Constant::getNullValue(cast<VectorType>(getType())->getElementType()); } Constant *ConstantAggregateZero::getStructElement(unsigned Elt) const { @@ -925,13 +1013,13 @@ Constant *ConstantAggregateZero::getStructElement(unsigned Elt) const { } Constant *ConstantAggregateZero::getElementValue(Constant *C) const { - if (isa<SequentialType>(getType())) + if (isa<ArrayType>(getType()) || isa<VectorType>(getType())) return getSequentialElement(); return getStructElement(cast<ConstantInt>(C)->getZExtValue()); } Constant *ConstantAggregateZero::getElementValue(unsigned Idx) const { - if (isa<SequentialType>(getType())) + if (isa<ArrayType>(getType()) || isa<VectorType>(getType())) return getSequentialElement(); return getStructElement(Idx); } @@ -950,7 +1038,9 @@ unsigned ConstantAggregateZero::getNumElements() const { //===----------------------------------------------------------------------===// UndefValue *UndefValue::getSequentialElement() const { - return UndefValue::get(getType()->getSequentialElementType()); + if (ArrayType *ATy = dyn_cast<ArrayType>(getType())) + return UndefValue::get(ATy->getElementType()); + return UndefValue::get(cast<VectorType>(getType())->getElementType()); } UndefValue *UndefValue::getStructElement(unsigned Elt) const { @@ -958,21 +1048,23 @@ UndefValue *UndefValue::getStructElement(unsigned Elt) const { } UndefValue *UndefValue::getElementValue(Constant *C) const { - if (isa<SequentialType>(getType())) + if (isa<ArrayType>(getType()) || isa<VectorType>(getType())) return getSequentialElement(); return getStructElement(cast<ConstantInt>(C)->getZExtValue()); } UndefValue *UndefValue::getElementValue(unsigned Idx) const { - if (isa<SequentialType>(getType())) + if (isa<ArrayType>(getType()) || isa<VectorType>(getType())) return getSequentialElement(); return getStructElement(Idx); } unsigned UndefValue::getNumElements() const { Type *Ty = getType(); - if (auto *ST = dyn_cast<SequentialType>(Ty)) - return ST->getNumElements(); + if (auto *AT = dyn_cast<ArrayType>(Ty)) + return AT->getNumElements(); + if (auto *VT = dyn_cast<VectorType>(Ty)) + return VT->getNumElements(); return Ty->getStructNumElements(); } @@ -1011,7 +1103,7 @@ static Constant *getFPSequenceIfElementsMatch(ArrayRef<Constant *> V) { Elts.push_back(CFP->getValueAPF().bitcastToAPInt().getLimitedValue()); else return nullptr; - return SequentialTy::getFP(V[0]->getContext(), Elts); + return SequentialTy::getFP(V[0]->getType(), Elts); } template <typename SequenceTy> @@ -1030,7 +1122,7 @@ static Constant *getSequenceIfElementsMatch(Constant *C, else if (CI->getType()->isIntegerTy(64)) return getIntSequenceIfElementsMatch<SequenceTy, uint64_t>(V); } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { - if (CFP->getType()->isHalfTy()) + if (CFP->getType()->isHalfTy() || CFP->getType()->isBFloatTy()) return getFPSequenceIfElementsMatch<SequenceTy, uint16_t>(V); else if (CFP->getType()->isFloatTy()) return getFPSequenceIfElementsMatch<SequenceTy, uint32_t>(V); @@ -1041,19 +1133,20 @@ static Constant *getSequenceIfElementsMatch(Constant *C, return nullptr; } -ConstantAggregate::ConstantAggregate(CompositeType *T, ValueTy VT, +ConstantAggregate::ConstantAggregate(Type *T, ValueTy VT, ArrayRef<Constant *> V) : Constant(T, VT, OperandTraits<ConstantAggregate>::op_end(this) - V.size(), V.size()) { llvm::copy(V, op_begin()); // Check that types match, unless this is an opaque struct. - if (auto *ST = dyn_cast<StructType>(T)) + if (auto *ST = dyn_cast<StructType>(T)) { if (ST->isOpaque()) return; - for (unsigned I = 0, E = V.size(); I != E; ++I) - assert(V[I]->getType() == T->getTypeAtIndex(I) && - "Initializer for composite element doesn't match!"); + for (unsigned I = 0, E = V.size(); I != E; ++I) + assert(V[I]->getType() == ST->getTypeAtIndex(I) && + "Initializer for struct element doesn't match!"); + } } ConstantArray::ConstantArray(ArrayType *T, ArrayRef<Constant *> V) @@ -1161,13 +1254,13 @@ ConstantVector::ConstantVector(VectorType *T, ArrayRef<Constant *> V) Constant *ConstantVector::get(ArrayRef<Constant*> V) { if (Constant *C = getImpl(V)) return C; - VectorType *Ty = VectorType::get(V.front()->getType(), V.size()); + auto *Ty = FixedVectorType::get(V.front()->getType(), V.size()); return Ty->getContext().pImpl->VectorConstants.getOrCreate(Ty, V); } Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) { assert(!V.empty() && "Vectors can't be empty"); - VectorType *T = VectorType::get(V.front()->getType(), V.size()); + auto *T = FixedVectorType::get(V.front()->getType(), V.size()); // If this is an all-undef or all-zero vector, return a // ConstantAggregateZero or UndefValue. @@ -1198,15 +1291,34 @@ Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) { return nullptr; } -Constant *ConstantVector::getSplat(unsigned NumElts, Constant *V) { - // If this splat is compatible with ConstantDataVector, use it instead of - // ConstantVector. - if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) && - ConstantDataSequential::isElementTypeCompatible(V->getType())) - return ConstantDataVector::getSplat(NumElts, V); +Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) { + if (!EC.Scalable) { + // If this splat is compatible with ConstantDataVector, use it instead of + // ConstantVector. + if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) && + ConstantDataSequential::isElementTypeCompatible(V->getType())) + return ConstantDataVector::getSplat(EC.Min, V); + + SmallVector<Constant *, 32> Elts(EC.Min, V); + return get(Elts); + } + + Type *VTy = VectorType::get(V->getType(), EC); + + if (V->isNullValue()) + return ConstantAggregateZero::get(VTy); + else if (isa<UndefValue>(V)) + return UndefValue::get(VTy); - SmallVector<Constant*, 32> Elts(NumElts, V); - return get(Elts); + Type *I32Ty = Type::getInt32Ty(VTy->getContext()); + + // Move scalar into vector. + Constant *UndefV = UndefValue::get(VTy); + V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0)); + // Build shuffle mask to perform the splat. + SmallVector<int, 8> Zeros(EC.Min, 0); + // Splat. + return ConstantExpr::getShuffleVector(V, UndefV, Zeros); } ConstantTokenNone *ConstantTokenNone::get(LLVMContext &Context) { @@ -1271,6 +1383,14 @@ unsigned ConstantExpr::getPredicate() const { return cast<CompareConstantExpr>(this)->predicate; } +ArrayRef<int> ConstantExpr::getShuffleMask() const { + return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask; +} + +Constant *ConstantExpr::getShuffleMaskForBitcode() const { + return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode; +} + Constant * ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const { assert(Op->getType() == getOperand(OpNo)->getType() && @@ -1322,7 +1442,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty, case Instruction::ExtractValue: return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy); case Instruction::ShuffleVector: - return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2], + return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(), OnlyIfReducedTy); case Instruction::GetElementPtr: { auto *GEPO = cast<GEPOperator>(this); @@ -1375,6 +1495,12 @@ bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) { Val2.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &losesInfo); return !losesInfo; } + case Type::BFloatTyID: { + if (&Val2.getSemantics() == &APFloat::BFloat()) + return true; + Val2.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &losesInfo); + return !losesInfo; + } case Type::FloatTyID: { if (&Val2.getSemantics() == &APFloat::IEEEsingle()) return true; @@ -1383,6 +1509,7 @@ bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) { } case Type::DoubleTyID: { if (&Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::BFloat() || &Val2.getSemantics() == &APFloat::IEEEsingle() || &Val2.getSemantics() == &APFloat::IEEEdouble()) return true; @@ -1391,16 +1518,19 @@ bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) { } case Type::X86_FP80TyID: return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::BFloat() || &Val2.getSemantics() == &APFloat::IEEEsingle() || &Val2.getSemantics() == &APFloat::IEEEdouble() || &Val2.getSemantics() == &APFloat::x87DoubleExtended(); case Type::FP128TyID: return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::BFloat() || &Val2.getSemantics() == &APFloat::IEEEsingle() || &Val2.getSemantics() == &APFloat::IEEEdouble() || &Val2.getSemantics() == &APFloat::IEEEquad(); case Type::PPC_FP128TyID: return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::BFloat() || &Val2.getSemantics() == &APFloat::IEEEsingle() || &Val2.getSemantics() == &APFloat::IEEEdouble() || &Val2.getSemantics() == &APFloat::PPCDoubleDouble(); @@ -1450,11 +1580,32 @@ void ConstantVector::destroyConstantImpl() { Constant *Constant::getSplatValue(bool AllowUndefs) const { assert(this->getType()->isVectorTy() && "Only valid for vectors!"); if (isa<ConstantAggregateZero>(this)) - return getNullValue(this->getType()->getVectorElementType()); + return getNullValue(cast<VectorType>(getType())->getElementType()); if (const ConstantDataVector *CV = dyn_cast<ConstantDataVector>(this)) return CV->getSplatValue(); if (const ConstantVector *CV = dyn_cast<ConstantVector>(this)) return CV->getSplatValue(AllowUndefs); + + // Check if this is a constant expression splat of the form returned by + // ConstantVector::getSplat() + const auto *Shuf = dyn_cast<ConstantExpr>(this); + if (Shuf && Shuf->getOpcode() == Instruction::ShuffleVector && + isa<UndefValue>(Shuf->getOperand(1))) { + + const auto *IElt = dyn_cast<ConstantExpr>(Shuf->getOperand(0)); + if (IElt && IElt->getOpcode() == Instruction::InsertElement && + isa<UndefValue>(IElt->getOperand(0))) { + + ArrayRef<int> Mask = Shuf->getShuffleMask(); + Constant *SplatVal = IElt->getOperand(1); + ConstantInt *Index = dyn_cast<ConstantInt>(IElt->getOperand(2)); + + if (Index && Index->getValue() == 0 && + std::all_of(Mask.begin(), Mask.end(), [](int I) { return I == 0; })) + return SplatVal; + } + } + return nullptr; } @@ -1735,8 +1886,8 @@ Constant *ConstantExpr::getFPCast(Constant *C, Type *Ty) { Constant *ConstantExpr::getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isIntOrIntVectorTy() && "Trunc operand must be integer"); @@ -1749,8 +1900,8 @@ Constant *ConstantExpr::getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getSExt(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isIntOrIntVectorTy() && "SExt operand must be integral"); @@ -1763,8 +1914,8 @@ Constant *ConstantExpr::getSExt(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getZExt(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isIntOrIntVectorTy() && "ZEXt operand must be integral"); @@ -1777,8 +1928,8 @@ Constant *ConstantExpr::getZExt(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getFPTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() && @@ -1789,8 +1940,8 @@ Constant *ConstantExpr::getFPTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getFPExtend(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isFPOrFPVectorTy() && Ty->isFPOrFPVectorTy() && @@ -1801,8 +1952,8 @@ Constant *ConstantExpr::getFPExtend(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getUIToFP(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isIntOrIntVectorTy() && Ty->isFPOrFPVectorTy() && @@ -1812,8 +1963,8 @@ Constant *ConstantExpr::getUIToFP(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getSIToFP(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isIntOrIntVectorTy() && Ty->isFPOrFPVectorTy() && @@ -1823,8 +1974,8 @@ Constant *ConstantExpr::getSIToFP(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getFPToUI(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isFPOrFPVectorTy() && Ty->isIntOrIntVectorTy() && @@ -1834,8 +1985,8 @@ Constant *ConstantExpr::getFPToUI(Constant *C, Type *Ty, bool OnlyIfReduced) { Constant *ConstantExpr::getFPToSI(Constant *C, Type *Ty, bool OnlyIfReduced) { #ifndef NDEBUG - bool fromVec = C->getType()->getTypeID() == Type::VectorTyID; - bool toVec = Ty->getTypeID() == Type::VectorTyID; + bool fromVec = isa<VectorType>(C->getType()); + bool toVec = isa<VectorType>(Ty); #endif assert((fromVec == toVec) && "Cannot convert from scalar to/from vector"); assert(C->getType()->isFPOrFPVectorTy() && Ty->isIntOrIntVectorTy() && @@ -1851,7 +2002,8 @@ Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy, "PtrToInt destination must be integer or integer vector"); assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy)); if (isa<VectorType>(C->getType())) - assert(C->getType()->getVectorNumElements()==DstTy->getVectorNumElements()&& + assert(cast<VectorType>(C->getType())->getNumElements() == + cast<VectorType>(DstTy)->getNumElements() && "Invalid cast between a different number of vector elements"); return getFoldedCast(Instruction::PtrToInt, C, DstTy, OnlyIfReduced); } @@ -1864,7 +2016,8 @@ Constant *ConstantExpr::getIntToPtr(Constant *C, Type *DstTy, "IntToPtr destination must be a pointer or pointer vector"); assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy)); if (isa<VectorType>(C->getType())) - assert(C->getType()->getVectorNumElements()==DstTy->getVectorNumElements()&& + assert(cast<VectorType>(C->getType())->getNumElements() == + cast<VectorType>(DstTy)->getNumElements() && "Invalid cast between a different number of vector elements"); return getFoldedCast(Instruction::IntToPtr, C, DstTy, OnlyIfReduced); } @@ -1895,14 +2048,14 @@ Constant *ConstantExpr::getAddrSpaceCast(Constant *C, Type *DstTy, Type *MidTy = PointerType::get(DstElemTy, SrcScalarTy->getAddressSpace()); if (VectorType *VT = dyn_cast<VectorType>(DstTy)) { // Handle vectors of pointers. - MidTy = VectorType::get(MidTy, VT->getNumElements()); + MidTy = FixedVectorType::get(MidTy, VT->getNumElements()); } C = getBitCast(C, MidTy); } return getFoldedCast(Instruction::AddrSpaceCast, C, DstTy, OnlyIfReduced); } -Constant *ConstantExpr::get(unsigned Opcode, Constant *C, unsigned Flags, +Constant *ConstantExpr::get(unsigned Opcode, Constant *C, unsigned Flags, Type *OnlyIfReducedTy) { // Check the operands for consistency first. assert(Instruction::isUnaryOp(Opcode) && @@ -2092,15 +2245,16 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, unsigned AS = C->getType()->getPointerAddressSpace(); Type *ReqTy = DestTy->getPointerTo(AS); - unsigned NumVecElts = 0; - if (C->getType()->isVectorTy()) - NumVecElts = C->getType()->getVectorNumElements(); - else for (auto Idx : Idxs) - if (Idx->getType()->isVectorTy()) - NumVecElts = Idx->getType()->getVectorNumElements(); + ElementCount EltCount = {0, false}; + if (VectorType *VecTy = dyn_cast<VectorType>(C->getType())) + EltCount = VecTy->getElementCount(); + else + for (auto Idx : Idxs) + if (VectorType *VecTy = dyn_cast<VectorType>(Idx->getType())) + EltCount = VecTy->getElementCount(); - if (NumVecElts) - ReqTy = VectorType::get(ReqTy, NumVecElts); + if (EltCount.Min != 0) + ReqTy = VectorType::get(ReqTy, EltCount); if (OnlyIfReducedTy == ReqTy) return nullptr; @@ -2109,14 +2263,20 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, std::vector<Constant*> ArgVec; ArgVec.reserve(1 + Idxs.size()); ArgVec.push_back(C); - for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { - assert((!Idxs[i]->getType()->isVectorTy() || - Idxs[i]->getType()->getVectorNumElements() == NumVecElts) && - "getelementptr index type missmatch"); - - Constant *Idx = cast<Constant>(Idxs[i]); - if (NumVecElts && !Idxs[i]->getType()->isVectorTy()) - Idx = ConstantVector::getSplat(NumVecElts, Idx); + auto GTI = gep_type_begin(Ty, Idxs), GTE = gep_type_end(Ty, Idxs); + for (; GTI != GTE; ++GTI) { + auto *Idx = cast<Constant>(GTI.getOperand()); + assert( + (!isa<VectorType>(Idx->getType()) || + cast<VectorType>(Idx->getType())->getElementCount() == EltCount) && + "getelementptr index type missmatch"); + + if (GTI.isStruct() && Idx->getType()->isVectorTy()) { + Idx = Idx->getSplatValue(); + } else if (GTI.isSequential() && EltCount.Min != 0 && + !Idx->getType()->isVectorTy()) { + Idx = ConstantVector::getSplat(EltCount, Idx); + } ArgVec.push_back(Idx); } @@ -2124,7 +2284,7 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, if (InRangeIndex && *InRangeIndex < 63) SubClassOptionalData |= (*InRangeIndex + 1) << 1; const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0, - SubClassOptionalData, None, Ty); + SubClassOptionalData, None, None, Ty); LLVMContextImpl *pImpl = C->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ReqTy, Key); @@ -2149,7 +2309,7 @@ Constant *ConstantExpr::getICmp(unsigned short pred, Constant *LHS, Type *ResultTy = Type::getInt1Ty(LHS->getContext()); if (VectorType *VT = dyn_cast<VectorType>(LHS->getType())) - ResultTy = VectorType::get(ResultTy, VT->getNumElements()); + ResultTy = VectorType::get(ResultTy, VT->getElementCount()); LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ResultTy, Key); @@ -2174,7 +2334,7 @@ Constant *ConstantExpr::getFCmp(unsigned short pred, Constant *LHS, Type *ResultTy = Type::getInt1Ty(LHS->getContext()); if (VectorType *VT = dyn_cast<VectorType>(LHS->getType())) - ResultTy = VectorType::get(ResultTy, VT->getNumElements()); + ResultTy = VectorType::get(ResultTy, VT->getElementCount()); LLVMContextImpl *pImpl = LHS->getType()->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ResultTy, Key); @@ -2190,7 +2350,7 @@ Constant *ConstantExpr::getExtractElement(Constant *Val, Constant *Idx, if (Constant *FC = ConstantFoldExtractElementInstruction(Val, Idx)) return FC; // Fold a few common cases. - Type *ReqTy = Val->getType()->getVectorElementType(); + Type *ReqTy = cast<VectorType>(Val->getType())->getElementType(); if (OnlyIfReducedTy == ReqTy) return nullptr; @@ -2206,7 +2366,7 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt, Constant *Idx, Type *OnlyIfReducedTy) { assert(Val->getType()->isVectorTy() && "Tried to create insertelement operation on non-vector type!"); - assert(Elt->getType() == Val->getType()->getVectorElementType() && + assert(Elt->getType() == cast<VectorType>(Val->getType())->getElementType() && "Insertelement types must match!"); assert(Idx->getType()->isIntegerTy() && "Insertelement index must be i32 type!"); @@ -2226,23 +2386,26 @@ Constant *ConstantExpr::getInsertElement(Constant *Val, Constant *Elt, } Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2, - Constant *Mask, Type *OnlyIfReducedTy) { + ArrayRef<int> Mask, + Type *OnlyIfReducedTy) { assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) && "Invalid shuffle vector constant expr operands!"); if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask)) return FC; // Fold a few common cases. - ElementCount NElts = Mask->getType()->getVectorElementCount(); - Type *EltTy = V1->getType()->getVectorElementType(); - Type *ShufTy = VectorType::get(EltTy, NElts); + unsigned NElts = Mask.size(); + auto V1VTy = cast<VectorType>(V1->getType()); + Type *EltTy = V1VTy->getElementType(); + bool TypeIsScalable = isa<ScalableVectorType>(V1VTy); + Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable); if (OnlyIfReducedTy == ShufTy) return nullptr; // Look up the constant in the table first to ensure uniqueness - Constant *ArgVec[] = { V1, V2, Mask }; - const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec); + Constant *ArgVec[] = {V1, V2}; + ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask); LLVMContextImpl *pImpl = ShufTy->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ShufTy, Key); @@ -2499,7 +2662,9 @@ Type *GetElementPtrConstantExpr::getResultElementType() const { // ConstantData* implementations Type *ConstantDataSequential::getElementType() const { - return getType()->getElementType(); + if (ArrayType *ATy = dyn_cast<ArrayType>(getType())) + return ATy->getElementType(); + return cast<VectorType>(getType())->getElementType(); } StringRef ConstantDataSequential::getRawDataValues() const { @@ -2507,7 +2672,8 @@ StringRef ConstantDataSequential::getRawDataValues() const { } bool ConstantDataSequential::isElementTypeCompatible(Type *Ty) { - if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy()) return true; + if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() || Ty->isDoubleTy()) + return true; if (auto *IT = dyn_cast<IntegerType>(Ty)) { switch (IT->getBitWidth()) { case 8: @@ -2524,7 +2690,7 @@ bool ConstantDataSequential::isElementTypeCompatible(Type *Ty) { unsigned ConstantDataSequential::getNumElements() const { if (ArrayType *AT = dyn_cast<ArrayType>(getType())) return AT->getNumElements(); - return getType()->getVectorNumElements(); + return cast<VectorType>(getType())->getNumElements(); } @@ -2552,7 +2718,12 @@ static bool isAllZeros(StringRef Arr) { /// the correct element type. We take the bytes in as a StringRef because /// we *want* an underlying "char*" to avoid TBAA type punning violations. Constant *ConstantDataSequential::getImpl(StringRef Elements, Type *Ty) { - assert(isElementTypeCompatible(Ty->getSequentialElementType())); +#ifndef NDEBUG + if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) + assert(isElementTypeCompatible(ATy->getElementType())); + else + assert(isElementTypeCompatible(cast<VectorType>(Ty)->getElementType())); +#endif // If the elements are all zero or there are no elements, return a CAZ, which // is more dense and canonical. if (isAllZeros(Elements)) @@ -2620,26 +2791,29 @@ void ConstantDataSequential::destroyConstantImpl() { Next = nullptr; } -/// getFP() constructors - Return a constant with array type with an element -/// count and element type of float with precision matching the number of -/// bits in the ArrayRef passed in. (i.e. half for 16bits, float for 32bits, -/// double for 64bits) Note that this can return a ConstantAggregateZero -/// object. -Constant *ConstantDataArray::getFP(LLVMContext &Context, - ArrayRef<uint16_t> Elts) { - Type *Ty = ArrayType::get(Type::getHalfTy(Context), Elts.size()); +/// getFP() constructors - Return a constant of array type with a float +/// element type taken from argument `ElementType', and count taken from +/// argument `Elts'. The amount of bits of the contained type must match the +/// number of bits of the type contained in the passed in ArrayRef. +/// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note +/// that this can return a ConstantAggregateZero object. +Constant *ConstantDataArray::getFP(Type *ElementType, ArrayRef<uint16_t> Elts) { + assert((ElementType->isHalfTy() || ElementType->isBFloatTy()) && + "Element type is not a 16-bit float type"); + Type *Ty = ArrayType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 2), Ty); } -Constant *ConstantDataArray::getFP(LLVMContext &Context, - ArrayRef<uint32_t> Elts) { - Type *Ty = ArrayType::get(Type::getFloatTy(Context), Elts.size()); +Constant *ConstantDataArray::getFP(Type *ElementType, ArrayRef<uint32_t> Elts) { + assert(ElementType->isFloatTy() && "Element type is not a 32-bit float type"); + Type *Ty = ArrayType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 4), Ty); } -Constant *ConstantDataArray::getFP(LLVMContext &Context, - ArrayRef<uint64_t> Elts) { - Type *Ty = ArrayType::get(Type::getDoubleTy(Context), Elts.size()); +Constant *ConstantDataArray::getFP(Type *ElementType, ArrayRef<uint64_t> Elts) { + assert(ElementType->isDoubleTy() && + "Element type is not a 64-bit float type"); + Type *Ty = ArrayType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 8), Ty); } @@ -2661,56 +2835,62 @@ Constant *ConstantDataArray::getString(LLVMContext &Context, /// count and element type matching the ArrayRef passed in. Note that this /// can return a ConstantAggregateZero object. Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<uint8_t> Elts){ - Type *Ty = VectorType::get(Type::getInt8Ty(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getInt8Ty(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 1), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<uint16_t> Elts){ - Type *Ty = VectorType::get(Type::getInt16Ty(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getInt16Ty(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 2), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<uint32_t> Elts){ - Type *Ty = VectorType::get(Type::getInt32Ty(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getInt32Ty(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<uint64_t> Elts){ - Type *Ty = VectorType::get(Type::getInt64Ty(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getInt64Ty(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 8), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<float> Elts) { - Type *Ty = VectorType::get(Type::getFloatTy(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getFloatTy(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 4), Ty); } Constant *ConstantDataVector::get(LLVMContext &Context, ArrayRef<double> Elts) { - Type *Ty = VectorType::get(Type::getDoubleTy(Context), Elts.size()); + auto *Ty = FixedVectorType::get(Type::getDoubleTy(Context), Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 8), Ty); } -/// getFP() constructors - Return a constant with vector type with an element -/// count and element type of float with the precision matching the number of -/// bits in the ArrayRef passed in. (i.e. half for 16bits, float for 32bits, -/// double for 64bits) Note that this can return a ConstantAggregateZero -/// object. -Constant *ConstantDataVector::getFP(LLVMContext &Context, +/// getFP() constructors - Return a constant of vector type with a float +/// element type taken from argument `ElementType', and count taken from +/// argument `Elts'. The amount of bits of the contained type must match the +/// number of bits of the type contained in the passed in ArrayRef. +/// (i.e. half or bfloat for 16bits, float for 32bits, double for 64bits) Note +/// that this can return a ConstantAggregateZero object. +Constant *ConstantDataVector::getFP(Type *ElementType, ArrayRef<uint16_t> Elts) { - Type *Ty = VectorType::get(Type::getHalfTy(Context), Elts.size()); + assert((ElementType->isHalfTy() || ElementType->isBFloatTy()) && + "Element type is not a 16-bit float type"); + auto *Ty = FixedVectorType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 2), Ty); } -Constant *ConstantDataVector::getFP(LLVMContext &Context, +Constant *ConstantDataVector::getFP(Type *ElementType, ArrayRef<uint32_t> Elts) { - Type *Ty = VectorType::get(Type::getFloatTy(Context), Elts.size()); + assert(ElementType->isFloatTy() && "Element type is not a 32-bit float type"); + auto *Ty = FixedVectorType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 4), Ty); } -Constant *ConstantDataVector::getFP(LLVMContext &Context, +Constant *ConstantDataVector::getFP(Type *ElementType, ArrayRef<uint64_t> Elts) { - Type *Ty = VectorType::get(Type::getDoubleTy(Context), Elts.size()); + assert(ElementType->isDoubleTy() && + "Element type is not a 64-bit float type"); + auto *Ty = FixedVectorType::get(ElementType, Elts.size()); const char *Data = reinterpret_cast<const char *>(Elts.data()); return getImpl(StringRef(Data, Elts.size() * 8), Ty); } @@ -2740,20 +2920,25 @@ Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) { if (CFP->getType()->isHalfTy()) { SmallVector<uint16_t, 16> Elts( NumElts, CFP->getValueAPF().bitcastToAPInt().getLimitedValue()); - return getFP(V->getContext(), Elts); + return getFP(V->getType(), Elts); + } + if (CFP->getType()->isBFloatTy()) { + SmallVector<uint16_t, 16> Elts( + NumElts, CFP->getValueAPF().bitcastToAPInt().getLimitedValue()); + return getFP(V->getType(), Elts); } if (CFP->getType()->isFloatTy()) { SmallVector<uint32_t, 16> Elts( NumElts, CFP->getValueAPF().bitcastToAPInt().getLimitedValue()); - return getFP(V->getContext(), Elts); + return getFP(V->getType(), Elts); } if (CFP->getType()->isDoubleTy()) { SmallVector<uint64_t, 16> Elts( NumElts, CFP->getValueAPF().bitcastToAPInt().getLimitedValue()); - return getFP(V->getContext(), Elts); + return getFP(V->getType(), Elts); } } - return ConstantVector::getSplat(NumElts, V); + return ConstantVector::getSplat({NumElts, false}, V); } @@ -2815,6 +3000,10 @@ APFloat ConstantDataSequential::getElementAsAPFloat(unsigned Elt) const { auto EltVal = *reinterpret_cast<const uint16_t *>(EltPtr); return APFloat(APFloat::IEEEhalf(), APInt(16, EltVal)); } + case Type::BFloatTyID: { + auto EltVal = *reinterpret_cast<const uint16_t *>(EltPtr); + return APFloat(APFloat::BFloat(), APInt(16, EltVal)); + } case Type::FloatTyID: { auto EltVal = *reinterpret_cast<const uint32_t *>(EltPtr); return APFloat(APFloat::IEEEsingle(), APInt(32, EltVal)); @@ -2839,8 +3028,8 @@ double ConstantDataSequential::getElementAsDouble(unsigned Elt) const { } Constant *ConstantDataSequential::getElementAsConstant(unsigned Elt) const { - if (getElementType()->isHalfTy() || getElementType()->isFloatTy() || - getElementType()->isDoubleTy()) + if (getElementType()->isHalfTy() || getElementType()->isBFloatTy() || + getElementType()->isFloatTy() || getElementType()->isDoubleTy()) return ConstantFP::get(getContext(), getElementAsAPFloat(Elt)); return ConstantInt::get(getElementType(), getElementAsInteger(Elt)); @@ -2863,7 +3052,7 @@ bool ConstantDataSequential::isCString() const { return Str.drop_back().find(0) == StringRef::npos; } -bool ConstantDataVector::isSplat() const { +bool ConstantDataVector::isSplatData() const { const char *Base = getRawDataValues().data(); // Compare elements 1+ to the 0'th element. @@ -2875,6 +3064,14 @@ bool ConstantDataVector::isSplat() const { return true; } +bool ConstantDataVector::isSplat() const { + if (!IsSplatSet) { + IsSplatSet = true; + IsSplat = isSplatData(); + } + return IsSplat; +} + Constant *ConstantDataVector::getSplatValue() const { // If they're all the same, return the 0th one as a representative. return isSplat() ? getElementAsConstant(0) : nullptr; @@ -3081,7 +3278,7 @@ Instruction *ConstantExpr::getAsInstruction() const { case Instruction::ExtractValue: return ExtractValueInst::Create(Ops[0], getIndices()); case Instruction::ShuffleVector: - return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]); + return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask()); case Instruction::GetElementPtr: { const auto *GO = cast<GEPOperator>(this); diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index f5e2481f3903..95c5ab6d0ee4 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -26,6 +26,7 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/OperandTraits.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" @@ -42,7 +43,7 @@ namespace llvm { /// UnaryConstantExpr - This class is private to Constants.cpp, and is used /// behind the scenes to implement unary constant exprs. -class UnaryConstantExpr : public ConstantExpr { +class UnaryConstantExpr final : public ConstantExpr { public: UnaryConstantExpr(unsigned Opcode, Constant *C, Type *Ty) : ConstantExpr(Ty, Opcode, &Op<0>(), 1) { @@ -55,11 +56,19 @@ public: } DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return Instruction::isCast(CE->getOpcode()) || + Instruction::isUnaryOp(CE->getOpcode()); + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// BinaryConstantExpr - This class is private to Constants.cpp, and is used /// behind the scenes to implement binary constant exprs. -class BinaryConstantExpr : public ConstantExpr { +class BinaryConstantExpr final : public ConstantExpr { public: BinaryConstantExpr(unsigned Opcode, Constant *C1, Constant *C2, unsigned Flags) @@ -76,11 +85,18 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return Instruction::isBinaryOp(CE->getOpcode()); + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// SelectConstantExpr - This class is private to Constants.cpp, and is used /// behind the scenes to implement select constant exprs. -class SelectConstantExpr : public ConstantExpr { +class SelectConstantExpr final : public ConstantExpr { public: SelectConstantExpr(Constant *C1, Constant *C2, Constant *C3) : ConstantExpr(C2->getType(), Instruction::Select, &Op<0>(), 3) { @@ -96,12 +112,19 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return CE->getOpcode() == Instruction::Select; + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// ExtractElementConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// extractelement constant exprs. -class ExtractElementConstantExpr : public ConstantExpr { +class ExtractElementConstantExpr final : public ConstantExpr { public: ExtractElementConstantExpr(Constant *C1, Constant *C2) : ConstantExpr(cast<VectorType>(C1->getType())->getElementType(), @@ -117,12 +140,19 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return CE->getOpcode() == Instruction::ExtractElement; + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// InsertElementConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// insertelement constant exprs. -class InsertElementConstantExpr : public ConstantExpr { +class InsertElementConstantExpr final : public ConstantExpr { public: InsertElementConstantExpr(Constant *C1, Constant *C2, Constant *C3) : ConstantExpr(C1->getType(), Instruction::InsertElement, @@ -139,37 +169,54 @@ public: /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return CE->getOpcode() == Instruction::InsertElement; + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// ShuffleVectorConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// shufflevector constant exprs. -class ShuffleVectorConstantExpr : public ConstantExpr { +class ShuffleVectorConstantExpr final : public ConstantExpr { public: - ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3) - : ConstantExpr(VectorType::get( - cast<VectorType>(C1->getType())->getElementType(), - cast<VectorType>(C3->getType())->getElementCount()), - Instruction::ShuffleVector, - &Op<0>(), 3) { + ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask) + : ConstantExpr(VectorType::get( + cast<VectorType>(C1->getType())->getElementType(), + Mask.size(), isa<ScalableVectorType>(C1->getType())), + Instruction::ShuffleVector, &Op<0>(), 2) { + assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) && + "Invalid shuffle vector instruction operands!"); Op<0>() = C1; Op<1>() = C2; - Op<2>() = C3; + ShuffleMask.assign(Mask.begin(), Mask.end()); + ShuffleMaskForBitcode = + ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType()); } - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } + SmallVector<int, 4> ShuffleMask; + Constant *ShuffleMaskForBitcode; + + void *operator new(size_t s) { return User::operator new(s, 2); } /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + + static bool classof(const ConstantExpr *CE) { + return CE->getOpcode() == Instruction::ShuffleVector; + } + static bool classof(const Value *V) { + return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); + } }; /// ExtractValueConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// extractvalue constant exprs. -class ExtractValueConstantExpr : public ConstantExpr { +class ExtractValueConstantExpr final : public ConstantExpr { public: ExtractValueConstantExpr(Constant *Agg, ArrayRef<unsigned> IdxList, Type *DestTy) @@ -200,7 +247,7 @@ public: /// InsertValueConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// insertvalue constant exprs. -class InsertValueConstantExpr : public ConstantExpr { +class InsertValueConstantExpr final : public ConstantExpr { public: InsertValueConstantExpr(Constant *Agg, Constant *Val, ArrayRef<unsigned> IdxList, Type *DestTy) @@ -231,7 +278,7 @@ public: /// GetElementPtrConstantExpr - This class is private to Constants.cpp, and is /// used behind the scenes to implement getelementpr constant exprs. -class GetElementPtrConstantExpr : public ConstantExpr { +class GetElementPtrConstantExpr final : public ConstantExpr { Type *SrcElementTy; Type *ResElementTy; @@ -265,7 +312,7 @@ public: // CompareConstantExpr - This class is private to Constants.cpp, and is used // behind the scenes to implement ICmp and FCmp constant expressions. This is // needed in order to store the predicate value for these instructions. -class CompareConstantExpr : public ConstantExpr { +class CompareConstantExpr final : public ConstantExpr { public: unsigned short predicate; CompareConstantExpr(Type *ty, Instruction::OtherOps opc, @@ -319,7 +366,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementConstantExpr, Value) template <> struct OperandTraits<ShuffleVectorConstantExpr> - : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {}; + : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value) template <> @@ -454,42 +501,64 @@ struct InlineAsmKeyType { InlineAsm *create(TypeClass *Ty) const { assert(PointerType::getUnqual(FTy) == Ty); - return new InlineAsm(FTy, AsmString, Constraints, HasSideEffects, - IsAlignStack, AsmDialect); + return new InlineAsm(FTy, std::string(AsmString), std::string(Constraints), + HasSideEffects, IsAlignStack, AsmDialect); } }; struct ConstantExprKeyType { +private: uint8_t Opcode; uint8_t SubclassOptionalData; uint16_t SubclassData; ArrayRef<Constant *> Ops; ArrayRef<unsigned> Indexes; + ArrayRef<int> ShuffleMask; Type *ExplicitTy; + static ArrayRef<int> getShuffleMaskIfValid(const ConstantExpr *CE) { + if (CE->getOpcode() == Instruction::ShuffleVector) + return CE->getShuffleMask(); + return None; + } + + static ArrayRef<unsigned> getIndicesIfValid(const ConstantExpr *CE) { + if (CE->hasIndices()) + return CE->getIndices(); + return None; + } + + static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) { + if (auto *GEPCE = dyn_cast<GetElementPtrConstantExpr>(CE)) + return GEPCE->getSourceElementType(); + return nullptr; + } + +public: ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops, unsigned short SubclassData = 0, unsigned short SubclassOptionalData = 0, ArrayRef<unsigned> Indexes = None, + ArrayRef<int> ShuffleMask = None, Type *ExplicitTy = nullptr) : Opcode(Opcode), SubclassOptionalData(SubclassOptionalData), SubclassData(SubclassData), Ops(Ops), Indexes(Indexes), - ExplicitTy(ExplicitTy) {} + ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {} ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE) : Opcode(CE->getOpcode()), SubclassOptionalData(CE->getRawSubclassOptionalData()), SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands), - Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()), - ExplicitTy(nullptr) {} + Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)), + ExplicitTy(getSourceElementTypeIfValid(CE)) {} ConstantExprKeyType(const ConstantExpr *CE, SmallVectorImpl<Constant *> &Storage) : Opcode(CE->getOpcode()), SubclassOptionalData(CE->getRawSubclassOptionalData()), SubclassData(CE->isCompare() ? CE->getPredicate() : 0), - Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()), - ExplicitTy(nullptr) { + Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)), + ExplicitTy(getSourceElementTypeIfValid(CE)) { assert(Storage.empty() && "Expected empty storage"); for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I) Storage.push_back(CE->getOperand(I)); @@ -499,7 +568,8 @@ struct ConstantExprKeyType { bool operator==(const ConstantExprKeyType &X) const { return Opcode == X.Opcode && SubclassData == X.SubclassData && SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops && - Indexes == X.Indexes; + Indexes == X.Indexes && ShuffleMask == X.ShuffleMask && + ExplicitTy == X.ExplicitTy; } bool operator==(const ConstantExpr *CE) const { @@ -514,15 +584,21 @@ struct ConstantExprKeyType { for (unsigned I = 0, E = Ops.size(); I != E; ++I) if (Ops[I] != CE->getOperand(I)) return false; - if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>())) + if (Indexes != getIndicesIfValid(CE)) + return false; + if (ShuffleMask != getShuffleMaskIfValid(CE)) + return false; + if (ExplicitTy != getSourceElementTypeIfValid(CE)) return false; return true; } unsigned getHash() const { - return hash_combine(Opcode, SubclassOptionalData, SubclassData, - hash_combine_range(Ops.begin(), Ops.end()), - hash_combine_range(Indexes.begin(), Indexes.end())); + return hash_combine( + Opcode, SubclassOptionalData, SubclassData, + hash_combine_range(Ops.begin(), Ops.end()), + hash_combine_range(Indexes.begin(), Indexes.end()), + hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy); } using TypeClass = ConstantInfo<ConstantExpr>::TypeClass; @@ -546,17 +622,14 @@ struct ConstantExprKeyType { case Instruction::InsertElement: return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]); case Instruction::ShuffleVector: - return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]); + return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask); case Instruction::InsertValue: return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty); case Instruction::ExtractValue: return new ExtractValueConstantExpr(Ops[0], Indexes, Ty); case Instruction::GetElementPtr: - return GetElementPtrConstantExpr::Create( - ExplicitTy ? ExplicitTy - : cast<PointerType>(Ops[0]->getType()->getScalarType()) - ->getElementType(), - Ops[0], Ops.slice(1), Ty, SubclassOptionalData); + return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1), + Ty, SubclassOptionalData); case Instruction::ICmp: return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData, Ops[0], Ops[1]); @@ -567,6 +640,10 @@ struct ConstantExprKeyType { } }; +// Free memory for a given constant. Assumes the constant has already been +// removed from all relevant maps. +void deleteConstant(Constant *C); + template <class ConstantClass> class ConstantUniqueMap { public: using ValType = typename ConstantInfo<ConstantClass>::ValType; @@ -630,7 +707,7 @@ public: void freeConstants() { for (auto &I : Map) - delete I; // Asserts that use_empty(). + deleteConstant(I); } private: @@ -703,6 +780,11 @@ public: } }; +template <> inline void ConstantUniqueMap<InlineAsm>::freeConstants() { + for (auto &I : Map) + delete I; +} + } // end namespace llvm #endif // LLVM_LIB_IR_CONSTANTSCONTEXT_H diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 04e34a90a9bc..6f3bbc80d4fd 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -127,11 +127,8 @@ unsigned LLVMGetMDKindID(const char *Name, unsigned SLen) { return LLVMGetMDKindIDInContext(LLVMGetGlobalContext(), Name, SLen); } -#define GET_ATTR_KIND_FROM_NAME -#include "AttributesCompatFunc.inc" - unsigned LLVMGetEnumAttributeKindForName(const char *Name, size_t SLen) { - return getAttrKindFromName(StringRef(Name, SLen)); + return Attribute::getAttrKindFromName(StringRef(Name, SLen)); } unsigned LLVMGetLastEnumAttributeKind(void) { @@ -480,6 +477,8 @@ LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty) { return LLVMVoidTypeKind; case Type::HalfTyID: return LLVMHalfTypeKind; + case Type::BFloatTyID: + return LLVMBFloatTypeKind; case Type::FloatTyID: return LLVMFloatTypeKind; case Type::DoubleTyID: @@ -504,12 +503,14 @@ LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty) { return LLVMArrayTypeKind; case Type::PointerTyID: return LLVMPointerTypeKind; - case Type::VectorTyID: + case Type::FixedVectorTyID: return LLVMVectorTypeKind; case Type::X86_MMXTyID: return LLVMX86_MMXTypeKind; case Type::TokenTyID: return LLVMTokenTypeKind; + case Type::ScalableVectorTyID: + return LLVMScalableVectorTypeKind; } llvm_unreachable("Unhandled TypeID."); } @@ -596,6 +597,9 @@ unsigned LLVMGetIntTypeWidth(LLVMTypeRef IntegerTy) { LLVMTypeRef LLVMHalfTypeInContext(LLVMContextRef C) { return (LLVMTypeRef) Type::getHalfTy(*unwrap(C)); } +LLVMTypeRef LLVMBFloatTypeInContext(LLVMContextRef C) { + return (LLVMTypeRef) Type::getBFloatTy(*unwrap(C)); +} LLVMTypeRef LLVMFloatTypeInContext(LLVMContextRef C) { return (LLVMTypeRef) Type::getFloatTy(*unwrap(C)); } @@ -618,6 +622,9 @@ LLVMTypeRef LLVMX86MMXTypeInContext(LLVMContextRef C) { LLVMTypeRef LLVMHalfType(void) { return LLVMHalfTypeInContext(LLVMGetGlobalContext()); } +LLVMTypeRef LLVMBFloatType(void) { + return LLVMBFloatTypeInContext(LLVMGetGlobalContext()); +} LLVMTypeRef LLVMFloatType(void) { return LLVMFloatTypeInContext(LLVMGetGlobalContext()); } @@ -749,14 +756,16 @@ LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType, unsigned AddressSpace) { } LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount) { - return wrap(VectorType::get(unwrap(ElementType), ElementCount)); + return wrap(FixedVectorType::get(unwrap(ElementType), ElementCount)); } LLVMTypeRef LLVMGetElementType(LLVMTypeRef WrappedTy) { auto *Ty = unwrap<Type>(WrappedTy); if (auto *PTy = dyn_cast<PointerType>(Ty)) return wrap(PTy->getElementType()); - return wrap(cast<SequentialType>(Ty)->getElementType()); + if (auto *ATy = dyn_cast<ArrayType>(Ty)) + return wrap(ATy->getElementType()); + return wrap(cast<VectorType>(Ty)->getElementType()); } unsigned LLVMGetNumContainedTypes(LLVMTypeRef Tp) { @@ -1784,9 +1793,11 @@ LLVMValueRef LLVMConstInsertElement(LLVMValueRef VectorConstant, LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant, LLVMValueRef VectorBConstant, LLVMValueRef MaskConstant) { + SmallVector<int, 16> IntMask; + ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask); return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant), unwrap<Constant>(VectorBConstant), - unwrap<Constant>(MaskConstant))); + IntMask)); } LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList, @@ -1992,7 +2003,7 @@ LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global) { unsigned LLVMGetAlignment(LLVMValueRef V) { Value *P = unwrap<Value>(V); - if (GlobalValue *GV = dyn_cast<GlobalValue>(P)) + if (GlobalObject *GV = dyn_cast<GlobalObject>(P)) return GV->getAlignment(); if (AllocaInst *AI = dyn_cast<AllocaInst>(P)) return AI->getAlignment(); @@ -2002,7 +2013,7 @@ unsigned LLVMGetAlignment(LLVMValueRef V) { return SI->getAlignment(); llvm_unreachable( - "only GlobalValue, AllocaInst, LoadInst and StoreInst have alignment"); + "only GlobalObject, AllocaInst, LoadInst and StoreInst have alignment"); } void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) { @@ -2010,11 +2021,11 @@ void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) { if (GlobalObject *GV = dyn_cast<GlobalObject>(P)) GV->setAlignment(MaybeAlign(Bytes)); else if (AllocaInst *AI = dyn_cast<AllocaInst>(P)) - AI->setAlignment(MaybeAlign(Bytes)); + AI->setAlignment(Align(Bytes)); else if (LoadInst *LI = dyn_cast<LoadInst>(P)) - LI->setAlignment(MaybeAlign(Bytes)); + LI->setAlignment(Align(Bytes)); else if (StoreInst *SI = dyn_cast<StoreInst>(P)) - SI->setAlignment(MaybeAlign(Bytes)); + SI->setAlignment(Align(Bytes)); else llvm_unreachable( "only GlobalValue, AllocaInst, LoadInst and StoreInst have alignment"); @@ -2837,7 +2848,7 @@ void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, } LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr) { - return wrap(unwrap<CallBase>(Instr)->getCalledValue()); + return wrap(unwrap<CallBase>(Instr)->getCalledOperand()); } LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef Instr) { @@ -3439,14 +3450,14 @@ LLVMValueRef LLVMBuildArrayMalloc(LLVMBuilderRef B, LLVMTypeRef Ty, return wrap(unwrap(B)->Insert(Malloc, Twine(Name))); } -LLVMValueRef LLVMBuildMemSet(LLVMBuilderRef B, LLVMValueRef Ptr, +LLVMValueRef LLVMBuildMemSet(LLVMBuilderRef B, LLVMValueRef Ptr, LLVMValueRef Val, LLVMValueRef Len, unsigned Align) { return wrap(unwrap(B)->CreateMemSet(unwrap(Ptr), unwrap(Val), unwrap(Len), MaybeAlign(Align))); } -LLVMValueRef LLVMBuildMemCpy(LLVMBuilderRef B, +LLVMValueRef LLVMBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Src, unsigned SrcAlign, LLVMValueRef Size) { diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp index c89f404e4296..45cbbb3a6037 100644 --- a/llvm/lib/IR/DIBuilder.cpp +++ b/llvm/lib/IR/DIBuilder.cpp @@ -140,7 +140,8 @@ DICompileUnit *DIBuilder::createCompileUnit( StringRef Flags, unsigned RunTimeVer, StringRef SplitName, DICompileUnit::DebugEmissionKind Kind, uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling, - DICompileUnit::DebugNameTableKind NameTableKind, bool RangesBaseAddress) { + DICompileUnit::DebugNameTableKind NameTableKind, bool RangesBaseAddress, + StringRef SysRoot, StringRef SDK) { assert(((Lang <= dwarf::DW_LANG_Fortran08 && Lang >= dwarf::DW_LANG_C89) || (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) && @@ -151,7 +152,7 @@ DICompileUnit *DIBuilder::createCompileUnit( VMContext, Lang, File, Producer, isOptimized, Flags, RunTimeVer, SplitName, Kind, nullptr, nullptr, nullptr, nullptr, nullptr, DWOId, SplitDebugInlining, DebugInfoForProfiling, NameTableKind, - RangesBaseAddress); + RangesBaseAddress, SysRoot, SDK); // Create a named metadata so that it is easier to find cu in a module. NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.cu"); @@ -245,7 +246,8 @@ DIMacroFile *DIBuilder::createTempMacroFile(DIMacroFile *Parent, DIEnumerator *DIBuilder::createEnumerator(StringRef Name, int64_t Val, bool IsUnsigned) { assert(!Name.empty() && "Unable to create enumerator without name"); - return DIEnumerator::get(VMContext, Val, IsUnsigned, Name); + return DIEnumerator::get(VMContext, APInt(64, Val, !IsUnsigned), IsUnsigned, + Name); } DIBasicType *DIBuilder::createUnspecifiedType(StringRef Name) { @@ -405,25 +407,26 @@ DIBuilder::createObjCProperty(StringRef Name, DIFile *File, unsigned LineNumber, DITemplateTypeParameter * DIBuilder::createTemplateTypeParameter(DIScope *Context, StringRef Name, - DIType *Ty) { + DIType *Ty, bool isDefault) { assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit"); - return DITemplateTypeParameter::get(VMContext, Name, Ty); + return DITemplateTypeParameter::get(VMContext, Name, Ty, isDefault); } static DITemplateValueParameter * createTemplateValueParameterHelper(LLVMContext &VMContext, unsigned Tag, DIScope *Context, StringRef Name, DIType *Ty, - Metadata *MD) { + bool IsDefault, Metadata *MD) { assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit"); - return DITemplateValueParameter::get(VMContext, Tag, Name, Ty, MD); + return DITemplateValueParameter::get(VMContext, Tag, Name, Ty, IsDefault, MD); } DITemplateValueParameter * DIBuilder::createTemplateValueParameter(DIScope *Context, StringRef Name, - DIType *Ty, Constant *Val) { + DIType *Ty, bool isDefault, + Constant *Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_template_value_parameter, Context, Name, Ty, - getConstantOrNull(Val)); + isDefault, getConstantOrNull(Val)); } DITemplateValueParameter * @@ -431,7 +434,7 @@ DIBuilder::createTemplateTemplateParameter(DIScope *Context, StringRef Name, DIType *Ty, StringRef Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_GNU_template_template_param, Context, Name, Ty, - MDString::get(VMContext, Val)); + false, MDString::get(VMContext, Val)); } DITemplateValueParameter * @@ -439,7 +442,7 @@ DIBuilder::createTemplateParameterPack(DIScope *Context, StringRef Name, DIType *Ty, DINodeArray Val) { return createTemplateValueParameterHelper( VMContext, dwarf::DW_TAG_GNU_template_parameter_pack, Context, Name, Ty, - Val.get()); + false, Val.get()); } DICompositeType *DIBuilder::createClassType( @@ -622,11 +625,22 @@ DITypeRefArray DIBuilder::getOrCreateTypeArray(ArrayRef<Metadata *> Elements) { } DISubrange *DIBuilder::getOrCreateSubrange(int64_t Lo, int64_t Count) { - return DISubrange::get(VMContext, Count, Lo); + auto *LB = ConstantAsMetadata::get( + ConstantInt::getSigned(Type::getInt64Ty(VMContext), Lo)); + auto *CountNode = ConstantAsMetadata::get( + ConstantInt::getSigned(Type::getInt64Ty(VMContext), Count)); + return DISubrange::get(VMContext, CountNode, LB, nullptr, nullptr); } DISubrange *DIBuilder::getOrCreateSubrange(int64_t Lo, Metadata *CountNode) { - return DISubrange::get(VMContext, CountNode, Lo); + auto *LB = ConstantAsMetadata::get( + ConstantInt::getSigned(Type::getInt64Ty(VMContext), Lo)); + return DISubrange::get(VMContext, CountNode, LB, nullptr, nullptr); +} + +DISubrange *DIBuilder::getOrCreateSubrange(Metadata *CountNode, Metadata *LB, + Metadata *UB, Metadata *Stride) { + return DISubrange::get(VMContext, CountNode, LB, UB, Stride); } static void checkGlobalVariableScope(DIScope *Context) { @@ -829,10 +843,10 @@ DINamespace *DIBuilder::createNameSpace(DIScope *Scope, StringRef Name, DIModule *DIBuilder::createModule(DIScope *Scope, StringRef Name, StringRef ConfigurationMacros, - StringRef IncludePath, - StringRef SysRoot) { - return DIModule::get(VMContext, getNonCompileUnitScope(Scope), Name, - ConfigurationMacros, IncludePath, SysRoot); + StringRef IncludePath, StringRef APINotesFile, + DIFile *File, unsigned LineNo) { + return DIModule::get(VMContext, File, getNonCompileUnitScope(Scope), Name, + ConfigurationMacros, IncludePath, APINotesFile, LineNo); } DILexicalBlockFile *DIBuilder::createLexicalBlockFile(DIScope *Scope, @@ -895,18 +909,15 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, return insertDbgValueIntrinsic(V, VarInfo, Expr, DL, InsertAtEnd, nullptr); } -/// Return an IRBuilder for inserting dbg.declare and dbg.value intrinsics. This -/// abstracts over the various ways to specify an insert position. -static IRBuilder<> getIRBForDbgInsertion(const DILocation *DL, - BasicBlock *InsertBB, - Instruction *InsertBefore) { - IRBuilder<> B(DL->getContext()); +/// Initialize IRBuilder for inserting dbg.declare and dbg.value intrinsics. +/// This abstracts over the various ways to specify an insert position. +static void initIRBuilder(IRBuilder<> &Builder, const DILocation *DL, + BasicBlock *InsertBB, Instruction *InsertBefore) { if (InsertBefore) - B.SetInsertPoint(InsertBefore); + Builder.SetInsertPoint(InsertBefore); else if (InsertBB) - B.SetInsertPoint(InsertBB); - B.SetCurrentDebugLocation(DL); - return B; + Builder.SetInsertPoint(InsertBB); + Builder.SetCurrentDebugLocation(DL); } static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) { @@ -936,7 +947,8 @@ Instruction *DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo, MetadataAsValue::get(VMContext, VarInfo), MetadataAsValue::get(VMContext, Expr)}; - IRBuilder<> B = getIRBForDbgInsertion(DL, InsertBB, InsertBefore); + IRBuilder<> B(DL->getContext()); + initIRBuilder(B, DL, InsertBB, InsertBefore); return B.CreateCall(DeclareFn, Args); } @@ -958,7 +970,8 @@ Instruction *DIBuilder::insertDbgValueIntrinsic( MetadataAsValue::get(VMContext, VarInfo), MetadataAsValue::get(VMContext, Expr)}; - IRBuilder<> B = getIRBForDbgInsertion(DL, InsertBB, InsertBefore); + IRBuilder<> B(DL->getContext()); + initIRBuilder(B, DL, InsertBB, InsertBefore); return B.CreateCall(ValueFn, Args); } @@ -976,7 +989,8 @@ Instruction *DIBuilder::insertLabel( trackIfUnresolved(LabelInfo); Value *Args[] = {MetadataAsValue::get(VMContext, LabelInfo)}; - IRBuilder<> B = getIRBForDbgInsertion(DL, InsertBB, InsertBefore); + IRBuilder<> B(DL->getContext()); + initIRBuilder(B, DL, InsertBB, InsertBefore); return B.CreateCall(LabelFn, Args); } diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp index 94e0740663cc..c44737c5bfc2 100644 --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -52,7 +52,7 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { Type *Ty = ST->getElementType(i); - const Align TyAlign(ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty)); + const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty); // Add padding if necessary to align the data element properly. if (!isAligned(TyAlign, StructSize)) { @@ -153,6 +153,8 @@ const char *DataLayout::getManglingComponent(const Triple &T) { return "-m:o"; if (T.isOSWindows() && T.isOSBinFormatCOFF()) return T.getArch() == Triple::x86 ? "-m:x" : "-m:w"; + if (T.isOSBinFormatXCOFF()) + return "-m:a"; return "-m:e"; } @@ -162,7 +164,7 @@ static const LayoutAlignElem DefaultAlignments[] = { {INTEGER_ALIGN, 16, Align(2), Align(2)}, // i16 {INTEGER_ALIGN, 32, Align(4), Align(4)}, // i32 {INTEGER_ALIGN, 64, Align(4), Align(8)}, // i64 - {FLOAT_ALIGN, 16, Align(2), Align(2)}, // half + {FLOAT_ALIGN, 16, Align(2), Align(2)}, // half, bfloat {FLOAT_ALIGN, 32, Align(4), Align(4)}, // float {FLOAT_ALIGN, 64, Align(8), Align(8)}, // double {FLOAT_ALIGN, 128, Align(16), Align(16)}, // ppcf128, quad, ... @@ -229,7 +231,7 @@ static unsigned getAddrSpace(StringRef R) { } void DataLayout::parseSpecifier(StringRef Desc) { - StringRepresentation = Desc; + StringRepresentation = std::string(Desc); while (!Desc.empty()) { // Split at '-'. std::pair<StringRef, StringRef> Split = split(Desc, '-'); @@ -260,8 +262,8 @@ void DataLayout::parseSpecifier(StringRef Desc) { switch (Specifier) { case 's': - // Ignored for backward compatibility. - // FIXME: remove this on LLVM 4.0. + // Deprecated, but ignoring here to preserve loading older textual llvm + // ASM file break; case 'E': BigEndian = true; @@ -444,6 +446,9 @@ void DataLayout::parseSpecifier(StringRef Desc) { case 'x': ManglingMode = MM_WinCOFFX86; break; + case 'a': + ManglingMode = MM_XCOFF; + break; } break; default: @@ -559,7 +564,10 @@ Align DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, uint32_t BitWidth, // with what clang and llvm-gcc do. unsigned Alignment = getTypeAllocSize(cast<VectorType>(Ty)->getElementType()); - Alignment *= cast<VectorType>(Ty)->getNumElements(); + // We're only calculating a natural alignment, so it doesn't have to be + // based on the full size for scalable vectors. Using the minimum element + // count should be enough here. + Alignment *= cast<VectorType>(Ty)->getElementCount().Min; Alignment = PowerOf2Ceil(Alignment); return Align(Alignment); } @@ -718,7 +726,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { case Type::StructTyID: { // Packed structure types always have an ABI alignment of one. if (cast<StructType>(Ty)->isPacked() && abi_or_pref) - return Align::None(); + return Align(1); // Get the layout annotation... which is lazily created on demand. const StructLayout *Layout = getStructLayout(cast<StructType>(Ty)); @@ -729,6 +737,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { AlignType = INTEGER_ALIGN; break; case Type::HalfTyID: + case Type::BFloatTyID: case Type::FloatTyID: case Type::DoubleTyID: // PPC_FP128TyID and FP128TyID have different data contents, but the @@ -739,7 +748,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { AlignType = FLOAT_ALIGN; break; case Type::X86_MMXTyID: - case Type::VectorTyID: + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: AlignType = VECTOR_ALIGN; break; default: @@ -752,8 +762,13 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { abi_or_pref, Ty); } +/// TODO: Remove this function once the transition to Align is over. unsigned DataLayout::getABITypeAlignment(Type *Ty) const { - return getAlignment(Ty, true).value(); + return getABITypeAlign(Ty).value(); +} + +Align DataLayout::getABITypeAlign(Type *Ty) const { + return getAlignment(Ty, true); } /// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for @@ -762,8 +777,13 @@ Align DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const { return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, nullptr); } +/// TODO: Remove this function once the transition to Align is over. unsigned DataLayout::getPrefTypeAlignment(Type *Ty) const { - return getAlignment(Ty, false).value(); + return getPrefTypeAlign(Ty).value(); +} + +Align DataLayout::getPrefTypeAlign(Type *Ty) const { + return getAlignment(Ty, false); } IntegerType *DataLayout::getIntPtrType(LLVMContext &C, @@ -777,7 +797,7 @@ Type *DataLayout::getIntPtrType(Type *Ty) const { unsigned NumBits = getPointerTypeSizeInBits(Ty); IntegerType *IntTy = IntegerType::get(Ty->getContext(), NumBits); if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) - return VectorType::get(IntTy, VecTy->getNumElements()); + return VectorType::get(IntTy, VecTy); return IntTy; } @@ -799,7 +819,7 @@ Type *DataLayout::getIndexType(Type *Ty) const { unsigned NumBits = getIndexTypeSizeInBits(Ty); IntegerType *IntTy = IntegerType::get(Ty->getContext(), NumBits); if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) - return VectorType::get(IntTy, VecTy->getNumElements()); + return VectorType::get(IntTy, VecTy); return IntTy; } @@ -831,15 +851,14 @@ int64_t DataLayout::getIndexedOffsetInType(Type *ElemTy, return Result; } -/// getPreferredAlignment - Return the preferred alignment of the specified -/// global. This includes an explicitly requested alignment (if the global -/// has one). -unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const { - unsigned GVAlignment = GV->getAlignment(); +/// getPreferredAlign - Return the preferred alignment of the specified global. +/// This includes an explicitly requested alignment (if the global has one). +Align DataLayout::getPreferredAlign(const GlobalVariable *GV) const { + MaybeAlign GVAlignment = GV->getAlign(); // If a section is specified, always precisely honor explicit alignment, // so we don't insert padding into a section we don't control. if (GVAlignment && GV->hasSection()) - return GVAlignment; + return *GVAlignment; // If no explicit alignment is specified, compute the alignment based on // the IR type. If an alignment is specified, increase it to match the ABI @@ -848,30 +867,24 @@ unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const { // FIXME: Not sure it makes sense to use the alignment of the type if // there's already an explicit alignment specification. Type *ElemType = GV->getValueType(); - unsigned Alignment = getPrefTypeAlignment(ElemType); - if (GVAlignment >= Alignment) { - Alignment = GVAlignment; - } else if (GVAlignment != 0) { - Alignment = std::max(GVAlignment, getABITypeAlignment(ElemType)); + Align Alignment = getPrefTypeAlign(ElemType); + if (GVAlignment) { + if (*GVAlignment >= Alignment) + Alignment = *GVAlignment; + else + Alignment = std::max(*GVAlignment, getABITypeAlign(ElemType)); } // If no explicit alignment is specified, and the global is large, increase // the alignment to 16. // FIXME: Why 16, specifically? - if (GV->hasInitializer() && GVAlignment == 0) { - if (Alignment < 16) { + if (GV->hasInitializer() && !GVAlignment) { + if (Alignment < Align(16)) { // If the global is not external, see if it is large. If so, give it a // larger alignment. if (getTypeSizeInBits(ElemType) > 128) - Alignment = 16; // 16-byte alignment. + Alignment = Align(16); // 16-byte alignment. } } return Alignment; } - -/// getPreferredAlignmentLog - Return the preferred alignment of the -/// specified global, returned in log form. This includes an explicitly -/// requested alignment (if the global has one). -unsigned DataLayout::getPreferredAlignmentLog(const GlobalVariable *GV) const { - return Log2_32(getPreferredAlignment(GV)); -} diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index fe8311923109..190b220dc9aa 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -105,10 +105,8 @@ void DebugInfoFinder::processCompileUnit(DICompileUnit *CU) { void DebugInfoFinder::processInstruction(const Module &M, const Instruction &I) { - if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) - processDeclare(M, DDI); - else if (auto *DVI = dyn_cast<DbgValueInst>(&I)) - processValue(M, DVI); + if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) + processVariable(M, *DVI); if (auto DbgLoc = I.getDebugLoc()) processLocation(M, DbgLoc.get()); @@ -194,24 +192,9 @@ void DebugInfoFinder::processSubprogram(DISubprogram *SP) { } } -void DebugInfoFinder::processDeclare(const Module &M, - const DbgDeclareInst *DDI) { - auto *N = dyn_cast<MDNode>(DDI->getVariable()); - if (!N) - return; - - auto *DV = dyn_cast<DILocalVariable>(N); - if (!DV) - return; - - if (!NodesSeen.insert(DV).second) - return; - processScope(DV->getScope()); - processType(DV->getType()); -} - -void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) { - auto *N = dyn_cast<MDNode>(DVI->getVariable()); +void DebugInfoFinder::processVariable(const Module &M, + const DbgVariableIntrinsic &DVI) { + auto *N = dyn_cast<MDNode>(DVI.getVariable()); if (!N) return; @@ -278,6 +261,41 @@ bool DebugInfoFinder::addScope(DIScope *Scope) { return true; } +static MDNode *updateLoopMetadataDebugLocationsImpl( + MDNode *OrigLoopID, + function_ref<DILocation *(const DILocation &)> Updater) { + assert(OrigLoopID && OrigLoopID->getNumOperands() > 0 && + "Loop ID needs at least one operand"); + assert(OrigLoopID && OrigLoopID->getOperand(0).get() == OrigLoopID && + "Loop ID should refer to itself"); + + // Save space for the self-referential LoopID. + SmallVector<Metadata *, 4> MDs = {nullptr}; + + for (unsigned i = 1; i < OrigLoopID->getNumOperands(); ++i) { + Metadata *MD = OrigLoopID->getOperand(i); + if (DILocation *DL = dyn_cast<DILocation>(MD)) { + if (DILocation *NewDL = Updater(*DL)) + MDs.push_back(NewDL); + } else + MDs.push_back(MD); + } + + MDNode *NewLoopID = MDNode::getDistinct(OrigLoopID->getContext(), MDs); + // Insert the self-referential LoopID. + NewLoopID->replaceOperandWith(0, NewLoopID); + return NewLoopID; +} + +void llvm::updateLoopMetadataDebugLocations( + Instruction &I, function_ref<DILocation *(const DILocation &)> Updater) { + MDNode *OrigLoopID = I.getMetadata(LLVMContext::MD_loop); + if (!OrigLoopID) + return; + MDNode *NewLoopID = updateLoopMetadataDebugLocationsImpl(OrigLoopID, Updater); + I.setMetadata(LLVMContext::MD_loop, NewLoopID); +} + static MDNode *stripDebugLocFromLoopID(MDNode *N) { assert(!N->operands().empty() && "Missing self reference?"); @@ -294,20 +312,10 @@ static MDNode *stripDebugLocFromLoopID(MDNode *N) { })) return nullptr; - SmallVector<Metadata *, 4> Args; - // Reserve operand 0 for loop id self reference. - auto TempNode = MDNode::getTemporary(N->getContext(), None); - Args.push_back(TempNode.get()); - // Add all non-debug location operands back. - for (auto Op = N->op_begin() + 1; Op != N->op_end(); Op++) { - if (!isa<DILocation>(*Op)) - Args.push_back(*Op); - } - - // Set the first operand to itself. - MDNode *LoopID = MDNode::get(N->getContext(), Args); - LoopID->replaceOperandWith(0, LoopID); - return LoopID; + auto dropDebugLoc = [](const DILocation &) -> DILocation * { + return nullptr; + }; + return updateLoopMetadataDebugLocationsImpl(N, dropDebugLoc); } bool llvm::stripDebugInfo(Function &F) { @@ -489,7 +497,7 @@ private: RetainedTypes, GlobalVariables, ImportedEntities, CU->getMacros(), CU->getDWOId(), CU->getSplitDebugInlining(), CU->getDebugInfoForProfiling(), CU->getNameTableKind(), - CU->getRangesBaseAddress()); + CU->getRangesBaseAddress(), CU->getSysRoot(), CU->getSDK()); } DILocation *getReplacementMDLocation(DILocation *MLD) { @@ -598,7 +606,9 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { Changed = true; } }; + RemoveUses("llvm.dbg.addr"); RemoveUses("llvm.dbg.declare"); + RemoveUses("llvm.dbg.label"); RemoveUses("llvm.dbg.value"); // Delete non-CU debug info named metadata nodes. @@ -637,7 +647,7 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { } for (auto &BB : F) { for (auto &I : BB) { - auto remapDebugLoc = [&](DebugLoc DL) -> DebugLoc { + auto remapDebugLoc = [&](const DebugLoc &DL) -> DebugLoc { auto *Scope = DL.getScope(); MDNode *InlinedAt = DL.getInlinedAt(); Scope = remap(Scope); @@ -648,15 +658,10 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { if (I.getDebugLoc() != DebugLoc()) I.setDebugLoc(remapDebugLoc(I.getDebugLoc())); - // Remap DILocations in untyped MDNodes (e.g., llvm.loop). - SmallVector<std::pair<unsigned, MDNode *>, 2> MDs; - I.getAllMetadata(MDs); - for (auto Attachment : MDs) - if (auto *T = dyn_cast_or_null<MDTuple>(Attachment.second)) - for (unsigned N = 0; N < T->getNumOperands(); ++N) - if (auto *Loc = dyn_cast_or_null<DILocation>(T->getOperand(N))) - if (Loc != DebugLoc()) - T->replaceOperandWith(N, remapDebugLoc(Loc)); + // Remap DILocations in llvm.loop attachments. + updateLoopMetadataDebugLocations(I, [&](const DILocation &Loc) { + return remapDebugLoc(&Loc).get(); + }); } } } @@ -757,16 +762,18 @@ LLVMMetadataRef LLVMDIBuilderCreateCompileUnit( LLVMBool isOptimized, const char *Flags, size_t FlagsLen, unsigned RuntimeVer, const char *SplitName, size_t SplitNameLen, LLVMDWARFEmissionKind Kind, unsigned DWOId, LLVMBool SplitDebugInlining, - LLVMBool DebugInfoForProfiling) { + LLVMBool DebugInfoForProfiling, const char *SysRoot, size_t SysRootLen, + const char *SDK, size_t SDKLen) { auto File = unwrapDI<DIFile>(FileRef); return wrap(unwrap(Builder)->createCompileUnit( - map_from_llvmDWARFsourcelanguage(Lang), File, - StringRef(Producer, ProducerLen), isOptimized, - StringRef(Flags, FlagsLen), RuntimeVer, - StringRef(SplitName, SplitNameLen), - static_cast<DICompileUnit::DebugEmissionKind>(Kind), DWOId, - SplitDebugInlining, DebugInfoForProfiling)); + map_from_llvmDWARFsourcelanguage(Lang), File, + StringRef(Producer, ProducerLen), isOptimized, StringRef(Flags, FlagsLen), + RuntimeVer, StringRef(SplitName, SplitNameLen), + static_cast<DICompileUnit::DebugEmissionKind>(Kind), DWOId, + SplitDebugInlining, DebugInfoForProfiling, + DICompileUnit::DebugNameTableKind::Default, false, + StringRef(SysRoot, SysRootLen), StringRef(SDK, SDKLen))); } LLVMMetadataRef @@ -782,12 +789,12 @@ LLVMDIBuilderCreateModule(LLVMDIBuilderRef Builder, LLVMMetadataRef ParentScope, const char *Name, size_t NameLen, const char *ConfigMacros, size_t ConfigMacrosLen, const char *IncludePath, size_t IncludePathLen, - const char *SysRoot, size_t SysRootLen) { + const char *APINotesFile, size_t APINotesFileLen) { return wrap(unwrap(Builder)->createModule( unwrapDI<DIScope>(ParentScope), StringRef(Name, NameLen), StringRef(ConfigMacros, ConfigMacrosLen), StringRef(IncludePath, IncludePathLen), - StringRef(SysRoot, SysRootLen))); + StringRef(APINotesFile, APINotesFileLen))); } LLVMMetadataRef LLVMDIBuilderCreateNameSpace(LLVMDIBuilderRef Builder, diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp index d3ecd9b0e03d..110d94116f10 100644 --- a/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/llvm/lib/IR/DebugInfoMetadata.cpp @@ -75,6 +75,21 @@ DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line, Storage, Context.pImpl->DILocations); } +const +DILocation *DILocation::getMergedLocations(ArrayRef<const DILocation *> Locs) { + if (Locs.empty()) + return nullptr; + if (Locs.size() == 1) + return Locs[0]; + auto *Merged = Locs[0]; + for (auto I = std::next(Locs.begin()), E = Locs.end(); I != E; ++I) { + Merged = getMergedLocation(Merged, *I); + if (Merged == nullptr) + break; + } + return Merged; +} + const DILocation *DILocation::getMergedLocation(const DILocation *LocA, const DILocation *LocB) { if (!LocA || !LocB) @@ -321,18 +336,106 @@ DISubrange *DISubrange::getImpl(LLVMContext &Context, int64_t Count, int64_t Lo, StorageType Storage, bool ShouldCreate) { auto *CountNode = ConstantAsMetadata::get( ConstantInt::getSigned(Type::getInt64Ty(Context), Count)); - return getImpl(Context, CountNode, Lo, Storage, ShouldCreate); + auto *LB = ConstantAsMetadata::get( + ConstantInt::getSigned(Type::getInt64Ty(Context), Lo)); + return getImpl(Context, CountNode, LB, nullptr, nullptr, Storage, + ShouldCreate); } DISubrange *DISubrange::getImpl(LLVMContext &Context, Metadata *CountNode, int64_t Lo, StorageType Storage, bool ShouldCreate) { - DEFINE_GETIMPL_LOOKUP(DISubrange, (CountNode, Lo)); - Metadata *Ops[] = { CountNode }; - DEFINE_GETIMPL_STORE(DISubrange, (CountNode, Lo), Ops); + auto *LB = ConstantAsMetadata::get( + ConstantInt::getSigned(Type::getInt64Ty(Context), Lo)); + return getImpl(Context, CountNode, LB, nullptr, nullptr, Storage, + ShouldCreate); +} + +DISubrange *DISubrange::getImpl(LLVMContext &Context, Metadata *CountNode, + Metadata *LB, Metadata *UB, Metadata *Stride, + StorageType Storage, bool ShouldCreate) { + DEFINE_GETIMPL_LOOKUP(DISubrange, (CountNode, LB, UB, Stride)); + Metadata *Ops[] = {CountNode, LB, UB, Stride}; + DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DISubrange, Ops); +} + +DISubrange::CountType DISubrange::getCount() const { + if (!getRawCountNode()) + return CountType(); + + if (auto *MD = dyn_cast<ConstantAsMetadata>(getRawCountNode())) + return CountType(cast<ConstantInt>(MD->getValue())); + + if (auto *DV = dyn_cast<DIVariable>(getRawCountNode())) + return CountType(DV); + + return CountType(); +} + +DISubrange::BoundType DISubrange::getLowerBound() const { + Metadata *LB = getRawLowerBound(); + if (!LB) + return BoundType(); + + assert((isa<ConstantAsMetadata>(LB) || isa<DIVariable>(LB) || + isa<DIExpression>(LB)) && + "LowerBound must be signed constant or DIVariable or DIExpression"); + + if (auto *MD = dyn_cast<ConstantAsMetadata>(LB)) + return BoundType(cast<ConstantInt>(MD->getValue())); + + if (auto *MD = dyn_cast<DIVariable>(LB)) + return BoundType(MD); + + if (auto *MD = dyn_cast<DIExpression>(LB)) + return BoundType(MD); + + return BoundType(); +} + +DISubrange::BoundType DISubrange::getUpperBound() const { + Metadata *UB = getRawUpperBound(); + if (!UB) + return BoundType(); + + assert((isa<ConstantAsMetadata>(UB) || isa<DIVariable>(UB) || + isa<DIExpression>(UB)) && + "UpperBound must be signed constant or DIVariable or DIExpression"); + + if (auto *MD = dyn_cast<ConstantAsMetadata>(UB)) + return BoundType(cast<ConstantInt>(MD->getValue())); + + if (auto *MD = dyn_cast<DIVariable>(UB)) + return BoundType(MD); + + if (auto *MD = dyn_cast<DIExpression>(UB)) + return BoundType(MD); + + return BoundType(); } -DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, int64_t Value, +DISubrange::BoundType DISubrange::getStride() const { + Metadata *ST = getRawStride(); + if (!ST) + return BoundType(); + + assert((isa<ConstantAsMetadata>(ST) || isa<DIVariable>(ST) || + isa<DIExpression>(ST)) && + "Stride must be signed constant or DIVariable or DIExpression"); + + if (auto *MD = dyn_cast<ConstantAsMetadata>(ST)) + return BoundType(cast<ConstantInt>(MD->getValue())); + + if (auto *MD = dyn_cast<DIVariable>(ST)) + return BoundType(MD); + + if (auto *MD = dyn_cast<DIExpression>(ST)) + return BoundType(MD); + + return BoundType(); +} + +DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, const APInt &Value, bool IsUnsigned, MDString *Name, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); @@ -390,17 +493,18 @@ DICompositeType *DICompositeType::getImpl( uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, MDString *Identifier, Metadata *Discriminator, - StorageType Storage, bool ShouldCreate) { + Metadata *DataLocation, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); // Keep this in sync with buildODRType. - DEFINE_GETIMPL_LOOKUP( - DICompositeType, (Tag, Name, File, Line, Scope, BaseType, SizeInBits, - AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams, Identifier, Discriminator)); - Metadata *Ops[] = {File, Scope, Name, BaseType, - Elements, VTableHolder, TemplateParams, Identifier, - Discriminator}; + DEFINE_GETIMPL_LOOKUP(DICompositeType, + (Tag, Name, File, Line, Scope, BaseType, SizeInBits, + AlignInBits, OffsetInBits, Flags, Elements, + RuntimeLang, VTableHolder, TemplateParams, Identifier, + Discriminator, DataLocation)); + Metadata *Ops[] = {File, Scope, Name, BaseType, + Elements, VTableHolder, TemplateParams, Identifier, + Discriminator, DataLocation}; DEFINE_GETIMPL_STORE(DICompositeType, (Tag, Line, RuntimeLang, SizeInBits, AlignInBits, OffsetInBits, Flags), Ops); @@ -411,7 +515,8 @@ DICompositeType *DICompositeType::buildODRType( Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, - Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator) { + Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator, + Metadata *DataLocation) { assert(!Identifier.getString().empty() && "Expected valid identifier"); if (!Context.isODRUniquingDebugTypes()) return nullptr; @@ -420,7 +525,8 @@ DICompositeType *DICompositeType::buildODRType( return CT = DICompositeType::getDistinct( Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams, &Identifier, Discriminator); + VTableHolder, TemplateParams, &Identifier, Discriminator, + DataLocation); // Only mutate CT if it's a forward declaration and the new operands aren't. assert(CT->getRawIdentifier() == &Identifier && "Wrong ODR identifier?"); @@ -430,9 +536,9 @@ DICompositeType *DICompositeType::buildODRType( // Mutate CT in place. Keep this in sync with getImpl. CT->mutate(Tag, Line, RuntimeLang, SizeInBits, AlignInBits, OffsetInBits, Flags); - Metadata *Ops[] = {File, Scope, Name, BaseType, - Elements, VTableHolder, TemplateParams, &Identifier, - Discriminator}; + Metadata *Ops[] = {File, Scope, Name, BaseType, + Elements, VTableHolder, TemplateParams, &Identifier, + Discriminator, DataLocation}; assert((std::end(Ops) - std::begin(Ops)) == (int)CT->getNumOperands() && "Mismatched number of operands"); for (unsigned I = 0, E = CT->getNumOperands(); I != E; ++I) @@ -446,7 +552,8 @@ DICompositeType *DICompositeType::getODRType( Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, - Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator) { + Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator, + Metadata *DataLocation) { assert(!Identifier.getString().empty() && "Expected valid identifier"); if (!Context.isODRUniquingDebugTypes()) return nullptr; @@ -455,7 +562,7 @@ DICompositeType *DICompositeType::getODRType( CT = DICompositeType::getDistinct( Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, - TemplateParams, &Identifier, Discriminator); + TemplateParams, &Identifier, Discriminator, DataLocation); return CT; } @@ -479,8 +586,9 @@ DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, DIFlags Flags, // FIXME: Implement this string-enum correspondence with a .def file and macros, // so that the association is explicit rather than implied. static const char *ChecksumKindName[DIFile::CSK_Last] = { - "CSK_MD5", - "CSK_SHA1" + "CSK_MD5", + "CSK_SHA1", + "CSK_SHA256", }; StringRef DIFile::getChecksumKindAsString(ChecksumKind CSKind) { @@ -495,6 +603,7 @@ Optional<DIFile::ChecksumKind> DIFile::getChecksumKind(StringRef CSKindStr) { return StringSwitch<Optional<DIFile::ChecksumKind>>(CSKindStr) .Case("CSK_MD5", DIFile::CSK_MD5) .Case("CSK_SHA1", DIFile::CSK_SHA1) + .Case("CSK_SHA256", DIFile::CSK_SHA256) .Default(None); } @@ -520,17 +629,24 @@ DICompileUnit *DICompileUnit::getImpl( unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, bool SplitDebugInlining, bool DebugInfoForProfiling, - unsigned NameTableKind, bool RangesBaseAddress, StorageType Storage, - bool ShouldCreate) { + unsigned NameTableKind, bool RangesBaseAddress, MDString *SysRoot, + MDString *SDK, StorageType Storage, bool ShouldCreate) { assert(Storage != Uniqued && "Cannot unique DICompileUnit"); assert(isCanonical(Producer) && "Expected canonical MDString"); assert(isCanonical(Flags) && "Expected canonical MDString"); assert(isCanonical(SplitDebugFilename) && "Expected canonical MDString"); - Metadata *Ops[] = { - File, Producer, Flags, SplitDebugFilename, - EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, - Macros}; + Metadata *Ops[] = {File, + Producer, + Flags, + SplitDebugFilename, + EnumTypes, + RetainedTypes, + GlobalVariables, + ImportedEntities, + Macros, + SysRoot, + SDK}; return storeImpl(new (array_lengthof(Ops)) DICompileUnit( Context, Storage, SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind, DWOId, SplitDebugInlining, @@ -660,12 +776,7 @@ DISubprogram *DISubprogram::getImpl( bool DISubprogram::describes(const Function *F) const { assert(F && "Invalid function"); - if (F->getSubprogram() == this) - return true; - StringRef Name = getLinkageName(); - if (Name.empty()) - Name = getName(); - return F->getName() == Name; + return F->getSubprogram() == this; } DILexicalBlock *DILexicalBlock::getImpl(LLVMContext &Context, Metadata *Scope, @@ -713,35 +824,38 @@ DICommonBlock *DICommonBlock::getImpl(LLVMContext &Context, Metadata *Scope, DEFINE_GETIMPL_STORE(DICommonBlock, (LineNo), Ops); } -DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *Scope, - MDString *Name, MDString *ConfigurationMacros, - MDString *IncludePath, MDString *SysRoot, - StorageType Storage, bool ShouldCreate) { +DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *File, + Metadata *Scope, MDString *Name, + MDString *ConfigurationMacros, + MDString *IncludePath, MDString *APINotesFile, + unsigned LineNo, StorageType Storage, + bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP( - DIModule, (Scope, Name, ConfigurationMacros, IncludePath, SysRoot)); - Metadata *Ops[] = {Scope, Name, ConfigurationMacros, IncludePath, SysRoot}; - DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIModule, Ops); + DEFINE_GETIMPL_LOOKUP(DIModule, (File, Scope, Name, ConfigurationMacros, + IncludePath, APINotesFile, LineNo)); + Metadata *Ops[] = {File, Scope, Name, ConfigurationMacros, + IncludePath, APINotesFile}; + DEFINE_GETIMPL_STORE(DIModule, (LineNo), Ops); } -DITemplateTypeParameter *DITemplateTypeParameter::getImpl(LLVMContext &Context, - MDString *Name, - Metadata *Type, - StorageType Storage, - bool ShouldCreate) { +DITemplateTypeParameter * +DITemplateTypeParameter::getImpl(LLVMContext &Context, MDString *Name, + Metadata *Type, bool isDefault, + StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DITemplateTypeParameter, (Name, Type)); + DEFINE_GETIMPL_LOOKUP(DITemplateTypeParameter, (Name, Type, isDefault)); Metadata *Ops[] = {Name, Type}; - DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DITemplateTypeParameter, Ops); + DEFINE_GETIMPL_STORE(DITemplateTypeParameter, (isDefault), Ops); } DITemplateValueParameter *DITemplateValueParameter::getImpl( LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *Type, - Metadata *Value, StorageType Storage, bool ShouldCreate) { + bool isDefault, Metadata *Value, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DITemplateValueParameter, (Tag, Name, Type, Value)); + DEFINE_GETIMPL_LOOKUP(DITemplateValueParameter, + (Tag, Name, Type, isDefault, Value)); Metadata *Ops[] = {Name, Type, Value}; - DEFINE_GETIMPL_STORE(DITemplateValueParameter, (Tag), Ops); + DEFINE_GETIMPL_STORE(DITemplateValueParameter, (Tag, isDefault), Ops); } DIGlobalVariable * @@ -925,6 +1039,7 @@ bool DIExpression::isValid() const { case dwarf::DW_OP_dup: case dwarf::DW_OP_regx: case dwarf::DW_OP_bregx: + case dwarf::DW_OP_push_object_address: break; } } @@ -1107,7 +1222,9 @@ DIExpression *DIExpression::append(const DIExpression *Expr, } NewOps.append(Ops.begin(), Ops.end()); - return DIExpression::get(Expr->getContext(), NewOps); + auto *result = DIExpression::get(Expr->getContext(), NewOps); + assert(result->isValid() && "concatenated expression is not valid"); + return result; } DIExpression *DIExpression::appendToStack(const DIExpression *Expr, diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp index 14d1396f1543..e945cbcba782 100644 --- a/llvm/lib/IR/DebugLoc.cpp +++ b/llvm/lib/IR/DebugLoc.cpp @@ -79,7 +79,7 @@ DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope, const_cast<MDNode *>(InlinedAt), ImplicitCode); } -DebugLoc DebugLoc::appendInlinedAt(DebugLoc DL, DILocation *InlinedAt, +DebugLoc DebugLoc::appendInlinedAt(const DebugLoc &DL, DILocation *InlinedAt, LLVMContext &Ctx, DenseMap<const MDNode *, MDNode *> &Cache, bool ReplaceLast) { diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp index 99d5aec3f043..6528c723fbfa 100644 --- a/llvm/lib/IR/DiagnosticInfo.cpp +++ b/llvm/lib/IR/DiagnosticInfo.cpp @@ -119,7 +119,7 @@ DiagnosticLocation::DiagnosticLocation(const DebugLoc &DL) { DiagnosticLocation::DiagnosticLocation(const DISubprogram *SP) { if (!SP) return; - + File = SP->getFile(); Line = SP->getScopeLine(); Column = 0; @@ -132,7 +132,7 @@ StringRef DiagnosticLocation::getRelativePath() const { std::string DiagnosticLocation::getAbsolutePath() const { StringRef Name = File->getFilename(); if (sys::path::is_absolute(Name)) - return Name; + return std::string(Name); SmallString<128> Path; sys::path::append(Path, File->getDirectory(), Name); @@ -160,8 +160,9 @@ const std::string DiagnosticInfoWithLocationBase::getLocationStr() const { return (Filename + ":" + Twine(Line) + ":" + Twine(Column)).str(); } -DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, const Value *V) - : Key(Key) { +DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, + const Value *V) + : Key(std::string(Key)) { if (auto *F = dyn_cast<Function>(V)) { if (DISubprogram *SP = F->getSubprogram()) Loc = SP; @@ -172,7 +173,7 @@ DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, const Value *V // Only include names that correspond to user variables. FIXME: We should use // debug info if available to get the name of the user variable. if (isa<llvm::Argument>(V) || isa<GlobalValue>(V)) - Val = GlobalValue::dropLLVMManglingEscape(V->getName()); + Val = std::string(GlobalValue::dropLLVMManglingEscape(V->getName())); else if (isa<Constant>(V)) { raw_string_ostream OS(Val); V->printAsOperand(OS, /*PrintType=*/false); @@ -181,39 +182,39 @@ DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, const Value *V } DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, const Type *T) - : Key(Key) { + : Key(std::string(Key)) { raw_string_ostream OS(Val); OS << *T; } DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, StringRef S) - : Key(Key), Val(S.str()) {} + : Key(std::string(Key)), Val(S.str()) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, int N) - : Key(Key), Val(itostr(N)) {} + : Key(std::string(Key)), Val(itostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, float N) - : Key(Key), Val(llvm::to_string(N)) {} + : Key(std::string(Key)), Val(llvm::to_string(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, long N) - : Key(Key), Val(itostr(N)) {} + : Key(std::string(Key)), Val(itostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, long long N) - : Key(Key), Val(itostr(N)) {} + : Key(std::string(Key)), Val(itostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, unsigned N) - : Key(Key), Val(utostr(N)) {} + : Key(std::string(Key)), Val(utostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, unsigned long N) - : Key(Key), Val(utostr(N)) {} + : Key(std::string(Key)), Val(utostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, unsigned long long N) - : Key(Key), Val(utostr(N)) {} + : Key(std::string(Key)), Val(utostr(N)) {} DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, DebugLoc Loc) - : Key(Key), Loc(Loc) { + : Key(std::string(Key)), Loc(Loc) { if (Loc) { Val = (Loc->getFilename() + ":" + Twine(Loc.getLine()) + ":" + Twine(Loc.getCol())).str(); @@ -243,11 +244,8 @@ OptimizationRemark::OptimizationRemark(const char *PassName, RemarkName, *Inst->getParent()->getParent(), Inst->getDebugLoc(), Inst->getParent()) {} -// Helper to allow for an assert before attempting to return an invalid -// reference. -static const BasicBlock &getFirstFunctionBlock(const Function *Func) { - assert(!Func->empty() && "Function does not have a body"); - return Func->front(); +static const BasicBlock *getFirstFunctionBlock(const Function *Func) { + return Func->empty() ? nullptr : &Func->front(); } OptimizationRemark::OptimizationRemark(const char *PassName, @@ -255,7 +253,7 @@ OptimizationRemark::OptimizationRemark(const char *PassName, const Function *Func) : DiagnosticInfoIROptimization(DK_OptimizationRemark, DS_Remark, PassName, RemarkName, *Func, Func->getSubprogram(), - &getFirstFunctionBlock(Func)) {} + getFirstFunctionBlock(Func)) {} bool OptimizationRemark::isEnabled() const { const Function &Fn = getFunction(); diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp index 03dc4da273a3..bb1cc347dcb1 100644 --- a/llvm/lib/IR/Dominators.cpp +++ b/llvm/lib/IR/Dominators.cpp @@ -134,18 +134,13 @@ bool DominatorTree::dominates(const Instruction *Def, // dominates every instruction in UseBB. // A PHI is dominated only if the instruction dominates every possible use in // the UseBB. - if (isa<InvokeInst>(Def) || isa<PHINode>(User)) + if (isa<InvokeInst>(Def) || isa<CallBrInst>(Def) || isa<PHINode>(User)) return dominates(Def, UseBB); if (DefBB != UseBB) return dominates(DefBB, UseBB); - // Loop through the basic block until we find Def or User. - BasicBlock::const_iterator I = DefBB->begin(); - for (; &*I != Def && &*I != User; ++I) - /*empty*/; - - return &*I == Def; + return Def->comesBefore(User); } // true if Def would dominate a use in any instruction in UseBB. @@ -173,6 +168,13 @@ bool DominatorTree::dominates(const Instruction *Def, return dominates(E, UseBB); } + // Callbr results are similarly only usable in the default destination. + if (const auto *CBI = dyn_cast<CallBrInst>(Def)) { + BasicBlock *NormalDest = CBI->getDefaultDest(); + BasicBlockEdge E(DefBB, NormalDest); + return dominates(E, UseBB); + } + return dominates(DefBB, UseBB); } @@ -278,6 +280,13 @@ bool DominatorTree::dominates(const Instruction *Def, const Use &U) const { return dominates(E, U); } + // Callbr results are similarly only usable in the default destination. + if (const auto *CBI = dyn_cast<CallBrInst>(Def)) { + BasicBlock *NormalDest = CBI->getDefaultDest(); + BasicBlockEdge E(DefBB, NormalDest); + return dominates(E, U); + } + // If the def and use are in different blocks, do a simple CFG dominator // tree query. if (DefBB != UseBB) @@ -289,12 +298,7 @@ bool DominatorTree::dominates(const Instruction *Def, const Use &U) const { if (isa<PHINode>(UserInst)) return true; - // Otherwise, just loop through the basic block until we find Def or User. - BasicBlock::const_iterator I = DefBB->begin(); - for (; &*I != Def && &*I != UserInst; ++I) - /*empty*/; - - return &*I != UserInst; + return Def->comesBefore(UserInst); } bool DominatorTree::isReachableFromEntry(const Use &U) const { @@ -312,6 +316,14 @@ bool DominatorTree::isReachableFromEntry(const Use &U) const { return isReachableFromEntry(I->getParent()); } +// Edge BBE1 dominates edge BBE2 if they match or BBE1 dominates start of BBE2. +bool DominatorTree::dominates(const BasicBlockEdge &BBE1, + const BasicBlockEdge &BBE2) const { + if (BBE1.getStart() == BBE2.getStart() && BBE1.getEnd() == BBE2.getEnd()) + return true; + return dominates(BBE1, BBE2.getStart()); +} + //===----------------------------------------------------------------------===// // DominatorTreeAnalysis and related pass implementations //===----------------------------------------------------------------------===// @@ -381,4 +393,3 @@ void DominatorTreeWrapperPass::verifyAnalysis() const { void DominatorTreeWrapperPass::print(raw_ostream &OS, const Module *) const { DT.print(OS); } - diff --git a/llvm/lib/IR/FPEnv.cpp b/llvm/lib/IR/FPEnv.cpp index 008852658232..516c702acec7 100644 --- a/llvm/lib/IR/FPEnv.cpp +++ b/llvm/lib/IR/FPEnv.cpp @@ -12,41 +12,47 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/StringSwitch.h" #include "llvm/IR/FPEnv.h" +#include "llvm/ADT/StringSwitch.h" namespace llvm { -Optional<fp::RoundingMode> StrToRoundingMode(StringRef RoundingArg) { +Optional<RoundingMode> StrToRoundingMode(StringRef RoundingArg) { // For dynamic rounding mode, we use round to nearest but we will set the // 'exact' SDNodeFlag so that the value will not be rounded. - return StringSwitch<Optional<fp::RoundingMode>>(RoundingArg) - .Case("round.dynamic", fp::rmDynamic) - .Case("round.tonearest", fp::rmToNearest) - .Case("round.downward", fp::rmDownward) - .Case("round.upward", fp::rmUpward) - .Case("round.towardzero", fp::rmTowardZero) + return StringSwitch<Optional<RoundingMode>>(RoundingArg) + .Case("round.dynamic", RoundingMode::Dynamic) + .Case("round.tonearest", RoundingMode::NearestTiesToEven) + .Case("round.tonearestaway", RoundingMode::NearestTiesToAway) + .Case("round.downward", RoundingMode::TowardNegative) + .Case("round.upward", RoundingMode::TowardPositive) + .Case("round.towardzero", RoundingMode::TowardZero) .Default(None); } -Optional<StringRef> RoundingModeToStr(fp::RoundingMode UseRounding) { +Optional<StringRef> RoundingModeToStr(RoundingMode UseRounding) { Optional<StringRef> RoundingStr = None; switch (UseRounding) { - case fp::rmDynamic: + case RoundingMode::Dynamic: RoundingStr = "round.dynamic"; break; - case fp::rmToNearest: + case RoundingMode::NearestTiesToEven: RoundingStr = "round.tonearest"; break; - case fp::rmDownward: + case RoundingMode::NearestTiesToAway: + RoundingStr = "round.tonearestaway"; + break; + case RoundingMode::TowardNegative: RoundingStr = "round.downward"; break; - case fp::rmUpward: + case RoundingMode::TowardPositive: RoundingStr = "round.upward"; break; - case fp::rmTowardZero: + case RoundingMode::TowardZero: RoundingStr = "round.towardzero"; break; + default: + break; } return RoundingStr; } @@ -74,5 +80,4 @@ Optional<StringRef> ExceptionBehaviorToStr(fp::ExceptionBehavior UseExcept) { } return ExceptStr; } - -}
\ No newline at end of file +} // namespace llvm diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 54612250b0d6..10d535e3ab11 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -20,6 +20,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/AbstractCallSite.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -114,11 +115,39 @@ bool Argument::hasInAllocaAttr() const { return hasAttribute(Attribute::InAlloca); } -bool Argument::hasByValOrInAllocaAttr() const { +bool Argument::hasPreallocatedAttr() const { + if (!getType()->isPointerTy()) + return false; + return hasAttribute(Attribute::Preallocated); +} + +bool Argument::hasPassPointeeByValueAttr() const { if (!getType()->isPointerTy()) return false; AttributeList Attrs = getParent()->getAttributes(); return Attrs.hasParamAttribute(getArgNo(), Attribute::ByVal) || - Attrs.hasParamAttribute(getArgNo(), Attribute::InAlloca); + Attrs.hasParamAttribute(getArgNo(), Attribute::InAlloca) || + Attrs.hasParamAttribute(getArgNo(), Attribute::Preallocated); +} + +uint64_t Argument::getPassPointeeByValueCopySize(const DataLayout &DL) const { + AttributeSet ParamAttrs + = getParent()->getAttributes().getParamAttributes(getArgNo()); + + // FIXME: All the type carrying attributes are mutually exclusive, so there + // should be a single query to get the stored type that handles any of them. + if (Type *ByValTy = ParamAttrs.getByValType()) + return DL.getTypeAllocSize(ByValTy); + if (Type *PreAllocTy = ParamAttrs.getPreallocatedType()) + return DL.getTypeAllocSize(PreAllocTy); + + // FIXME: inalloca always depends on pointee element type. It's also possible + // for byval to miss it. + if (ParamAttrs.hasAttribute(Attribute::InAlloca) || + ParamAttrs.hasAttribute(Attribute::ByVal) || + ParamAttrs.hasAttribute(Attribute::Preallocated)) + return DL.getTypeAllocSize(cast<PointerType>(getType())->getElementType()); + + return 0; } unsigned Argument::getParamAlignment() const { @@ -320,6 +349,18 @@ static MutableArrayRef<Argument> makeArgArray(Argument *Args, size_t Count) { return MutableArrayRef<Argument>(Args, Count); } +bool Function::isConstrainedFPIntrinsic() const { + switch (getIntrinsicID()) { +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: +#include "llvm/IR/ConstrainedOps.def" + return true; +#undef INSTRUCTION + default: + return false; + } +} + void Function::clearArguments() { for (Argument &A : makeArgArray(Arguments, NumArgs)) { A.setName(""); @@ -632,16 +673,17 @@ static std::string getMangledTypeStr(Type* Ty) { // Ensure nested function types are distinguishable. Result += "f"; } else if (VectorType* VTy = dyn_cast<VectorType>(Ty)) { - if (VTy->isScalable()) + ElementCount EC = VTy->getElementCount(); + if (EC.Scalable) Result += "nx"; - Result += "v" + utostr(VTy->getVectorNumElements()) + - getMangledTypeStr(VTy->getVectorElementType()); + Result += "v" + utostr(EC.Min) + getMangledTypeStr(VTy->getElementType()); } else if (Ty) { switch (Ty->getTypeID()) { default: llvm_unreachable("Unhandled type"); case Type::VoidTyID: Result += "isVoid"; break; case Type::MetadataTyID: Result += "Metadata"; break; case Type::HalfTyID: Result += "f16"; break; + case Type::BFloatTyID: Result += "bf16"; break; case Type::FloatTyID: Result += "f32"; break; case Type::DoubleTyID: Result += "f64"; break; case Type::X86_FP80TyID: Result += "f80"; break; @@ -726,13 +768,18 @@ enum IIT_Info { IIT_SCALABLE_VEC = 43, IIT_SUBDIVIDE2_ARG = 44, IIT_SUBDIVIDE4_ARG = 45, - IIT_VEC_OF_BITCASTS_TO_INT = 46 + IIT_VEC_OF_BITCASTS_TO_INT = 46, + IIT_V128 = 47, + IIT_BF16 = 48 }; static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, + IIT_Info LastInfo, SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) { using namespace Intrinsic; + bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC); + IIT_Info Info = IIT_Info(Infos[NextElt++]); unsigned StructElts = 2; @@ -755,6 +802,9 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, case IIT_F16: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Half, 0)); return; + case IIT_BF16: + OutputTable.push_back(IITDescriptor::get(IITDescriptor::BFloat, 0)); + return; case IIT_F32: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Float, 0)); return; @@ -783,49 +833,53 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 128)); return; case IIT_V1: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 1)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(1, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V2: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 2)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(2, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V4: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 4)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(4, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V8: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 8)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(8, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V16: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 16)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(16, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V32: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 32)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(32, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V64: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 64)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(64, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); + return; + case IIT_V128: + OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V512: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 512)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_V1024: - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Vector, 1024)); - DecodeIITType(NextElt, Infos, OutputTable); + OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector)); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_PTR: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 0)); - DecodeIITType(NextElt, Infos, OutputTable); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; case IIT_ANYPTR: { // [ANYPTR addrspace, subtype] OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, Infos[NextElt++])); - DecodeIITType(NextElt, Infos, OutputTable); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; } case IIT_ARG: { @@ -888,7 +942,7 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct,StructElts)); for (unsigned i = 0; i != StructElts; ++i) - DecodeIITType(NextElt, Infos, OutputTable); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; } case IIT_SUBDIVIDE2_ARG: { @@ -910,9 +964,7 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, return; } case IIT_SCALABLE_VEC: { - OutputTable.push_back(IITDescriptor::get(IITDescriptor::ScalableVecArgument, - 0)); - DecodeIITType(NextElt, Infos, OutputTable); + DecodeIITType(NextElt, Infos, Info, OutputTable); return; } case IIT_VEC_OF_BITCASTS_TO_INT: { @@ -957,9 +1009,9 @@ void Intrinsic::getIntrinsicInfoTableEntries(ID id, } // Okay, decode the table into the output vector of IITDescriptors. - DecodeIITType(NextElt, IITEntries, T); + DecodeIITType(NextElt, IITEntries, IIT_Done, T); while (NextElt != IITEntries.size() && IITEntries[NextElt] != 0) - DecodeIITType(NextElt, IITEntries, T); + DecodeIITType(NextElt, IITEntries, IIT_Done, T); } static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, @@ -976,6 +1028,7 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, case IITDescriptor::Token: return Type::getTokenTy(Context); case IITDescriptor::Metadata: return Type::getMetadataTy(Context); case IITDescriptor::Half: return Type::getHalfTy(Context); + case IITDescriptor::BFloat: return Type::getBFloatTy(Context); case IITDescriptor::Float: return Type::getFloatTy(Context); case IITDescriptor::Double: return Type::getDoubleTy(Context); case IITDescriptor::Quad: return Type::getFP128Ty(Context); @@ -983,7 +1036,8 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, case IITDescriptor::Integer: return IntegerType::get(Context, D.Integer_Width); case IITDescriptor::Vector: - return VectorType::get(DecodeFixedType(Infos, Tys, Context),D.Vector_Width); + return VectorType::get(DecodeFixedType(Infos, Tys, Context), + D.Vector_Width); case IITDescriptor::Pointer: return PointerType::get(DecodeFixedType(Infos, Tys, Context), D.Pointer_AddressSpace); @@ -1038,7 +1092,7 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, VectorType *VTy = dyn_cast<VectorType>(Ty); if (!VTy) llvm_unreachable("Expected an argument of Vector Type"); - Type *EltTy = VTy->getVectorElementType(); + Type *EltTy = VTy->getElementType(); return PointerType::getUnqual(EltTy); } case IITDescriptor::VecElementArgument: { @@ -1056,11 +1110,6 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, case IITDescriptor::VecOfAnyPtrsToElt: // Return the overloaded type (which determines the pointers address space) return Tys[D.getOverloadArgNumber()]; - case IITDescriptor::ScalableVecArgument: { - Type *Ty = DecodeFixedType(Infos, Tys, Context); - return VectorType::get(Ty->getVectorElementType(), - { Ty->getVectorNumElements(), true }); - } } llvm_unreachable("unhandled"); } @@ -1158,13 +1207,14 @@ static bool matchIntrinsicType( case IITDescriptor::Token: return !Ty->isTokenTy(); case IITDescriptor::Metadata: return !Ty->isMetadataTy(); case IITDescriptor::Half: return !Ty->isHalfTy(); + case IITDescriptor::BFloat: return !Ty->isBFloatTy(); case IITDescriptor::Float: return !Ty->isFloatTy(); case IITDescriptor::Double: return !Ty->isDoubleTy(); case IITDescriptor::Quad: return !Ty->isFP128Ty(); case IITDescriptor::Integer: return !Ty->isIntegerTy(D.Integer_Width); case IITDescriptor::Vector: { VectorType *VT = dyn_cast<VectorType>(Ty); - return !VT || VT->getNumElements() != D.Vector_Width || + return !VT || VT->getElementCount() != D.Vector_Width || matchIntrinsicType(VT->getElementType(), Infos, ArgTys, DeferredChecks, IsDeferredCheck); } @@ -1264,7 +1314,7 @@ static bool matchIntrinsicType( if (ReferenceType->getElementCount() != ThisArgType->getElementCount()) return true; - EltTy = ThisArgType->getVectorElementType(); + EltTy = ThisArgType->getElementType(); } return matchIntrinsicType(EltTy, Infos, ArgTys, DeferredChecks, IsDeferredCheck); @@ -1309,15 +1359,13 @@ static bool matchIntrinsicType( VectorType *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]); VectorType *ThisArgVecTy = dyn_cast<VectorType>(Ty); if (!ThisArgVecTy || !ReferenceType || - (ReferenceType->getVectorNumElements() != - ThisArgVecTy->getVectorNumElements())) + (ReferenceType->getNumElements() != ThisArgVecTy->getNumElements())) return true; PointerType *ThisArgEltTy = - dyn_cast<PointerType>(ThisArgVecTy->getVectorElementType()); + dyn_cast<PointerType>(ThisArgVecTy->getElementType()); if (!ThisArgEltTy) return true; - return ThisArgEltTy->getElementType() != - ReferenceType->getVectorElementType(); + return ThisArgEltTy->getElementType() != ReferenceType->getElementType(); } case IITDescriptor::VecElementArgument: { if (D.getArgumentNumber() >= ArgTys.size()) @@ -1339,13 +1387,6 @@ static bool matchIntrinsicType( } return true; } - case IITDescriptor::ScalableVecArgument: { - VectorType *VTy = dyn_cast<VectorType>(Ty); - if (!VTy || !VTy->isScalable()) - return true; - return matchIntrinsicType(VTy, Infos, ArgTys, DeferredChecks, - IsDeferredCheck); - } case IITDescriptor::VecOfBitcastsToInt: { if (D.getArgumentNumber() >= ArgTys.size()) return IsDeferredCheck || DeferCheck(Ty); @@ -1405,42 +1446,60 @@ Intrinsic::matchIntrinsicVarArg(bool isVarArg, return true; } -Optional<Function*> Intrinsic::remangleIntrinsicFunction(Function *F) { +bool Intrinsic::getIntrinsicSignature(Function *F, + SmallVectorImpl<Type *> &ArgTys) { Intrinsic::ID ID = F->getIntrinsicID(); if (!ID) - return None; + return false; - FunctionType *FTy = F->getFunctionType(); - // Accumulate an array of overloaded types for the given intrinsic - SmallVector<Type *, 4> ArgTys; - { - SmallVector<Intrinsic::IITDescriptor, 8> Table; - getIntrinsicInfoTableEntries(ID, Table); - ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; - - if (Intrinsic::matchIntrinsicSignature(FTy, TableRef, ArgTys)) - return None; - if (Intrinsic::matchIntrinsicVarArg(FTy->isVarArg(), TableRef)) - return None; + SmallVector<Intrinsic::IITDescriptor, 8> Table; + getIntrinsicInfoTableEntries(ID, Table); + ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; + + if (Intrinsic::matchIntrinsicSignature(F->getFunctionType(), TableRef, + ArgTys) != + Intrinsic::MatchIntrinsicTypesResult::MatchIntrinsicTypes_Match) { + return false; } + if (Intrinsic::matchIntrinsicVarArg(F->getFunctionType()->isVarArg(), + TableRef)) + return false; + return true; +} + +Optional<Function *> Intrinsic::remangleIntrinsicFunction(Function *F) { + SmallVector<Type *, 4> ArgTys; + if (!getIntrinsicSignature(F, ArgTys)) + return None; + Intrinsic::ID ID = F->getIntrinsicID(); StringRef Name = F->getName(); if (Name == Intrinsic::getName(ID, ArgTys)) return None; auto NewDecl = Intrinsic::getDeclaration(F->getParent(), ID, ArgTys); NewDecl->setCallingConv(F->getCallingConv()); - assert(NewDecl->getFunctionType() == FTy && "Shouldn't change the signature"); + assert(NewDecl->getFunctionType() == F->getFunctionType() && + "Shouldn't change the signature"); return NewDecl; } /// hasAddressTaken - returns true if there are any uses of this function -/// other than direct calls or invokes to it. -bool Function::hasAddressTaken(const User* *PutOffender) const { +/// other than direct calls or invokes to it. Optionally ignores callback +/// uses. +bool Function::hasAddressTaken(const User **PutOffender, + bool IgnoreCallbackUses) const { for (const Use &U : uses()) { const User *FU = U.getUser(); if (isa<BlockAddress>(FU)) continue; + + if (IgnoreCallbackUses) { + AbstractCallSite ACS(&U); + if (ACS && ACS.isCallbackCall()) + continue; + } + const auto *Call = dyn_cast<CallBase>(FU); if (!Call) { if (PutOffender) @@ -1620,9 +1679,7 @@ Optional<StringRef> Function::getSectionPrefix() const { } bool Function::nullPointerIsDefined() const { - return getFnAttribute("null-pointer-is-valid") - .getValueAsString() - .equals("true"); + return hasFnAttribute(Attribute::NullPointerIsValid); } bool llvm::NullPointerIsDefined(const Function *F, unsigned AS) { diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index 46a9696b2944..dd8e62164de1 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -65,6 +65,7 @@ Value *GlobalValue::handleOperandChangeImpl(Value *From, Value *To) { void GlobalValue::copyAttributesFrom(const GlobalValue *Src) { setVisibility(Src->getVisibility()); setUnnamedAddr(Src->getUnnamedAddr()); + setThreadLocalMode(Src->getThreadLocalMode()); setDLLStorageClass(Src->getDLLStorageClass()); setDSOLocal(Src->isDSOLocal()); setPartition(Src->getPartition()); @@ -94,18 +95,17 @@ void GlobalValue::eraseFromParent() { llvm_unreachable("not a global"); } -unsigned GlobalValue::getAlignment() const { - if (auto *GA = dyn_cast<GlobalAlias>(this)) { - // In general we cannot compute this at the IR level, but we try. - if (const GlobalObject *GO = GA->getBaseObject()) - return GO->getAlignment(); +bool GlobalValue::isInterposable() const { + if (isInterposableLinkage(getLinkage())) + return true; + return getParent() && getParent()->getSemanticInterposition() && + !isDSOLocal(); +} - // FIXME: we should also be able to handle: - // Alias = Global + Offset - // Alias = Absolute - return 0; - } - return cast<GlobalObject>(this)->getAlignment(); +bool GlobalValue::canBenefitFromLocalAlias() const { + // See AsmPrinter::getSymbolPreferLocal(). + return GlobalObject::isExternalLinkage(getLinkage()) && !isDeclaration() && + !isa<GlobalIFunc>(this) && !hasComdat(); } unsigned GlobalValue::getAddressSpace() const { @@ -113,12 +113,8 @@ unsigned GlobalValue::getAddressSpace() const { return PtrTy->getAddressSpace(); } -void GlobalObject::setAlignment(unsigned Align) { - setAlignment(MaybeAlign(Align)); -} - void GlobalObject::setAlignment(MaybeAlign Align) { - assert((!Align || Align <= MaximumAlignment) && + assert((!Align || *Align <= MaximumAlignment) && "Alignment is greater than MaximumAlignment!"); unsigned AlignmentData = encode(Align); unsigned OldData = getGlobalValueSubClassData(); @@ -143,7 +139,7 @@ std::string GlobalValue::getGlobalIdentifier(StringRef Name, if (Name[0] == '\1') Name = Name.substr(1); - std::string NewName = Name; + std::string NewName = std::string(Name); if (llvm::GlobalValue::isLocalLinkage(Linkage)) { // For local symbols, prepend the main file name to distinguish them. // Do not include the full path in the file name since there's no guarantee @@ -242,7 +238,7 @@ bool GlobalValue::isDeclaration() const { return false; } -bool GlobalValue::canIncreaseAlignment() const { +bool GlobalObject::canIncreaseAlignment() const { // Firstly, can only increase the alignment of a global if it // is a strong definition. if (!isStrongDefinitionForLinker()) @@ -410,7 +406,6 @@ void GlobalVariable::setInitializer(Constant *InitVal) { /// from the GlobalVariable Src to this one. void GlobalVariable::copyAttributesFrom(const GlobalVariable *Src) { GlobalObject::copyAttributesFrom(Src); - setThreadLocalMode(Src->getThreadLocalMode()); setExternallyInitialized(Src->isExternallyInitialized()); setAttributes(Src->getAttributes()); } diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index 30b558a655cb..b87dfe1c8df6 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/NoFolder.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" @@ -49,7 +50,7 @@ GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, nullptr, GlobalVariable::NotThreadLocal, AddressSpace); GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - GV->setAlignment(Align::None()); + GV->setAlignment(Align(1)); return GV; } @@ -64,38 +65,20 @@ Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) { return Ptr; // Otherwise, we need to insert a bitcast. - PT = getInt8PtrTy(PT->getAddressSpace()); - BitCastInst *BCI = new BitCastInst(Ptr, PT, ""); - BB->getInstList().insert(InsertPt, BCI); - SetInstDebugLocation(BCI); - return BCI; + return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace())); } static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops, IRBuilderBase *Builder, const Twine &Name = "", - Instruction *FMFSource = nullptr) { - CallInst *CI = CallInst::Create(Callee, Ops, Name); + Instruction *FMFSource = nullptr, + ArrayRef<OperandBundleDef> OpBundles = {}) { + CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name); if (FMFSource) CI->copyFastMathFlags(FMFSource); - Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI); - Builder->SetInstDebugLocation(CI); return CI; } -static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest, - BasicBlock *UnwindDest, - ArrayRef<Value *> Ops, - IRBuilderBase *Builder, - const Twine &Name = "") { - InvokeInst *II = - InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name); - Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(), - II); - Builder->SetInstDebugLocation(II); - return II; -} - CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, @@ -152,16 +135,6 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet( return CI; } -CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, - unsigned SrcAlign, Value *Size, - bool isVolatile, MDNode *TBAATag, - MDNode *TBAAStructTag, MDNode *ScopeTag, - MDNode *NoAliasTag) { - return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), - Size, isVolatile, TBAATag, TBAAStructTag, ScopeTag, - NoAliasTag); -} - CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile, @@ -200,8 +173,32 @@ CallInst *IRBuilderBase::CreateMemCpy(Value *Dst, MaybeAlign DstAlign, return CI; } +CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, + Value *Src, MaybeAlign SrcAlign, + Value *Size) { + Dst = getCastedInt8PtrValue(Dst); + Src = getCastedInt8PtrValue(Src); + Value *IsVolatile = getInt1(false); + + Value *Ops[] = {Dst, Src, Size, IsVolatile}; + Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; + Function *F = BB->getParent(); + Module *M = F->getParent(); + Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys); + + CallInst *CI = createCallHelper(TheFn, Ops, this); + + auto *MCI = cast<MemCpyInlineInst>(CI); + if (DstAlign) + MCI->setDestAlignment(*DstAlign); + if (SrcAlign) + MCI->setSourceAlignment(*SrcAlign); + + return CI; +} + CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( - Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, + Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { assert(DstAlign >= ElementSize && @@ -276,7 +273,7 @@ CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign, } CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( - Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, + Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { assert(DstAlign >= ElementSize && @@ -295,10 +292,8 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( CallInst *CI = createCallHelper(TheFn, Ops, this); // Set the alignment of the pointer args. - CI->addParamAttr( - 0, Attribute::getWithAlignment(CI->getContext(), Align(DstAlign))); - CI->addParamAttr( - 1, Attribute::getWithAlignment(CI->getContext(), Align(SrcAlign))); + CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign)); + CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign)); // Set the TBAA info if present. if (TBAATag) @@ -455,25 +450,27 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { return createCallHelper(TheFn, Ops, this); } -CallInst *IRBuilderBase::CreateAssumption(Value *Cond) { +CallInst * +IRBuilderBase::CreateAssumption(Value *Cond, + ArrayRef<OperandBundleDef> OpBundles) { assert(Cond->getType() == getInt1Ty() && "an assumption condition must be of type i1"); Value *Ops[] = { Cond }; Module *M = BB->getParent()->getParent(); Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume); - return createCallHelper(FnAssume, Ops, this); + return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles); } /// Create a call to a Masked Load intrinsic. -/// \p Ptr - base pointer for the load -/// \p Align - alignment of the source location -/// \p Mask - vector of booleans which indicates what vector lanes should -/// be accessed in memory -/// \p PassThru - pass-through value that is used to fill the masked-off lanes -/// of the result -/// \p Name - name of the result variable -CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align, +/// \p Ptr - base pointer for the load +/// \p Alignment - alignment of the source location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// \p PassThru - pass-through value that is used to fill the masked-off lanes +/// of the result +/// \p Name - name of the result variable +CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment, Value *Mask, Value *PassThru, const Twine &Name) { auto *PtrTy = cast<PointerType>(Ptr->getType()); @@ -483,25 +480,25 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align, if (!PassThru) PassThru = UndefValue::get(DataTy); Type *OverloadedTypes[] = { DataTy, PtrTy }; - Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru}; + Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru}; return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name); } /// Create a call to a Masked Store intrinsic. -/// \p Val - data to be stored, -/// \p Ptr - base pointer for the store -/// \p Align - alignment of the destination location -/// \p Mask - vector of booleans which indicates what vector lanes should -/// be accessed in memory +/// \p Val - data to be stored, +/// \p Ptr - base pointer for the store +/// \p Alignment - alignment of the destination location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, - unsigned Align, Value *Mask) { + Align Alignment, Value *Mask) { auto *PtrTy = cast<PointerType>(Ptr->getType()); Type *DataTy = PtrTy->getElementType(); assert(DataTy->isVectorTy() && "Ptr should point to a vector"); assert(Mask && "Mask should not be all-ones (null)"); Type *OverloadedTypes[] = { DataTy, PtrTy }; - Value *Ops[] = { Val, Ptr, getInt32(Align), Mask }; + Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask}; return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes); } @@ -525,23 +522,23 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, /// \p PassThru - pass-through value that is used to fill the masked-off lanes /// of the result /// \p Name - name of the result variable -CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align, - Value *Mask, Value *PassThru, - const Twine& Name) { +CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment, + Value *Mask, Value *PassThru, + const Twine &Name) { auto PtrsTy = cast<VectorType>(Ptrs->getType()); auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); - unsigned NumElts = PtrsTy->getVectorNumElements(); - Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts); + unsigned NumElts = PtrsTy->getNumElements(); + auto *DataTy = FixedVectorType::get(PtrTy->getElementType(), NumElts); if (!Mask) - Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context), - NumElts)); + Mask = Constant::getAllOnesValue( + FixedVectorType::get(Type::getInt1Ty(Context), NumElts)); if (!PassThru) PassThru = UndefValue::get(DataTy); Type *OverloadedTypes[] = {DataTy, PtrsTy}; - Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru}; + Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru}; // We specify only one type when we create this intrinsic. Types of other // arguments are derived from this type. @@ -557,36 +554,34 @@ CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align, /// \p Mask - vector of booleans which indicates what vector lanes should /// be accessed in memory CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, - unsigned Align, Value *Mask) { + Align Alignment, Value *Mask) { auto PtrsTy = cast<VectorType>(Ptrs->getType()); auto DataTy = cast<VectorType>(Data->getType()); - unsigned NumElts = PtrsTy->getVectorNumElements(); + unsigned NumElts = PtrsTy->getNumElements(); #ifndef NDEBUG auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); - assert(NumElts == DataTy->getVectorNumElements() && + assert(NumElts == DataTy->getNumElements() && PtrTy->getElementType() == DataTy->getElementType() && "Incompatible pointer and data types"); #endif if (!Mask) - Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context), - NumElts)); + Mask = Constant::getAllOnesValue( + FixedVectorType::get(Type::getInt1Ty(Context), NumElts)); Type *OverloadedTypes[] = {DataTy, PtrsTy}; - Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask}; + Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask}; // We specify only one type when we create this intrinsic. Types of other // arguments are derived from this type. return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes); } -template <typename T0, typename T1, typename T2, typename T3> +template <typename T0> static std::vector<Value *> getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, - Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs, - ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, - ArrayRef<T3> GCArgs) { + Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) { std::vector<Value *> Args; Args.push_back(B.getInt64(ID)); Args.push_back(B.getInt32(NumPatchBytes)); @@ -594,20 +589,45 @@ getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, Args.push_back(B.getInt32(CallArgs.size())); Args.push_back(B.getInt32(Flags)); Args.insert(Args.end(), CallArgs.begin(), CallArgs.end()); - Args.push_back(B.getInt32(TransitionArgs.size())); - Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end()); - Args.push_back(B.getInt32(DeoptArgs.size())); - Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end()); - Args.insert(Args.end(), GCArgs.begin(), GCArgs.end()); - + // GC Transition and Deopt args are now always handled via operand bundle. + // They will be removed from the signature of gc.statepoint shortly. + Args.push_back(B.getInt32(0)); + Args.push_back(B.getInt32(0)); + // GC args are now encoded in the gc-live operand bundle return Args; } +template<typename T1, typename T2, typename T3> +static std::vector<OperandBundleDef> +getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs, + Optional<ArrayRef<T2>> DeoptArgs, + ArrayRef<T3> GCArgs) { + std::vector<OperandBundleDef> Rval; + if (DeoptArgs) { + SmallVector<Value*, 16> DeoptValues; + DeoptValues.insert(DeoptValues.end(), DeoptArgs->begin(), DeoptArgs->end()); + Rval.emplace_back("deopt", DeoptValues); + } + if (TransitionArgs) { + SmallVector<Value*, 16> TransitionValues; + TransitionValues.insert(TransitionValues.end(), + TransitionArgs->begin(), TransitionArgs->end()); + Rval.emplace_back("gc-transition", TransitionValues); + } + if (GCArgs.size()) { + SmallVector<Value*, 16> LiveValues; + LiveValues.insert(LiveValues.end(), GCArgs.begin(), GCArgs.end()); + Rval.emplace_back("gc-live", LiveValues); + } + return Rval; +} + template <typename T0, typename T1, typename T2, typename T3> static CallInst *CreateGCStatepointCallCommon( IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs, - ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, + Optional<ArrayRef<T1>> TransitionArgs, + Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) { // Extract out the type of the callee. auto *FuncPtrType = cast<PointerType>(ActualCallee->getType()); @@ -623,13 +643,17 @@ static CallInst *CreateGCStatepointCallCommon( std::vector<Value *> Args = getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags, - CallArgs, TransitionArgs, DeoptArgs, GCArgs); - return createCallHelper(FnStatepoint, Args, Builder, Name); + CallArgs); + + return Builder->CreateCall(FnStatepoint, Args, + getStatepointBundles(TransitionArgs, DeoptArgs, + GCArgs), + Name); } CallInst *IRBuilderBase::CreateGCStatepointCall( uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, - ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs, + ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>( this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), @@ -638,8 +662,9 @@ CallInst *IRBuilderBase::CreateGCStatepointCall( CallInst *IRBuilderBase::CreateGCStatepointCall( uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags, - ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs, - ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { + ArrayRef<Use> CallArgs, Optional<ArrayRef<Use>> TransitionArgs, + Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, + const Twine &Name) { return CreateGCStatepointCallCommon<Use, Use, Use, Value *>( this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs, Name); @@ -647,7 +672,7 @@ CallInst *IRBuilderBase::CreateGCStatepointCall( CallInst *IRBuilderBase::CreateGCStatepointCall( uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, - ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs, + ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>( this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), @@ -658,8 +683,9 @@ template <typename T0, typename T1, typename T2, typename T3> static InvokeInst *CreateGCStatepointInvokeCommon( IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, - uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs, - ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) { + uint32_t Flags, ArrayRef<T0> InvokeArgs, + Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs, + ArrayRef<T3> GCArgs, const Twine &Name) { // Extract out the type of the callee. auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType()); assert(isa<FunctionType>(FuncPtrType->getElementType()) && @@ -672,15 +698,18 @@ static InvokeInst *CreateGCStatepointInvokeCommon( std::vector<Value *> Args = getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags, - InvokeArgs, TransitionArgs, DeoptArgs, GCArgs); - return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder, - Name); + InvokeArgs); + + return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args, + getStatepointBundles(TransitionArgs, DeoptArgs, + GCArgs), + Name); } InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, - ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs, + ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>( this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, @@ -691,8 +720,8 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, - ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs, - ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { + ArrayRef<Use> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs, + Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>( this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags, InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name); @@ -701,7 +730,7 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, - ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { + Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) { return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>( this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs, @@ -762,3 +791,360 @@ CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID, Function *Fn = Intrinsic::getDeclaration(M, ID, Types); return createCallHelper(Fn, Args, this, Name, FMFSource); } + +CallInst *IRBuilderBase::CreateConstrainedFPBinOp( + Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, + const Twine &Name, MDNode *FPMathTag, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + Value *RoundingV = getConstrainedFPRounding(Rounding); + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, RoundingV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + setFPAttrs(C, FPMathTag, UseFMF); + return C; +} + +Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, + const Twine &Name, MDNode *FPMathTag) { + if (Instruction::isBinaryOp(Opc)) { + assert(Ops.size() == 2 && "Invalid number of operands!"); + return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), + Ops[0], Ops[1], Name, FPMathTag); + } + if (Instruction::isUnaryOp(Opc)) { + assert(Ops.size() == 1 && "Invalid number of operands!"); + return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), + Ops[0], Name, FPMathTag); + } + llvm_unreachable("Unexpected opcode!"); +} + +CallInst *IRBuilderBase::CreateConstrainedFPCast( + Intrinsic::ID ID, Value *V, Type *DestTy, + Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C; + bool HasRoundingMD = false; + switch (ID) { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) { + Value *RoundingV = getConstrainedFPRounding(Rounding); + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, + nullptr, Name); + } else + C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, + Name); + + setConstrainedFPCallAttr(C); + + if (isa<FPMathOperator>(C)) + setFPAttrs(C, FPMathTag, UseFMF); + return C; +} + +Value *IRBuilderBase::CreateFCmpHelper( + CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name, + MDNode *FPMathTag, bool IsSignaling) { + if (IsFPConstrained) { + auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps + : Intrinsic::experimental_constrained_fcmp; + return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name); + } + + if (auto *LC = dyn_cast<Constant>(LHS)) + if (auto *RC = dyn_cast<Constant>(RHS)) + return Insert(Folder.CreateFCmp(P, LC, RC), Name); + return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); +} + +CallInst *IRBuilderBase::CreateConstrainedFPCmp( + Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, + const Twine &Name, Optional<fp::ExceptionBehavior> Except) { + Value *PredicateV = getConstrainedFPPredicate(P); + Value *ExceptV = getConstrainedFPExcept(Except); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, PredicateV, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + return C; +} + +CallInst *IRBuilderBase::CreateConstrainedFPCall( + Function *Callee, ArrayRef<Value *> Args, const Twine &Name, + Optional<RoundingMode> Rounding, + Optional<fp::ExceptionBehavior> Except) { + llvm::SmallVector<Value *, 6> UseArgs; + + for (auto *OneArg : Args) + UseArgs.push_back(OneArg); + bool HasRoundingMD = false; + switch (Callee->getIntrinsicID()) { + default: + break; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + HasRoundingMD = ROUND_MODE; \ + break; +#include "llvm/IR/ConstrainedOps.def" + } + if (HasRoundingMD) + UseArgs.push_back(getConstrainedFPRounding(Rounding)); + UseArgs.push_back(getConstrainedFPExcept(Except)); + + CallInst *C = CreateCall(Callee, UseArgs, Name); + setConstrainedFPCallAttr(C); + return C; +} + +Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False, + const Twine &Name, Instruction *MDFrom) { + if (auto *CC = dyn_cast<Constant>(C)) + if (auto *TC = dyn_cast<Constant>(True)) + if (auto *FC = dyn_cast<Constant>(False)) + return Insert(Folder.CreateSelect(CC, TC, FC), Name); + + SelectInst *Sel = SelectInst::Create(C, True, False); + if (MDFrom) { + MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); + MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); + Sel = addBranchMetadata(Sel, Prof, Unpred); + } + if (isa<FPMathOperator>(Sel)) + setFPAttrs(Sel, nullptr /* MDNode* */, FMF); + return Insert(Sel, Name); +} + +Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, + const Twine &Name) { + assert(LHS->getType() == RHS->getType() && + "Pointer subtraction operand types must match!"); + auto *ArgType = cast<PointerType>(LHS->getType()); + Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); + Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); + Value *Difference = CreateSub(LHS_int, RHS_int); + return CreateExactSDiv(Difference, + ConstantExpr::getSizeOf(ArgType->getElementType()), + Name); +} + +Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) { + assert(isa<PointerType>(Ptr->getType()) && + "launder.invariant.group only applies to pointers."); + // FIXME: we could potentially avoid casts to/from i8*. + auto *PtrType = Ptr->getType(); + auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); + if (PtrType != Int8PtrTy) + Ptr = CreateBitCast(Ptr, Int8PtrTy); + Module *M = BB->getParent()->getParent(); + Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( + M, Intrinsic::launder_invariant_group, {Int8PtrTy}); + + assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && + FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == + Int8PtrTy && + "LaunderInvariantGroup should take and return the same type"); + + CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); + + if (PtrType != Int8PtrTy) + return CreateBitCast(Fn, PtrType); + return Fn; +} + +Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { + assert(isa<PointerType>(Ptr->getType()) && + "strip.invariant.group only applies to pointers."); + + // FIXME: we could potentially avoid casts to/from i8*. + auto *PtrType = Ptr->getType(); + auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); + if (PtrType != Int8PtrTy) + Ptr = CreateBitCast(Ptr, Int8PtrTy); + Module *M = BB->getParent()->getParent(); + Function *FnStripInvariantGroup = Intrinsic::getDeclaration( + M, Intrinsic::strip_invariant_group, {Int8PtrTy}); + + assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy && + FnStripInvariantGroup->getFunctionType()->getParamType(0) == + Int8PtrTy && + "StripInvariantGroup should take and return the same type"); + + CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); + + if (PtrType != Int8PtrTy) + return CreateBitCast(Fn, PtrType); + return Fn; +} + +Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V, + const Twine &Name) { + assert(NumElts > 0 && "Cannot splat to an empty vector!"); + + // First insert it into an undef vector so we can shuffle it. + Type *I32Ty = getInt32Ty(); + Value *Undef = UndefValue::get(FixedVectorType::get(V->getType(), NumElts)); + V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0), + Name + ".splatinsert"); + + // Shuffle the value across the desired number of elements. + Value *Zeros = + ConstantAggregateZero::get(FixedVectorType::get(I32Ty, NumElts)); + return CreateShuffleVector(V, Undef, Zeros, Name + ".splat"); +} + +Value *IRBuilderBase::CreateExtractInteger( + const DataLayout &DL, Value *From, IntegerType *ExtractedTy, + uint64_t Offset, const Twine &Name) { + auto *IntTy = cast<IntegerType>(From->getType()); + assert(DL.getTypeStoreSize(ExtractedTy) + Offset <= + DL.getTypeStoreSize(IntTy) && + "Element extends past full value"); + uint64_t ShAmt = 8 * Offset; + Value *V = From; + if (DL.isBigEndian()) + ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - + DL.getTypeStoreSize(ExtractedTy) - Offset); + if (ShAmt) { + V = CreateLShr(V, ShAmt, Name + ".shift"); + } + assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && + "Cannot extract to a larger integer!"); + if (ExtractedTy != IntTy) { + V = CreateTrunc(V, ExtractedTy, Name + ".trunc"); + } + return V; +} + +Value *IRBuilderBase::CreatePreserveArrayAccessIndex( + Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, + MDNode *DbgInfo) { + assert(isa<PointerType>(Base->getType()) && + "Invalid Base ptr type for preserve.array.access.index."); + auto *BaseType = Base->getType(); + + Value *LastIndexV = getInt32(LastIndex); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + SmallVector<Value *, 4> IdxList; + for (unsigned I = 0; I < Dimension; ++I) + IdxList.push_back(Zero); + IdxList.push_back(LastIndexV); + + Type *ResultType = + GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); + + Value *DimV = getInt32(Dimension); + CallInst *Fn = + CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +Value *IRBuilderBase::CreatePreserveUnionAccessIndex( + Value *Base, unsigned FieldIndex, MDNode *DbgInfo) { + assert(isa<PointerType>(Base->getType()) && + "Invalid Base ptr type for preserve.union.access.index."); + auto *BaseType = Base->getType(); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = + CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +Value *IRBuilderBase::CreatePreserveStructAccessIndex( + Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, + MDNode *DbgInfo) { + assert(isa<PointerType>(Base->getType()) && + "Invalid Base ptr type for preserve.struct.access.index."); + auto *BaseType = Base->getType(); + + Value *GEPIndex = getInt32(Index); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + Type *ResultType = + GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, + {Base, GEPIndex, DIIndex}); + if (DbgInfo) + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; +} + +CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL, + Value *PtrValue, + Value *AlignValue, + Value *OffsetValue) { + SmallVector<Value *, 4> Vals({PtrValue, AlignValue}); + if (OffsetValue) + Vals.push_back(OffsetValue); + OperandBundleDefT<Value *> AlignOpB("align", Vals); + return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB}); +} + +CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, + Value *PtrValue, + unsigned Alignment, + Value *OffsetValue) { + assert(isa<PointerType>(PtrValue->getType()) && + "trying to create an alignment assumption on a non-pointer?"); + assert(Alignment != 0 && "Invalid Alignment"); + auto *PtrTy = cast<PointerType>(PtrValue->getType()); + Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); + Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment); + return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue); +} + +CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, + Value *PtrValue, + Value *Alignment, + Value *OffsetValue) { + assert(isa<PointerType>(PtrValue->getType()) && + "trying to create an alignment assumption on a non-pointer?"); + return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue); +} + +IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {} +IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {} +IRBuilderFolder::~IRBuilderFolder() {} +void ConstantFolder::anchor() {} +void NoFolder::anchor() {} diff --git a/llvm/lib/IR/InlineAsm.cpp b/llvm/lib/IR/InlineAsm.cpp index fd732f9eda8b..ee30b92522d0 100644 --- a/llvm/lib/IR/InlineAsm.cpp +++ b/llvm/lib/IR/InlineAsm.cpp @@ -136,14 +136,14 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, // Find the end of the register name. StringRef::iterator ConstraintEnd = std::find(I+1, E, '}'); if (ConstraintEnd == E) return true; // "{foo" - pCodes->push_back(StringRef(I, ConstraintEnd+1 - I)); + pCodes->push_back(std::string(StringRef(I, ConstraintEnd + 1 - I))); I = ConstraintEnd+1; } else if (isdigit(static_cast<unsigned char>(*I))) { // Matching Constraint // Maximal munch numbers. StringRef::iterator NumStart = I; while (I != E && isdigit(static_cast<unsigned char>(*I))) ++I; - pCodes->push_back(StringRef(NumStart, I - NumStart)); + pCodes->push_back(std::string(StringRef(NumStart, I - NumStart))); unsigned N = atoi(pCodes->back().c_str()); // Check that this is a valid matching constraint! if (N >= ConstraintsSoFar.size() || ConstraintsSoFar[N].Type != isOutput|| @@ -179,7 +179,7 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, } else if (*I == '^') { // Multi-letter constraint // FIXME: For now assuming these are 2-character constraints. - pCodes->push_back(StringRef(I+1, 2)); + pCodes->push_back(std::string(StringRef(I + 1, 2))); I += 3; } else if (*I == '@') { // Multi-letter constraint @@ -189,11 +189,11 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, int N = C - '0'; assert(N > 0 && "Found a zero letter constraint!"); ++I; - pCodes->push_back(StringRef(I, N)); + pCodes->push_back(std::string(StringRef(I, N))); I += N; } else { // Single letter constraint. - pCodes->push_back(StringRef(I, 1)); + pCodes->push_back(std::string(StringRef(I, 1))); ++I; } } diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index 7da169712896..bfbd801cb7a7 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -43,6 +43,19 @@ Instruction::Instruction(Type *ty, unsigned it, Use *Ops, unsigned NumOps, Instruction::~Instruction() { assert(!Parent && "Instruction still linked in the program!"); + + // Replace any extant metadata uses of this instruction with undef to + // preserve debug info accuracy. Some alternatives include: + // - Treat Instruction like any other Value, and point its extant metadata + // uses to an empty ValueAsMetadata node. This makes extant dbg.value uses + // trivially dead (i.e. fair game for deletion in many passes), leading to + // stale dbg.values being in effect for too long. + // - Call salvageDebugInfoOrMarkUndef. Not needed to make instruction removal + // correct. OTOH results in wasted work in some common cases (e.g. when all + // instructions in a BasicBlock are deleted). + if (isUsedByMetadata()) + ValueAsMetadata::handleRAUW(this, UndefValue::get(getType())); + if (hasMetadataHashEntry()) clearMetadataHashEntries(); } @@ -97,6 +110,15 @@ void Instruction::moveBefore(BasicBlock &BB, BB.getInstList().splice(I, getParent()->getInstList(), getIterator()); } +bool Instruction::comesBefore(const Instruction *Other) const { + assert(Parent && Other->Parent && + "instructions without BB parents have no order"); + assert(Parent == Other->Parent && "cross-BB instruction order comparison"); + if (!Parent->isInstrOrderValid()) + Parent->renumberInstructions(); + return Order < Other->Order; +} + void Instruction::setHasNoUnsignedWrap(bool b) { cast<OverflowingBinaryOperator>(this)->setHasNoUnsignedWrap(b); } @@ -176,6 +198,11 @@ void Instruction::setHasAllowReciprocal(bool B) { cast<FPMathOperator>(this)->setHasAllowReciprocal(B); } +void Instruction::setHasAllowContract(bool B) { + assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); + cast<FPMathOperator>(this)->setHasAllowContract(B); +} + void Instruction::setHasApproxFunc(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasApproxFunc(B); @@ -434,6 +461,9 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() && RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() && RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID(); + if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1)) + return SVI->getShuffleMask() == + cast<ShuffleVectorInst>(I2)->getShuffleMask(); return true; } @@ -744,12 +774,3 @@ Instruction *Instruction::clone() const { New->copyMetadata(*this); return New; } - -void Instruction::setProfWeight(uint64_t W) { - assert(isa<CallBase>(this) && - "Can only set weights for call like instructions"); - SmallVector<uint32_t, 1> Weights; - Weights.push_back(W); - MDBuilder MDB(getContext()); - setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights)); -} diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index c264277fa53c..2f17a0d73af4 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -18,7 +18,6 @@ #include "llvm/ADT/Twine.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -54,7 +53,7 @@ Optional<uint64_t> AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const { uint64_t Size = DL.getTypeAllocSizeInBits(getAllocatedType()); if (isArrayAllocation()) { - auto C = dyn_cast<ConstantInt>(getArraySize()); + auto *C = dyn_cast<ConstantInt>(getArraySize()); if (!C) return None; Size *= C->getZExtValue(); @@ -63,14 +62,6 @@ AllocaInst::getAllocationSizeInBits(const DataLayout &DL) const { } //===----------------------------------------------------------------------===// -// CallSite Class -//===----------------------------------------------------------------------===// - -User::op_iterator CallSite::getCallee() const { - return cast<CallBase>(getInstruction())->op_end() - 1; -} - -//===----------------------------------------------------------------------===// // SelectInst Class //===----------------------------------------------------------------------===// @@ -90,7 +81,7 @@ const char *SelectInst::areInvalidOperands(Value *Op0, Value *Op1, Value *Op2) { VectorType *ET = dyn_cast<VectorType>(Op1->getType()); if (!ET) return "selected values for vector select must be vectors"; - if (ET->getNumElements() != VT->getNumElements()) + if (ET->getElementCount() != VT->getElementCount()) return "vector select requires selected vectors to have " "the same vector length as select condition"; } else if (Op0->getType() != Type::getInt1Ty(Op0->getContext())) { @@ -256,6 +247,20 @@ void LandingPadInst::addClause(Constant *Val) { // CallBase Implementation //===----------------------------------------------------------------------===// +CallBase *CallBase::Create(CallBase *CB, ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertPt) { + switch (CB->getOpcode()) { + case Instruction::Call: + return CallInst::Create(cast<CallInst>(CB), Bundles, InsertPt); + case Instruction::Invoke: + return InvokeInst::Create(cast<InvokeInst>(CB), Bundles, InsertPt); + case Instruction::CallBr: + return CallBrInst::Create(cast<CallBrInst>(CB), Bundles, InsertPt); + default: + llvm_unreachable("Unknown CallBase sub-class!"); + } +} + Function *CallBase::getCaller() { return getParent()->getParent(); } unsigned CallBase::getNumSubclassExtraOperandsDynamic() const { @@ -264,13 +269,10 @@ unsigned CallBase::getNumSubclassExtraOperandsDynamic() const { } bool CallBase::isIndirectCall() const { - const Value *V = getCalledValue(); + const Value *V = getCalledOperand(); if (isa<Function>(V) || isa<Constant>(V)) return false; - if (const CallInst *CI = dyn_cast<CallInst>(this)) - if (CI->isInlineAsm()) - return false; - return true; + return !isInlineAsm(); } /// Tests if this call site must be tail call optimized. Only a CallInst can @@ -342,16 +344,22 @@ bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const { bool CallBase::hasFnAttrOnCalledFunction(Attribute::AttrKind Kind) const { if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(AttributeList::FunctionIndex, Kind); + return F->getAttributes().hasFnAttribute(Kind); return false; } bool CallBase::hasFnAttrOnCalledFunction(StringRef Kind) const { if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(AttributeList::FunctionIndex, Kind); + return F->getAttributes().hasFnAttribute(Kind); return false; } +void CallBase::getOperandBundlesAsDefs( + SmallVectorImpl<OperandBundleDef> &Defs) const { + for (unsigned i = 0, e = getNumOperandBundles(); i != e; ++i) + Defs.emplace_back(getOperandBundleAt(i)); +} + CallBase::op_iterator CallBase::populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles, const unsigned BeginIndex) { @@ -378,6 +386,53 @@ CallBase::populateBundleOperandInfos(ArrayRef<OperandBundleDef> Bundles, return It; } +CallBase::BundleOpInfo &CallBase::getBundleOpInfoForOperand(unsigned OpIdx) { + /// When there isn't many bundles, we do a simple linear search. + /// Else fallback to a binary-search that use the fact that bundles usually + /// have similar number of argument to get faster convergence. + if (bundle_op_info_end() - bundle_op_info_begin() < 8) { + for (auto &BOI : bundle_op_infos()) + if (BOI.Begin <= OpIdx && OpIdx < BOI.End) + return BOI; + + llvm_unreachable("Did not find operand bundle for operand!"); + } + + assert(OpIdx >= arg_size() && "the Idx is not in the operand bundles"); + assert(bundle_op_info_end() - bundle_op_info_begin() > 0 && + OpIdx < std::prev(bundle_op_info_end())->End && + "The Idx isn't in the operand bundle"); + + /// We need a decimal number below and to prevent using floating point numbers + /// we use an intergal value multiplied by this constant. + constexpr unsigned NumberScaling = 1024; + + bundle_op_iterator Begin = bundle_op_info_begin(); + bundle_op_iterator End = bundle_op_info_end(); + bundle_op_iterator Current; + + while (Begin != End) { + unsigned ScaledOperandPerBundle = + NumberScaling * (std::prev(End)->End - Begin->Begin) / (End - Begin); + Current = Begin + (((OpIdx - Begin->Begin) * NumberScaling) / + ScaledOperandPerBundle); + if (Current >= End) + Current = std::prev(End); + assert(Current < End && Current >= Begin && + "the operand bundle doesn't cover every value in the range"); + if (OpIdx >= Current->Begin && OpIdx < Current->End) + break; + if (OpIdx >= Current->End) + Begin = Current + 1; + else + End = Current; + } + + assert(OpIdx >= Current->Begin && OpIdx < Current->End && + "the operand bundle doesn't cover every value in the range"); + return *Current; +} + //===----------------------------------------------------------------------===// // CallInst Implementation //===----------------------------------------------------------------------===// @@ -450,7 +505,7 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB, Instruction *InsertPt) { std::vector<Value *> Args(CI->arg_begin(), CI->arg_end()); - auto *NewCI = CallInst::Create(CI->getFunctionType(), CI->getCalledValue(), + auto *NewCI = CallInst::Create(CI->getFunctionType(), CI->getCalledOperand(), Args, OpB, CI->getName(), InsertPt); NewCI->setTailCallKind(CI->getTailCallKind()); NewCI->setCallingConv(CI->getCallingConv()); @@ -761,9 +816,9 @@ InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB, Instruction *InsertPt) { std::vector<Value *> Args(II->arg_begin(), II->arg_end()); - auto *NewII = InvokeInst::Create(II->getFunctionType(), II->getCalledValue(), - II->getNormalDest(), II->getUnwindDest(), - Args, OpB, II->getName(), InsertPt); + auto *NewII = InvokeInst::Create( + II->getFunctionType(), II->getCalledOperand(), II->getNormalDest(), + II->getUnwindDest(), Args, OpB, II->getName(), InsertPt); NewII->setCallingConv(II->getCallingConv()); NewII->SubclassOptionalData = II->SubclassOptionalData; NewII->setAttributes(II->getAttributes()); @@ -844,11 +899,9 @@ CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB, Instruction *InsertPt) { std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end()); - auto *NewCBI = CallBrInst::Create(CBI->getFunctionType(), - CBI->getCalledValue(), - CBI->getDefaultDest(), - CBI->getIndirectDests(), - Args, OpB, CBI->getName(), InsertPt); + auto *NewCBI = CallBrInst::Create( + CBI->getFunctionType(), CBI->getCalledOperand(), CBI->getDefaultDest(), + CBI->getIndirectDests(), Args, OpB, CBI->getName(), InsertPt); NewCBI->setCallingConv(CBI->getCallingConv()); NewCBI->SubclassOptionalData = CBI->SubclassOptionalData; NewCBI->setAttributes(CBI->getAttributes()); @@ -921,7 +974,8 @@ CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI) OperandTraits<CleanupReturnInst>::op_end(this) - CRI.getNumOperands(), CRI.getNumOperands()) { - setInstructionSubclassData(CRI.getSubclassDataFromInstruction()); + setSubclassData<Instruction::OpaqueField>( + CRI.getSubclassData<Instruction::OpaqueField>()); Op<0>() = CRI.Op<0>(); if (CRI.hasUnwindDest()) Op<1>() = CRI.Op<1>(); @@ -929,7 +983,7 @@ CleanupReturnInst::CleanupReturnInst(const CleanupReturnInst &CRI) void CleanupReturnInst::init(Value *CleanupPad, BasicBlock *UnwindBB) { if (UnwindBB) - setInstructionSubclassData(getSubclassDataFromInstruction() | 1); + setSubclassData<UnwindDestField>(true); Op<0>() = CleanupPad; if (UnwindBB) @@ -1033,7 +1087,7 @@ void CatchSwitchInst::init(Value *ParentPad, BasicBlock *UnwindDest, Op<0>() = ParentPad; if (UnwindDest) { - setInstructionSubclassData(getSubclassDataFromInstruction() | 1); + setSubclassData<UnwindDestField>(true); setUnwindDest(UnwindDest); } } @@ -1207,6 +1261,19 @@ static Value *getAISize(LLVMContext &Context, Value *Amt) { return Amt; } +static Align computeAllocaDefaultAlign(Type *Ty, BasicBlock *BB) { + assert(BB && "Insertion BB cannot be null when alignment not provided!"); + assert(BB->getParent() && + "BB must be in a Function when alignment not provided!"); + const DataLayout &DL = BB->getModule()->getDataLayout(); + return DL.getPrefTypeAlign(Ty); +} + +static Align computeAllocaDefaultAlign(Type *Ty, Instruction *I) { + assert(I && "Insertion position cannot be null when alignment not provided!"); + return computeAllocaDefaultAlign(Ty, I->getParent()); +} + AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name, Instruction *InsertBefore) : AllocaInst(Ty, AddrSpace, /*ArraySize=*/nullptr, Name, InsertBefore) {} @@ -1217,27 +1284,29 @@ AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name, AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, const Twine &Name, Instruction *InsertBefore) - : AllocaInst(Ty, AddrSpace, ArraySize, /*Align=*/None, Name, InsertBefore) { -} + : AllocaInst(Ty, AddrSpace, ArraySize, + computeAllocaDefaultAlign(Ty, InsertBefore), Name, + InsertBefore) {} AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, const Twine &Name, BasicBlock *InsertAtEnd) - : AllocaInst(Ty, AddrSpace, ArraySize, /*Align=*/None, Name, InsertAtEnd) {} + : AllocaInst(Ty, AddrSpace, ArraySize, + computeAllocaDefaultAlign(Ty, InsertAtEnd), Name, + InsertAtEnd) {} AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, - MaybeAlign Align, const Twine &Name, + Align Align, const Twine &Name, Instruction *InsertBefore) : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca, getAISize(Ty->getContext(), ArraySize), InsertBefore), AllocatedType(Ty) { - setAlignment(MaybeAlign(Align)); + setAlignment(Align); assert(!Ty->isVoidTy() && "Cannot allocate void!"); setName(Name); } AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, - MaybeAlign Align, const Twine &Name, - BasicBlock *InsertAtEnd) + Align Align, const Twine &Name, BasicBlock *InsertAtEnd) : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca, getAISize(Ty->getContext(), ArraySize), InsertAtEnd), AllocatedType(Ty) { @@ -1246,17 +1315,6 @@ AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, setName(Name); } -void AllocaInst::setAlignment(MaybeAlign Align) { - assert((!Align || *Align <= MaximumAlignment) && - "Alignment is greater than MaximumAlignment!"); - setInstructionSubclassData((getSubclassDataFromInstruction() & ~31) | - encode(Align)); - if (Align) - assert(getAlignment() == Align->value() && - "Alignment representation error!"); - else - assert(getAlignment() == 0 && "Alignment representation error!"); -} bool AllocaInst::isArrayAllocation() const { if (ConstantInt *CI = dyn_cast<ConstantInt>(getOperand(0))) @@ -1287,6 +1345,19 @@ void LoadInst::AssertOK() { "Alignment required for atomic load"); } +static Align computeLoadStoreDefaultAlign(Type *Ty, BasicBlock *BB) { + assert(BB && "Insertion BB cannot be null when alignment not provided!"); + assert(BB->getParent() && + "BB must be in a Function when alignment not provided!"); + const DataLayout &DL = BB->getModule()->getDataLayout(); + return DL.getABITypeAlign(Ty); +} + +static Align computeLoadStoreDefaultAlign(Type *Ty, Instruction *I) { + assert(I && "Insertion position cannot be null when alignment not provided!"); + return computeLoadStoreDefaultAlign(Ty, I->getParent()); +} + LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, Instruction *InsertBef) : LoadInst(Ty, Ptr, Name, /*isVolatile=*/false, InsertBef) {} @@ -1297,36 +1368,38 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, Instruction *InsertBef) - : LoadInst(Ty, Ptr, Name, isVolatile, /*Align=*/None, InsertBef) {} + : LoadInst(Ty, Ptr, Name, isVolatile, + computeLoadStoreDefaultAlign(Ty, InsertBef), InsertBef) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, BasicBlock *InsertAE) - : LoadInst(Ty, Ptr, Name, isVolatile, /*Align=*/None, InsertAE) {} + : LoadInst(Ty, Ptr, Name, isVolatile, + computeLoadStoreDefaultAlign(Ty, InsertAE), InsertAE) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, - MaybeAlign Align, Instruction *InsertBef) + Align Align, Instruction *InsertBef) : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, SyncScope::System, InsertBef) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, - MaybeAlign Align, BasicBlock *InsertAE) + Align Align, BasicBlock *InsertAE) : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, SyncScope::System, InsertAE) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, - MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, + Align Align, AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBef) : UnaryInstruction(Ty, Load, Ptr, InsertBef) { assert(Ty == cast<PointerType>(Ptr->getType())->getElementType()); setVolatile(isVolatile); - setAlignment(MaybeAlign(Align)); + setAlignment(Align); setAtomic(Order, SSID); AssertOK(); setName(Name); } LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, - MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, + Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAE) : UnaryInstruction(Ty, Load, Ptr, InsertAE) { assert(Ty == cast<PointerType>(Ptr->getType())->getElementType()); @@ -1337,14 +1410,6 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, setName(Name); } -void LoadInst::setAlignment(MaybeAlign Align) { - assert((!Align || *Align <= MaximumAlignment) && - "Alignment is greater than MaximumAlignment!"); - setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) | - (encode(Align) << 1)); - assert(getAlign() == Align && "Alignment representation error!"); -} - //===----------------------------------------------------------------------===// // StoreInst Implementation //===----------------------------------------------------------------------===// @@ -1368,23 +1433,27 @@ StoreInst::StoreInst(Value *val, Value *addr, BasicBlock *InsertAtEnd) StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Instruction *InsertBefore) - : StoreInst(val, addr, isVolatile, /*Align=*/None, InsertBefore) {} + : StoreInst(val, addr, isVolatile, + computeLoadStoreDefaultAlign(val->getType(), InsertBefore), + InsertBefore) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, BasicBlock *InsertAtEnd) - : StoreInst(val, addr, isVolatile, /*Align=*/None, InsertAtEnd) {} + : StoreInst(val, addr, isVolatile, + computeLoadStoreDefaultAlign(val->getType(), InsertAtEnd), + InsertAtEnd) {} -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, Instruction *InsertBefore) : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, SyncScope::System, InsertBefore) {} -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, BasicBlock *InsertAtEnd) : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, SyncScope::System, InsertAtEnd) {} -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Type::getVoidTy(val->getContext()), Store, @@ -1398,7 +1467,7 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, AssertOK(); } -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Type::getVoidTy(val->getContext()), Store, @@ -1412,20 +1481,13 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, MaybeAlign Align, AssertOK(); } -void StoreInst::setAlignment(MaybeAlign Alignment) { - assert((!Alignment || *Alignment <= MaximumAlignment) && - "Alignment is greater than MaximumAlignment!"); - setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) | - (encode(Alignment) << 1)); - assert(getAlign() == Alignment && "Alignment representation error!"); -} //===----------------------------------------------------------------------===// // AtomicCmpXchgInst Implementation //===----------------------------------------------------------------------===// void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, - AtomicOrdering SuccessOrdering, + Align Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID) { Op<0>() = Ptr; @@ -1434,6 +1496,7 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, setSuccessOrdering(SuccessOrdering); setFailureOrdering(FailureOrdering); setSyncScopeID(SSID); + setAlignment(Alignment); assert(getOperand(0) && getOperand(1) && getOperand(2) && "All operands must be non-null!"); @@ -1458,6 +1521,7 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, } AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, + Align Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID, @@ -1466,10 +1530,11 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())), AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this), OperandTraits<AtomicCmpXchgInst>::operands(this), InsertBefore) { - Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID); + Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID); } AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, + Align Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID, @@ -1478,7 +1543,7 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, StructType::get(Cmp->getType(), Type::getInt1Ty(Cmp->getContext())), AtomicCmpXchg, OperandTraits<AtomicCmpXchgInst>::op_begin(this), OperandTraits<AtomicCmpXchgInst>::operands(this), InsertAtEnd) { - Init(Ptr, Cmp, NewVal, SuccessOrdering, FailureOrdering, SSID); + Init(Ptr, Cmp, NewVal, Alignment, SuccessOrdering, FailureOrdering, SSID); } //===----------------------------------------------------------------------===// @@ -1486,13 +1551,14 @@ AtomicCmpXchgInst::AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, //===----------------------------------------------------------------------===// void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, + Align Alignment, AtomicOrdering Ordering, SyncScope::ID SSID) { Op<0>() = Ptr; Op<1>() = Val; setOperation(Operation); setOrdering(Ordering); setSyncScopeID(SSID); + setAlignment(Alignment); assert(getOperand(0) && getOperand(1) && "All operands must be non-null!"); @@ -1506,25 +1572,21 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, } AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, - SyncScope::ID SSID, - Instruction *InsertBefore) - : Instruction(Val->getType(), AtomicRMW, - OperandTraits<AtomicRMWInst>::op_begin(this), - OperandTraits<AtomicRMWInst>::operands(this), - InsertBefore) { - Init(Operation, Ptr, Val, Ordering, SSID); + Align Alignment, AtomicOrdering Ordering, + SyncScope::ID SSID, Instruction *InsertBefore) + : Instruction(Val->getType(), AtomicRMW, + OperandTraits<AtomicRMWInst>::op_begin(this), + OperandTraits<AtomicRMWInst>::operands(this), InsertBefore) { + Init(Operation, Ptr, Val, Alignment, Ordering, SSID); } AtomicRMWInst::AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, - AtomicOrdering Ordering, - SyncScope::ID SSID, - BasicBlock *InsertAtEnd) - : Instruction(Val->getType(), AtomicRMW, - OperandTraits<AtomicRMWInst>::op_begin(this), - OperandTraits<AtomicRMWInst>::operands(this), - InsertAtEnd) { - Init(Operation, Ptr, Val, Ordering, SSID); + Align Alignment, AtomicOrdering Ordering, + SyncScope::ID SSID, BasicBlock *InsertAtEnd) + : Instruction(Val->getType(), AtomicRMW, + OperandTraits<AtomicRMWInst>::op_begin(this), + OperandTraits<AtomicRMWInst>::operands(this), InsertAtEnd) { + Init(Operation, Ptr, Val, Alignment, Ordering, SSID); } StringRef AtomicRMWInst::getOperationName(BinOp Op) { @@ -1606,35 +1668,44 @@ GetElementPtrInst::GetElementPtrInst(const GetElementPtrInst &GEPI) SubclassOptionalData = GEPI.SubclassOptionalData; } -/// getIndexedType - Returns the type of the element that would be accessed with -/// a gep instruction with the specified parameters. -/// -/// The Idxs pointer should point to a continuous piece of memory containing the -/// indices, either as Value* or uint64_t. -/// -/// A null type is returned if the indices are invalid for the specified -/// pointer type. -/// -template <typename IndexTy> -static Type *getIndexedTypeInternal(Type *Agg, ArrayRef<IndexTy> IdxList) { - // Handle the special case of the empty set index set, which is always valid. - if (IdxList.empty()) - return Agg; - - // If there is at least one index, the top level type must be sized, otherwise - // it cannot be 'stepped over'. - if (!Agg->isSized()) +Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, Value *Idx) { + if (auto *Struct = dyn_cast<StructType>(Ty)) { + if (!Struct->indexValid(Idx)) + return nullptr; + return Struct->getTypeAtIndex(Idx); + } + if (!Idx->getType()->isIntOrIntVectorTy()) return nullptr; + if (auto *Array = dyn_cast<ArrayType>(Ty)) + return Array->getElementType(); + if (auto *Vector = dyn_cast<VectorType>(Ty)) + return Vector->getElementType(); + return nullptr; +} + +Type *GetElementPtrInst::getTypeAtIndex(Type *Ty, uint64_t Idx) { + if (auto *Struct = dyn_cast<StructType>(Ty)) { + if (Idx >= Struct->getNumElements()) + return nullptr; + return Struct->getElementType(Idx); + } + if (auto *Array = dyn_cast<ArrayType>(Ty)) + return Array->getElementType(); + if (auto *Vector = dyn_cast<VectorType>(Ty)) + return Vector->getElementType(); + return nullptr; +} - unsigned CurIdx = 1; - for (; CurIdx != IdxList.size(); ++CurIdx) { - CompositeType *CT = dyn_cast<CompositeType>(Agg); - if (!CT || CT->isPointerTy()) return nullptr; - IndexTy Index = IdxList[CurIdx]; - if (!CT->indexValid(Index)) return nullptr; - Agg = CT->getTypeAtIndex(Index); +template <typename IndexTy> +static Type *getIndexedTypeInternal(Type *Ty, ArrayRef<IndexTy> IdxList) { + if (IdxList.empty()) + return Ty; + for (IndexTy V : IdxList.slice(1)) { + Ty = GetElementPtrInst::getTypeAtIndex(Ty, V); + if (!Ty) + return Ty; } - return CurIdx == IdxList.size() ? Agg : nullptr; + return Ty; } Type *GetElementPtrInst::getIndexedType(Type *Ty, ArrayRef<Value *> IdxList) { @@ -1781,66 +1852,120 @@ bool InsertElementInst::isValidOperands(const Value *Vec, const Value *Elt, ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask, const Twine &Name, Instruction *InsertBefore) -: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(), - cast<VectorType>(Mask->getType())->getElementCount()), - ShuffleVector, - OperandTraits<ShuffleVectorInst>::op_begin(this), - OperandTraits<ShuffleVectorInst>::operands(this), - InsertBefore) { + : Instruction( + VectorType::get(cast<VectorType>(V1->getType())->getElementType(), + cast<VectorType>(Mask->getType())->getElementCount()), + ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this), + OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) { assert(isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"); + Op<0>() = V1; Op<1>() = V2; - Op<2>() = Mask; + SmallVector<int, 16> MaskArr; + getShuffleMask(cast<Constant>(Mask), MaskArr); + setShuffleMask(MaskArr); setName(Name); } ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask, + const Twine &Name, BasicBlock *InsertAtEnd) + : Instruction( + VectorType::get(cast<VectorType>(V1->getType())->getElementType(), + cast<VectorType>(Mask->getType())->getElementCount()), + ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this), + OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) { + assert(isValidOperands(V1, V2, Mask) && + "Invalid shuffle vector instruction operands!"); + + Op<0>() = V1; + Op<1>() = V2; + SmallVector<int, 16> MaskArr; + getShuffleMask(cast<Constant>(Mask), MaskArr); + setShuffleMask(MaskArr); + setName(Name); +} + +ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask, const Twine &Name, - BasicBlock *InsertAtEnd) -: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(), - cast<VectorType>(Mask->getType())->getElementCount()), - ShuffleVector, - OperandTraits<ShuffleVectorInst>::op_begin(this), - OperandTraits<ShuffleVectorInst>::operands(this), - InsertAtEnd) { + Instruction *InsertBefore) + : Instruction( + VectorType::get(cast<VectorType>(V1->getType())->getElementType(), + Mask.size(), isa<ScalableVectorType>(V1->getType())), + ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this), + OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) { + assert(isValidOperands(V1, V2, Mask) && + "Invalid shuffle vector instruction operands!"); + Op<0>() = V1; + Op<1>() = V2; + setShuffleMask(Mask); + setName(Name); +} + +ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask, + const Twine &Name, BasicBlock *InsertAtEnd) + : Instruction( + VectorType::get(cast<VectorType>(V1->getType())->getElementType(), + Mask.size(), isa<ScalableVectorType>(V1->getType())), + ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this), + OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) { assert(isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"); Op<0>() = V1; Op<1>() = V2; - Op<2>() = Mask; + setShuffleMask(Mask); setName(Name); } void ShuffleVectorInst::commute() { - int NumOpElts = Op<0>()->getType()->getVectorNumElements(); - int NumMaskElts = getMask()->getType()->getVectorNumElements(); - SmallVector<Constant*, 16> NewMask(NumMaskElts); - Type *Int32Ty = Type::getInt32Ty(getContext()); + int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements(); + int NumMaskElts = ShuffleMask.size(); + SmallVector<int, 16> NewMask(NumMaskElts); for (int i = 0; i != NumMaskElts; ++i) { int MaskElt = getMaskValue(i); - if (MaskElt == -1) { - NewMask[i] = UndefValue::get(Int32Ty); + if (MaskElt == UndefMaskElem) { + NewMask[i] = UndefMaskElem; continue; } assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask"); MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts; - NewMask[i] = ConstantInt::get(Int32Ty, MaskElt); + NewMask[i] = MaskElt; } - Op<2>() = ConstantVector::get(NewMask); + setShuffleMask(NewMask); Op<0>().swap(Op<1>()); } bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, + ArrayRef<int> Mask) { + // V1 and V2 must be vectors of the same type. + if (!isa<VectorType>(V1->getType()) || V1->getType() != V2->getType()) + return false; + + // Make sure the mask elements make sense. + int V1Size = cast<VectorType>(V1->getType())->getElementCount().Min; + for (int Elem : Mask) + if (Elem != UndefMaskElem && Elem >= V1Size * 2) + return false; + + if (isa<ScalableVectorType>(V1->getType())) + if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask)) + return false; + + return true; +} + +bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, const Value *Mask) { // V1 and V2 must be vectors of the same type. if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType()) return false; - // Mask must be vector of i32. + // Mask must be vector of i32, and must be the same kind of vector as the + // input vectors auto *MaskTy = dyn_cast<VectorType>(Mask->getType()); - if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32)) + if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) || + isa<ScalableVectorType>(MaskTy) != isa<ScalableVectorType>(V1->getType())) return false; // Check to see if Mask is valid. @@ -1868,31 +1993,17 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, return true; } - // The bitcode reader can create a place holder for a forward reference - // used as the shuffle mask. When this occurs, the shuffle mask will - // fall into this case and fail. To avoid this error, do this bit of - // ugliness to allow such a mask pass. - if (const auto *CE = dyn_cast<ConstantExpr>(Mask)) - if (CE->getOpcode() == Instruction::UserOp1) - return true; - return false; } -int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) { - assert(i < Mask->getType()->getVectorNumElements() && "Index out of range"); - if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) - return CDS->getElementAsInteger(i); - Constant *C = Mask->getAggregateElement(i); - if (isa<UndefValue>(C)) - return -1; - return cast<ConstantInt>(C)->getZExtValue(); -} - void ShuffleVectorInst::getShuffleMask(const Constant *Mask, SmallVectorImpl<int> &Result) { - unsigned NumElts = Mask->getType()->getVectorNumElements(); - + unsigned NumElts = cast<VectorType>(Mask->getType())->getElementCount().Min; + if (isa<ConstantAggregateZero>(Mask)) { + Result.resize(NumElts, 0); + return; + } + Result.reserve(NumElts); if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask)) { for (unsigned i = 0; i != NumElts; ++i) Result.push_back(CDS->getElementAsInteger(i)); @@ -1905,6 +2016,30 @@ void ShuffleVectorInst::getShuffleMask(const Constant *Mask, } } +void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) { + ShuffleMask.assign(Mask.begin(), Mask.end()); + ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType()); +} +Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask, + Type *ResultTy) { + Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext()); + if (isa<ScalableVectorType>(ResultTy)) { + assert(is_splat(Mask) && "Unexpected shuffle"); + Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true); + if (Mask[0] == 0) + return Constant::getNullValue(VecTy); + return UndefValue::get(VecTy); + } + SmallVector<Constant *, 16> MaskConst; + for (int Elem : Mask) { + if (Elem == UndefMaskElem) + MaskConst.push_back(UndefValue::get(Int32Ty)); + else + MaskConst.push_back(ConstantInt::get(Int32Ty, Elem)); + } + return ConstantVector::get(MaskConst); +} + static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) { assert(!Mask.empty() && "Shuffle mask must contain elements"); bool UsesLHS = false; @@ -1919,8 +2054,8 @@ static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) { if (UsesLHS && UsesRHS) return false; } - assert((UsesLHS ^ UsesRHS) && "Should have selected from exactly 1 source"); - return true; + // Allow for degenerate case: completely undef mask means neither source is used. + return UsesLHS || UsesRHS; } bool ShuffleVectorInst::isSingleSourceMask(ArrayRef<int> Mask) { @@ -2048,13 +2183,15 @@ bool ShuffleVectorInst::isExtractSubvectorMask(ArrayRef<int> Mask, } bool ShuffleVectorInst::isIdentityWithPadding() const { - int NumOpElts = Op<0>()->getType()->getVectorNumElements(); - int NumMaskElts = getType()->getVectorNumElements(); + if (isa<UndefValue>(Op<2>())) + return false; + int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements(); + int NumMaskElts = cast<VectorType>(getType())->getNumElements(); if (NumMaskElts <= NumOpElts) return false; // The first part of the mask must choose elements from exactly 1 source op. - SmallVector<int, 16> Mask = getShuffleMask(); + ArrayRef<int> Mask = getShuffleMask(); if (!isIdentityMaskImpl(Mask, NumOpElts)) return false; @@ -2067,8 +2204,16 @@ bool ShuffleVectorInst::isIdentityWithPadding() const { } bool ShuffleVectorInst::isIdentityWithExtract() const { - int NumOpElts = Op<0>()->getType()->getVectorNumElements(); - int NumMaskElts = getType()->getVectorNumElements(); + if (isa<UndefValue>(Op<2>())) + return false; + + // FIXME: Not currently possible to express a shuffle mask for a scalable + // vector for this case + if (isa<ScalableVectorType>(getType())) + return false; + + int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements(); + int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements(); if (NumMaskElts >= NumOpElts) return false; @@ -2077,11 +2222,12 @@ bool ShuffleVectorInst::isIdentityWithExtract() const { bool ShuffleVectorInst::isConcat() const { // Vector concatenation is differentiated from identity with padding. - if (isa<UndefValue>(Op<0>()) || isa<UndefValue>(Op<1>())) + if (isa<UndefValue>(Op<0>()) || isa<UndefValue>(Op<1>()) || + isa<UndefValue>(Op<2>())) return false; - int NumOpElts = Op<0>()->getType()->getVectorNumElements(); - int NumMaskElts = getType()->getVectorNumElements(); + int NumOpElts = cast<VectorType>(Op<0>()->getType())->getNumElements(); + int NumMaskElts = getType()->getNumElements(); if (NumMaskElts != NumOpElts * 2) return false; @@ -2163,15 +2309,15 @@ Type *ExtractValueInst::getIndexedType(Type *Agg, if (ArrayType *AT = dyn_cast<ArrayType>(Agg)) { if (Index >= AT->getNumElements()) return nullptr; + Agg = AT->getElementType(); } else if (StructType *ST = dyn_cast<StructType>(Agg)) { if (Index >= ST->getNumElements()) return nullptr; + Agg = ST->getElementType(Index); } else { // Not a valid type to index into. return nullptr; } - - Agg = cast<CompositeType>(Agg)->getTypeAtIndex(Index); } return const_cast<Type*>(Agg); } @@ -2383,20 +2529,6 @@ BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name, return BinaryOperator::CreateNUWSub(zero, Op, Name, InsertAtEnd); } -BinaryOperator *BinaryOperator::CreateFNeg(Value *Op, const Twine &Name, - Instruction *InsertBefore) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return new BinaryOperator(Instruction::FSub, zero, Op, - Op->getType(), Name, InsertBefore); -} - -BinaryOperator *BinaryOperator::CreateFNeg(Value *Op, const Twine &Name, - BasicBlock *InsertAtEnd) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return new BinaryOperator(Instruction::FSub, zero, Op, - Op->getType(), Name, InsertAtEnd); -} - BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name, Instruction *InsertBefore) { Constant *C = Constant::getAllOnesValue(Op->getType()); @@ -2836,7 +2968,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty, "Invalid cast"); assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast"); assert((!Ty->isVectorTy() || - Ty->getVectorNumElements() == S->getType()->getVectorNumElements()) && + cast<VectorType>(Ty)->getNumElements() == + cast<VectorType>(S->getType())->getNumElements()) && "Invalid cast"); if (Ty->isIntOrIntVectorTy()) @@ -2854,7 +2987,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty, "Invalid cast"); assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast"); assert((!Ty->isVectorTy() || - Ty->getVectorNumElements() == S->getType()->getVectorNumElements()) && + cast<VectorType>(Ty)->getNumElements() == + cast<VectorType>(S->getType())->getNumElements()) && "Invalid cast"); if (Ty->isIntOrIntVectorTy()) @@ -3185,57 +3319,54 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) { SrcTy->isAggregateType() || DstTy->isAggregateType()) return false; - // Get the size of the types in bits, we'll need this later - unsigned SrcBitSize = SrcTy->getScalarSizeInBits(); - unsigned DstBitSize = DstTy->getScalarSizeInBits(); + // Get the size of the types in bits, and whether we are dealing + // with vector types, we'll need this later. + bool SrcIsVec = isa<VectorType>(SrcTy); + bool DstIsVec = isa<VectorType>(DstTy); + unsigned SrcScalarBitSize = SrcTy->getScalarSizeInBits(); + unsigned DstScalarBitSize = DstTy->getScalarSizeInBits(); // If these are vector types, get the lengths of the vectors (using zero for // scalar types means that checking that vector lengths match also checks that // scalars are not being converted to vectors or vectors to scalars). - unsigned SrcLength = SrcTy->isVectorTy() ? - cast<VectorType>(SrcTy)->getNumElements() : 0; - unsigned DstLength = DstTy->isVectorTy() ? - cast<VectorType>(DstTy)->getNumElements() : 0; + ElementCount SrcEC = SrcIsVec ? cast<VectorType>(SrcTy)->getElementCount() + : ElementCount(0, false); + ElementCount DstEC = DstIsVec ? cast<VectorType>(DstTy)->getElementCount() + : ElementCount(0, false); // Switch on the opcode provided switch (op) { default: return false; // This is an input error case Instruction::Trunc: return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() && - SrcLength == DstLength && SrcBitSize > DstBitSize; + SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize; case Instruction::ZExt: return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() && - SrcLength == DstLength && SrcBitSize < DstBitSize; + SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize; case Instruction::SExt: return SrcTy->isIntOrIntVectorTy() && DstTy->isIntOrIntVectorTy() && - SrcLength == DstLength && SrcBitSize < DstBitSize; + SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize; case Instruction::FPTrunc: return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() && - SrcLength == DstLength && SrcBitSize > DstBitSize; + SrcEC == DstEC && SrcScalarBitSize > DstScalarBitSize; case Instruction::FPExt: return SrcTy->isFPOrFPVectorTy() && DstTy->isFPOrFPVectorTy() && - SrcLength == DstLength && SrcBitSize < DstBitSize; + SrcEC == DstEC && SrcScalarBitSize < DstScalarBitSize; case Instruction::UIToFP: case Instruction::SIToFP: return SrcTy->isIntOrIntVectorTy() && DstTy->isFPOrFPVectorTy() && - SrcLength == DstLength; + SrcEC == DstEC; case Instruction::FPToUI: case Instruction::FPToSI: return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() && - SrcLength == DstLength; + SrcEC == DstEC; case Instruction::PtrToInt: - if (isa<VectorType>(SrcTy) != isa<VectorType>(DstTy)) + if (SrcEC != DstEC) return false; - if (VectorType *VT = dyn_cast<VectorType>(SrcTy)) - if (VT->getNumElements() != cast<VectorType>(DstTy)->getNumElements()) - return false; return SrcTy->isPtrOrPtrVectorTy() && DstTy->isIntOrIntVectorTy(); case Instruction::IntToPtr: - if (isa<VectorType>(SrcTy) != isa<VectorType>(DstTy)) + if (SrcEC != DstEC) return false; - if (VectorType *VT = dyn_cast<VectorType>(SrcTy)) - if (VT->getNumElements() != cast<VectorType>(DstTy)->getNumElements()) - return false; return SrcTy->isIntOrIntVectorTy() && DstTy->isPtrOrPtrVectorTy(); case Instruction::BitCast: { PointerType *SrcPtrTy = dyn_cast<PointerType>(SrcTy->getScalarType()); @@ -3256,14 +3387,12 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) { return false; // A vector of pointers must have the same number of elements. - VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy); - VectorType *DstVecTy = dyn_cast<VectorType>(DstTy); - if (SrcVecTy && DstVecTy) - return (SrcVecTy->getNumElements() == DstVecTy->getNumElements()); - if (SrcVecTy) - return SrcVecTy->getNumElements() == 1; - if (DstVecTy) - return DstVecTy->getNumElements() == 1; + if (SrcIsVec && DstIsVec) + return SrcEC == DstEC; + if (SrcIsVec) + return SrcEC == ElementCount(1, false); + if (DstIsVec) + return DstEC == ElementCount(1, false); return true; } @@ -3279,14 +3408,7 @@ CastInst::castIsValid(Instruction::CastOps op, Value *S, Type *DstTy) { if (SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace()) return false; - if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy)) { - if (VectorType *DstVecTy = dyn_cast<VectorType>(DstTy)) - return (SrcVecTy->getNumElements() == DstVecTy->getNumElements()); - - return false; - } - - return true; + return SrcEC == DstEC; } } } @@ -4137,7 +4259,7 @@ InsertValueInst *InsertValueInst::cloneImpl() const { AllocaInst *AllocaInst::cloneImpl() const { AllocaInst *Result = new AllocaInst(getAllocatedType(), getType()->getAddressSpace(), - (Value *)getOperand(0), MaybeAlign(getAlignment())); + getOperand(0), getAlign()); Result->setUsedWithInAlloca(isUsedWithInAlloca()); Result->setSwiftError(isSwiftError()); return Result; @@ -4145,21 +4267,18 @@ AllocaInst *AllocaInst::cloneImpl() const { LoadInst *LoadInst::cloneImpl() const { return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), - MaybeAlign(getAlignment()), getOrdering(), - getSyncScopeID()); + getAlign(), getOrdering(), getSyncScopeID()); } StoreInst *StoreInst::cloneImpl() const { - return new StoreInst(getOperand(0), getOperand(1), isVolatile(), - MaybeAlign(getAlignment()), getOrdering(), - getSyncScopeID()); + return new StoreInst(getOperand(0), getOperand(1), isVolatile(), getAlign(), + getOrdering(), getSyncScopeID()); } AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { - AtomicCmpXchgInst *Result = - new AtomicCmpXchgInst(getOperand(0), getOperand(1), getOperand(2), - getSuccessOrdering(), getFailureOrdering(), - getSyncScopeID()); + AtomicCmpXchgInst *Result = new AtomicCmpXchgInst( + getOperand(0), getOperand(1), getOperand(2), getAlign(), + getSuccessOrdering(), getFailureOrdering(), getSyncScopeID()); Result->setVolatile(isVolatile()); Result->setWeak(isWeak()); return Result; @@ -4167,8 +4286,8 @@ AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { AtomicRMWInst *AtomicRMWInst::cloneImpl() const { AtomicRMWInst *Result = - new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1), - getOrdering(), getSyncScopeID()); + new AtomicRMWInst(getOperation(), getOperand(0), getOperand(1), + getAlign(), getOrdering(), getSyncScopeID()); Result->setVolatile(isVolatile()); return Result; } @@ -4254,7 +4373,7 @@ InsertElementInst *InsertElementInst::cloneImpl() const { } ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const { - return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2)); + return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask()); } PHINode *PHINode::cloneImpl() const { return new PHINode(*this); } diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index b23742b83c12..c4e06cd979ed 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -21,13 +21,15 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Operator.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/Operator.h" +#include "llvm/IR/PatternMatch.h" + #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -102,7 +104,7 @@ Value *InstrProfIncrementInst::getStep() const { return ConstantInt::get(Type::getInt64Ty(Context), 1); } -Optional<fp::RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const { +Optional<RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const { unsigned NumOperands = getNumArgOperands(); Metadata *MD = cast<MetadataAsValue>(getArgOperand(NumOperands - 2))->getMetadata(); @@ -121,55 +123,53 @@ ConstrainedFPIntrinsic::getExceptionBehavior() const { return StrToExceptionBehavior(cast<MDString>(MD)->getString()); } -FCmpInst::Predicate -ConstrainedFPCmpIntrinsic::getPredicate() const { - Metadata *MD = - cast<MetadataAsValue>(getArgOperand(2))->getMetadata(); +FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const { + Metadata *MD = cast<MetadataAsValue>(getArgOperand(2))->getMetadata(); if (!MD || !isa<MDString>(MD)) return FCmpInst::BAD_FCMP_PREDICATE; return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString()) - .Case("oeq", FCmpInst::FCMP_OEQ) - .Case("ogt", FCmpInst::FCMP_OGT) - .Case("oge", FCmpInst::FCMP_OGE) - .Case("olt", FCmpInst::FCMP_OLT) - .Case("ole", FCmpInst::FCMP_OLE) - .Case("one", FCmpInst::FCMP_ONE) - .Case("ord", FCmpInst::FCMP_ORD) - .Case("uno", FCmpInst::FCMP_UNO) - .Case("ueq", FCmpInst::FCMP_UEQ) - .Case("ugt", FCmpInst::FCMP_UGT) - .Case("uge", FCmpInst::FCMP_UGE) - .Case("ult", FCmpInst::FCMP_ULT) - .Case("ule", FCmpInst::FCMP_ULE) - .Case("une", FCmpInst::FCMP_UNE) - .Default(FCmpInst::BAD_FCMP_PREDICATE); + .Case("oeq", FCmpInst::FCMP_OEQ) + .Case("ogt", FCmpInst::FCMP_OGT) + .Case("oge", FCmpInst::FCMP_OGE) + .Case("olt", FCmpInst::FCMP_OLT) + .Case("ole", FCmpInst::FCMP_OLE) + .Case("one", FCmpInst::FCMP_ONE) + .Case("ord", FCmpInst::FCMP_ORD) + .Case("uno", FCmpInst::FCMP_UNO) + .Case("ueq", FCmpInst::FCMP_UEQ) + .Case("ugt", FCmpInst::FCMP_UGT) + .Case("uge", FCmpInst::FCMP_UGE) + .Case("ult", FCmpInst::FCMP_ULT) + .Case("ule", FCmpInst::FCMP_ULE) + .Case("une", FCmpInst::FCMP_UNE) + .Default(FCmpInst::BAD_FCMP_PREDICATE); } bool ConstrainedFPIntrinsic::isUnaryOp() const { switch (getIntrinsicID()) { - default: - return false; -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ - case Intrinsic::INTRINSIC: \ - return NARG == 1; + default: + return false; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + return NARG == 1; #include "llvm/IR/ConstrainedOps.def" } } bool ConstrainedFPIntrinsic::isTernaryOp() const { switch (getIntrinsicID()) { - default: - return false; -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ - case Intrinsic::INTRINSIC: \ - return NARG == 3; + default: + return false; +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: \ + return NARG == 3; #include "llvm/IR/ConstrainedOps.def" } } bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) { switch (I->getIntrinsicID()) { -#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC, DAGN) \ +#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" return true; @@ -178,36 +178,165 @@ bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) { } } +ElementCount VPIntrinsic::getStaticVectorLength() const { + auto GetVectorLengthOfType = [](const Type *T) -> ElementCount { + auto VT = cast<VectorType>(T); + auto ElemCount = VT->getElementCount(); + return ElemCount; + }; + + auto VPMask = getMaskParam(); + return GetVectorLengthOfType(VPMask->getType()); +} + +Value *VPIntrinsic::getMaskParam() const { + auto maskPos = GetMaskParamPos(getIntrinsicID()); + if (maskPos) + return getArgOperand(maskPos.getValue()); + return nullptr; +} + +Value *VPIntrinsic::getVectorLengthParam() const { + auto vlenPos = GetVectorLengthParamPos(getIntrinsicID()); + if (vlenPos) + return getArgOperand(vlenPos.getValue()); + return nullptr; +} + +Optional<int> VPIntrinsic::GetMaskParamPos(Intrinsic::ID IntrinsicID) { + switch (IntrinsicID) { + default: + return None; + +#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \ + case Intrinsic::VPID: \ + return MASKPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +Optional<int> VPIntrinsic::GetVectorLengthParamPos(Intrinsic::ID IntrinsicID) { + switch (IntrinsicID) { + default: + return None; + +#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \ + case Intrinsic::VPID: \ + return VLENPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) { + switch (ID) { + default: + return false; + +#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \ + case Intrinsic::VPID: \ + break; +#include "llvm/IR/VPIntrinsics.def" + } + return true; +} + +// Equivalent non-predicated opcode +unsigned VPIntrinsic::GetFunctionalOpcodeForVP(Intrinsic::ID ID) { + switch (ID) { + default: + return Instruction::Call; + +#define HANDLE_VP_TO_OC(VPID, OC) \ + case Intrinsic::VPID: \ + return Instruction::OC; +#include "llvm/IR/VPIntrinsics.def" + } +} + +Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned OC) { + switch (OC) { + default: + return Intrinsic::not_intrinsic; + +#define HANDLE_VP_TO_OC(VPID, OC) \ + case Instruction::OC: \ + return Intrinsic::VPID; +#include "llvm/IR/VPIntrinsics.def" + } +} + +bool VPIntrinsic::canIgnoreVectorLengthParam() const { + using namespace PatternMatch; + + ElementCount EC = getStaticVectorLength(); + + // No vlen param - no lanes masked-off by it. + auto *VLParam = getVectorLengthParam(); + if (!VLParam) + return true; + + // Note that the VP intrinsic causes undefined behavior if the Explicit Vector + // Length parameter is strictly greater-than the number of vector elements of + // the operation. This function returns true when this is detected statically + // in the IR. + + // Check whether "W == vscale * EC.Min" + if (EC.Scalable) { + // Undig the DL + auto ParMod = this->getModule(); + if (!ParMod) + return false; + const auto &DL = ParMod->getDataLayout(); + + // Compare vscale patterns + uint64_t VScaleFactor; + if (match(VLParam, m_c_Mul(m_ConstantInt(VScaleFactor), m_VScale(DL)))) + return VScaleFactor >= EC.Min; + return (EC.Min == 1) && match(VLParam, m_VScale(DL)); + } + + // standard SIMD operation + auto VLConst = dyn_cast<ConstantInt>(VLParam); + if (!VLConst) + return false; + + uint64_t VLNum = VLConst->getZExtValue(); + if (VLNum >= EC.Min) + return true; + + return false; +} + Instruction::BinaryOps BinaryOpIntrinsic::getBinaryOp() const { switch (getIntrinsicID()) { - case Intrinsic::uadd_with_overflow: - case Intrinsic::sadd_with_overflow: - case Intrinsic::uadd_sat: - case Intrinsic::sadd_sat: - return Instruction::Add; - case Intrinsic::usub_with_overflow: - case Intrinsic::ssub_with_overflow: - case Intrinsic::usub_sat: - case Intrinsic::ssub_sat: - return Instruction::Sub; - case Intrinsic::umul_with_overflow: - case Intrinsic::smul_with_overflow: - return Instruction::Mul; - default: - llvm_unreachable("Invalid intrinsic"); + case Intrinsic::uadd_with_overflow: + case Intrinsic::sadd_with_overflow: + case Intrinsic::uadd_sat: + case Intrinsic::sadd_sat: + return Instruction::Add; + case Intrinsic::usub_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::usub_sat: + case Intrinsic::ssub_sat: + return Instruction::Sub; + case Intrinsic::umul_with_overflow: + case Intrinsic::smul_with_overflow: + return Instruction::Mul; + default: + llvm_unreachable("Invalid intrinsic"); } } bool BinaryOpIntrinsic::isSigned() const { switch (getIntrinsicID()) { - case Intrinsic::sadd_with_overflow: - case Intrinsic::ssub_with_overflow: - case Intrinsic::smul_with_overflow: - case Intrinsic::sadd_sat: - case Intrinsic::ssub_sat: - return true; - default: - return false; + case Intrinsic::sadd_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::smul_with_overflow: + case Intrinsic::sadd_sat: + case Intrinsic::ssub_sat: + return true; + default: + return false; } } diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index cb13b27aa50f..7ebca5274369 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -19,9 +19,10 @@ #include "llvm/ADT/Twine.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" -#include "llvm/IR/RemarkStreamer.h" +#include "llvm/Remarks/RemarkStreamer.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -67,6 +68,16 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { "cfguardtarget operand bundle id drifted!"); (void)CFGuardTargetEntry; + auto *PreallocatedEntry = pImpl->getOrInsertBundleTag("preallocated"); + assert(PreallocatedEntry->second == LLVMContext::OB_preallocated && + "preallocated operand bundle id drifted!"); + (void)PreallocatedEntry; + + auto *GCLiveEntry = pImpl->getOrInsertBundleTag("gc-live"); + assert(GCLiveEntry->second == LLVMContext::OB_gc_live && + "gc-transition operand bundle id drifted!"); + (void)GCLiveEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && @@ -142,15 +153,26 @@ uint64_t LLVMContext::getDiagnosticsHotnessThreshold() const { return pImpl->DiagnosticsHotnessThreshold; } -RemarkStreamer *LLVMContext::getRemarkStreamer() { - return pImpl->RemarkDiagStreamer.get(); +remarks::RemarkStreamer *LLVMContext::getMainRemarkStreamer() { + return pImpl->MainRemarkStreamer.get(); } -const RemarkStreamer *LLVMContext::getRemarkStreamer() const { - return const_cast<LLVMContext *>(this)->getRemarkStreamer(); +const remarks::RemarkStreamer *LLVMContext::getMainRemarkStreamer() const { + return const_cast<LLVMContext *>(this)->getMainRemarkStreamer(); } -void LLVMContext::setRemarkStreamer( - std::unique_ptr<RemarkStreamer> RemarkStreamer) { - pImpl->RemarkDiagStreamer = std::move(RemarkStreamer); +void LLVMContext::setMainRemarkStreamer( + std::unique_ptr<remarks::RemarkStreamer> RemarkStreamer) { + pImpl->MainRemarkStreamer = std::move(RemarkStreamer); +} + +LLVMRemarkStreamer *LLVMContext::getLLVMRemarkStreamer() { + return pImpl->LLVMRS.get(); +} +const LLVMRemarkStreamer *LLVMContext::getLLVMRemarkStreamer() const { + return const_cast<LLVMContext *>(this)->getLLVMRemarkStreamer(); +} +void LLVMContext::setLLVMRemarkStreamer( + std::unique_ptr<LLVMRemarkStreamer> RemarkStreamer) { + pImpl->LLVMRS = std::move(RemarkStreamer); } DiagnosticHandler::DiagnosticHandlerTy @@ -214,7 +236,7 @@ LLVMContext::getDiagnosticMessagePrefix(DiagnosticSeverity Severity) { void LLVMContext::diagnose(const DiagnosticInfo &DI) { if (auto *OptDiagBase = dyn_cast<DiagnosticInfoOptimizationBase>(&DI)) - if (RemarkStreamer *RS = getRemarkStreamer()) + if (LLVMRemarkStreamer *RS = getLLVMRemarkStreamer()) RS->emit(*OptDiagBase); // If there is a report handler, use it. @@ -265,6 +287,11 @@ void LLVMContext::getOperandBundleTags(SmallVectorImpl<StringRef> &Tags) const { pImpl->getOperandBundleTags(Tags); } +StringMapEntry<uint32_t> * +LLVMContext::getOrInsertBundleTag(StringRef TagName) const { + return pImpl->getOrInsertBundleTag(TagName); +} + uint32_t LLVMContext::getOperandBundleTagID(StringRef Tag) const { return pImpl->getOperandBundleTagID(Tag); } diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp index 5f9782714170..f197b3e67d30 100644 --- a/llvm/lib/IR/LLVMContextImpl.cpp +++ b/llvm/lib/IR/LLVMContextImpl.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "LLVMContextImpl.h" +#include "llvm/ADT/SetVector.h" #include "llvm/IR/Module.h" #include "llvm/IR/OptBisect.h" #include "llvm/IR/Type.h" @@ -25,6 +26,7 @@ LLVMContextImpl::LLVMContextImpl(LLVMContext &C) VoidTy(C, Type::VoidTyID), LabelTy(C, Type::LabelTyID), HalfTy(C, Type::HalfTyID), + BFloatTy(C, Type::BFloatTyID), FloatTy(C, Type::FloatTyID), DoubleTy(C, Type::DoubleTyID), MetadataTy(C, Type::MetadataTyID), @@ -103,21 +105,6 @@ LLVMContextImpl::~LLVMContextImpl() { delete CDSConstant.second; CDSConstants.clear(); - // Destroy attributes. - for (FoldingSetIterator<AttributeImpl> I = AttrsSet.begin(), - E = AttrsSet.end(); I != E; ) { - FoldingSetIterator<AttributeImpl> Elem = I++; - delete &*Elem; - } - - // Destroy attribute lists. - for (FoldingSetIterator<AttributeListImpl> I = AttrsLists.begin(), - E = AttrsLists.end(); - I != E;) { - FoldingSetIterator<AttributeListImpl> Elem = I++; - delete &*Elem; - } - // Destroy attribute node lists. for (FoldingSetIterator<AttributeSetNode> I = AttrsSetNodes.begin(), E = AttrsSetNodes.end(); I != E; ) { @@ -142,18 +129,19 @@ LLVMContextImpl::~LLVMContextImpl() { } void LLVMContextImpl::dropTriviallyDeadConstantArrays() { - bool Changed; - do { - Changed = false; - - for (auto I = ArrayConstants.begin(), E = ArrayConstants.end(); I != E;) { - auto *C = *I++; - if (C->use_empty()) { - Changed = true; - C->destroyConstant(); + SmallSetVector<ConstantArray *, 4> WorkList(ArrayConstants.begin(), + ArrayConstants.end()); + + while (!WorkList.empty()) { + ConstantArray *C = WorkList.pop_back_val(); + if (C->use_empty()) { + for (const Use &Op : C->operands()) { + if (auto *COp = dyn_cast<ConstantArray>(Op)) + WorkList.insert(COp); } + C->destroyConstant(); } - } while (Changed); + } } void Module::dropTriviallyDeadConstantArrays() { diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index 6f5d5752b38d..1c7d8746d242 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -29,14 +29,13 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/Metadata.h" -#include "llvm/IR/RemarkStreamer.h" #include "llvm/IR/TrackingMDRef.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Casting.h" @@ -53,8 +52,7 @@ namespace llvm { -class ConstantFP; -class ConstantInt; +class StringRef; class Type; class Value; class ValueHandleBase; @@ -325,49 +323,66 @@ template <> struct MDNodeKeyImpl<GenericDINode> : MDNodeOpsKey { template <> struct MDNodeKeyImpl<DISubrange> { Metadata *CountNode; - int64_t LowerBound; - - MDNodeKeyImpl(Metadata *CountNode, int64_t LowerBound) - : CountNode(CountNode), LowerBound(LowerBound) {} + Metadata *LowerBound; + Metadata *UpperBound; + Metadata *Stride; + + MDNodeKeyImpl(Metadata *CountNode, Metadata *LowerBound, Metadata *UpperBound, + Metadata *Stride) + : CountNode(CountNode), LowerBound(LowerBound), UpperBound(UpperBound), + Stride(Stride) {} MDNodeKeyImpl(const DISubrange *N) - : CountNode(N->getRawCountNode()), - LowerBound(N->getLowerBound()) {} + : CountNode(N->getRawCountNode()), LowerBound(N->getRawLowerBound()), + UpperBound(N->getRawUpperBound()), Stride(N->getRawStride()) {} bool isKeyOf(const DISubrange *RHS) const { - if (LowerBound != RHS->getLowerBound()) - return false; - - if (auto *RHSCount = RHS->getCount().dyn_cast<ConstantInt*>()) - if (auto *MD = dyn_cast<ConstantAsMetadata>(CountNode)) - if (RHSCount->getSExtValue() == - cast<ConstantInt>(MD->getValue())->getSExtValue()) + auto BoundsEqual = [=](Metadata *Node1, Metadata *Node2) -> bool { + if (Node1 == Node2) + return true; + + ConstantAsMetadata *MD1 = dyn_cast_or_null<ConstantAsMetadata>(Node1); + ConstantAsMetadata *MD2 = dyn_cast_or_null<ConstantAsMetadata>(Node2); + if (MD1 && MD2) { + ConstantInt *CV1 = cast<ConstantInt>(MD1->getValue()); + ConstantInt *CV2 = cast<ConstantInt>(MD2->getValue()); + if (CV1->getSExtValue() == CV2->getSExtValue()) return true; + } + return false; + }; - return CountNode == RHS->getRawCountNode(); + return BoundsEqual(CountNode, RHS->getRawCountNode()) && + BoundsEqual(LowerBound, RHS->getRawLowerBound()) && + BoundsEqual(UpperBound, RHS->getRawUpperBound()) && + BoundsEqual(Stride, RHS->getRawStride()); } unsigned getHashValue() const { - if (auto *MD = dyn_cast<ConstantAsMetadata>(CountNode)) - return hash_combine(cast<ConstantInt>(MD->getValue())->getSExtValue(), - LowerBound); - return hash_combine(CountNode, LowerBound); + if (CountNode) + if (auto *MD = dyn_cast<ConstantAsMetadata>(CountNode)) + return hash_combine(cast<ConstantInt>(MD->getValue())->getSExtValue(), + LowerBound, UpperBound, Stride); + return hash_combine(CountNode, LowerBound, UpperBound, Stride); } }; template <> struct MDNodeKeyImpl<DIEnumerator> { - int64_t Value; + APInt Value; MDString *Name; bool IsUnsigned; - MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name) + MDNodeKeyImpl(APInt Value, bool IsUnsigned, MDString *Name) : Value(Value), Name(Name), IsUnsigned(IsUnsigned) {} + MDNodeKeyImpl(int64_t Value, bool IsUnsigned, MDString *Name) + : Value(APInt(64, Value, !IsUnsigned)), Name(Name), + IsUnsigned(IsUnsigned) {} MDNodeKeyImpl(const DIEnumerator *N) : Value(N->getValue()), Name(N->getRawName()), IsUnsigned(N->isUnsigned()) {} bool isKeyOf(const DIEnumerator *RHS) const { - return Value == RHS->getValue() && IsUnsigned == RHS->isUnsigned() && - Name == RHS->getRawName(); + return APInt::isSameValue(Value, RHS->getValue()) && + IsUnsigned == RHS->isUnsigned() && Name == RHS->getRawName(); } unsigned getHashValue() const { return hash_combine(Value, Name); } @@ -509,19 +524,21 @@ template <> struct MDNodeKeyImpl<DICompositeType> { Metadata *TemplateParams; MDString *Identifier; Metadata *Discriminator; + Metadata *DataLocation; MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, - MDString *Identifier, Metadata *Discriminator) + MDString *Identifier, Metadata *Discriminator, + Metadata *DataLocation) : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope), BaseType(BaseType), SizeInBits(SizeInBits), OffsetInBits(OffsetInBits), AlignInBits(AlignInBits), Flags(Flags), Elements(Elements), RuntimeLang(RuntimeLang), VTableHolder(VTableHolder), TemplateParams(TemplateParams), Identifier(Identifier), - Discriminator(Discriminator) {} + Discriminator(Discriminator), DataLocation(DataLocation) {} MDNodeKeyImpl(const DICompositeType *N) : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Scope(N->getRawScope()), @@ -531,7 +548,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> { RuntimeLang(N->getRuntimeLang()), VTableHolder(N->getRawVTableHolder()), TemplateParams(N->getRawTemplateParams()), Identifier(N->getRawIdentifier()), - Discriminator(N->getRawDiscriminator()) {} + Discriminator(N->getRawDiscriminator()), + DataLocation(N->getRawDataLocation()) {} bool isKeyOf(const DICompositeType *RHS) const { return Tag == RHS->getTag() && Name == RHS->getRawName() && @@ -545,7 +563,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> { VTableHolder == RHS->getRawVTableHolder() && TemplateParams == RHS->getRawTemplateParams() && Identifier == RHS->getRawIdentifier() && - Discriminator == RHS->getRawDiscriminator(); + Discriminator == RHS->getRawDiscriminator() && + DataLocation == RHS->getRawDataLocation(); } unsigned getHashValue() const { @@ -815,67 +834,81 @@ template <> struct MDNodeKeyImpl<DICommonBlock> { }; template <> struct MDNodeKeyImpl<DIModule> { + Metadata *File; Metadata *Scope; MDString *Name; MDString *ConfigurationMacros; MDString *IncludePath; - MDString *SysRoot; + MDString *APINotesFile; + unsigned LineNo; - MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *ConfigurationMacros, - MDString *IncludePath, MDString *SysRoot) - : Scope(Scope), Name(Name), ConfigurationMacros(ConfigurationMacros), - IncludePath(IncludePath), SysRoot(SysRoot) {} + MDNodeKeyImpl(Metadata *File, Metadata *Scope, MDString *Name, + MDString *ConfigurationMacros, MDString *IncludePath, + MDString *APINotesFile, unsigned LineNo) + : File(File), Scope(Scope), Name(Name), + ConfigurationMacros(ConfigurationMacros), IncludePath(IncludePath), + APINotesFile(APINotesFile), LineNo(LineNo) {} MDNodeKeyImpl(const DIModule *N) - : Scope(N->getRawScope()), Name(N->getRawName()), + : File(N->getRawFile()), Scope(N->getRawScope()), Name(N->getRawName()), ConfigurationMacros(N->getRawConfigurationMacros()), - IncludePath(N->getRawIncludePath()), SysRoot(N->getRawSysRoot()) {} + IncludePath(N->getRawIncludePath()), + APINotesFile(N->getRawAPINotesFile()), LineNo(N->getLineNo()) {} bool isKeyOf(const DIModule *RHS) const { return Scope == RHS->getRawScope() && Name == RHS->getRawName() && ConfigurationMacros == RHS->getRawConfigurationMacros() && IncludePath == RHS->getRawIncludePath() && - SysRoot == RHS->getRawSysRoot(); + APINotesFile == RHS->getRawAPINotesFile() && + File == RHS->getRawFile() && LineNo == RHS->getLineNo(); } unsigned getHashValue() const { - return hash_combine(Scope, Name, - ConfigurationMacros, IncludePath, SysRoot); + return hash_combine(Scope, Name, ConfigurationMacros, IncludePath); } }; template <> struct MDNodeKeyImpl<DITemplateTypeParameter> { MDString *Name; Metadata *Type; + bool IsDefault; - MDNodeKeyImpl(MDString *Name, Metadata *Type) : Name(Name), Type(Type) {} + MDNodeKeyImpl(MDString *Name, Metadata *Type, bool IsDefault) + : Name(Name), Type(Type), IsDefault(IsDefault) {} MDNodeKeyImpl(const DITemplateTypeParameter *N) - : Name(N->getRawName()), Type(N->getRawType()) {} + : Name(N->getRawName()), Type(N->getRawType()), + IsDefault(N->isDefault()) {} bool isKeyOf(const DITemplateTypeParameter *RHS) const { - return Name == RHS->getRawName() && Type == RHS->getRawType(); + return Name == RHS->getRawName() && Type == RHS->getRawType() && + IsDefault == RHS->isDefault(); } - unsigned getHashValue() const { return hash_combine(Name, Type); } + unsigned getHashValue() const { return hash_combine(Name, Type, IsDefault); } }; template <> struct MDNodeKeyImpl<DITemplateValueParameter> { unsigned Tag; MDString *Name; Metadata *Type; + bool IsDefault; Metadata *Value; - MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *Type, Metadata *Value) - : Tag(Tag), Name(Name), Type(Type), Value(Value) {} + MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *Type, bool IsDefault, + Metadata *Value) + : Tag(Tag), Name(Name), Type(Type), IsDefault(IsDefault), Value(Value) {} MDNodeKeyImpl(const DITemplateValueParameter *N) : Tag(N->getTag()), Name(N->getRawName()), Type(N->getRawType()), - Value(N->getValue()) {} + IsDefault(N->isDefault()), Value(N->getValue()) {} bool isKeyOf(const DITemplateValueParameter *RHS) const { return Tag == RHS->getTag() && Name == RHS->getRawName() && - Type == RHS->getRawType() && Value == RHS->getValue(); + Type == RHS->getRawType() && IsDefault == RHS->isDefault() && + Value == RHS->getValue(); } - unsigned getHashValue() const { return hash_combine(Tag, Name, Type, Value); } + unsigned getHashValue() const { + return hash_combine(Tag, Name, Type, IsDefault, Value); + } }; template <> struct MDNodeKeyImpl<DIGlobalVariable> { @@ -1248,11 +1281,17 @@ public: LLVMContext::InlineAsmDiagHandlerTy InlineAsmDiagHandler = nullptr; void *InlineAsmDiagContext = nullptr; + /// The main remark streamer used by all the other streamers (e.g. IR, MIR, + /// frontends, etc.). This should only be used by the specific streamers, and + /// never directly. + std::unique_ptr<remarks::RemarkStreamer> MainRemarkStreamer; + std::unique_ptr<DiagnosticHandler> DiagHandler; bool RespectDiagnosticFilters = false; bool DiagnosticsHotnessRequested = false; uint64_t DiagnosticsHotnessThreshold = 0; - std::unique_ptr<RemarkStreamer> RemarkDiagStreamer; + /// The specialized remark streamer used by LLVM's OptimizationRemarkEmitter. + std::unique_ptr<LLVMRemarkStreamer> LLVMRS; LLVMContext::YieldCallbackTy YieldCallback = nullptr; void *YieldOpaqueHandle = nullptr; @@ -1317,7 +1356,8 @@ public: std::unique_ptr<ConstantTokenNone> TheNoneToken; // Basic type instances. - Type VoidTy, LabelTy, HalfTy, FloatTy, DoubleTy, MetadataTy, TokenTy; + Type VoidTy, LabelTy, HalfTy, BFloatTy, FloatTy, DoubleTy, MetadataTy, + TokenTy; Type X86_FP80Ty, FP128Ty, PPC_FP128Ty, X86_MMXTy; IntegerType Int1Ty, Int8Ty, Int16Ty, Int32Ty, Int64Ty, Int128Ty; @@ -1364,9 +1404,6 @@ public: /// instructions in different blocks at the same location. DenseMap<std::pair<const char *, unsigned>, unsigned> DiscriminatorTable; - int getOrAddScopeRecordIdxEntry(MDNode *N, int ExistingIdx); - int getOrAddScopeInlinedAtIdxEntry(MDNode *Scope, MDNode *IA,int ExistingIdx); - /// A set of interned tags for operand bundles. The StringMap maps /// bundle tags to their IDs. /// diff --git a/llvm/lib/IR/RemarkStreamer.cpp b/llvm/lib/IR/LLVMRemarkStreamer.cpp index cdbcc4f456c5..96001ab42c38 100644 --- a/llvm/lib/IR/RemarkStreamer.cpp +++ b/llvm/lib/IR/LLVMRemarkStreamer.cpp @@ -1,4 +1,4 @@ -//===- llvm/IR/RemarkStreamer.cpp - Remark Streamer -*- C++ -------------*-===// +//===- llvm/IR/LLVMRemarkStreamer.cpp - Remark Streamer -*- C++ ---------*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,45 +6,19 @@ // //===----------------------------------------------------------------------===// // -// This file contains the implementation of the remark outputting as part of -// LLVMContext. +// This file contains the implementation of the conversion between IR +// Diagnostics and serializable remarks::Remark objects. // //===----------------------------------------------------------------------===// -#include "llvm/IR/RemarkStreamer.h" +#include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/Remarks/BitstreamRemarkSerializer.h" -#include "llvm/Remarks/RemarkFormat.h" -#include "llvm/Remarks/RemarkSerializer.h" -#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" using namespace llvm; -static cl::opt<cl::boolOrDefault> EnableRemarksSection( - "remarks-section", - cl::desc( - "Emit a section containing remark diagnostics metadata. By default, " - "this is enabled for the following formats: yaml-strtab, bitstream."), - cl::init(cl::BOU_UNSET), cl::Hidden); - -RemarkStreamer::RemarkStreamer( - std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer, - Optional<StringRef> FilenameIn) - : PassFilter(), RemarkSerializer(std::move(RemarkSerializer)), - Filename(FilenameIn ? Optional<std::string>(FilenameIn->str()) : None) {} - -Error RemarkStreamer::setFilter(StringRef Filter) { - Regex R = Regex(Filter); - std::string RegexError; - if (!R.isValid(RegexError)) - return createStringError(std::make_error_code(std::errc::invalid_argument), - RegexError.data()); - PassFilter = std::move(R); - return Error::success(); -} - /// DiagnosticKind -> remarks::Type static remarks::Type toRemarkType(enum DiagnosticKind Kind) { switch (Kind) { @@ -81,7 +55,7 @@ toRemarkLocation(const DiagnosticLocation &DL) { /// LLVM Diagnostic -> Remark remarks::Remark -RemarkStreamer::toRemark(const DiagnosticInfoOptimizationBase &Diag) { +LLVMRemarkStreamer::toRemark(const DiagnosticInfoOptimizationBase &Diag) const { remarks::Remark R; // The result. R.RemarkType = toRemarkType(static_cast<DiagnosticKind>(Diag.getKind())); R.PassName = Diag.getPassName(); @@ -101,51 +75,24 @@ RemarkStreamer::toRemark(const DiagnosticInfoOptimizationBase &Diag) { return R; } -void RemarkStreamer::emit(const DiagnosticInfoOptimizationBase &Diag) { - if (Optional<Regex> &Filter = PassFilter) - if (!Filter->match(Diag.getPassName())) +void LLVMRemarkStreamer::emit(const DiagnosticInfoOptimizationBase &Diag) { + if (!RS.matchesFilter(Diag.getPassName())) return; // First, convert the diagnostic to a remark. remarks::Remark R = toRemark(Diag); // Then, emit the remark through the serializer. - RemarkSerializer->emit(R); + RS.getSerializer().emit(R); } -bool RemarkStreamer::needsSection() const { - if (EnableRemarksSection == cl::BOU_TRUE) - return true; - - if (EnableRemarksSection == cl::BOU_FALSE) - return false; +char LLVMRemarkSetupFileError::ID = 0; +char LLVMRemarkSetupPatternError::ID = 0; +char LLVMRemarkSetupFormatError::ID = 0; - assert(EnableRemarksSection == cl::BOU_UNSET); - - // We only need a section if we're in separate mode. - if (RemarkSerializer->Mode != remarks::SerializerMode::Separate) - return false; - - // Only some formats need a section: - // * bitstream - // * yaml-strtab - switch (RemarkSerializer->SerializerFormat) { - case remarks::Format::YAMLStrTab: - case remarks::Format::Bitstream: - return true; - default: - return false; - } -} - -char RemarkSetupFileError::ID = 0; -char RemarkSetupPatternError::ID = 0; -char RemarkSetupFormatError::ID = 0; - -Expected<std::unique_ptr<ToolOutputFile>> -llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, - StringRef RemarksPasses, StringRef RemarksFormat, - bool RemarksWithHotness, - unsigned RemarksHotnessThreshold) { +Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks( + LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, + StringRef RemarksFormat, bool RemarksWithHotness, + unsigned RemarksHotnessThreshold) { if (RemarksWithHotness) Context.setDiagnosticsHotnessRequested(true); @@ -157,7 +104,7 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, Expected<remarks::Format> Format = remarks::parseFormat(RemarksFormat); if (Error E = Format.takeError()) - return make_error<RemarkSetupFormatError>(std::move(E)); + return make_error<LLVMRemarkSetupFormatError>(std::move(E)); std::error_code EC; auto Flags = *Format == remarks::Format::YAML ? sys::fs::OF_Text @@ -167,29 +114,34 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, // We don't use llvm::FileError here because some diagnostics want the file // name separately. if (EC) - return make_error<RemarkSetupFileError>(errorCodeToError(EC)); + return make_error<LLVMRemarkSetupFileError>(errorCodeToError(EC)); Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer = remarks::createRemarkSerializer( *Format, remarks::SerializerMode::Separate, RemarksFile->os()); if (Error E = RemarkSerializer.takeError()) - return make_error<RemarkSetupFormatError>(std::move(E)); + return make_error<LLVMRemarkSetupFormatError>(std::move(E)); - Context.setRemarkStreamer(std::make_unique<RemarkStreamer>( + // Create the main remark streamer. + Context.setMainRemarkStreamer(std::make_unique<remarks::RemarkStreamer>( std::move(*RemarkSerializer), RemarksFilename)); + // Create LLVM's optimization remarks streamer. + Context.setLLVMRemarkStreamer( + std::make_unique<LLVMRemarkStreamer>(*Context.getMainRemarkStreamer())); + if (!RemarksPasses.empty()) - if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses)) - return make_error<RemarkSetupPatternError>(std::move(E)); + if (Error E = Context.getMainRemarkStreamer()->setFilter(RemarksPasses)) + return make_error<LLVMRemarkSetupPatternError>(std::move(E)); return std::move(RemarksFile); } -Error llvm::setupOptimizationRemarks(LLVMContext &Context, raw_ostream &OS, - StringRef RemarksPasses, - StringRef RemarksFormat, - bool RemarksWithHotness, - unsigned RemarksHotnessThreshold) { +Error llvm::setupLLVMOptimizationRemarks(LLVMContext &Context, raw_ostream &OS, + StringRef RemarksPasses, + StringRef RemarksFormat, + bool RemarksWithHotness, + unsigned RemarksHotnessThreshold) { if (RemarksWithHotness) Context.setDiagnosticsHotnessRequested(true); @@ -198,20 +150,25 @@ Error llvm::setupOptimizationRemarks(LLVMContext &Context, raw_ostream &OS, Expected<remarks::Format> Format = remarks::parseFormat(RemarksFormat); if (Error E = Format.takeError()) - return make_error<RemarkSetupFormatError>(std::move(E)); + return make_error<LLVMRemarkSetupFormatError>(std::move(E)); Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer = remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, OS); if (Error E = RemarkSerializer.takeError()) - return make_error<RemarkSetupFormatError>(std::move(E)); + return make_error<LLVMRemarkSetupFormatError>(std::move(E)); + + // Create the main remark streamer. + Context.setMainRemarkStreamer( + std::make_unique<remarks::RemarkStreamer>(std::move(*RemarkSerializer))); - Context.setRemarkStreamer( - std::make_unique<RemarkStreamer>(std::move(*RemarkSerializer))); + // Create LLVM's optimization remarks streamer. + Context.setLLVMRemarkStreamer( + std::make_unique<LLVMRemarkStreamer>(*Context.getMainRemarkStreamer())); if (!RemarksPasses.empty()) - if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses)) - return make_error<RemarkSetupPatternError>(std::move(E)); + if (Error E = Context.getMainRemarkStreamer()->setFilter(RemarksPasses)) + return make_error<LLVMRemarkSetupPatternError>(std::move(E)); return Error::success(); } diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp index 90239bb76298..74869fa62c66 100644 --- a/llvm/lib/IR/LegacyPassManager.cpp +++ b/llvm/lib/IR/LegacyPassManager.cpp @@ -33,7 +33,6 @@ #include <algorithm> #include <unordered_set> using namespace llvm; -using namespace llvm::legacy; // See PassManagers.h for Pass Manager infrastructure overview. @@ -132,7 +131,8 @@ bool llvm::forcePrintModuleIR() { return PrintModuleScope; } bool llvm::isFunctionInPrintList(StringRef FunctionName) { static std::unordered_set<std::string> PrintFuncNames(PrintFuncsList.begin(), PrintFuncsList.end()); - return PrintFuncNames.empty() || PrintFuncNames.count(FunctionName); + return PrintFuncNames.empty() || + PrintFuncNames.count(std::string(FunctionName)); } /// isPassDebuggingExecutionsOrMore - Return true if -debug-pass=Executions /// or higher is specified. @@ -239,7 +239,7 @@ void PMDataManager::emitInstrCountChangedRemark( // Helper lambda that emits a remark when the size of a function has changed. auto EmitFunctionSizeChangedRemark = [&FunctionToInstrCount, &F, &BB, - &PassName](const std::string &Fname) { + &PassName](StringRef Fname) { unsigned FnCountBefore, FnCountAfter; std::pair<unsigned, unsigned> &Change = FunctionToInstrCount[Fname]; std::tie(FnCountBefore, FnCountAfter) = Change; @@ -386,8 +386,68 @@ public: void FunctionPassManagerImpl::anchor() {} char FunctionPassManagerImpl::ID = 0; -} // End of legacy namespace -} // End of llvm namespace + +//===----------------------------------------------------------------------===// +// FunctionPassManagerImpl implementation +// +bool FunctionPassManagerImpl::doInitialization(Module &M) { + bool Changed = false; + + dumpArguments(); + dumpPasses(); + + for (ImmutablePass *ImPass : getImmutablePasses()) + Changed |= ImPass->doInitialization(M); + + for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) + Changed |= getContainedManager(Index)->doInitialization(M); + + return Changed; +} + +bool FunctionPassManagerImpl::doFinalization(Module &M) { + bool Changed = false; + + for (int Index = getNumContainedManagers() - 1; Index >= 0; --Index) + Changed |= getContainedManager(Index)->doFinalization(M); + + for (ImmutablePass *ImPass : getImmutablePasses()) + Changed |= ImPass->doFinalization(M); + + return Changed; +} + +void FunctionPassManagerImpl::releaseMemoryOnTheFly() { + if (!wasRun) + return; + for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + FPPassManager *FPPM = getContainedManager(Index); + for (unsigned Index = 0; Index < FPPM->getNumContainedPasses(); ++Index) { + FPPM->getContainedPass(Index)->releaseMemory(); + } + } + wasRun = false; +} + +// Execute all the passes managed by this top level manager. +// Return true if any function is modified by a pass. +bool FunctionPassManagerImpl::run(Function &F) { + bool Changed = false; + + initializeAllAnalysisInfo(); + for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + Changed |= getContainedManager(Index)->runOnFunction(F); + F.getContext().yield(); + } + + for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) + getContainedManager(Index)->cleanup(); + + wasRun = true; + return Changed; +} +} // namespace legacy +} // namespace llvm namespace { //===----------------------------------------------------------------------===// @@ -405,7 +465,7 @@ public: // Delete on the fly managers. ~MPPassManager() override { for (auto &OnTheFlyManager : OnTheFlyManagers) { - FunctionPassManagerImpl *FPP = OnTheFlyManager.second; + legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second; delete FPP; } } @@ -436,7 +496,8 @@ public: /// Return function pass corresponding to PassInfo PI, that is /// required by module pass MP. Instantiate analysis pass, by using /// its runOnFunction() for function F. - Pass* getOnTheFlyPass(Pass *MP, AnalysisID PI, Function &F) override; + std::tuple<Pass *, bool> getOnTheFlyPass(Pass *MP, AnalysisID PI, + Function &F) override; StringRef getPassName() const override { return "Module Pass Manager"; } @@ -449,7 +510,7 @@ public: for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { ModulePass *MP = getContainedPass(Index); MP->dumpPassStructure(Offset + 1); - MapVector<Pass *, FunctionPassManagerImpl *>::const_iterator I = + MapVector<Pass *, legacy::FunctionPassManagerImpl *>::const_iterator I = OnTheFlyManagers.find(MP); if (I != OnTheFlyManagers.end()) I->second->dumpPassStructure(Offset + 2); @@ -469,7 +530,7 @@ public: private: /// Collection of on the fly FPPassManagers. These managers manage /// function passes that are required by module passes. - MapVector<Pass *, FunctionPassManagerImpl *> OnTheFlyManagers; + MapVector<Pass *, legacy::FunctionPassManagerImpl *> OnTheFlyManagers; }; char MPPassManager::ID = 0; @@ -532,8 +593,35 @@ public: void PassManagerImpl::anchor() {} char PassManagerImpl::ID = 0; -} // End of legacy namespace -} // End of llvm namespace + +//===----------------------------------------------------------------------===// +// PassManagerImpl implementation + +// +/// run - Execute all of the passes scheduled for execution. Keep track of +/// whether any of the passes modifies the module, and if so, return true. +bool PassManagerImpl::run(Module &M) { + bool Changed = false; + + dumpArguments(); + dumpPasses(); + + for (ImmutablePass *ImPass : getImmutablePasses()) + Changed |= ImPass->doInitialization(M); + + initializeAllAnalysisInfo(); + for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { + Changed |= getContainedManager(Index)->runOnModule(M); + M.getContext().yield(); + } + + for (ImmutablePass *ImPass : getImmutablePasses()) + Changed |= ImPass->doFinalization(M); + + return Changed; +} +} // namespace legacy +} // namespace llvm //===----------------------------------------------------------------------===// // PMTopLevelManager implementation @@ -1289,7 +1377,8 @@ void PMDataManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) { llvm_unreachable("Unable to schedule pass"); } -Pass *PMDataManager::getOnTheFlyPass(Pass *P, AnalysisID PI, Function &F) { +std::tuple<Pass *, bool> PMDataManager::getOnTheFlyPass(Pass *P, AnalysisID PI, + Function &F) { llvm_unreachable("Unable to find on the fly pass"); } @@ -1306,17 +1395,20 @@ Pass *AnalysisResolver::getAnalysisIfAvailable(AnalysisID ID, bool dir) const { return PM.findAnalysisPass(ID, dir); } -Pass *AnalysisResolver::findImplPass(Pass *P, AnalysisID AnalysisPI, - Function &F) { +std::tuple<Pass *, bool> +AnalysisResolver::findImplPass(Pass *P, AnalysisID AnalysisPI, Function &F) { return PM.getOnTheFlyPass(P, AnalysisPI, F); } +namespace llvm { +namespace legacy { + //===----------------------------------------------------------------------===// // FunctionPassManager implementation /// Create new Function pass manager FunctionPassManager::FunctionPassManager(Module *m) : M(m) { - FPM = new FunctionPassManagerImpl(); + FPM = new legacy::FunctionPassManagerImpl(); // FPM is the top level manager. FPM->setTopLevelManager(FPM); @@ -1355,36 +1447,8 @@ bool FunctionPassManager::doInitialization() { bool FunctionPassManager::doFinalization() { return FPM->doFinalization(*M); } - -//===----------------------------------------------------------------------===// -// FunctionPassManagerImpl implementation -// -bool FunctionPassManagerImpl::doInitialization(Module &M) { - bool Changed = false; - - dumpArguments(); - dumpPasses(); - - for (ImmutablePass *ImPass : getImmutablePasses()) - Changed |= ImPass->doInitialization(M); - - for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) - Changed |= getContainedManager(Index)->doInitialization(M); - - return Changed; -} - -bool FunctionPassManagerImpl::doFinalization(Module &M) { - bool Changed = false; - - for (int Index = getNumContainedManagers() - 1; Index >= 0; --Index) - Changed |= getContainedManager(Index)->doFinalization(M); - - for (ImmutablePass *ImPass : getImmutablePasses()) - Changed |= ImPass->doFinalization(M); - - return Changed; -} +} // namespace legacy +} // namespace llvm /// cleanup - After running all passes, clean up pass manager cache. void FPPassManager::cleanup() { @@ -1396,35 +1460,6 @@ void FPPassManager::cleanup() { } } -void FunctionPassManagerImpl::releaseMemoryOnTheFly() { - if (!wasRun) - return; - for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { - FPPassManager *FPPM = getContainedManager(Index); - for (unsigned Index = 0; Index < FPPM->getNumContainedPasses(); ++Index) { - FPPM->getContainedPass(Index)->releaseMemory(); - } - } - wasRun = false; -} - -// Execute all the passes managed by this top level manager. -// Return true if any function is modified by a pass. -bool FunctionPassManagerImpl::run(Function &F) { - bool Changed = false; - - initializeAllAnalysisInfo(); - for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { - Changed |= getContainedManager(Index)->runOnFunction(F); - F.getContext().yield(); - } - - for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) - getContainedManager(Index)->cleanup(); - - wasRun = true; - return Changed; -} //===----------------------------------------------------------------------===// // FPPassManager implementation @@ -1440,6 +1475,74 @@ void FPPassManager::dumpPassStructure(unsigned Offset) { } } +#ifdef EXPENSIVE_CHECKS +namespace { +namespace details { + +// Basic hashing mechanism to detect structural change to the IR, used to verify +// pass return status consistency with actual change. Loosely copied from +// llvm/lib/Transforms/Utils/FunctionComparator.cpp + +class StructuralHash { + uint64_t Hash = 0x6acaa36bef8325c5ULL; + + void update(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); } + +public: + StructuralHash() = default; + + void update(Function &F) { + if (F.empty()) + return; + + update(F.isVarArg()); + update(F.arg_size()); + + SmallVector<const BasicBlock *, 8> BBs; + SmallPtrSet<const BasicBlock *, 16> VisitedBBs; + + BBs.push_back(&F.getEntryBlock()); + VisitedBBs.insert(BBs[0]); + while (!BBs.empty()) { + const BasicBlock *BB = BBs.pop_back_val(); + update(45798); // Block header + for (auto &Inst : *BB) + update(Inst.getOpcode()); + + const Instruction *Term = BB->getTerminator(); + for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) { + if (!VisitedBBs.insert(Term->getSuccessor(i)).second) + continue; + BBs.push_back(Term->getSuccessor(i)); + } + } + } + + void update(Module &M) { + for (Function &F : M) + update(F); + } + + uint64_t getHash() const { return Hash; } +}; + +} // namespace details + +uint64_t StructuralHash(Function &F) { + details::StructuralHash H; + H.update(F); + return H.getHash(); +} + +uint64_t StructuralHash(Module &M) { + details::StructuralHash H; + H.update(M); + return H.getHash(); +} + +} // end anonymous namespace + +#endif /// Execute all of the passes scheduled for execution by invoking /// runOnFunction method. Keep track of whether any of the passes modifies @@ -1478,7 +1581,16 @@ bool FPPassManager::runOnFunction(Function &F) { { PassManagerPrettyStackEntry X(FP, F); TimeRegion PassTimer(getPassTimer(FP)); +#ifdef EXPENSIVE_CHECKS + uint64_t RefHash = StructuralHash(F); +#endif LocalChanged |= FP->runOnFunction(F); + +#ifdef EXPENSIVE_CHECKS + assert((LocalChanged || (RefHash == StructuralHash(F))) && + "Pass modifies its input and doesn't report it."); +#endif + if (EmitICRemark) { unsigned NewSize = F.getInstructionCount(); @@ -1551,7 +1663,7 @@ MPPassManager::runOnModule(Module &M) { // Initialize on-the-fly passes for (auto &OnTheFlyManager : OnTheFlyManagers) { - FunctionPassManagerImpl *FPP = OnTheFlyManager.second; + legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second; Changed |= FPP->doInitialization(M); } @@ -1579,7 +1691,17 @@ MPPassManager::runOnModule(Module &M) { PassManagerPrettyStackEntry X(MP, M); TimeRegion PassTimer(getPassTimer(MP)); +#ifdef EXPENSIVE_CHECKS + uint64_t RefHash = StructuralHash(M); +#endif + LocalChanged |= MP->runOnModule(M); + +#ifdef EXPENSIVE_CHECKS + assert((LocalChanged || (RefHash == StructuralHash(M))) && + "Pass modifies its input and doesn't report it."); +#endif + if (EmitICRemark) { // Update the size of the module. unsigned ModuleCount = M.getInstructionCount(); @@ -1612,7 +1734,7 @@ MPPassManager::runOnModule(Module &M) { // Finalize on-the-fly passes for (auto &OnTheFlyManager : OnTheFlyManagers) { - FunctionPassManagerImpl *FPP = OnTheFlyManager.second; + legacy::FunctionPassManagerImpl *FPP = OnTheFlyManager.second; // We don't know when is the last time an on-the-fly pass is run, // so we need to releaseMemory / finalize here FPP->releaseMemoryOnTheFly(); @@ -1633,9 +1755,9 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) { RequiredPass->getPotentialPassManagerType()) && "Unable to handle Pass that requires lower level Analysis pass"); - FunctionPassManagerImpl *FPP = OnTheFlyManagers[P]; + legacy::FunctionPassManagerImpl *FPP = OnTheFlyManagers[P]; if (!FPP) { - FPP = new FunctionPassManagerImpl(); + FPP = new legacy::FunctionPassManagerImpl(); // FPP is the top level manager. FPP->setTopLevelManager(FPP); @@ -1664,42 +1786,19 @@ void MPPassManager::addLowerLevelRequiredPass(Pass *P, Pass *RequiredPass) { /// Return function pass corresponding to PassInfo PI, that is /// required by module pass MP. Instantiate analysis pass, by using /// its runOnFunction() for function F. -Pass* MPPassManager::getOnTheFlyPass(Pass *MP, AnalysisID PI, Function &F){ - FunctionPassManagerImpl *FPP = OnTheFlyManagers[MP]; +std::tuple<Pass *, bool> MPPassManager::getOnTheFlyPass(Pass *MP, AnalysisID PI, + Function &F) { + legacy::FunctionPassManagerImpl *FPP = OnTheFlyManagers[MP]; assert(FPP && "Unable to find on the fly pass"); FPP->releaseMemoryOnTheFly(); - FPP->run(F); - return ((PMTopLevelManager*)FPP)->findAnalysisPass(PI); + bool Changed = FPP->run(F); + return std::make_tuple(((PMTopLevelManager *)FPP)->findAnalysisPass(PI), + Changed); } - -//===----------------------------------------------------------------------===// -// PassManagerImpl implementation - -// -/// run - Execute all of the passes scheduled for execution. Keep track of -/// whether any of the passes modifies the module, and if so, return true. -bool PassManagerImpl::run(Module &M) { - bool Changed = false; - - dumpArguments(); - dumpPasses(); - - for (ImmutablePass *ImPass : getImmutablePasses()) - Changed |= ImPass->doInitialization(M); - - initializeAllAnalysisInfo(); - for (unsigned Index = 0; Index < getNumContainedManagers(); ++Index) { - Changed |= getContainedManager(Index)->runOnModule(M); - M.getContext().yield(); - } - - for (ImmutablePass *ImPass : getImmutablePasses()) - Changed |= ImPass->doFinalization(M); - - return Changed; -} +namespace llvm { +namespace legacy { //===----------------------------------------------------------------------===// // PassManager implementation @@ -1724,6 +1823,8 @@ void PassManager::add(Pass *P) { bool PassManager::run(Module &M) { return PM->run(M); } +} // namespace legacy +} // namespace llvm //===----------------------------------------------------------------------===// // PMStack implementation @@ -1814,4 +1915,4 @@ void FunctionPass::assignPassManager(PMStack &PMS, PM->add(this); } -PassManagerBase::~PassManagerBase() {} +legacy::PassManagerBase::~PassManagerBase() {} diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index 7bdb85ace522..40d70f43132d 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -68,7 +68,7 @@ MDNode *MDBuilder::createFunctionEntryCount( Ops.push_back(createConstant(ConstantInt::get(Int64Ty, Count))); if (Imports) { SmallVector<GlobalValue::GUID, 2> OrderID(Imports->begin(), Imports->end()); - llvm::stable_sort(OrderID); + llvm::sort(OrderID); for (auto ID : OrderID) Ops.push_back(createConstant(ConstantInt::get(Int64Ty, ID))); } diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp index d73f748b0584..0d66e321c396 100644 --- a/llvm/lib/IR/Mangler.cpp +++ b/llvm/lib/IR/Mangler.cpp @@ -94,15 +94,18 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F, const DataLayout &DL) { // Calculate arguments size total. unsigned ArgWords = 0; + + const unsigned PtrSize = DL.getPointerSize(); + for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end(); AI != AE; ++AI) { - Type *Ty = AI->getType(); // 'Dereference' type in case of byval or inalloca parameter attribute. - if (AI->hasByValOrInAllocaAttr()) - Ty = cast<PointerType>(Ty)->getElementType(); + uint64_t AllocSize = AI->hasPassPointeeByValueAttr() ? + AI->getPassPointeeByValueCopySize(DL) : + DL.getTypeAllocSize(AI->getType()); + // Size should be aligned to pointer size. - unsigned PtrSize = DL.getPointerSize(); - ArgWords += alignTo(DL.getTypeAllocSize(Ty), PtrSize); + ArgWords += alignTo(AllocSize, PtrSize); } OS << '@' << ArgWords; diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index de092894d30c..ce89009e86eb 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -914,7 +914,7 @@ MDNode *MDNode::intersect(MDNode *A, MDNode *B) { SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); - MDs.remove_if([&](Metadata *MD) { return !is_contained(BSet, MD); }); + MDs.remove_if([&](Metadata *MD) { return !BSet.count(MD); }); // FIXME: This preserves long-standing behaviour, but is it really the right // behaviour? Or was that an unintended side-effect of node uniquing? @@ -934,7 +934,7 @@ MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF(); APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF(); - if (AVal.compare(BVal) == APFloat::cmpLessThan) + if (AVal < BVal) return A; return B; } @@ -1500,7 +1500,10 @@ void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { TypeID})); } -void GlobalObject::addVCallVisibilityMetadata(VCallVisibility Visibility) { +void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) { + // Remove any existing vcall visibility metadata first in case we are + // updating. + eraseMetadata(LLVMContext::MD_vcall_visibility); addMetadata(LLVMContext::MD_vcall_visibility, *MDNode::get(getContext(), {ConstantAsMetadata::get(ConstantInt::get( diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp index 271ae126d722..3ea181a9b48d 100644 --- a/llvm/lib/IR/Module.cpp +++ b/llvm/lib/IR/Module.cpp @@ -33,6 +33,7 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/SymbolTableListTraits.h" #include "llvm/IR/Type.h" #include "llvm/IR/TypeFinder.h" @@ -71,9 +72,9 @@ template class llvm::SymbolTableListTraits<GlobalIFunc>; // Module::Module(StringRef MID, LLVMContext &C) - : Context(C), Materializer(), ModuleID(MID), SourceFileName(MID), DL("") { - ValSymTab = new ValueSymbolTable(); - NamedMDSymTab = new StringMap<NamedMDNode *>(); + : Context(C), ValSymTab(std::make_unique<ValueSymbolTable>()), + Materializer(), ModuleID(std::string(MID)), + SourceFileName(std::string(MID)), DL("") { Context.addModule(this); } @@ -84,13 +85,11 @@ Module::~Module() { FunctionList.clear(); AliasList.clear(); IFuncList.clear(); - NamedMDList.clear(); - delete ValSymTab; - delete static_cast<StringMap<NamedMDNode *> *>(NamedMDSymTab); } -std::unique_ptr<RandomNumberGenerator> Module::createRNG(const Pass* P) const { - SmallString<32> Salt(P->getPassName()); +std::unique_ptr<RandomNumberGenerator> +Module::createRNG(const StringRef Name) const { + SmallString<32> Salt(Name); // This RNG is guaranteed to produce the same random stream only // when the Module ID and thus the input filename is the same. This @@ -104,7 +103,8 @@ std::unique_ptr<RandomNumberGenerator> Module::createRNG(const Pass* P) const { // store salt metadata from the Module constructor. Salt += sys::path::filename(getModuleIdentifier()); - return std::unique_ptr<RandomNumberGenerator>(new RandomNumberGenerator(Salt)); + return std::unique_ptr<RandomNumberGenerator>( + new RandomNumberGenerator(Salt)); } /// getNamedValue - Return the first global value in the module with @@ -250,15 +250,14 @@ GlobalIFunc *Module::getNamedIFunc(StringRef Name) const { NamedMDNode *Module::getNamedMetadata(const Twine &Name) const { SmallString<256> NameData; StringRef NameRef = Name.toStringRef(NameData); - return static_cast<StringMap<NamedMDNode*> *>(NamedMDSymTab)->lookup(NameRef); + return NamedMDSymTab.lookup(NameRef); } /// getOrInsertNamedMetadata - Return the first named MDNode in the module /// with the specified name. This method returns a new NamedMDNode if a /// NamedMDNode with the specified name is not found. NamedMDNode *Module::getOrInsertNamedMetadata(StringRef Name) { - NamedMDNode *&NMD = - (*static_cast<StringMap<NamedMDNode *> *>(NamedMDSymTab))[Name]; + NamedMDNode *&NMD = NamedMDSymTab[Name]; if (!NMD) { NMD = new NamedMDNode(Name); NMD->setParent(this); @@ -270,7 +269,7 @@ NamedMDNode *Module::getOrInsertNamedMetadata(StringRef Name) { /// eraseNamedMetadata - Remove the given NamedMDNode from this module and /// delete it. void Module::eraseNamedMetadata(NamedMDNode *NMD) { - static_cast<StringMap<NamedMDNode *> *>(NamedMDSymTab)->erase(NMD->getName()); + NamedMDSymTab.erase(NMD->getName()); NamedMDList.erase(NMD->getIterator()); } @@ -285,6 +284,20 @@ bool Module::isValidModFlagBehavior(Metadata *MD, ModFlagBehavior &MFB) { return false; } +bool Module::isValidModuleFlag(const MDNode &ModFlag, ModFlagBehavior &MFB, + MDString *&Key, Metadata *&Val) { + if (ModFlag.getNumOperands() < 3) + return false; + if (!isValidModFlagBehavior(ModFlag.getOperand(0), MFB)) + return false; + MDString *K = dyn_cast_or_null<MDString>(ModFlag.getOperand(1)); + if (!K) + return false; + Key = K; + Val = ModFlag.getOperand(2); + return true; +} + /// getModuleFlagsMetadata - Returns the module flags in the provided vector. void Module:: getModuleFlagsMetadata(SmallVectorImpl<ModuleFlagEntry> &Flags) const { @@ -293,13 +306,11 @@ getModuleFlagsMetadata(SmallVectorImpl<ModuleFlagEntry> &Flags) const { for (const MDNode *Flag : ModFlags->operands()) { ModFlagBehavior MFB; - if (Flag->getNumOperands() >= 3 && - isValidModFlagBehavior(Flag->getOperand(0), MFB) && - dyn_cast_or_null<MDString>(Flag->getOperand(1))) { + MDString *Key = nullptr; + Metadata *Val = nullptr; + if (isValidModuleFlag(*Flag, MFB, Key, Val)) { // Check the operands of the MDNode before accessing the operands. // The verifier will actually catch these failures. - MDString *Key = cast<MDString>(Flag->getOperand(1)); - Metadata *Val = Flag->getOperand(2); Flags.push_back(ModuleFlagEntry(MFB, Key, Val)); } } @@ -360,6 +371,23 @@ void Module::addModuleFlag(MDNode *Node) { getOrInsertModuleFlagsMetadata()->addOperand(Node); } +void Module::setModuleFlag(ModFlagBehavior Behavior, StringRef Key, + Metadata *Val) { + NamedMDNode *ModFlags = getOrInsertModuleFlagsMetadata(); + // Replace the flag if it already exists. + for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { + MDNode *Flag = ModFlags->getOperand(I); + ModFlagBehavior MFB; + MDString *K = nullptr; + Metadata *V = nullptr; + if (isValidModuleFlag(*Flag, MFB, K, V) && K->getString() == Key) { + Flag->replaceOperandWith(2, Val); + return; + } + } + addModuleFlag(Behavior, Key, Val); +} + void Module::setDataLayout(StringRef Desc) { DL.reset(Desc); } @@ -549,9 +577,9 @@ void Module::setCodeModel(CodeModel::Model CL) { void Module::setProfileSummary(Metadata *M, ProfileSummary::Kind Kind) { if (Kind == ProfileSummary::PSK_CSInstr) - addModuleFlag(ModFlagBehavior::Error, "CSProfileSummary", M); + setModuleFlag(ModFlagBehavior::Error, "CSProfileSummary", M); else - addModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M); + setModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M); } Metadata *Module::getProfileSummary(bool IsCS) { @@ -559,6 +587,27 @@ Metadata *Module::getProfileSummary(bool IsCS) { : getModuleFlag("ProfileSummary")); } +bool Module::getSemanticInterposition() const { + Metadata *MF = getModuleFlag("SemanticInterposition"); + + auto *Val = cast_or_null<ConstantAsMetadata>(MF); + if (!Val) + return false; + + return cast<ConstantInt>(Val->getValue())->getZExtValue(); +} + +void Module::setSemanticInterposition(bool SI) { + addModuleFlag(ModFlagBehavior::Error, "SemanticInterposition", SI); +} + +bool Module::noSemanticInterposition() const { + // Conservatively require an explicit zero value for now. + Metadata *MF = getModuleFlag("SemanticInterposition"); + auto *Val = cast_or_null<ConstantAsMetadata>(MF); + return Val && cast<ConstantInt>(Val->getValue())->getZExtValue() == 0; +} + void Module::setOwnedMemoryBuffer(std::unique_ptr<MemoryBuffer> MB) { OwnedMemoryBuffer = std::move(MB); } @@ -625,3 +674,23 @@ GlobalVariable *llvm::collectUsedGlobalVariables( } return GV; } + +void Module::setPartialSampleProfileRatio(const ModuleSummaryIndex &Index) { + if (auto *SummaryMD = getProfileSummary(/*IsCS*/ false)) { + std::unique_ptr<ProfileSummary> ProfileSummary( + ProfileSummary::getFromMD(SummaryMD)); + if (ProfileSummary) { + if (ProfileSummary->getKind() != ProfileSummary::PSK_Sample || + !ProfileSummary->isPartialProfile()) + return; + uint64_t BlockCount = Index.getBlockCount(); + uint32_t NumCounts = ProfileSummary->getNumCounts(); + if (!NumCounts) + return; + double Ratio = (double)BlockCount / NumCounts; + ProfileSummary->setPartialProfileRatio(Ratio); + setProfileSummary(ProfileSummary->getMD(getContext()), + ProfileSummary::PSK_Sample); + } + } +} diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp index 180f96269a13..91612eafada7 100644 --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -31,6 +31,12 @@ static cl::opt<bool> PropagateAttrs("propagate-attrs", cl::init(true), cl::Hidden, cl::desc("Propagate attributes in index")); +static cl::opt<bool> ImportConstantsWithRefs( + "import-constants-with-refs", cl::init(true), cl::Hidden, + cl::desc("Import constant global variables with references")); + +constexpr uint32_t FunctionSummary::ParamAccess::RangeWidth; + FunctionSummary FunctionSummary::ExternalNode = FunctionSummary::makeDummyFunctionSummary({}); @@ -68,6 +74,52 @@ std::pair<unsigned, unsigned> FunctionSummary::specialRefCounts() const { constexpr uint64_t ModuleSummaryIndex::BitcodeSummaryVersion; +uint64_t ModuleSummaryIndex::getFlags() const { + uint64_t Flags = 0; + if (withGlobalValueDeadStripping()) + Flags |= 0x1; + if (skipModuleByDistributedBackend()) + Flags |= 0x2; + if (hasSyntheticEntryCounts()) + Flags |= 0x4; + if (enableSplitLTOUnit()) + Flags |= 0x8; + if (partiallySplitLTOUnits()) + Flags |= 0x10; + if (withAttributePropagation()) + Flags |= 0x20; + return Flags; +} + +void ModuleSummaryIndex::setFlags(uint64_t Flags) { + assert(Flags <= 0x3f && "Unexpected bits in flag"); + // 1 bit: WithGlobalValueDeadStripping flag. + // Set on combined index only. + if (Flags & 0x1) + setWithGlobalValueDeadStripping(); + // 1 bit: SkipModuleByDistributedBackend flag. + // Set on combined index only. + if (Flags & 0x2) + setSkipModuleByDistributedBackend(); + // 1 bit: HasSyntheticEntryCounts flag. + // Set on combined index only. + if (Flags & 0x4) + setHasSyntheticEntryCounts(); + // 1 bit: DisableSplitLTOUnit flag. + // Set on per module indexes. It is up to the client to validate + // the consistency of this flag across modules being linked. + if (Flags & 0x8) + setEnableSplitLTOUnit(); + // 1 bit: PartiallySplitLTOUnits flag. + // Set on combined index only. + if (Flags & 0x10) + setPartiallySplitLTOUnits(); + // 1 bit: WithAttributePropagation flag. + // Set on combined index only. + if (Flags & 0x20) + setWithAttributePropagation(); +} + // Collect for the given module the list of function it defines // (GUID -> Summary). void ModuleSummaryIndex::collectDefinedFunctionsForModule( @@ -221,7 +273,8 @@ bool ModuleSummaryIndex::canImportGlobalVar(GlobalValueSummary *S, // c) Link error (external declaration with internal definition). // However we do not promote objects referenced by writeonly GV // initializer by means of converting it to 'zeroinitializer' - return !isReadOnly(GVS) && !isWriteOnly(GVS) && GVS->refs().size(); + return !(ImportConstantsWithRefs && GVS->isConstant()) && + !isReadOnly(GVS) && !isWriteOnly(GVS) && GVS->refs().size(); }; auto *GVS = cast<GlobalVarSummary>(S->getBaseObject()); @@ -249,7 +302,7 @@ void ModuleSummaryIndex::dumpSCCs(raw_ostream &O) { if (V.getSummaryList().size()) F = cast<FunctionSummary>(V.getSummaryList().front().get()); O << " " << (F == nullptr ? "External" : "") << " " << utostr(V.getGUID()) - << (I.hasLoop() ? " (has loop)" : "") << "\n"; + << (I.hasCycle() ? " (has cycle)" : "") << "\n"; } O << "}\n"; } @@ -405,6 +458,12 @@ static bool hasWriteOnlyFlag(const GlobalValueSummary *S) { return false; } +static bool hasConstantFlag(const GlobalValueSummary *S) { + if (auto *GVS = dyn_cast<GlobalVarSummary>(S)) + return GVS->isConstant(); + return false; +} + void ModuleSummaryIndex::exportToDot( raw_ostream &OS, const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const { @@ -482,6 +541,8 @@ void ModuleSummaryIndex::exportToDot( A.addComment("immutable"); if (Flags.Live && hasWriteOnlyFlag(SummaryIt.second)) A.addComment("writeOnly"); + if (Flags.Live && hasConstantFlag(SummaryIt.second)) + A.addComment("constant"); } if (Flags.DSOLocal) A.addComment("dsoLocal"); diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp index 8ba68674d50e..0f70fc37dee2 100644 --- a/llvm/lib/IR/Operator.cpp +++ b/llvm/lib/IR/Operator.cpp @@ -31,33 +31,107 @@ Type *GEPOperator::getResultElementType() const { return cast<GetElementPtrConstantExpr>(this)->getResultElementType(); } -bool GEPOperator::accumulateConstantOffset(const DataLayout &DL, - APInt &Offset) const { - assert(Offset.getBitWidth() == - DL.getIndexSizeInBits(getPointerAddressSpace()) && - "The offset bit width does not match DL specification."); +Align GEPOperator::getMaxPreservedAlignment(const DataLayout &DL) const { + /// compute the worse possible offset for every level of the GEP et accumulate + /// the minimum alignment into Result. + Align Result = Align(llvm::Value::MaximumAlignment); for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this); GTI != GTE; ++GTI) { + int64_t Offset = 1; ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand()); - if (!OpC) - return false; - if (OpC->isZero()) - continue; - // Handle a struct index, which adds its field offset to the pointer. if (StructType *STy = GTI.getStructTypeOrNull()) { - unsigned ElementIdx = OpC->getZExtValue(); const StructLayout *SL = DL.getStructLayout(STy); - Offset += APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); - continue; + Offset = SL->getElementOffset(OpC->getZExtValue()); + } else { + assert(GTI.isSequential() && "should be sequencial"); + /// If the index isn't know we take 1 because it is the index that will + /// give the worse alignment of the offset. + int64_t ElemCount = 1; + if (OpC) + ElemCount = OpC->getZExtValue(); + Offset = DL.getTypeAllocSize(GTI.getIndexedType()) * ElemCount; } + Result = Align(MinAlign(Offset, Result.value())); + } + return Result; +} +bool GEPOperator::accumulateConstantOffset( + const DataLayout &DL, APInt &Offset, + function_ref<bool(Value &, APInt &)> ExternalAnalysis) const { + assert(Offset.getBitWidth() == + DL.getIndexSizeInBits(getPointerAddressSpace()) && + "The offset bit width does not match DL specification."); + + bool UsedExternalAnalysis = false; + auto AccumulateOffset = [&](APInt Index, uint64_t Size) -> bool { + Index = Index.sextOrTrunc(Offset.getBitWidth()); + APInt IndexedSize = APInt(Offset.getBitWidth(), Size); // For array or vector indices, scale the index by the size of the type. - APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth()); - Offset += Index * APInt(Offset.getBitWidth(), - DL.getTypeAllocSize(GTI.getIndexedType())); + if (!UsedExternalAnalysis) { + Offset += Index * IndexedSize; + } else { + // External Analysis can return a result higher/lower than the value + // represents. We need to detect overflow/underflow. + bool Overflow = false; + APInt OffsetPlus = Index.smul_ov(IndexedSize, Overflow); + if (Overflow) + return false; + Offset = Offset.sadd_ov(OffsetPlus, Overflow); + if (Overflow) + return false; + } + return true; + }; + + for (gep_type_iterator GTI = gep_type_begin(this), GTE = gep_type_end(this); + GTI != GTE; ++GTI) { + // Scalable vectors are multiplied by a runtime constant. + bool ScalableType = false; + if (isa<ScalableVectorType>(GTI.getIndexedType())) + ScalableType = true; + + Value *V = GTI.getOperand(); + StructType *STy = GTI.getStructTypeOrNull(); + // Handle ConstantInt if possible. + if (auto ConstOffset = dyn_cast<ConstantInt>(V)) { + if (ConstOffset->isZero()) + continue; + // if the type is scalable and the constant is not zero (vscale * n * 0 = + // 0) bailout. + if (ScalableType) + return false; + // Handle a struct index, which adds its field offset to the pointer. + if (STy) { + unsigned ElementIdx = ConstOffset->getZExtValue(); + const StructLayout *SL = DL.getStructLayout(STy); + // Element offset is in bytes. + if (!AccumulateOffset( + APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)), + 1)) + return false; + continue; + } + if (!AccumulateOffset(ConstOffset->getValue(), + DL.getTypeAllocSize(GTI.getIndexedType()))) + return false; + continue; + } + + // The operand is not constant, check if an external analysis was provided. + // External analsis is not applicable to a struct type. + if (!ExternalAnalysis || STy || ScalableType) + return false; + APInt AnalysisIndex; + if (!ExternalAnalysis(*V, AnalysisIndex)) + return false; + UsedExternalAnalysis = true; + if (!AccumulateOffset(AnalysisIndex, + DL.getTypeAllocSize(GTI.getIndexedType()))) + return false; } return true; } -} +} // namespace llvm diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp index dbdbbf4cf35e..a815da2bdc51 100644 --- a/llvm/lib/IR/Pass.cpp +++ b/llvm/lib/IR/Pass.cpp @@ -14,8 +14,6 @@ #include "llvm/Pass.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/Attributes.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LLVMContext.h" @@ -24,7 +22,6 @@ #include "llvm/IR/OptBisect.h" #include "llvm/PassInfo.h" #include "llvm/PassRegistry.h" -#include "llvm/PassSupport.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp index cde9b873795e..624827ff8cd9 100644 --- a/llvm/lib/IR/PassManager.cpp +++ b/llvm/lib/IR/PassManager.cpp @@ -9,6 +9,7 @@ #include "llvm/IR/PassManager.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/PassManagerImpl.h" using namespace llvm; @@ -88,7 +89,7 @@ bool FunctionAnalysisManagerModuleProxy::Result::invalidate( // Return false to indicate that this result is still a valid proxy. return false; } -} +} // namespace llvm AnalysisSetKey CFGAnalyses::SetKey; diff --git a/llvm/lib/IR/PassRegistry.cpp b/llvm/lib/IR/PassRegistry.cpp index 92c188b11898..0572c4fe5237 100644 --- a/llvm/lib/IR/PassRegistry.cpp +++ b/llvm/lib/IR/PassRegistry.cpp @@ -13,8 +13,8 @@ #include "llvm/PassRegistry.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/Pass.h" #include "llvm/PassInfo.h" -#include "llvm/PassSupport.h" #include "llvm/Support/ManagedStatic.h" #include <cassert> #include <memory> diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp index 9cc44ea05fee..25275e5733ac 100644 --- a/llvm/lib/IR/PassTimingInfo.cpp +++ b/llvm/lib/IR/PassTimingInfo.cpp @@ -168,17 +168,18 @@ void reportAndResetTimings(raw_ostream *OutStream) { /// Returns the timer for the specified pass invocation of \p PassID. /// Each time it creates a new timer. Timer &TimePassesHandler::getPassTimer(StringRef PassID) { - // Bump counts for each request of the timer. - unsigned Count = nextPassID(PassID); + // Take a vector of Timers created for this \p PassID and append + // one more timer to it. + TimerVector &Timers = TimingData[PassID]; + unsigned Count = Timers.size() + 1; - // Unconditionally appending description with a pass-invocation number. std::string FullDesc = formatv("{0} #{1}", PassID, Count).str(); - PassInvocationID UID{PassID, Count}; Timer *T = new Timer(PassID, FullDesc, TG); - auto Pair = TimingData.try_emplace(UID, T); - assert(Pair.second && "should always create a new timer"); - return *(Pair.first->second.get()); + Timers.emplace_back(T); + assert(Count == Timers.size() && "sanity check"); + + return *T; } TimePassesHandler::TimePassesHandler(bool Enabled) @@ -198,17 +199,23 @@ LLVM_DUMP_METHOD void TimePassesHandler::dump() const { dbgs() << "Dumping timers for " << getTypeName<TimePassesHandler>() << ":\n\tRunning:\n"; for (auto &I : TimingData) { - const Timer *MyTimer = I.second.get(); - if (!MyTimer || MyTimer->isRunning()) - dbgs() << "\tTimer " << MyTimer << " for pass " << I.first.first << "(" - << I.first.second << ")\n"; + StringRef PassID = I.getKey(); + const TimerVector& MyTimers = I.getValue(); + for (unsigned idx = 0; idx < MyTimers.size(); idx++) { + const Timer* MyTimer = MyTimers[idx].get(); + if (MyTimer && MyTimer->isRunning()) + dbgs() << "\tTimer " << MyTimer << " for pass " << PassID << "(" << idx << ")\n"; + } } dbgs() << "\tTriggered:\n"; for (auto &I : TimingData) { - const Timer *MyTimer = I.second.get(); - if (!MyTimer || (MyTimer->hasTriggered() && !MyTimer->isRunning())) - dbgs() << "\tTimer " << MyTimer << " for pass " << I.first.first << "(" - << I.first.second << ")\n"; + StringRef PassID = I.getKey(); + const TimerVector& MyTimers = I.getValue(); + for (unsigned idx = 0; idx < MyTimers.size(); idx++) { + const Timer* MyTimer = MyTimers[idx].get(); + if (MyTimer && MyTimer->hasTriggered() && !MyTimer->isRunning()) + dbgs() << "\tTimer " << MyTimer << " for pass " << PassID << "(" << idx << ")\n"; + } } } diff --git a/llvm/lib/IR/ProfileSummary.cpp b/llvm/lib/IR/ProfileSummary.cpp index 11d95ac19be6..ac6bcd9fe3af 100644 --- a/llvm/lib/IR/ProfileSummary.cpp +++ b/llvm/lib/IR/ProfileSummary.cpp @@ -18,6 +18,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/Format.h" using namespace llvm; @@ -31,6 +32,14 @@ static Metadata *getKeyValMD(LLVMContext &Context, const char *Key, return MDTuple::get(Context, Ops); } +static Metadata *getKeyFPValMD(LLVMContext &Context, const char *Key, + double Val) { + Type *DoubleTy = Type::getDoubleTy(Context); + Metadata *Ops[2] = {MDString::get(Context, Key), + ConstantAsMetadata::get(ConstantFP::get(DoubleTy, Val))}; + return MDTuple::get(Context, Ops); +} + // Return an MDTuple with two elements. The first element is a string Key and // the second is a string Value. static Metadata *getKeyValMD(LLVMContext &Context, const char *Key, @@ -65,35 +74,63 @@ Metadata *ProfileSummary::getDetailedSummaryMD(LLVMContext &Context) { // "ProfileFormat" and a string representing the format ("InstrProf" or // "SampleProfile"). The rest of the elements of the outer MDTuple are specific // to the kind of profile summary as returned by getFormatSpecificMD. -Metadata *ProfileSummary::getMD(LLVMContext &Context) { +// IsPartialProfile is an optional field and \p AddPartialField will decide +// whether to add a field for it. +// PartialProfileRatio is an optional field and \p AddPartialProfileRatioField +// will decide whether to add a field for it. +Metadata *ProfileSummary::getMD(LLVMContext &Context, bool AddPartialField, + bool AddPartialProfileRatioField) { const char *KindStr[3] = {"InstrProf", "CSInstrProf", "SampleProfile"}; - Metadata *Components[] = { - getKeyValMD(Context, "ProfileFormat", KindStr[PSK]), - getKeyValMD(Context, "TotalCount", getTotalCount()), - getKeyValMD(Context, "MaxCount", getMaxCount()), - getKeyValMD(Context, "MaxInternalCount", getMaxInternalCount()), - getKeyValMD(Context, "MaxFunctionCount", getMaxFunctionCount()), - getKeyValMD(Context, "NumCounts", getNumCounts()), - getKeyValMD(Context, "NumFunctions", getNumFunctions()), - getDetailedSummaryMD(Context), - }; + SmallVector<Metadata *, 16> Components; + Components.push_back(getKeyValMD(Context, "ProfileFormat", KindStr[PSK])); + Components.push_back(getKeyValMD(Context, "TotalCount", getTotalCount())); + Components.push_back(getKeyValMD(Context, "MaxCount", getMaxCount())); + Components.push_back( + getKeyValMD(Context, "MaxInternalCount", getMaxInternalCount())); + Components.push_back( + getKeyValMD(Context, "MaxFunctionCount", getMaxFunctionCount())); + Components.push_back(getKeyValMD(Context, "NumCounts", getNumCounts())); + Components.push_back(getKeyValMD(Context, "NumFunctions", getNumFunctions())); + if (AddPartialField) + Components.push_back( + getKeyValMD(Context, "IsPartialProfile", isPartialProfile())); + if (AddPartialProfileRatioField) + Components.push_back(getKeyFPValMD(Context, "PartialProfileRatio", + getPartialProfileRatio())); + Components.push_back(getDetailedSummaryMD(Context)); return MDTuple::get(Context, Components); } -// Parse an MDTuple representing (Key, Val) pair. -static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val) { +// Get the value metadata for the input MD/Key. +static ConstantAsMetadata *getValMD(MDTuple *MD, const char *Key) { if (!MD) - return false; + return nullptr; if (MD->getNumOperands() != 2) - return false; + return nullptr; MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0)); ConstantAsMetadata *ValMD = dyn_cast<ConstantAsMetadata>(MD->getOperand(1)); if (!KeyMD || !ValMD) - return false; + return nullptr; if (!KeyMD->getString().equals(Key)) - return false; - Val = cast<ConstantInt>(ValMD->getValue())->getZExtValue(); - return true; + return nullptr; + return ValMD; +} + +// Parse an MDTuple representing (Key, Val) pair. +static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val) { + if (auto *ValMD = getValMD(MD, Key)) { + Val = cast<ConstantInt>(ValMD->getValue())->getZExtValue(); + return true; + } + return false; +} + +static bool getVal(MDTuple *MD, const char *Key, double &Val) { + if (auto *ValMD = getValMD(MD, Key)) { + Val = cast<ConstantFP>(ValMD->getValue())->getValueAPF().convertToDouble(); + return true; + } + return false; } // Check if an MDTuple represents a (Key, Val) pair. @@ -139,12 +176,29 @@ static bool getSummaryFromMD(MDTuple *MD, SummaryEntryVector &Summary) { return true; } +// Get the value of an optional field. Increment 'Idx' if it was present. Return +// true if we can move onto the next field. +template <typename ValueType> +static bool getOptionalVal(MDTuple *Tuple, unsigned &Idx, const char *Key, + ValueType &Value) { + if (getVal(dyn_cast<MDTuple>(Tuple->getOperand(Idx)), Key, Value)) { + Idx++; + // Need to make sure when the key is present, we won't step over the bound + // of Tuple operand array. Since (non-optional) DetailedSummary always comes + // last, the next entry in the tuple operand array must exist. + return Idx < Tuple->getNumOperands(); + } + // It was absent, keep going. + return true; +} + ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) { MDTuple *Tuple = dyn_cast_or_null<MDTuple>(MD); - if (!Tuple || Tuple->getNumOperands() != 8) + if (!Tuple || Tuple->getNumOperands() < 8 || Tuple->getNumOperands() > 10) return nullptr; - auto &FormatMD = Tuple->getOperand(0); + unsigned I = 0; + auto &FormatMD = Tuple->getOperand(I++); ProfileSummary::Kind SummaryKind; if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat", "SampleProfile")) @@ -160,27 +214,55 @@ ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) { uint64_t NumCounts, TotalCount, NumFunctions, MaxFunctionCount, MaxCount, MaxInternalCount; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(1)), "TotalCount", + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "TotalCount", TotalCount)) return nullptr; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(2)), "MaxCount", MaxCount)) + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxCount", MaxCount)) return nullptr; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(3)), "MaxInternalCount", + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxInternalCount", MaxInternalCount)) return nullptr; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(4)), "MaxFunctionCount", + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "MaxFunctionCount", MaxFunctionCount)) return nullptr; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(5)), "NumCounts", NumCounts)) + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "NumCounts", + NumCounts)) return nullptr; - if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(6)), "NumFunctions", + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(I++)), "NumFunctions", NumFunctions)) return nullptr; + // Optional fields. Need to initialize because the fields are optional. + uint64_t IsPartialProfile = 0; + if (!getOptionalVal(Tuple, I, "IsPartialProfile", IsPartialProfile)) + return nullptr; + double PartialProfileRatio = 0; + if (!getOptionalVal(Tuple, I, "PartialProfileRatio", PartialProfileRatio)) + return nullptr; + SummaryEntryVector Summary; - if (!getSummaryFromMD(dyn_cast<MDTuple>(Tuple->getOperand(7)), Summary)) + if (!getSummaryFromMD(dyn_cast<MDTuple>(Tuple->getOperand(I++)), Summary)) return nullptr; return new ProfileSummary(SummaryKind, std::move(Summary), TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount, - NumCounts, NumFunctions); + NumCounts, NumFunctions, IsPartialProfile, + PartialProfileRatio); +} + +void ProfileSummary::printSummary(raw_ostream &OS) { + OS << "Total functions: " << NumFunctions << "\n"; + OS << "Maximum function count: " << MaxFunctionCount << "\n"; + OS << "Maximum block count: " << MaxCount << "\n"; + OS << "Total number of blocks: " << NumCounts << "\n"; + OS << "Total count: " << TotalCount << "\n"; +} + +void ProfileSummary::printDetailedSummary(raw_ostream &OS) { + OS << "Detailed summary:\n"; + for (auto Entry : DetailedSummary) { + OS << Entry.NumCounts << " blocks with count >= " << Entry.MinCount + << " account for " + << format("%0.6g", (float)Entry.Cutoff / Scale * 100) + << " percentage of the total counts.\n"; + } } diff --git a/llvm/lib/IR/SafepointIRVerifier.cpp b/llvm/lib/IR/SafepointIRVerifier.cpp index f9578394a827..6bf7caa50a12 100644 --- a/llvm/lib/IR/SafepointIRVerifier.cpp +++ b/llvm/lib/IR/SafepointIRVerifier.cpp @@ -45,6 +45,7 @@ #include "llvm/IR/Statepoint.h" #include "llvm/IR/Value.h" #include "llvm/InitializePasses.h" +#include "llvm/Support/Allocator.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -207,7 +208,7 @@ PreservedAnalyses SafepointIRVerifierPass::run(Function &F, Verify(F, DT, CD); return PreservedAnalyses::all(); } -} +} // namespace llvm namespace { @@ -782,7 +783,7 @@ void GCPtrTracker::transferBlock(const BasicBlock *BB, BasicBlockState &BBS, void GCPtrTracker::transferInstruction(const Instruction &I, bool &Cleared, AvailableValueSet &Available) { - if (isStatepoint(I)) { + if (isa<GCStatepointInst>(I)) { Cleared = true; Available.clear(); } else if (containsGCPtrType(I.getType())) diff --git a/llvm/lib/IR/Statepoint.cpp b/llvm/lib/IR/Statepoint.cpp index fce89b42e9bf..bbfbbe489bae 100644 --- a/llvm/lib/IR/Statepoint.cpp +++ b/llvm/lib/IR/Statepoint.cpp @@ -17,40 +17,6 @@ using namespace llvm; -bool llvm::isStatepoint(const CallBase *Call) { - if (auto *F = Call->getCalledFunction()) - return F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint; - return false; -} - -bool llvm::isStatepoint(const Value *V) { - if (auto *Call = dyn_cast<CallBase>(V)) - return isStatepoint(Call); - return false; -} - -bool llvm::isStatepoint(const Value &V) { - return isStatepoint(&V); -} - -bool llvm::isGCRelocate(const CallBase *Call) { - return isa<GCRelocateInst>(Call); -} - -bool llvm::isGCRelocate(const Value *V) { - if (auto *Call = dyn_cast<CallBase>(V)) - return isGCRelocate(Call); - return false; -} - -bool llvm::isGCResult(const CallBase *Call) { return isa<GCResultInst>(Call); } - -bool llvm::isGCResult(const Value *V) { - if (auto *Call = dyn_cast<CallBase>(V)) - return isGCResult(Call); - return false; -} - bool llvm::isStatepointDirectiveAttr(Attribute Attr) { return Attr.hasAttribute("statepoint-id") || Attr.hasAttribute("statepoint-num-patch-bytes"); diff --git a/llvm/lib/IR/SymbolTableListTraitsImpl.h b/llvm/lib/IR/SymbolTableListTraitsImpl.h index f399c823d6fb..4283744bd058 100644 --- a/llvm/lib/IR/SymbolTableListTraitsImpl.h +++ b/llvm/lib/IR/SymbolTableListTraitsImpl.h @@ -20,6 +20,11 @@ namespace llvm { +/// Notify basic blocks when an instruction is inserted. +template <typename ParentClass> +inline void invalidateParentIListOrdering(ParentClass *Parent) {} +template <> void invalidateParentIListOrdering(BasicBlock *BB); + /// setSymTabObject - This is called when (f.e.) the parent of a basic block /// changes. This requires us to remove all the instruction symtab entries from /// the current function and reinsert them into the new function. @@ -64,6 +69,7 @@ void SymbolTableListTraits<ValueSubClass>::addNodeToList(ValueSubClass *V) { assert(!V->getParent() && "Value already in a container!!"); ItemParentClass *Owner = getListOwner(); V->setParent(Owner); + invalidateParentIListOrdering(Owner); if (V->hasName()) if (ValueSymbolTable *ST = getSymTab(Owner)) ST->reinsertValue(V); @@ -81,8 +87,13 @@ void SymbolTableListTraits<ValueSubClass>::removeNodeFromList( template <typename ValueSubClass> void SymbolTableListTraits<ValueSubClass>::transferNodesFromList( SymbolTableListTraits &L2, iterator first, iterator last) { - // We only have to do work here if transferring instructions between BBs - ItemParentClass *NewIP = getListOwner(), *OldIP = L2.getListOwner(); + // Transfering nodes, even within the same BB, invalidates the ordering. The + // list that we removed the nodes from still has a valid ordering. + ItemParentClass *NewIP = getListOwner(); + invalidateParentIListOrdering(NewIP); + + // Nothing else needs to be done if we're reording nodes within the same list. + ItemParentClass *OldIP = L2.getListOwner(); if (NewIP == OldIP) return; diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 3eab5042b542..d869a6e07cca 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -40,6 +40,7 @@ Type *Type::getPrimitiveType(LLVMContext &C, TypeID IDNumber) { switch (IDNumber) { case VoidTyID : return getVoidTy(C); case HalfTyID : return getHalfTy(C); + case BFloatTyID : return getBFloatTy(C); case FloatTyID : return getFloatTy(C); case DoubleTyID : return getDoubleTy(C); case X86_FP80TyID : return getX86_FP80Ty(C); @@ -68,20 +69,17 @@ bool Type::canLosslesslyBitCastTo(Type *Ty) const { return false; // Vector -> Vector conversions are always lossless if the two vector types - // have the same size, otherwise not. Also, 64-bit vector types can be - // converted to x86mmx. - if (auto *thisPTy = dyn_cast<VectorType>(this)) { - if (auto *thatPTy = dyn_cast<VectorType>(Ty)) - return thisPTy->getBitWidth() == thatPTy->getBitWidth(); - if (Ty->getTypeID() == Type::X86_MMXTyID && - thisPTy->getBitWidth() == 64) - return true; - } + // have the same size, otherwise not. + if (isa<VectorType>(this) && isa<VectorType>(Ty)) + return getPrimitiveSizeInBits() == Ty->getPrimitiveSizeInBits(); - if (this->getTypeID() == Type::X86_MMXTyID) - if (auto *thatPTy = dyn_cast<VectorType>(Ty)) - if (thatPTy->getBitWidth() == 64) - return true; + // 64-bit fixed width vector types can be losslessly converted to x86mmx. + if (((isa<FixedVectorType>(this)) && Ty->isX86_MMXTy()) && + getPrimitiveSizeInBits().getFixedSize() == 64) + return true; + if ((isX86_MMXTy() && isa<FixedVectorType>(Ty)) && + Ty->getPrimitiveSizeInBits().getFixedSize() == 64) + return true; // At this point we have only various mismatches of the first class types // remaining and ptr->ptr. Just select the lossless conversions. Everything @@ -115,6 +113,7 @@ bool Type::isEmptyTy() const { TypeSize Type::getPrimitiveSizeInBits() const { switch (getTypeID()) { case Type::HalfTyID: return TypeSize::Fixed(16); + case Type::BFloatTyID: return TypeSize::Fixed(16); case Type::FloatTyID: return TypeSize::Fixed(32); case Type::DoubleTyID: return TypeSize::Fixed(64); case Type::X86_FP80TyID: return TypeSize::Fixed(80); @@ -123,16 +122,21 @@ TypeSize Type::getPrimitiveSizeInBits() const { case Type::X86_MMXTyID: return TypeSize::Fixed(64); case Type::IntegerTyID: return TypeSize::Fixed(cast<IntegerType>(this)->getBitWidth()); - case Type::VectorTyID: { + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: { const VectorType *VTy = cast<VectorType>(this); - return TypeSize(VTy->getBitWidth(), VTy->isScalable()); + ElementCount EC = VTy->getElementCount(); + TypeSize ETS = VTy->getElementType()->getPrimitiveSizeInBits(); + assert(!ETS.isScalable() && "Vector type should have fixed-width elements"); + return {ETS.getFixedSize() * EC.Min, EC.Scalable}; } default: return TypeSize::Fixed(0); } } unsigned Type::getScalarSizeInBits() const { - return getScalarType()->getPrimitiveSizeInBits(); + // It is safe to assume that the scalar types have a fixed size. + return getScalarType()->getPrimitiveSizeInBits().getFixedSize(); } int Type::getFPMantissaWidth() const { @@ -140,6 +144,7 @@ int Type::getFPMantissaWidth() const { return VTy->getElementType()->getFPMantissaWidth(); assert(isFloatingPointTy() && "Not a floating point type!"); if (getTypeID() == HalfTyID) return 11; + if (getTypeID() == BFloatTyID) return 8; if (getTypeID() == FloatTyID) return 24; if (getTypeID() == DoubleTyID) return 53; if (getTypeID() == X86_FP80TyID) return 64; @@ -165,6 +170,7 @@ bool Type::isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited) const { Type *Type::getVoidTy(LLVMContext &C) { return &C.pImpl->VoidTy; } Type *Type::getLabelTy(LLVMContext &C) { return &C.pImpl->LabelTy; } Type *Type::getHalfTy(LLVMContext &C) { return &C.pImpl->HalfTy; } +Type *Type::getBFloatTy(LLVMContext &C) { return &C.pImpl->BFloatTy; } Type *Type::getFloatTy(LLVMContext &C) { return &C.pImpl->FloatTy; } Type *Type::getDoubleTy(LLVMContext &C) { return &C.pImpl->DoubleTy; } Type *Type::getMetadataTy(LLVMContext &C) { return &C.pImpl->MetadataTy; } @@ -189,6 +195,10 @@ PointerType *Type::getHalfPtrTy(LLVMContext &C, unsigned AS) { return getHalfTy(C)->getPointerTo(AS); } +PointerType *Type::getBFloatPtrTy(LLVMContext &C, unsigned AS) { + return getBFloatTy(C)->getPointerTo(AS); +} + PointerType *Type::getFloatPtrTy(LLVMContext &C, unsigned AS) { return getFloatTy(C)->getPointerTo(AS); } @@ -509,11 +519,9 @@ StringRef StructType::getName() const { } bool StructType::isValidElementType(Type *ElemTy) { - if (auto *VTy = dyn_cast<VectorType>(ElemTy)) - return !VTy->isScalable(); return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() && !ElemTy->isMetadataTy() && !ElemTy->isFunctionTy() && - !ElemTy->isTokenTy(); + !ElemTy->isTokenTy() && !isa<ScalableVectorType>(ElemTy); } bool StructType::isLayoutIdentical(StructType *Other) const { @@ -529,52 +537,24 @@ StructType *Module::getTypeByName(StringRef Name) const { return getContext().pImpl->NamedStructTypes.lookup(Name); } -//===----------------------------------------------------------------------===// -// CompositeType Implementation -//===----------------------------------------------------------------------===// - -Type *CompositeType::getTypeAtIndex(const Value *V) const { - if (auto *STy = dyn_cast<StructType>(this)) { - unsigned Idx = - (unsigned)cast<Constant>(V)->getUniqueInteger().getZExtValue(); - assert(indexValid(Idx) && "Invalid structure index!"); - return STy->getElementType(Idx); - } - - return cast<SequentialType>(this)->getElementType(); +Type *StructType::getTypeAtIndex(const Value *V) const { + unsigned Idx = (unsigned)cast<Constant>(V)->getUniqueInteger().getZExtValue(); + assert(indexValid(Idx) && "Invalid structure index!"); + return getElementType(Idx); } -Type *CompositeType::getTypeAtIndex(unsigned Idx) const{ - if (auto *STy = dyn_cast<StructType>(this)) { - assert(indexValid(Idx) && "Invalid structure index!"); - return STy->getElementType(Idx); - } - - return cast<SequentialType>(this)->getElementType(); -} - -bool CompositeType::indexValid(const Value *V) const { - if (auto *STy = dyn_cast<StructType>(this)) { - // Structure indexes require (vectors of) 32-bit integer constants. In the - // vector case all of the indices must be equal. - if (!V->getType()->isIntOrIntVectorTy(32)) - return false; - const Constant *C = dyn_cast<Constant>(V); - if (C && V->getType()->isVectorTy()) - C = C->getSplatValue(); - const ConstantInt *CU = dyn_cast_or_null<ConstantInt>(C); - return CU && CU->getZExtValue() < STy->getNumElements(); - } - - // Sequential types can be indexed by any integer. - return V->getType()->isIntOrIntVectorTy(); -} - -bool CompositeType::indexValid(unsigned Idx) const { - if (auto *STy = dyn_cast<StructType>(this)) - return Idx < STy->getNumElements(); - // Sequential types can be indexed by any integer. - return true; +bool StructType::indexValid(const Value *V) const { + // Structure indexes require (vectors of) 32-bit integer constants. In the + // vector case all of the indices must be equal. + if (!V->getType()->isIntOrIntVectorTy(32)) + return false; + if (isa<ScalableVectorType>(V->getType())) + return false; + const Constant *C = dyn_cast<Constant>(V); + if (C && V->getType()->isVectorTy()) + C = C->getSplatValue(); + const ConstantInt *CU = dyn_cast_or_null<ConstantInt>(C); + return CU && CU->getZExtValue() < getNumElements(); } //===----------------------------------------------------------------------===// @@ -582,7 +562,11 @@ bool CompositeType::indexValid(unsigned Idx) const { //===----------------------------------------------------------------------===// ArrayType::ArrayType(Type *ElType, uint64_t NumEl) - : SequentialType(ArrayTyID, ElType, NumEl) {} + : Type(ElType->getContext(), ArrayTyID), ContainedType(ElType), + NumElements(NumEl) { + ContainedTys = &ContainedType; + NumContainedTys = 1; +} ArrayType *ArrayType::get(Type *ElementType, uint64_t NumElements) { assert(isValidElementType(ElementType) && "Invalid type for array element!"); @@ -597,37 +581,75 @@ ArrayType *ArrayType::get(Type *ElementType, uint64_t NumElements) { } bool ArrayType::isValidElementType(Type *ElemTy) { - if (auto *VTy = dyn_cast<VectorType>(ElemTy)) - return !VTy->isScalable(); return !ElemTy->isVoidTy() && !ElemTy->isLabelTy() && !ElemTy->isMetadataTy() && !ElemTy->isFunctionTy() && - !ElemTy->isTokenTy(); + !ElemTy->isTokenTy() && !isa<ScalableVectorType>(ElemTy); } //===----------------------------------------------------------------------===// // VectorType Implementation //===----------------------------------------------------------------------===// -VectorType::VectorType(Type *ElType, ElementCount EC) - : SequentialType(VectorTyID, ElType, EC.Min), Scalable(EC.Scalable) {} +VectorType::VectorType(Type *ElType, unsigned EQ, Type::TypeID TID) + : Type(ElType->getContext(), TID), ContainedType(ElType), + ElementQuantity(EQ) { + ContainedTys = &ContainedType; + NumContainedTys = 1; +} VectorType *VectorType::get(Type *ElementType, ElementCount EC) { - assert(EC.Min > 0 && "#Elements of a VectorType must be greater than 0"); + if (EC.Scalable) + return ScalableVectorType::get(ElementType, EC.Min); + else + return FixedVectorType::get(ElementType, EC.Min); +} + +bool VectorType::isValidElementType(Type *ElemTy) { + return ElemTy->isIntegerTy() || ElemTy->isFloatingPointTy() || + ElemTy->isPointerTy(); +} + +//===----------------------------------------------------------------------===// +// FixedVectorType Implementation +//===----------------------------------------------------------------------===// + +FixedVectorType *FixedVectorType::get(Type *ElementType, unsigned NumElts) { + assert(NumElts > 0 && "#Elements of a VectorType must be greater than 0"); assert(isValidElementType(ElementType) && "Element type of a VectorType must " "be an integer, floating point, or " "pointer type."); + ElementCount EC(NumElts, false); + LLVMContextImpl *pImpl = ElementType->getContext().pImpl; - VectorType *&Entry = ElementType->getContext().pImpl - ->VectorTypes[std::make_pair(ElementType, EC)]; + VectorType *&Entry = ElementType->getContext() + .pImpl->VectorTypes[std::make_pair(ElementType, EC)]; + if (!Entry) - Entry = new (pImpl->Alloc) VectorType(ElementType, EC); - return Entry; + Entry = new (pImpl->Alloc) FixedVectorType(ElementType, NumElts); + return cast<FixedVectorType>(Entry); } -bool VectorType::isValidElementType(Type *ElemTy) { - return ElemTy->isIntegerTy() || ElemTy->isFloatingPointTy() || - ElemTy->isPointerTy(); +//===----------------------------------------------------------------------===// +// ScalableVectorType Implementation +//===----------------------------------------------------------------------===// + +ScalableVectorType *ScalableVectorType::get(Type *ElementType, + unsigned MinNumElts) { + assert(MinNumElts > 0 && "#Elements of a VectorType must be greater than 0"); + assert(isValidElementType(ElementType) && "Element type of a VectorType must " + "be an integer, floating point, or " + "pointer type."); + + ElementCount EC(MinNumElts, true); + + LLVMContextImpl *pImpl = ElementType->getContext().pImpl; + VectorType *&Entry = ElementType->getContext() + .pImpl->VectorTypes[std::make_pair(ElementType, EC)]; + + if (!Entry) + Entry = new (pImpl->Alloc) ScalableVectorType(ElementType, MinNumElts); + return cast<ScalableVectorType>(Entry); } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/IR/Use.cpp b/llvm/lib/IR/Use.cpp index 18c61757ee84..dc0716b85372 100644 --- a/llvm/lib/IR/Use.cpp +++ b/llvm/lib/IR/Use.cpp @@ -37,52 +37,10 @@ void Use::swap(Use &RHS) { } } -User *Use::getUser() const { - const Use *End = getImpliedUser(); - const UserRef *ref = reinterpret_cast<const UserRef *>(End); - return ref->getInt() ? ref->getPointer() - : reinterpret_cast<User *>(const_cast<Use *>(End)); -} - unsigned Use::getOperandNo() const { return this - getUser()->op_begin(); } -// Sets up the waymarking algorithm's tags for a series of Uses. See the -// algorithm details here: -// -// http://www.llvm.org/docs/ProgrammersManual.html#the-waymarking-algorithm -// -Use *Use::initTags(Use *const Start, Use *Stop) { - ptrdiff_t Done = 0; - while (Done < 20) { - if (Start == Stop--) - return Start; - static const PrevPtrTag tags[20] = { - fullStopTag, oneDigitTag, stopTag, oneDigitTag, oneDigitTag, - stopTag, zeroDigitTag, oneDigitTag, oneDigitTag, stopTag, - zeroDigitTag, oneDigitTag, zeroDigitTag, oneDigitTag, stopTag, - oneDigitTag, oneDigitTag, oneDigitTag, oneDigitTag, stopTag}; - new (Stop) Use(tags[Done++]); - } - - ptrdiff_t Count = Done; - while (Start != Stop) { - --Stop; - if (!Count) { - new (Stop) Use(stopTag); - ++Done; - Count = Done; - } else { - new (Stop) Use(PrevPtrTag(Count & 1)); - Count >>= 1; - ++Done; - } - } - - return Start; -} - void Use::zap(Use *Start, const Use *Stop, bool del) { while (Start != Stop) (--Stop)->~Use(); @@ -90,37 +48,4 @@ void Use::zap(Use *Start, const Use *Stop, bool del) { ::operator delete(Start); } -const Use *Use::getImpliedUser() const { - const Use *Current = this; - - while (true) { - unsigned Tag = (Current++)->Prev.getInt(); - switch (Tag) { - case zeroDigitTag: - case oneDigitTag: - continue; - - case stopTag: { - ++Current; - ptrdiff_t Offset = 1; - while (true) { - unsigned Tag = Current->Prev.getInt(); - switch (Tag) { - case zeroDigitTag: - case oneDigitTag: - ++Current; - Offset = (Offset << 1) + Tag; - continue; - default: - return Current + Offset; - } - } - } - - case fullStopTag: - return Current; - } - } -} - -} // End llvm namespace +} // namespace llvm diff --git a/llvm/lib/IR/User.cpp b/llvm/lib/IR/User.cpp index 4a3eba9e8cf7..7da592f40127 100644 --- a/llvm/lib/IR/User.cpp +++ b/llvm/lib/IR/User.cpp @@ -9,6 +9,7 @@ #include "llvm/IR/User.h" #include "llvm/IR/Constant.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/IntrinsicInst.h" namespace llvm { class BasicBlock; @@ -39,20 +40,18 @@ void User::replaceUsesOfWith(Value *From, Value *To) { void User::allocHungoffUses(unsigned N, bool IsPhi) { assert(HasHungOffUses && "alloc must have hung off uses"); - static_assert(alignof(Use) >= alignof(Use::UserRef), - "Alignment is insufficient for 'hung-off-uses' pieces"); - static_assert(alignof(Use::UserRef) >= alignof(BasicBlock *), + static_assert(alignof(Use) >= alignof(BasicBlock *), "Alignment is insufficient for 'hung-off-uses' pieces"); - // Allocate the array of Uses, followed by a pointer (with bottom bit set) to - // the User. - size_t size = N * sizeof(Use) + sizeof(Use::UserRef); + // Allocate the array of Uses + size_t size = N * sizeof(Use); if (IsPhi) size += N * sizeof(BasicBlock *); Use *Begin = static_cast<Use*>(::operator new(size)); Use *End = Begin + N; - (void) new(End) Use::UserRef(const_cast<User*>(this), 1); - setOperandList(Use::initTags(Begin, End)); + setOperandList(Begin); + for (; Begin != End; Begin++) + new (Begin) Use(this); } void User::growHungoffUses(unsigned NewNumUses, bool IsPhi) { @@ -73,10 +72,8 @@ void User::growHungoffUses(unsigned NewNumUses, bool IsPhi) { // If this is a Phi, then we need to copy the BB pointers too. if (IsPhi) { - auto *OldPtr = - reinterpret_cast<char *>(OldOps + OldNumUses) + sizeof(Use::UserRef); - auto *NewPtr = - reinterpret_cast<char *>(NewOps + NewNumUses) + sizeof(Use::UserRef); + auto *OldPtr = reinterpret_cast<char *>(OldOps + OldNumUses); + auto *NewPtr = reinterpret_cast<char *>(NewOps + NewNumUses); std::copy(OldPtr, OldPtr + (OldNumUses * sizeof(BasicBlock *)), NewPtr); } Use::zap(OldOps, OldOps + OldNumUses, true); @@ -105,6 +102,12 @@ MutableArrayRef<uint8_t> User::getDescriptor() { reinterpret_cast<uint8_t *>(DI) - DI->SizeInBytes, DI->SizeInBytes); } +bool User::isDroppable() const { + if (const auto *Intr = dyn_cast<IntrinsicInst>(this)) + return Intr->getIntrinsicID() == Intrinsic::assume; + return false; +} + //===----------------------------------------------------------------------===// // User operator new Implementations //===----------------------------------------------------------------------===// @@ -128,7 +131,8 @@ void *User::allocateFixedOperandUser(size_t Size, unsigned Us, Obj->NumUserOperands = Us; Obj->HasHungOffUses = false; Obj->HasDescriptor = DescBytes != 0; - Use::initTags(Start, End); + for (; Start != End; Start++) + new (Start) Use(Obj); if (DescBytes != 0) { auto *DescInfo = reinterpret_cast<DescriptorInfo *>(Storage + DescBytes); @@ -191,4 +195,4 @@ LLVM_NO_SANITIZE_MEMORY_ATTRIBUTE void User::operator delete(void *Usr) { } } -} // End llvm namespace +} // namespace llvm diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index cf9d08f6fc02..efb8d53e8964 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -83,13 +83,17 @@ Value::~Value() { // reference and something is wrong. This code is here to print out where // the value is still being referenced. // - if (!use_empty()) { + // Note that use_empty() cannot be called here, as it eventually downcasts + // 'this' to GlobalValue (derived class of Value), but GlobalValue has already + // been destructed, so accessing it is UB. + // + if (!materialized_use_empty()) { dbgs() << "While deleting: " << *VTy << " %" << getName() << "\n"; for (auto *U : users()) dbgs() << "Use still stuck around after Def is destroyed:" << *U << "\n"; } #endif - assert(use_empty() && "Uses remain when a value is destroyed!"); + assert(materialized_use_empty() && "Uses remain when a value is destroyed!"); // If this value is named, destroy the name. This should not be in a symtab // at this point. @@ -107,6 +111,10 @@ void Value::deleteValue() { static_cast<DerivedUser *>(this)->DeleteValue( \ static_cast<DerivedUser *>(this)); \ break; +#define HANDLE_CONSTANT(Name) \ + case Value::Name##Val: \ + llvm_unreachable("constants should be destroyed with destroyConstant"); \ + break; #define HANDLE_INSTRUCTION(Name) /* nothing */ #include "llvm/IR/Value.def" @@ -124,8 +132,10 @@ void Value::deleteValue() { void Value::destroyValueName() { ValueName *Name = getValueName(); - if (Name) - Name->Destroy(); + if (Name) { + MallocAllocator Allocator; + Name->Destroy(Allocator); + } setValueName(nullptr); } @@ -137,6 +147,51 @@ bool Value::hasNUsesOrMore(unsigned N) const { return hasNItemsOrMore(use_begin(), use_end(), N); } +static bool isUnDroppableUser(const User *U) { return !U->isDroppable(); } + +Use *Value::getSingleUndroppableUse() { + Use *Result = nullptr; + for (Use &U : uses()) { + if (!U.getUser()->isDroppable()) { + if (Result) + return nullptr; + Result = &U; + } + } + return Result; +} + +bool Value::hasNUndroppableUses(unsigned int N) const { + return hasNItems(user_begin(), user_end(), N, isUnDroppableUser); +} + +bool Value::hasNUndroppableUsesOrMore(unsigned int N) const { + return hasNItemsOrMore(user_begin(), user_end(), N, isUnDroppableUser); +} + +void Value::dropDroppableUses( + llvm::function_ref<bool(const Use *)> ShouldDrop) { + SmallVector<Use *, 8> ToBeEdited; + for (Use &U : uses()) + if (U.getUser()->isDroppable() && ShouldDrop(&U)) + ToBeEdited.push_back(&U); + for (Use *U : ToBeEdited) { + U->removeFromList(); + if (auto *Assume = dyn_cast<IntrinsicInst>(U->getUser())) { + assert(Assume->getIntrinsicID() == Intrinsic::assume); + unsigned OpNo = U->getOperandNo(); + if (OpNo == 0) + Assume->setOperand(0, ConstantInt::getTrue(Assume->getContext())); + else { + Assume->setOperand(OpNo, UndefValue::get(U->get()->getType())); + CallInst::BundleOpInfo &BOI = Assume->getBundleOpInfoForOperand(OpNo); + BOI.Tag = getContext().pImpl->getOrInsertBundleTag("ignore"); + } + } else + llvm_unreachable("unkown droppable use"); + } +} + bool Value::isUsedInBasicBlock(const BasicBlock *BB) const { // This can be computed either by scanning the instructions in BB, or by // scanning the use list of this Value. Both lists can be very long, but @@ -263,7 +318,8 @@ void Value::setNameImpl(const Twine &NewName) { destroyValueName(); // Create the new name. - setValueName(ValueName::Create(NameRef)); + MallocAllocator Allocator; + setValueName(ValueName::Create(NameRef, Allocator)); getValueName()->setValue(this); return; } @@ -463,8 +519,12 @@ enum PointerStripKind { PSK_InBounds }; +template <PointerStripKind StripKind> static void NoopCallback(const Value *) {} + template <PointerStripKind StripKind> -static const Value *stripPointerCastsAndOffsets(const Value *V) { +static const Value *stripPointerCastsAndOffsets( + const Value *V, + function_ref<void(const Value *)> Func = NoopCallback<StripKind>) { if (!V->getType()->isPointerTy()) return V; @@ -474,6 +534,7 @@ static const Value *stripPointerCastsAndOffsets(const Value *V) { Visited.insert(V); do { + Func(V); if (auto *GEP = dyn_cast<GEPOperator>(V)) { switch (StripKind) { case PSK_ZeroIndices: @@ -495,6 +556,8 @@ static const Value *stripPointerCastsAndOffsets(const Value *V) { V = GEP->getPointerOperand(); } else if (Operator::getOpcode(V) == Instruction::BitCast) { V = cast<Operator>(V)->getOperand(0); + if (!V->getType()->isPointerTy()) + return V; } else if (StripKind != PSK_ZeroIndicesSameRepresentation && Operator::getOpcode(V) == Instruction::AddrSpaceCast) { // TODO: If we know an address space cast will not change the @@ -547,9 +610,9 @@ const Value *Value::stripPointerCastsAndInvariantGroups() const { return stripPointerCastsAndOffsets<PSK_ZeroIndicesAndInvariantGroups>(this); } -const Value * -Value::stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, - bool AllowNonInbounds) const { +const Value *Value::stripAndAccumulateConstantOffsets( + const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, + function_ref<bool(Value &, APInt &)> ExternalAnalysis) const { if (!getType()->isPtrOrPtrVectorTy()) return this; @@ -575,7 +638,7 @@ Value::stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, // of GEP's pointer type rather than the size of the original // pointer type. APInt GEPOffset(DL.getIndexTypeSizeInBits(V->getType()), 0); - if (!GEP->accumulateConstantOffset(DL, GEPOffset)) + if (!GEP->accumulateConstantOffset(DL, GEPOffset, ExternalAnalysis)) return V; // Stop traversal if the pointer offset wouldn't fit in the bit-width @@ -584,7 +647,20 @@ Value::stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, if (GEPOffset.getMinSignedBits() > BitWidth) return V; - Offset += GEPOffset.sextOrTrunc(BitWidth); + // External Analysis can return a result higher/lower than the value + // represents. We need to detect overflow/underflow. + APInt GEPOffsetST = GEPOffset.sextOrTrunc(BitWidth); + if (!ExternalAnalysis) { + Offset += GEPOffsetST; + } else { + bool Overflow = false; + APInt OldOffset = Offset; + Offset = Offset.sadd_ov(GEPOffsetST, Overflow); + if (Overflow) { + Offset = OldOffset; + return V; + } + } V = GEP->getPointerOperand(); } else if (Operator::getOpcode(V) == Instruction::BitCast || Operator::getOpcode(V) == Instruction::AddrSpaceCast) { @@ -602,8 +678,9 @@ Value::stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, return V; } -const Value *Value::stripInBoundsOffsets() const { - return stripPointerCastsAndOffsets<PSK_InBounds>(this); +const Value * +Value::stripInBoundsOffsets(function_ref<void(const Value *)> Func) const { + return stripPointerCastsAndOffsets<PSK_InBounds>(this, Func); } uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, @@ -617,7 +694,7 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, if (DerefBytes == 0 && (A->hasByValAttr() || A->hasStructRetAttr())) { Type *PT = cast<PointerType>(A->getType())->getElementType(); if (PT->isSized()) - DerefBytes = DL.getTypeStoreSize(PT); + DerefBytes = DL.getTypeStoreSize(PT).getKnownMinSize(); } if (DerefBytes == 0) { DerefBytes = A->getDereferenceableOrNullBytes(); @@ -658,30 +735,31 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL, } } else if (auto *AI = dyn_cast<AllocaInst>(this)) { if (!AI->isArrayAllocation()) { - DerefBytes = DL.getTypeStoreSize(AI->getAllocatedType()); + DerefBytes = + DL.getTypeStoreSize(AI->getAllocatedType()).getKnownMinSize(); CanBeNull = false; } } else if (auto *GV = dyn_cast<GlobalVariable>(this)) { if (GV->getValueType()->isSized() && !GV->hasExternalWeakLinkage()) { // TODO: Don't outright reject hasExternalWeakLinkage but set the // CanBeNull flag. - DerefBytes = DL.getTypeStoreSize(GV->getValueType()); + DerefBytes = DL.getTypeStoreSize(GV->getValueType()).getFixedSize(); CanBeNull = false; } } return DerefBytes; } -MaybeAlign Value::getPointerAlignment(const DataLayout &DL) const { +Align Value::getPointerAlignment(const DataLayout &DL) const { assert(getType()->isPointerTy() && "must be pointer"); if (auto *GO = dyn_cast<GlobalObject>(this)) { if (isa<Function>(GO)) { - const MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign(); + Align FunctionPtrAlign = DL.getFunctionPtrAlign().valueOrOne(); switch (DL.getFunctionPtrAlignType()) { case DataLayout::FunctionPtrAlignType::Independent: return FunctionPtrAlign; case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign: - return std::max(FunctionPtrAlign, MaybeAlign(GO->getAlignment())); + return std::max(FunctionPtrAlign, GO->getAlign().valueOrOne()); } llvm_unreachable("Unhandled FunctionPtrAlignType"); } @@ -694,43 +772,47 @@ MaybeAlign Value::getPointerAlignment(const DataLayout &DL) const { // it the preferred alignment. Otherwise, we have to assume that it // may only have the minimum ABI alignment. if (GVar->isStrongDefinitionForLinker()) - return MaybeAlign(DL.getPreferredAlignment(GVar)); + return DL.getPreferredAlign(GVar); else - return Align(DL.getABITypeAlignment(ObjectType)); + return DL.getABITypeAlign(ObjectType); } } } - return Alignment; + return Alignment.valueOrOne(); } else if (const Argument *A = dyn_cast<Argument>(this)) { - const MaybeAlign Alignment(A->getParamAlignment()); + const MaybeAlign Alignment = A->getParamAlign(); if (!Alignment && A->hasStructRetAttr()) { // An sret parameter has at least the ABI alignment of the return type. Type *EltTy = cast<PointerType>(A->getType())->getElementType(); if (EltTy->isSized()) - return Align(DL.getABITypeAlignment(EltTy)); + return DL.getABITypeAlign(EltTy); } - return Alignment; + return Alignment.valueOrOne(); } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) { - const MaybeAlign Alignment(AI->getAlignment()); - if (!Alignment) { - Type *AllocatedType = AI->getAllocatedType(); - if (AllocatedType->isSized()) - return MaybeAlign(DL.getPrefTypeAlignment(AllocatedType)); - } - return Alignment; + return AI->getAlign(); } else if (const auto *Call = dyn_cast<CallBase>(this)) { - const MaybeAlign Alignment(Call->getRetAlignment()); + MaybeAlign Alignment = Call->getRetAlign(); if (!Alignment && Call->getCalledFunction()) - return MaybeAlign( - Call->getCalledFunction()->getAttributes().getRetAlignment()); - return Alignment; + Alignment = Call->getCalledFunction()->getAttributes().getRetAlignment(); + return Alignment.valueOrOne(); } else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) { if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) { ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0)); - return MaybeAlign(CI->getLimitedValue()); + return Align(CI->getLimitedValue()); + } + } else if (auto *CstPtr = dyn_cast<Constant>(this)) { + if (auto *CstInt = dyn_cast_or_null<ConstantInt>(ConstantExpr::getPtrToInt( + const_cast<Constant *>(CstPtr), DL.getIntPtrType(getType()), + /*OnlyIfReduced=*/true))) { + size_t TrailingZeros = CstInt->getValue().countTrailingZeros(); + // While the actual alignment may be large, elsewhere we have + // an arbitrary upper alignmet limit, so let's clamp to it. + return Align(TrailingZeros < Value::MaxAlignmentExponent + ? uint64_t(1) << TrailingZeros + : Value::MaximumAlignment); } } - return llvm::None; + return Align(1); } const Value *Value::DoPHITranslation(const BasicBlock *CurBB, @@ -754,12 +836,12 @@ void Value::reverseUseList() { while (Current) { Use *Next = Current->Next; Current->Next = Head; - Head->setPrev(&Current->Next); + Head->Prev = &Current->Next; Head = Current; Current = Next; } UseList = Head; - Head->setPrev(&UseList); + Head->Prev = &UseList; } bool Value::isSwiftError() const { diff --git a/llvm/lib/IR/ValueSymbolTable.cpp b/llvm/lib/IR/ValueSymbolTable.cpp index 417ec045071d..b49842315f36 100644 --- a/llvm/lib/IR/ValueSymbolTable.cpp +++ b/llvm/lib/IR/ValueSymbolTable.cpp @@ -31,7 +31,7 @@ using namespace llvm; // Class destructor ValueSymbolTable::~ValueSymbolTable() { -#ifndef NDEBUG // Only do this in -g mode... +#ifndef NDEBUG // Only do this in -g mode... for (const auto &VI : vmap) dbgs() << "Value still in symbol table! Type = '" << *VI.getValue()->getType() << "' Name = '" << VI.getKeyData() @@ -69,7 +69,7 @@ ValueName *ValueSymbolTable::makeUniqueName(Value *V, // Insert a value into the symbol table with the specified name... // -void ValueSymbolTable::reinsertValue(Value* V) { +void ValueSymbolTable::reinsertValue(Value *V) { assert(V->hasName() && "Can't insert nameless Value into symbol table"); // Try inserting the name, assuming it won't conflict. @@ -83,7 +83,8 @@ void ValueSymbolTable::reinsertValue(Value* V) { SmallString<256> UniqueName(V->getName().begin(), V->getName().end()); // The name is too already used, just free it so we can allocate a new name. - V->getValueName()->Destroy(); + MallocAllocator Allocator; + V->getValueName()->Destroy(Allocator); ValueName *VN = makeUniqueName(V, UniqueName); V->setValueName(VN); @@ -116,11 +117,11 @@ ValueName *ValueSymbolTable::createValueName(StringRef Name, Value *V) { // dump - print out the symbol table // LLVM_DUMP_METHOD void ValueSymbolTable::dump() const { - //dbgs() << "ValueSymbolTable:\n"; + // dbgs() << "ValueSymbolTable:\n"; for (const auto &I : *this) { - //dbgs() << " '" << I->getKeyData() << "' = "; + // dbgs() << " '" << I->getKeyData() << "' = "; I.getValue()->dump(); - //dbgs() << "\n"; + // dbgs() << "\n"; } } #endif diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index d15b70d71b47..6df1072925f9 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -397,6 +397,9 @@ public: } private: + /// Whether a metadata node is allowed to be, or contain, a DILocation. + enum class AreDebugLocsAllowed { No, Yes }; + // Verification methods... void visitGlobalValue(const GlobalValue &GV); void visitGlobalVariable(const GlobalVariable &GV); @@ -405,7 +408,7 @@ private: void visitAliaseeSubExpr(SmallPtrSetImpl<const GlobalAlias *> &Visited, const GlobalAlias &A, const Constant &C); void visitNamedMDNode(const NamedMDNode &NMD); - void visitMDNode(const MDNode &MD); + void visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs); void visitMetadataAsValue(const MetadataAsValue &MD, Function *F); void visitValueAsMetadata(const ValueAsMetadata &MD, Function *F); void visitComdat(const Comdat &C); @@ -567,8 +570,9 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(), "Global is external, but doesn't have external or weak linkage!", &GV); - Assert(GV.getAlignment() <= Value::MaximumAlignment, - "huge alignment values are unsupported", &GV); + if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV)) + Assert(GO->getAlignment() <= Value::MaximumAlignment, + "huge alignment values are unsupported", GO); Assert(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV), "Only global variables can have appending linkage!", &GV); @@ -590,15 +594,12 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { "Global is marked as dllimport, but not external", &GV); } - if (GV.hasLocalLinkage()) + if (GV.isImplicitDSOLocal()) Assert(GV.isDSOLocal(), - "GlobalValue with private or internal linkage must be dso_local!", + "GlobalValue with local linkage or non-default " + "visibility must be dso_local!", &GV); - if (!GV.hasDefaultVisibility() && !GV.hasExternalWeakLinkage()) - Assert(GV.isDSOLocal(), - "GlobalValue with non default visibility must be dso_local!", &GV); - forEachUser(&GV, GlobalValueVisited, [&](const Value *V) -> bool { if (const Instruction *I = dyn_cast<Instruction>(V)) { if (!I->getParent() || !I->getParent()->getParent()) @@ -701,8 +702,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { // the runtime size. If the global is a struct or an array containing // scalable vectors, that will be caught by the isValidElementType methods // in StructType or ArrayType instead. - if (auto *VTy = dyn_cast<VectorType>(GV.getValueType())) - Assert(!VTy->isScalable(), "Globals cannot contain scalable vectors", &GV); + Assert(!isa<ScalableVectorType>(GV.getValueType()), + "Globals cannot contain scalable vectors", &GV); if (!GV.hasInitializer()) { visitGlobalValue(GV); @@ -783,11 +784,11 @@ void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { if (!MD) continue; - visitMDNode(*MD); + visitMDNode(*MD, AreDebugLocsAllowed::Yes); } } -void Verifier::visitMDNode(const MDNode &MD) { +void Verifier::visitMDNode(const MDNode &MD, AreDebugLocsAllowed AllowLocs) { // Only visit each node once. Metadata can be mutually recursive, so this // avoids infinite recursion here, as well as being an optimization. if (!MDNodes.insert(&MD).second) @@ -810,8 +811,10 @@ void Verifier::visitMDNode(const MDNode &MD) { continue; Assert(!isa<LocalAsMetadata>(Op), "Invalid operand for global metadata!", &MD, Op); + AssertDI(!isa<DILocation>(Op) || AllowLocs == AreDebugLocsAllowed::Yes, + "DILocation not allowed within this metadata node", &MD, Op); if (auto *N = dyn_cast<MDNode>(Op)) { - visitMDNode(*N); + visitMDNode(*N, AllowLocs); continue; } if (auto *V = dyn_cast<ValueAsMetadata>(Op)) { @@ -854,7 +857,7 @@ void Verifier::visitValueAsMetadata(const ValueAsMetadata &MD, Function *F) { void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { Metadata *MD = MDV.getMetadata(); if (auto *N = dyn_cast<MDNode>(MD)) { - visitMDNode(*N); + visitMDNode(*N, AreDebugLocsAllowed::No); return; } @@ -891,12 +894,30 @@ void Verifier::visitDIScope(const DIScope &N) { void Verifier::visitDISubrange(const DISubrange &N) { AssertDI(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); + AssertDI(N.getRawCountNode() || N.getRawUpperBound(), + "Subrange must contain count or upperBound", &N); + AssertDI(!N.getRawCountNode() || !N.getRawUpperBound(), + "Subrange can have any one of count or upperBound", &N); + AssertDI(!N.getRawCountNode() || N.getCount(), + "Count must either be a signed constant or a DIVariable", &N); auto Count = N.getCount(); - AssertDI(Count, "Count must either be a signed constant or a DIVariable", - &N); - AssertDI(!Count.is<ConstantInt*>() || - Count.get<ConstantInt*>()->getSExtValue() >= -1, + AssertDI(!Count || !Count.is<ConstantInt *>() || + Count.get<ConstantInt *>()->getSExtValue() >= -1, "invalid subrange count", &N); + auto *LBound = N.getRawLowerBound(); + AssertDI(!LBound || isa<ConstantAsMetadata>(LBound) || + isa<DIVariable>(LBound) || isa<DIExpression>(LBound), + "LowerBound must be signed constant or DIVariable or DIExpression", + &N); + auto *UBound = N.getRawUpperBound(); + AssertDI(!UBound || isa<ConstantAsMetadata>(UBound) || + isa<DIVariable>(UBound) || isa<DIExpression>(UBound), + "UpperBound must be signed constant or DIVariable or DIExpression", + &N); + auto *Stride = N.getRawStride(); + AssertDI(!Stride || isa<ConstantAsMetadata>(Stride) || + isa<DIVariable>(Stride) || isa<DIExpression>(Stride), + "Stride must be signed constant or DIVariable or DIExpression", &N); } void Verifier::visitDIEnumerator(const DIEnumerator &N) { @@ -1009,6 +1030,11 @@ void Verifier::visitDICompositeType(const DICompositeType &N) { AssertDI(isa<DIDerivedType>(D) && N.getTag() == dwarf::DW_TAG_variant_part, "discriminator can only appear on variant part"); } + + if (N.getRawDataLocation()) { + AssertDI(N.getTag() == dwarf::DW_TAG_array_type, + "dataLocation can only appear in array type"); + } } void Verifier::visitDISubroutineType(const DISubroutineType &N) { @@ -1037,6 +1063,9 @@ void Verifier::visitDIFile(const DIFile &N) { case DIFile::CSK_SHA1: Size = 40; break; + case DIFile::CSK_SHA256: + Size = 64; + break; } AssertDI(Checksum->Value.size() == Size, "invalid checksum length", &N); AssertDI(Checksum->Value.find_if_not(llvm::isHexDigit) == StringRef::npos, @@ -1250,7 +1279,9 @@ void Verifier::visitDIGlobalVariable(const DIGlobalVariable &N) { AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); AssertDI(isType(N.getRawType()), "invalid type ref", &N, N.getRawType()); - AssertDI(N.getType(), "missing global variable type", &N); + // Assert only if the global variable is not an extern + if (N.isDefinition()) + AssertDI(N.getType(), "missing global variable type", &N); if (auto *Member = N.getRawStaticDataMemberDeclaration()) { AssertDI(isa<DIDerivedType>(Member), "invalid static data member declaration", &N, Member); @@ -1476,6 +1507,13 @@ Verifier::visitModuleFlag(const MDNode *Op, "'Linker Options' named metadata no longer supported"); } + if (ID->getString() == "SemanticInterposition") { + ConstantInt *Value = + mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(2)); + Assert(Value, + "SemanticInterposition metadata requires constant integer argument"); + } + if (ID->getString() == "CG Profile") { for (const MDOperand &MDO : cast<MDNode>(Op->getOperand(2))->operands()) visitModuleFlagCGProfileEntry(MDO); @@ -1502,6 +1540,7 @@ void Verifier::visitModuleFlagCGProfileEntry(const MDOperand &MDO) { /// Return true if this attribute kind only applies to functions. static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { switch (Kind) { + case Attribute::NoMerge: case Attribute::NoReturn: case Attribute::NoSync: case Attribute::WillReturn: @@ -1545,6 +1584,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { case Attribute::SpeculativeLoadHardening: case Attribute::Speculatable: case Attribute::StrictFP: + case Attribute::NullPointerIsValid: return true; default: break; @@ -1556,7 +1596,8 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { /// arguments. static bool isFuncOrArgAttr(Attribute::AttrKind Kind) { return Kind == Attribute::ReadOnly || Kind == Attribute::WriteOnly || - Kind == Attribute::ReadNone || Kind == Attribute::NoFree; + Kind == Attribute::ReadNone || Kind == Attribute::NoFree || + Kind == Attribute::Preallocated; } void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, @@ -1565,6 +1606,13 @@ void Verifier::verifyAttributeTypes(AttributeSet Attrs, bool IsFunction, if (A.isStringAttribute()) continue; + if (A.isIntAttribute() != + Attribute::doesAttrKindHaveArgument(A.getKindAsEnum())) { + CheckFailed("Attribute '" + A.getAsString() + "' should have an Argument", + V); + return; + } + if (isFuncOnlyAttr(A.getKindAsEnum())) { if (!IsFunction) { CheckFailed("Attribute '" + A.getAsString() + @@ -1600,11 +1648,13 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, unsigned AttrCount = 0; AttrCount += Attrs.hasAttribute(Attribute::ByVal); AttrCount += Attrs.hasAttribute(Attribute::InAlloca); + AttrCount += Attrs.hasAttribute(Attribute::Preallocated); AttrCount += Attrs.hasAttribute(Attribute::StructRet) || Attrs.hasAttribute(Attribute::InReg); AttrCount += Attrs.hasAttribute(Attribute::Nest); - Assert(AttrCount <= 1, "Attributes 'byval', 'inalloca', 'inreg', 'nest', " - "and 'sret' are incompatible!", + Assert(AttrCount <= 1, + "Attributes 'byval', 'inalloca', 'preallocated', 'inreg', 'nest', " + "and 'sret' are incompatible!", V); Assert(!(Attrs.hasAttribute(Attribute::InAlloca) && @@ -1654,6 +1704,12 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, "Attribute 'byval' type does not match parameter!", V); } + if (Attrs.hasAttribute(Attribute::Preallocated)) { + Assert(Attrs.getPreallocatedType() == + cast<PointerType>(Ty)->getElementType(), + "Attribute 'preallocated' type does not match parameter!", V); + } + AttrBuilder IncompatibleAttrs = AttributeFuncs::typeIncompatible(Ty); Assert(!AttrBuilder(Attrs).overlaps(IncompatibleAttrs), "Wrong types for attribute: " + @@ -1664,8 +1720,10 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, SmallPtrSet<Type*, 4> Visited; if (!PTy->getElementType()->isSized(&Visited)) { Assert(!Attrs.hasAttribute(Attribute::ByVal) && - !Attrs.hasAttribute(Attribute::InAlloca), - "Attributes 'byval' and 'inalloca' do not support unsized types!", + !Attrs.hasAttribute(Attribute::InAlloca) && + !Attrs.hasAttribute(Attribute::Preallocated), + "Attributes 'byval', 'inalloca', and 'preallocated' do not " + "support unsized types!", V); } if (!isa<PointerType>(PTy->getElementType())) @@ -1706,9 +1764,11 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, !RetAttrs.hasAttribute(Attribute::NoFree) && !RetAttrs.hasAttribute(Attribute::Returned) && !RetAttrs.hasAttribute(Attribute::InAlloca) && + !RetAttrs.hasAttribute(Attribute::Preallocated) && !RetAttrs.hasAttribute(Attribute::SwiftSelf) && !RetAttrs.hasAttribute(Attribute::SwiftError)), - "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', 'nofree'" + "Attributes 'byval', 'inalloca', 'preallocated', 'nest', 'sret', " + "'nocapture', 'nofree', " "'returned', 'swiftself', and 'swifterror' do not apply to return " "values!", V); @@ -1852,16 +1912,25 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, CheckFailed("invalid value for 'frame-pointer' attribute: " + FP, V); } + if (Attrs.hasFnAttribute("patchable-function-prefix")) { + StringRef S = Attrs + .getAttribute(AttributeList::FunctionIndex, + "patchable-function-prefix") + .getValueAsString(); + unsigned N; + if (S.getAsInteger(10, N)) + CheckFailed( + "\"patchable-function-prefix\" takes an unsigned integer: " + S, V); + } if (Attrs.hasFnAttribute("patchable-function-entry")) { - StringRef S0 = Attrs - .getAttribute(AttributeList::FunctionIndex, - "patchable-function-entry") - .getValueAsString(); - StringRef S = S0; + StringRef S = Attrs + .getAttribute(AttributeList::FunctionIndex, + "patchable-function-entry") + .getValueAsString(); unsigned N; if (S.getAsInteger(10, N)) CheckFailed( - "\"patchable-function-entry\" takes an unsigned integer: " + S0, V); + "\"patchable-function-entry\" takes an unsigned integer: " + S, V); } } @@ -2037,6 +2106,13 @@ void Verifier::verifyStatepoint(const CallBase &Call) { "gc.statepoint number of transition arguments must be positive", Call); const int EndTransitionArgsInx = EndCallArgsInx + 1 + NumTransitionArgs; + // We're migrating away from inline operands to operand bundles, enforce + // the either/or property during transition. + if (Call.getOperandBundle(LLVMContext::OB_gc_transition)) { + Assert(NumTransitionArgs == 0, + "can't use both deopt operands and deopt bundle on a statepoint"); + } + const Value *NumDeoptArgsV = Call.getArgOperand(EndTransitionArgsInx + 1); Assert(isa<ConstantInt>(NumDeoptArgsV), "gc.statepoint number of deoptimization arguments " @@ -2048,6 +2124,13 @@ void Verifier::verifyStatepoint(const CallBase &Call) { "must be positive", Call); + // We're migrating away from inline operands to operand bundles, enforce + // the either/or property during transition. + if (Call.getOperandBundle(LLVMContext::OB_deopt)) { + Assert(NumDeoptArgs == 0, + "can't use both deopt operands and deopt bundle on a statepoint"); + } + const int ExpectedNumArgs = 7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs; Assert(ExpectedNumArgs <= (int)Call.arg_size(), @@ -2277,7 +2360,7 @@ void Verifier::visitFunction(const Function &F) { "function declaration may not have a !prof attachment", &F); // Verify the metadata itself. - visitMDNode(*I.second); + visitMDNode(*I.second, AreDebugLocsAllowed::Yes); } Assert(!F.hasPersonalityFn(), "Function declaration shouldn't have a personality routine", &F); @@ -2301,6 +2384,7 @@ void Verifier::visitFunction(const Function &F) { // Visit metadata attachments. for (const auto &I : MDs) { // Verify that the attachment is legal. + auto AllowLocs = AreDebugLocsAllowed::No; switch (I.first) { default: break; @@ -2315,6 +2399,7 @@ void Verifier::visitFunction(const Function &F) { AssertDI(!AttachedTo || AttachedTo == &F, "DISubprogram attached to more than one function", SP, &F); AttachedTo = &F; + AllowLocs = AreDebugLocsAllowed::Yes; break; } case LLVMContext::MD_prof: @@ -2325,7 +2410,7 @@ void Verifier::visitFunction(const Function &F) { } // Verify the metadata itself. - visitMDNode(*I.second); + visitMDNode(*I.second, AllowLocs); } } @@ -2344,8 +2429,7 @@ void Verifier::visitFunction(const Function &F) { if (!HasDebugInfo) return; - // Check that all !dbg attachments lead to back to N (or, at least, another - // subprogram that describes the same function). + // Check that all !dbg attachments lead to back to N. // // FIXME: Check this incrementally while visiting !dbg attachments. // FIXME: Only check when N is the canonical subprogram for F. @@ -2363,18 +2447,20 @@ void Verifier::visitFunction(const Function &F) { AssertDI(Parent && isa<DILocalScope>(Parent), "DILocation's scope must be a DILocalScope", N, &F, &I, DL, Parent); + DILocalScope *Scope = DL->getInlinedAtScope(); - if (Scope && !Seen.insert(Scope).second) + Assert(Scope, "Failed to find DILocalScope", DL); + + if (!Seen.insert(Scope).second) return; - DISubprogram *SP = Scope ? Scope->getSubprogram() : nullptr; + DISubprogram *SP = Scope->getSubprogram(); // Scope and SP could be the same MDNode and we don't want to skip // validation in that case if (SP && ((Scope != SP) && !Seen.insert(SP).second)) return; - // FIXME: Once N is canonical, check "SP == &N". AssertDI(SP->describes(&F), "!dbg attachment points at wrong subprogram for function", N, &F, &I, DL, Scope, SP); @@ -2513,8 +2599,6 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { void Verifier::visitCallBrInst(CallBrInst &CBI) { Assert(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI); - Assert(CBI.getType()->isVoidTy(), "Callbr return value is not supported!", - &CBI); for (unsigned i = 0, e = CBI.getNumSuccessors(); i != e; ++i) Assert(CBI.getSuccessor(i)->getType()->isLabelTy(), "Callbr successors must all have pointer type!", &CBI); @@ -2532,8 +2616,7 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) { if (auto *BA = dyn_cast<BlockAddress>(V)) ArgBBs.insert(BA->getBasicBlock()); for (BasicBlock *BB : CBI.getIndirectDests()) - Assert(ArgBBs.find(BB) != ArgBBs.end(), - "Indirect label missing from arglist.", &CBI); + Assert(ArgBBs.count(BB), "Indirect label missing from arglist.", &CBI); } visitTerminator(CBI); @@ -2661,8 +2744,8 @@ void Verifier::visitUIToFPInst(UIToFPInst &I) { &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "UIToFP source and dest vector length mismatch", &I); visitInstruction(I); @@ -2684,8 +2767,8 @@ void Verifier::visitSIToFPInst(SIToFPInst &I) { &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "SIToFP source and dest vector length mismatch", &I); visitInstruction(I); @@ -2707,8 +2790,8 @@ void Verifier::visitFPToUIInst(FPToUIInst &I) { "FPToUI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "FPToUI source and dest vector length mismatch", &I); visitInstruction(I); @@ -2730,8 +2813,8 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) { "FPToSI result must be integer or integer vector", &I); if (SrcVec && DstVec) - Assert(cast<VectorType>(SrcTy)->getNumElements() == - cast<VectorType>(DestTy)->getNumElements(), + Assert(cast<VectorType>(SrcTy)->getElementCount() == + cast<VectorType>(DestTy)->getElementCount(), "FPToSI source and dest vector length mismatch", &I); visitInstruction(I); @@ -2753,9 +2836,9 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) { &I); if (SrcTy->isVectorTy()) { - VectorType *VSrc = cast<VectorType>(SrcTy); - VectorType *VDest = cast<VectorType>(DestTy); - Assert(VSrc->getNumElements() == VDest->getNumElements(), + auto *VSrc = cast<VectorType>(SrcTy); + auto *VDest = cast<VectorType>(DestTy); + Assert(VSrc->getElementCount() == VDest->getElementCount(), "PtrToInt Vector width mismatch", &I); } @@ -2778,9 +2861,9 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) { Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "IntToPtr type mismatch", &I); if (SrcTy->isVectorTy()) { - VectorType *VSrc = cast<VectorType>(SrcTy); - VectorType *VDest = cast<VectorType>(DestTy); - Assert(VSrc->getNumElements() == VDest->getNumElements(), + auto *VSrc = cast<VectorType>(SrcTy); + auto *VDest = cast<VectorType>(DestTy); + Assert(VSrc->getElementCount() == VDest->getElementCount(), "IntToPtr Vector width mismatch", &I); } visitInstruction(I); @@ -2803,8 +2886,9 @@ void Verifier::visitAddrSpaceCastInst(AddrSpaceCastInst &I) { &I); Assert(SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace(), "AddrSpaceCast must be between different address spaces", &I); - if (SrcTy->isVectorTy()) - Assert(SrcTy->getVectorNumElements() == DestTy->getVectorNumElements(), + if (auto *SrcVTy = dyn_cast<VectorType>(SrcTy)) + Assert(SrcVTy->getNumElements() == + cast<VectorType>(DestTy)->getNumElements(), "AddrSpaceCast vector pointer number of elements mismatch", &I); visitInstruction(I); } @@ -2836,9 +2920,9 @@ void Verifier::visitPHINode(PHINode &PN) { } void Verifier::visitCallBase(CallBase &Call) { - Assert(Call.getCalledValue()->getType()->isPointerTy(), + Assert(Call.getCalledOperand()->getType()->isPointerTy(), "Called function must be a pointer!", Call); - PointerType *FPTy = cast<PointerType>(Call.getCalledValue()->getType()); + PointerType *FPTy = cast<PointerType>(Call.getCalledOperand()->getType()); Assert(FPTy->getElementType()->isFunctionTy(), "Called function is not pointer to function type!", Call); @@ -2871,16 +2955,23 @@ void Verifier::visitCallBase(CallBase &Call) { bool IsIntrinsic = Call.getCalledFunction() && Call.getCalledFunction()->getName().startswith("llvm."); - Function *Callee - = dyn_cast<Function>(Call.getCalledValue()->stripPointerCasts()); + Function *Callee = + dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts()); - if (Attrs.hasAttribute(AttributeList::FunctionIndex, Attribute::Speculatable)) { + if (Attrs.hasFnAttribute(Attribute::Speculatable)) { // Don't allow speculatable on call sites, unless the underlying function // declaration is also speculatable. Assert(Callee && Callee->isSpeculatable(), "speculatable attribute may not apply to call sites", Call); } + if (Attrs.hasFnAttribute(Attribute::Preallocated)) { + Assert(Call.getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_arg, + "preallocated as a call site attribute can only be on " + "llvm.call.preallocated.arg"); + } + // Verify call attributes. verifyFunctionAttrs(FTy, Attrs, &Call, IsIntrinsic); @@ -2927,6 +3018,17 @@ void Verifier::visitCallBase(CallBase &Call) { Assert(isa<ConstantInt>(ArgVal) || isa<ConstantFP>(ArgVal), "immarg operand has non-immediate parameter", ArgVal, Call); } + + if (Call.paramHasAttr(i, Attribute::Preallocated)) { + Value *ArgVal = Call.getArgOperand(i); + bool hasOB = + Call.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0; + bool isMustTail = Call.isMustTailCall(); + Assert(hasOB != isMustTail, + "preallocated operand either requires a preallocated bundle or " + "the call to be musttail (but not both)", + ArgVal, Call); + } } if (FTy->isVarArg()) { @@ -2997,9 +3099,11 @@ void Verifier::visitCallBase(CallBase &Call) { visitIntrinsicCall(ID, Call); // Verify that a callsite has at most one "deopt", at most one "funclet", at - // most one "gc-transition", and at most one "cfguardtarget" operand bundle. + // most one "gc-transition", at most one "cfguardtarget", + // and at most one "preallocated" operand bundle. bool FoundDeoptBundle = false, FoundFuncletBundle = false, - FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false; + FoundGCTransitionBundle = false, FoundCFGuardTargetBundle = false, + FoundPreallocatedBundle = false, FoundGCLiveBundle = false;; for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = Call.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); @@ -3024,6 +3128,22 @@ void Verifier::visitCallBase(CallBase &Call) { FoundCFGuardTargetBundle = true; Assert(BU.Inputs.size() == 1, "Expected exactly one cfguardtarget bundle operand", Call); + } else if (Tag == LLVMContext::OB_preallocated) { + Assert(!FoundPreallocatedBundle, "Multiple preallocated operand bundles", + Call); + FoundPreallocatedBundle = true; + Assert(BU.Inputs.size() == 1, + "Expected exactly one preallocated bundle operand", Call); + auto Input = dyn_cast<IntrinsicInst>(BU.Inputs.front()); + Assert(Input && + Input->getIntrinsicID() == Intrinsic::call_preallocated_setup, + "\"preallocated\" argument must be a token from " + "llvm.call.preallocated.setup", + Call); + } else if (Tag == LLVMContext::OB_gc_live) { + Assert(!FoundGCLiveBundle, "Multiple gc-live operand bundles", + Call); + FoundGCLiveBundle = true; } } @@ -3054,15 +3174,17 @@ static bool isTypeCongruent(Type *L, Type *R) { static AttrBuilder getParameterABIAttributes(int I, AttributeList Attrs) { static const Attribute::AttrKind ABIAttrs[] = { - Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, - Attribute::InReg, Attribute::Returned, Attribute::SwiftSelf, - Attribute::SwiftError}; + Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, + Attribute::InReg, Attribute::SwiftSelf, Attribute::SwiftError, + Attribute::Preallocated}; AttrBuilder Copy; for (auto AK : ABIAttrs) { if (Attrs.hasParamAttribute(I, AK)) Copy.addAttribute(AK); } - if (Attrs.hasParamAttribute(I, Attribute::Alignment)) + // `align` is ABI-affecting only in combination with `byval`. + if (Attrs.hasParamAttribute(I, Attribute::Alignment) && + Attrs.hasParamAttribute(I, Attribute::ByVal)) Copy.addAlignmentAttr(Attrs.getParamAlignment(I)); return Copy; } @@ -3096,7 +3218,7 @@ void Verifier::verifyMustTailCall(CallInst &CI) { "cannot guarantee tail call due to mismatched calling conv", &CI); // - All ABI-impacting function attributes, such as sret, byval, inreg, - // returned, and inalloca, must match. + // returned, preallocated, and inalloca, must match. AttributeList CallerAttrs = F->getAttributes(); AttributeList CalleeAttrs = CI.getAttributes(); for (int I = 0, E = CallerTy->getNumParams(); I != E; ++I) { @@ -3154,7 +3276,7 @@ void Verifier::visitInvokeInst(InvokeInst &II) { /// visitUnaryOperator - Check the argument to the unary operator. /// void Verifier::visitUnaryOperator(UnaryOperator &U) { - Assert(U.getType() == U.getOperand(0)->getType(), + Assert(U.getType() == U.getOperand(0)->getType(), "Unary operators must have same type for" "operands and result!", &U); @@ -3286,7 +3408,7 @@ void Verifier::visitInsertElementInst(InsertElementInst &IE) { void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) { Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1), - SV.getOperand(2)), + SV.getShuffleMask()), "Invalid shufflevector operands!", &SV); visitInstruction(SV); } @@ -3310,16 +3432,18 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { GEP.getResultElementType() == ElTy, "GEP is not of right type for indices!", &GEP, ElTy); - if (GEP.getType()->isVectorTy()) { + if (auto *GEPVTy = dyn_cast<VectorType>(GEP.getType())) { // Additional checks for vector GEPs. - unsigned GEPWidth = GEP.getType()->getVectorNumElements(); + ElementCount GEPWidth = GEPVTy->getElementCount(); if (GEP.getPointerOperandType()->isVectorTy()) - Assert(GEPWidth == GEP.getPointerOperandType()->getVectorNumElements(), - "Vector GEP result width doesn't match operand's", &GEP); + Assert( + GEPWidth == + cast<VectorType>(GEP.getPointerOperandType())->getElementCount(), + "Vector GEP result width doesn't match operand's", &GEP); for (Value *Idx : Idxs) { Type *IndexTy = Idx->getType(); - if (IndexTy->isVectorTy()) { - unsigned IndexWidth = IndexTy->getVectorNumElements(); + if (auto *IndexVTy = dyn_cast<VectorType>(IndexTy)) { + ElementCount IndexWidth = IndexVTy->getElementCount(); Assert(IndexWidth == GEPWidth, "Invalid GEP index vector width", &GEP); } Assert(IndexTy->isIntOrIntVectorTy(), @@ -4050,23 +4174,28 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) { // Check consistency of !prof branch_weights metadata. if (ProfName.equals("branch_weights")) { - unsigned ExpectedNumOperands = 0; - if (BranchInst *BI = dyn_cast<BranchInst>(&I)) - ExpectedNumOperands = BI->getNumSuccessors(); - else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I)) - ExpectedNumOperands = SI->getNumSuccessors(); - else if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) - ExpectedNumOperands = 1; - else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I)) - ExpectedNumOperands = IBI->getNumDestinations(); - else if (isa<SelectInst>(&I)) - ExpectedNumOperands = 2; - else - CheckFailed("!prof branch_weights are not allowed for this instruction", - MD); + if (isa<InvokeInst>(&I)) { + Assert(MD->getNumOperands() == 2 || MD->getNumOperands() == 3, + "Wrong number of InvokeInst branch_weights operands", MD); + } else { + unsigned ExpectedNumOperands = 0; + if (BranchInst *BI = dyn_cast<BranchInst>(&I)) + ExpectedNumOperands = BI->getNumSuccessors(); + else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I)) + ExpectedNumOperands = SI->getNumSuccessors(); + else if (isa<CallInst>(&I)) + ExpectedNumOperands = 1; + else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I)) + ExpectedNumOperands = IBI->getNumDestinations(); + else if (isa<SelectInst>(&I)) + ExpectedNumOperands = 2; + else + CheckFailed("!prof branch_weights are not allowed for this instruction", + MD); - Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, - "Wrong number of operands", MD); + Assert(MD->getNumOperands() == 1 + ExpectedNumOperands, + "Wrong number of operands", MD); + } for (unsigned i = 1; i < MD->getNumOperands(); ++i) { auto &MDO = MD->getOperand(i); Assert(MDO, "second operand should not be null", MD); @@ -4238,7 +4367,7 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *N = I.getDebugLoc().getAsMDNode()) { AssertDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N); - visitMDNode(*N); + visitMDNode(*N, AreDebugLocsAllowed::Yes); } if (auto *DII = dyn_cast<DbgVariableIntrinsic>(&I)) { @@ -4246,6 +4375,17 @@ void Verifier::visitInstruction(Instruction &I) { verifyNotEntryValue(*DII); } + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + I.getAllMetadata(MDs); + for (auto Attachment : MDs) { + unsigned Kind = Attachment.first; + auto AllowLocs = + (Kind == LLVMContext::MD_dbg || Kind == LLVMContext::MD_loop) + ? AreDebugLocsAllowed::Yes + : AreDebugLocsAllowed::No; + visitMDNode(*Attachment.second, AllowLocs); + } + InstsInThisBlock.insert(&I); } @@ -4304,6 +4444,41 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { switch (ID) { default: break; + case Intrinsic::assume: { + for (auto &Elem : Call.bundle_op_infos()) { + Assert(Elem.Tag->getKey() == "ignore" || + Attribute::isExistingAttribute(Elem.Tag->getKey()), + "tags must be valid attribute names"); + Attribute::AttrKind Kind = + Attribute::getAttrKindFromName(Elem.Tag->getKey()); + unsigned ArgCount = Elem.End - Elem.Begin; + if (Kind == Attribute::Alignment) { + Assert(ArgCount <= 3 && ArgCount >= 2, + "alignment assumptions should have 2 or 3 arguments"); + Assert(Call.getOperand(Elem.Begin)->getType()->isPointerTy(), + "first argument should be a pointer"); + Assert(Call.getOperand(Elem.Begin + 1)->getType()->isIntegerTy(), + "second argument should be an integer"); + if (ArgCount == 3) + Assert(Call.getOperand(Elem.Begin + 2)->getType()->isIntegerTy(), + "third argument should be an integer if present"); + return; + } + Assert(ArgCount <= 2, "to many arguments"); + if (Kind == Attribute::None) + break; + if (Attribute::doesAttrKindHaveArgument(Kind)) { + Assert(ArgCount == 2, "this attribute should have 2 arguments"); + Assert(isa<ConstantInt>(Call.getOperand(Elem.Begin + 1)), + "the second argument should be a constant integral value"); + } else if (isFuncOnlyAttr(Kind)) { + Assert((ArgCount) == 0, "this attribute has no argument"); + } else if (!isFuncOrArgAttr(Kind)) { + Assert((ArgCount) == 1, "this attribute should have one argument"); + } + } + break; + } case Intrinsic::coro_id: { auto *InfoArg = Call.getArgOperand(3)->stripPointerCasts(); if (isa<ConstantPointerNull>(InfoArg)) @@ -4318,7 +4493,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "an array"); break; } -#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC, DAGN) \ +#define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(Call)); @@ -4338,6 +4513,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { visitDbgLabelIntrinsic("label", cast<DbgLabelInst>(Call)); break; case Intrinsic::memcpy: + case Intrinsic::memcpy_inline: case Intrinsic::memmove: case Intrinsic::memset: { const auto *MI = cast<MemIntrinsic>(&Call); @@ -4368,15 +4544,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "must be a power of 2", Call); - if (auto *LengthCI = dyn_cast<ConstantInt>(AMI->getLength())) { - uint64_t Length = LengthCI->getZExtValue(); - uint64_t ElementSize = AMI->getElementSizeInBytes(); - Assert((Length % ElementSize) == 0, - "constant length must be a multiple of the element size in the " - "element-wise atomic memory intrinsic", - Call); - } - auto IsValidAlignment = [&](uint64_t Alignment) { return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); }; @@ -4390,6 +4557,85 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { } break; } + case Intrinsic::call_preallocated_setup: { + auto *NumArgs = dyn_cast<ConstantInt>(Call.getArgOperand(0)); + Assert(NumArgs != nullptr, + "llvm.call.preallocated.setup argument must be a constant"); + bool FoundCall = false; + for (User *U : Call.users()) { + auto *UseCall = dyn_cast<CallBase>(U); + Assert(UseCall != nullptr, + "Uses of llvm.call.preallocated.setup must be calls"); + const Function *Fn = UseCall->getCalledFunction(); + if (Fn && Fn->getIntrinsicID() == Intrinsic::call_preallocated_arg) { + auto *AllocArgIndex = dyn_cast<ConstantInt>(UseCall->getArgOperand(1)); + Assert(AllocArgIndex != nullptr, + "llvm.call.preallocated.alloc arg index must be a constant"); + auto AllocArgIndexInt = AllocArgIndex->getValue(); + Assert(AllocArgIndexInt.sge(0) && + AllocArgIndexInt.slt(NumArgs->getValue()), + "llvm.call.preallocated.alloc arg index must be between 0 and " + "corresponding " + "llvm.call.preallocated.setup's argument count"); + } else if (Fn && Fn->getIntrinsicID() == + Intrinsic::call_preallocated_teardown) { + // nothing to do + } else { + Assert(!FoundCall, "Can have at most one call corresponding to a " + "llvm.call.preallocated.setup"); + FoundCall = true; + size_t NumPreallocatedArgs = 0; + for (unsigned i = 0; i < UseCall->getNumArgOperands(); i++) { + if (UseCall->paramHasAttr(i, Attribute::Preallocated)) { + ++NumPreallocatedArgs; + } + } + Assert(NumPreallocatedArgs != 0, + "cannot use preallocated intrinsics on a call without " + "preallocated arguments"); + Assert(NumArgs->equalsInt(NumPreallocatedArgs), + "llvm.call.preallocated.setup arg size must be equal to number " + "of preallocated arguments " + "at call site", + Call, *UseCall); + // getOperandBundle() cannot be called if more than one of the operand + // bundle exists. There is already a check elsewhere for this, so skip + // here if we see more than one. + if (UseCall->countOperandBundlesOfType(LLVMContext::OB_preallocated) > + 1) { + return; + } + auto PreallocatedBundle = + UseCall->getOperandBundle(LLVMContext::OB_preallocated); + Assert(PreallocatedBundle, + "Use of llvm.call.preallocated.setup outside intrinsics " + "must be in \"preallocated\" operand bundle"); + Assert(PreallocatedBundle->Inputs.front().get() == &Call, + "preallocated bundle must have token from corresponding " + "llvm.call.preallocated.setup"); + } + } + break; + } + case Intrinsic::call_preallocated_arg: { + auto *Token = dyn_cast<CallBase>(Call.getArgOperand(0)); + Assert(Token && Token->getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_setup, + "llvm.call.preallocated.arg token argument must be a " + "llvm.call.preallocated.setup"); + Assert(Call.hasFnAttr(Attribute::Preallocated), + "llvm.call.preallocated.arg must be called with a \"preallocated\" " + "call site attribute"); + break; + } + case Intrinsic::call_preallocated_teardown: { + auto *Token = dyn_cast<CallBase>(Call.getArgOperand(0)); + Assert(Token && Token->getCalledFunction()->getIntrinsicID() == + Intrinsic::call_preallocated_setup, + "llvm.call.preallocated.teardown token argument must be a " + "llvm.call.preallocated.setup"); + break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: @@ -4506,20 +4752,20 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { LandingPad->getParent()); Assert(InvokeBB->getTerminator(), "safepoint block should be well formed", InvokeBB); - Assert(isStatepoint(InvokeBB->getTerminator()), + Assert(isa<GCStatepointInst>(InvokeBB->getTerminator()), "gc relocate should be linked to a statepoint", InvokeBB); } else { // In all other cases relocate should be tied to the statepoint directly. // This covers relocates on a normal return path of invoke statepoint and // relocates of a call statepoint. auto Token = Call.getArgOperand(0); - Assert(isa<Instruction>(Token) && isStatepoint(cast<Instruction>(Token)), + Assert(isa<GCStatepointInst>(Token), "gc relocate is incorrectly tied to the statepoint", Call, Token); } // Verify rest of the relocate arguments. const CallBase &StatepointCall = - *cast<CallBase>(cast<GCRelocateInst>(Call).getStatepoint()); + *cast<GCRelocateInst>(Call).getStatepoint(); // Both the base and derived must be piped through the safepoint. Value *Base = Call.getArgOperand(1); @@ -4530,47 +4776,55 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Assert(isa<ConstantInt>(Derived), "gc.relocate operand #3 must be integer offset", Call); - const int BaseIndex = cast<ConstantInt>(Base)->getZExtValue(); - const int DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue(); + const uint64_t BaseIndex = cast<ConstantInt>(Base)->getZExtValue(); + const uint64_t DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue(); + // Check the bounds - Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint base index out of bounds", Call); - Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCall.arg_size(), - "gc.relocate: statepoint derived index out of bounds", Call); - - // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' - // section of the statepoint's argument. - Assert(StatepointCall.arg_size() > 0, - "gc.statepoint: insufficient arguments"); - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)), - "gc.statement: number of call arguments must be constant integer"); - const unsigned NumCallArgs = + if (auto Opt = StatepointCall.getOperandBundle(LLVMContext::OB_gc_live)) { + Assert(BaseIndex < Opt->Inputs.size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(DerivedIndex < Opt->Inputs.size(), + "gc.relocate: statepoint derived index out of bounds", Call); + } else { + Assert(BaseIndex < StatepointCall.arg_size(), + "gc.relocate: statepoint base index out of bounds", Call); + Assert(DerivedIndex < StatepointCall.arg_size(), + "gc.relocate: statepoint derived index out of bounds", Call); + + // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' + // section of the statepoint's argument. + Assert(StatepointCall.arg_size() > 0, + "gc.statepoint: insufficient arguments"); + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)), + "gc.statement: number of call arguments must be constant integer"); + const uint64_t NumCallArgs = cast<ConstantInt>(StatepointCall.getArgOperand(3))->getZExtValue(); - Assert(StatepointCall.arg_size() > NumCallArgs + 5, - "gc.statepoint: mismatch in number of call arguments"); - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)), - "gc.statepoint: number of transition arguments must be " - "a constant integer"); - const int NumTransitionArgs = - cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)) - ->getZExtValue(); - const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; - Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)), - "gc.statepoint: number of deoptimization arguments must be " - "a constant integer"); - const int NumDeoptArgs = - cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)) - ->getZExtValue(); - const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; - const int GCParamArgsEnd = StatepointCall.arg_size(); - Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, - "gc.relocate: statepoint base index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); - Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, - "gc.relocate: statepoint derived index doesn't fall within the " - "'gc parameters' section of the statepoint call", - Call); + Assert(StatepointCall.arg_size() > NumCallArgs + 5, + "gc.statepoint: mismatch in number of call arguments"); + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)), + "gc.statepoint: number of transition arguments must be " + "a constant integer"); + const uint64_t NumTransitionArgs = + cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)) + ->getZExtValue(); + const uint64_t DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1; + Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)), + "gc.statepoint: number of deoptimization arguments must be " + "a constant integer"); + const uint64_t NumDeoptArgs = + cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)) + ->getZExtValue(); + const uint64_t GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; + const uint64_t GCParamArgsEnd = StatepointCall.arg_size(); + Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, + "gc.relocate: statepoint base index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd, + "gc.relocate: statepoint derived index doesn't fall within the " + "'gc parameters' section of the statepoint call", + Call); + } // Relocated value must be either a pointer type or vector-of-pointer type, // but gc_relocate does not need to return the same pointer type as the @@ -4598,6 +4852,14 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "eh.exceptionpointer argument must be a catchpad", Call); break; } + case Intrinsic::get_active_lane_mask: { + Assert(Call.getType()->isVectorTy(), "get_active_lane_mask: must return a " + "vector", Call); + auto *ElemTy = Call.getType()->getScalarType(); + Assert(ElemTy->isIntegerTy(1), "get_active_lane_mask: element type is not " + "i1", Call); + break; + } case Intrinsic::masked_load: { Assert(Call.getType()->isVectorTy(), "masked_load: must return a vector", Call); @@ -4617,8 +4879,8 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "masked_load: return must match pointer type", Call); Assert(PassThru->getType() == DataTy, "masked_load: pass through and data type must match", Call); - Assert(Mask->getType()->getVectorNumElements() == - DataTy->getVectorNumElements(), + Assert(cast<VectorType>(Mask->getType())->getElementCount() == + cast<VectorType>(DataTy)->getElementCount(), "masked_load: vector mask must be same length as data", Call); break; } @@ -4636,12 +4898,27 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType(); Assert(DataTy == Val->getType(), "masked_store: storee must match pointer type", Call); - Assert(Mask->getType()->getVectorNumElements() == - DataTy->getVectorNumElements(), + Assert(cast<VectorType>(Mask->getType())->getElementCount() == + cast<VectorType>(DataTy)->getElementCount(), "masked_store: vector mask must be same length as data", Call); break; } + case Intrinsic::masked_gather: { + const APInt &Alignment = + cast<ConstantInt>(Call.getArgOperand(1))->getValue(); + Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), + "masked_gather: alignment must be 0 or a power of 2", Call); + break; + } + case Intrinsic::masked_scatter: { + const APInt &Alignment = + cast<ConstantInt>(Call.getArgOperand(2))->getValue(); + Assert(Alignment.isNullValue() || Alignment.isPowerOf2(), + "masked_scatter: alignment must be 0 or a power of 2", Call); + break; + } + case Intrinsic::experimental_guard: { Assert(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call); Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, @@ -4691,7 +4968,9 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { case Intrinsic::umul_fix: case Intrinsic::umul_fix_sat: case Intrinsic::sdiv_fix: - case Intrinsic::udiv_fix: { + case Intrinsic::sdiv_fix_sat: + case Intrinsic::udiv_fix: + case Intrinsic::udiv_fix_sat: { Value *Op1 = Call.getArgOperand(0); Value *Op2 = Call.getArgOperand(1); Assert(Op1->getType()->isIntOrIntVectorTy(), @@ -4706,7 +4985,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "third argument of [us][mul|div]_fix[_sat] must fit within 32 bits"); if (ID == Intrinsic::smul_fix || ID == Intrinsic::smul_fix_sat || - ID == Intrinsic::sdiv_fix) { + ID == Intrinsic::sdiv_fix || ID == Intrinsic::sdiv_fix_sat) { Assert( Op3->getZExtValue() < Op1->getType()->getScalarSizeInBits(), "the scale of s[mul|div]_fix[_sat] must be less than the width of " @@ -4728,6 +5007,85 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "Intrinsic does not support vectors", &Call); break; } + case Intrinsic::bswap: { + Type *Ty = Call.getType(); + unsigned Size = Ty->getScalarSizeInBits(); + Assert(Size % 16 == 0, "bswap must be an even number of bytes", &Call); + break; + } + case Intrinsic::matrix_multiply: + case Intrinsic::matrix_transpose: + case Intrinsic::matrix_column_major_load: + case Intrinsic::matrix_column_major_store: { + Function *IF = Call.getCalledFunction(); + ConstantInt *Stride = nullptr; + ConstantInt *NumRows; + ConstantInt *NumColumns; + VectorType *ResultTy; + Type *Op0ElemTy = nullptr; + Type *Op1ElemTy = nullptr; + switch (ID) { + case Intrinsic::matrix_multiply: + NumRows = cast<ConstantInt>(Call.getArgOperand(2)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + Op1ElemTy = + cast<VectorType>(Call.getArgOperand(1)->getType())->getElementType(); + break; + case Intrinsic::matrix_transpose: + NumRows = cast<ConstantInt>(Call.getArgOperand(1)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(2)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + break; + case Intrinsic::matrix_column_major_load: + Stride = dyn_cast<ConstantInt>(Call.getArgOperand(1)); + NumRows = cast<ConstantInt>(Call.getArgOperand(3)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); + ResultTy = cast<VectorType>(Call.getType()); + Op0ElemTy = + cast<PointerType>(Call.getArgOperand(0)->getType())->getElementType(); + break; + case Intrinsic::matrix_column_major_store: + Stride = dyn_cast<ConstantInt>(Call.getArgOperand(2)); + NumRows = cast<ConstantInt>(Call.getArgOperand(4)); + NumColumns = cast<ConstantInt>(Call.getArgOperand(5)); + ResultTy = cast<VectorType>(Call.getArgOperand(0)->getType()); + Op0ElemTy = + cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); + Op1ElemTy = + cast<PointerType>(Call.getArgOperand(1)->getType())->getElementType(); + break; + default: + llvm_unreachable("unexpected intrinsic"); + } + + Assert(ResultTy->getElementType()->isIntegerTy() || + ResultTy->getElementType()->isFloatingPointTy(), + "Result type must be an integer or floating-point type!", IF); + + Assert(ResultTy->getElementType() == Op0ElemTy, + "Vector element type mismatch of the result and first operand " + "vector!", IF); + + if (Op1ElemTy) + Assert(ResultTy->getElementType() == Op1ElemTy, + "Vector element type mismatch of the result and second operand " + "vector!", IF); + + Assert(ResultTy->getNumElements() == + NumRows->getZExtValue() * NumColumns->getZExtValue(), + "Result of a matrix operation does not fit in the returned vector!"); + + if (Stride) + Assert(Stride->getZExtValue() >= NumRows->getZExtValue(), + "Stride must be greater or equal than the number of rows!", IF); + + break; + } }; } @@ -4754,7 +5112,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { unsigned NumOperands; bool HasRoundingMD; switch (FPI.getIntrinsicID()) { -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: \ NumOperands = NARG; \ HasRoundingMD = ROUND_MODE; \ @@ -4777,7 +5135,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { Type *ResultTy = FPI.getType(); Assert(!ValTy->isVectorTy() && !ResultTy->isVectorTy(), "Intrinsic does not support vectors", &FPI); - } + } break; case Intrinsic::experimental_constrained_lround: @@ -4787,7 +5145,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { Assert(!ValTy->isVectorTy() && !ResultTy->isVectorTy(), "Intrinsic does not support vectors", &FPI); break; - } + } case Intrinsic::experimental_constrained_fcmp: case Intrinsic::experimental_constrained_fcmps: { @@ -4798,7 +5156,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { } case Intrinsic::experimental_constrained_fptosi: - case Intrinsic::experimental_constrained_fptoui: { + case Intrinsic::experimental_constrained_fptoui: { Value *Operand = FPI.getArgOperand(0); uint64_t NumSrcElem = 0; Assert(Operand->getType()->isFPOrFPVectorTy(), @@ -4870,7 +5228,7 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { "Intrinsic first argument's type must be smaller than result type", &FPI); } - } + } break; default: @@ -5136,7 +5494,7 @@ struct VerifierLegacyPass : public FunctionPass { bool runOnFunction(Function &F) override { if (!V->verify(F) && FatalErrors) { - errs() << "in function " << F.getName() << '\n'; + errs() << "in function " << F.getName() << '\n'; report_fatal_error("Broken function found, compilation aborted!"); } return false; |