diff options
Diffstat (limited to 'contrib/llvm/lib/IR')
47 files changed, 3202 insertions, 1665 deletions
diff --git a/contrib/llvm/lib/IR/AsmWriter.cpp b/contrib/llvm/lib/IR/AsmWriter.cpp index 9b2399dd880c..eecef9423f2e 100644 --- a/contrib/llvm/lib/IR/AsmWriter.cpp +++ b/contrib/llvm/lib/IR/AsmWriter.cpp @@ -1,3 +1,4 @@ + //===-- AsmWriter.cpp - Printing LLVM as an assembly file -----------------===// // // The LLVM Compiler Infrastructure @@ -310,6 +311,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::X86_StdCall: Out << "x86_stdcallcc"; break; case CallingConv::X86_FastCall: Out << "x86_fastcallcc"; break; case CallingConv::X86_ThisCall: Out << "x86_thiscallcc"; break; + case CallingConv::X86_RegCall: Out << "x86_regcallcc"; break; case CallingConv::X86_VectorCall:Out << "x86_vectorcallcc"; break; case CallingConv::Intel_OCL_BI: Out << "intel_ocl_bicc"; break; case CallingConv::ARM_APCS: Out << "arm_apcscc"; break; @@ -336,9 +338,7 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { } } -// PrintEscapedString - Print each character of the specified string, escaping -// it if it is not printable or if it is an escape char. -static void PrintEscapedString(StringRef Name, raw_ostream &Out) { +void llvm::PrintEscapedString(StringRef Name, raw_ostream &Out) { for (unsigned i = 0, e = Name.size(); i != e; ++i) { unsigned char C = Name[i]; if (isprint(C) && C != '\\' && C != '"') @@ -1041,39 +1041,6 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Metadata *MD, SlotTracker *Machine, const Module *Context, bool FromValue = false); -static const char *getPredicateText(unsigned predicate) { - const char * pred = "unknown"; - switch (predicate) { - case FCmpInst::FCMP_FALSE: pred = "false"; break; - case FCmpInst::FCMP_OEQ: pred = "oeq"; break; - case FCmpInst::FCMP_OGT: pred = "ogt"; break; - case FCmpInst::FCMP_OGE: pred = "oge"; break; - case FCmpInst::FCMP_OLT: pred = "olt"; break; - case FCmpInst::FCMP_OLE: pred = "ole"; break; - case FCmpInst::FCMP_ONE: pred = "one"; break; - case FCmpInst::FCMP_ORD: pred = "ord"; break; - case FCmpInst::FCMP_UNO: pred = "uno"; break; - case FCmpInst::FCMP_UEQ: pred = "ueq"; break; - case FCmpInst::FCMP_UGT: pred = "ugt"; break; - case FCmpInst::FCMP_UGE: pred = "uge"; break; - case FCmpInst::FCMP_ULT: pred = "ult"; break; - case FCmpInst::FCMP_ULE: pred = "ule"; break; - case FCmpInst::FCMP_UNE: pred = "une"; break; - case FCmpInst::FCMP_TRUE: pred = "true"; break; - case ICmpInst::ICMP_EQ: pred = "eq"; break; - case ICmpInst::ICMP_NE: pred = "ne"; break; - case ICmpInst::ICMP_SGT: pred = "sgt"; break; - case ICmpInst::ICMP_SGE: pred = "sge"; break; - case ICmpInst::ICMP_SLT: pred = "slt"; break; - case ICmpInst::ICMP_SLE: pred = "sle"; break; - case ICmpInst::ICMP_UGT: pred = "ugt"; break; - case ICmpInst::ICMP_UGE: pred = "uge"; break; - case ICmpInst::ICMP_ULT: pred = "ult"; break; - case ICmpInst::ICMP_ULE: pred = "ule"; break; - } - return pred; -} - static void writeAtomicRMWOperation(raw_ostream &Out, AtomicRMWInst::BinOp Op) { switch (Op) { @@ -1139,15 +1106,15 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, } if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV)) { - if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEsingle || - &CFP->getValueAPF().getSemantics() == &APFloat::IEEEdouble) { + if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEsingle() || + &CFP->getValueAPF().getSemantics() == &APFloat::IEEEdouble()) { // We would like to output the FP constant value in exponential notation, // but we cannot do this if doing so will lose precision. Check here to // make sure that we only output it in exponential format if we can parse // the value back and get the same value. // bool ignored; - bool isDouble = &CFP->getValueAPF().getSemantics()==&APFloat::IEEEdouble; + bool isDouble = &CFP->getValueAPF().getSemantics()==&APFloat::IEEEdouble(); bool isInf = CFP->getValueAPF().isInfinity(); bool isNaN = CFP->getValueAPF().isNaN(); if (!isInf && !isNaN) { @@ -1164,7 +1131,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, ((StrVal[0] == '-' || StrVal[0] == '+') && (StrVal[1] >= '0' && StrVal[1] <= '9'))) { // Reparse stringized version! - if (APFloat(APFloat::IEEEdouble, StrVal).convertToDouble() == Val) { + if (APFloat(APFloat::IEEEdouble(), StrVal).convertToDouble() == Val) { Out << StrVal; return; } @@ -1179,7 +1146,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, APFloat apf = CFP->getValueAPF(); // Floats are represented in ASCII IR as double, convert. if (!isDouble) - apf.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, + apf.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, &ignored); Out << format_hex(apf.bitcastToAPInt().getZExtValue(), 0, /*Upper=*/true); return; @@ -1190,26 +1157,26 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, // fixed number of hex digits. Out << "0x"; APInt API = CFP->getValueAPF().bitcastToAPInt(); - if (&CFP->getValueAPF().getSemantics() == &APFloat::x87DoubleExtended) { + if (&CFP->getValueAPF().getSemantics() == &APFloat::x87DoubleExtended()) { Out << 'K'; Out << format_hex_no_prefix(API.getHiBits(16).getZExtValue(), 4, /*Upper=*/true); Out << format_hex_no_prefix(API.getLoBits(64).getZExtValue(), 16, /*Upper=*/true); return; - } else if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEquad) { + } else if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEquad()) { Out << 'L'; Out << format_hex_no_prefix(API.getLoBits(64).getZExtValue(), 16, /*Upper=*/true); Out << format_hex_no_prefix(API.getHiBits(64).getZExtValue(), 16, /*Upper=*/true); - } else if (&CFP->getValueAPF().getSemantics() == &APFloat::PPCDoubleDouble) { + } else if (&CFP->getValueAPF().getSemantics() == &APFloat::PPCDoubleDouble()) { Out << 'M'; Out << format_hex_no_prefix(API.getLoBits(64).getZExtValue(), 16, /*Upper=*/true); Out << format_hex_no_prefix(API.getHiBits(64).getZExtValue(), 16, /*Upper=*/true); - } else if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEhalf) { + } else if (&CFP->getValueAPF().getSemantics() == &APFloat::IEEEhalf()) { Out << 'H'; Out << format_hex_no_prefix(API.getZExtValue(), 4, /*Upper=*/true); @@ -1349,15 +1316,22 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << CE->getOpcodeName(); WriteOptimizationInfo(Out, CE); if (CE->isCompare()) - Out << ' ' << getPredicateText(CE->getPredicate()); + Out << ' ' << CmpInst::getPredicateName( + static_cast<CmpInst::Predicate>(CE->getPredicate())); Out << " ("; + Optional<unsigned> InRangeOp; if (const GEPOperator *GEP = dyn_cast<GEPOperator>(CE)) { TypePrinter.print(GEP->getSourceElementType(), Out); Out << ", "; + InRangeOp = GEP->getInRangeIndex(); + if (InRangeOp) + ++*InRangeOp; } for (User::const_op_iterator OI=CE->op_begin(); OI != CE->op_end(); ++OI) { + if (InRangeOp && unsigned(OI - CE->op_begin()) == *InRangeOp) + Out << "inrange "; TypePrinter.print((*OI)->getType(), Out); Out << ' '; WriteAsOperandInternal(Out, *OI, &TypePrinter, Machine, Context); @@ -1434,14 +1408,15 @@ struct MDFieldPrinter { } void printTag(const DINode *N); void printMacinfoType(const DIMacroNode *N); + void printChecksumKind(const DIFile *N); void printString(StringRef Name, StringRef Value, bool ShouldSkipEmpty = true); void printMetadata(StringRef Name, const Metadata *MD, bool ShouldSkipNull = true); template <class IntTy> void printInt(StringRef Name, IntTy Int, bool ShouldSkipZero = true); - void printBool(StringRef Name, bool Value); - void printDIFlags(StringRef Name, unsigned Flags); + void printBool(StringRef Name, bool Value, Optional<bool> Default = None); + void printDIFlags(StringRef Name, DINode::DIFlags Flags); template <class IntTy, class Stringifier> void printDwarfEnum(StringRef Name, IntTy Value, Stringifier toString, bool ShouldSkipZero = true); @@ -1451,7 +1426,8 @@ struct MDFieldPrinter { void MDFieldPrinter::printTag(const DINode *N) { Out << FS << "tag: "; - if (const char *Tag = dwarf::TagString(N->getTag())) + auto Tag = dwarf::TagString(N->getTag()); + if (!Tag.empty()) Out << Tag; else Out << N->getTag(); @@ -1459,12 +1435,20 @@ void MDFieldPrinter::printTag(const DINode *N) { void MDFieldPrinter::printMacinfoType(const DIMacroNode *N) { Out << FS << "type: "; - if (const char *Type = dwarf::MacinfoString(N->getMacinfoType())) + auto Type = dwarf::MacinfoString(N->getMacinfoType()); + if (!Type.empty()) Out << Type; else Out << N->getMacinfoType(); } +void MDFieldPrinter::printChecksumKind(const DIFile *N) { + if (N->getChecksumKind() == DIFile::CSK_None) + // Skip CSK_None checksum kind. + return; + Out << FS << "checksumkind: " << N->getChecksumKindAsString(); +} + void MDFieldPrinter::printString(StringRef Name, StringRef Value, bool ShouldSkipEmpty) { if (ShouldSkipEmpty && Value.empty()) @@ -1503,23 +1487,26 @@ void MDFieldPrinter::printInt(StringRef Name, IntTy Int, bool ShouldSkipZero) { Out << FS << Name << ": " << Int; } -void MDFieldPrinter::printBool(StringRef Name, bool Value) { +void MDFieldPrinter::printBool(StringRef Name, bool Value, + Optional<bool> Default) { + if (Default && Value == *Default) + return; Out << FS << Name << ": " << (Value ? "true" : "false"); } -void MDFieldPrinter::printDIFlags(StringRef Name, unsigned Flags) { +void MDFieldPrinter::printDIFlags(StringRef Name, DINode::DIFlags Flags) { if (!Flags) return; Out << FS << Name << ": "; - SmallVector<unsigned, 8> SplitFlags; - unsigned Extra = DINode::splitFlags(Flags, SplitFlags); + SmallVector<DINode::DIFlags, 8> SplitFlags; + auto Extra = DINode::splitFlags(Flags, SplitFlags); FieldSeparator FlagsFS(" | "); - for (unsigned F : SplitFlags) { - const char *StringF = DINode::getFlagString(F); - assert(StringF && "Expected valid flag"); + for (auto F : SplitFlags) { + auto StringF = DINode::getFlagString(F); + assert(!StringF.empty() && "Expected valid flag"); Out << FlagsFS << StringF; } if (Extra || SplitFlags.empty()) @@ -1539,7 +1526,8 @@ void MDFieldPrinter::printDwarfEnum(StringRef Name, IntTy Value, return; Out << FS << Name << ": "; - if (const char *S = toString(Value)) + auto S = toString(Value); + if (!S.empty()) Out << S; else Out << Value; @@ -1673,6 +1661,8 @@ static void writeDIFile(raw_ostream &Out, const DIFile *N, TypePrinting *, /* ShouldSkipEmpty */ false); Printer.printString("directory", N->getDirectory(), /* ShouldSkipEmpty */ false); + Printer.printChecksumKind(N); + Printer.printString("checksum", N->getChecksum(), /* ShouldSkipEmpty */ true); Out << ")"; } @@ -1697,6 +1687,7 @@ static void writeDICompileUnit(raw_ostream &Out, const DICompileUnit *N, Printer.printMetadata("imports", N->getRawImportedEntities()); Printer.printMetadata("macros", N->getRawMacros()); Printer.printInt("dwoId", N->getDWOId()); + Printer.printBool("splitDebugInlining", N->getSplitDebugInlining(), true); Out << ")"; } @@ -1765,6 +1756,7 @@ static void writeDINamespace(raw_ostream &Out, const DINamespace *N, Printer.printMetadata("scope", N->getRawScope(), /* ShouldSkipNull */ false); Printer.printMetadata("file", N->getRawFile()); Printer.printInt("line", N->getLine()); + Printer.printBool("exportSymbols", N->getExportSymbols(), false); Out << ")"; } @@ -1845,8 +1837,8 @@ static void writeDIGlobalVariable(raw_ostream &Out, const DIGlobalVariable *N, Printer.printMetadata("type", N->getRawType()); Printer.printBool("isLocal", N->isLocalToUnit()); Printer.printBool("isDefinition", N->isDefinition()); - Printer.printMetadata("variable", N->getRawVariable()); Printer.printMetadata("declaration", N->getRawStaticDataMemberDeclaration()); + Printer.printInt("align", N->getAlignInBits()); Out << ")"; } @@ -1862,6 +1854,7 @@ static void writeDILocalVariable(raw_ostream &Out, const DILocalVariable *N, Printer.printInt("line", N->getLine()); Printer.printMetadata("type", N->getRawType()); Printer.printDIFlags("flags", N->getFlags()); + Printer.printInt("align", N->getAlignInBits()); Out << ")"; } @@ -1872,8 +1865,8 @@ static void writeDIExpression(raw_ostream &Out, const DIExpression *N, FieldSeparator FS; if (N->isValid()) { for (auto I = N->expr_op_begin(), E = N->expr_op_end(); I != E; ++I) { - const char *OpStr = dwarf::OperationEncodingString(I->getOp()); - assert(OpStr && "Expected valid opcode"); + auto OpStr = dwarf::OperationEncodingString(I->getOp()); + assert(!OpStr.empty() && "Expected valid opcode"); Out << FS << OpStr; for (unsigned A = 0, AE = I->getNumArgs(); A != AE; ++A) @@ -1886,6 +1879,18 @@ static void writeDIExpression(raw_ostream &Out, const DIExpression *N, Out << ")"; } +static void writeDIGlobalVariableExpression(raw_ostream &Out, + const DIGlobalVariableExpression *N, + TypePrinting *TypePrinter, + SlotTracker *Machine, + const Module *Context) { + Out << "!DIGlobalVariableExpression("; + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printMetadata("var", N->getVariable()); + Printer.printMetadata("expr", N->getExpression()); + Out << ")"; +} + static void writeDIObjCProperty(raw_ostream &Out, const DIObjCProperty *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { @@ -2869,7 +2874,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { // Print out the compare instruction predicates if (const CmpInst *CI = dyn_cast<CmpInst>(&I)) - Out << ' ' << getPredicateText(CI->getPredicate()); + Out << ' ' << CmpInst::getPredicateName(CI->getPredicate()); // Print out the atomicrmw operation if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I)) @@ -3008,7 +3013,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { } Operand = CI->getCalledValue(); - FunctionType *FTy = cast<FunctionType>(CI->getFunctionType()); + FunctionType *FTy = CI->getFunctionType(); Type *RetTy = FTy->getReturnType(); const AttributeSet &PAL = CI->getAttributes(); @@ -3045,7 +3050,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { } else if (const InvokeInst *II = dyn_cast<InvokeInst>(&I)) { Operand = II->getCalledValue(); - FunctionType *FTy = cast<FunctionType>(II->getFunctionType()); + FunctionType *FTy = II->getFunctionType(); Type *RetTy = FTy->getReturnType(); const AttributeSet &PAL = II->getAttributes(); diff --git a/contrib/llvm/lib/IR/AttributeImpl.h b/contrib/llvm/lib/IR/AttributeImpl.h index d58bff56576d..d0d27101aa86 100644 --- a/contrib/llvm/lib/IR/AttributeImpl.h +++ b/contrib/llvm/lib/IR/AttributeImpl.h @@ -16,17 +16,22 @@ #ifndef LLVM_LIB_IR_ATTRIBUTEIMPL_H #define LLVM_LIB_IR_ATTRIBUTEIMPL_H +#include "AttributeSetNode.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/FoldingSet.h" -#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" #include "llvm/IR/Attributes.h" -#include "AttributeSetNode.h" -#include "llvm/Support/DataTypes.h" +#include "llvm/Support/TrailingObjects.h" +#include <algorithm> +#include <cassert> #include <climits> +#include <cstddef> +#include <cstdint> #include <string> +#include <utility> namespace llvm { -class Constant; class LLVMContext; //===----------------------------------------------------------------------===// @@ -36,10 +41,6 @@ class LLVMContext; class AttributeImpl : public FoldingSetNode { unsigned char KindID; ///< Holds the AttrEntryKind of the attribute - // AttributesImpl is uniqued, these should not be publicly available. - void operator=(const AttributeImpl &) = delete; - AttributeImpl(const AttributeImpl &) = delete; - protected: enum AttrEntryKind { EnumAttrEntry, @@ -50,6 +51,10 @@ protected: AttributeImpl(AttrEntryKind KindID) : KindID(KindID) {} public: + // AttributesImpl is uniqued, these should not be available. + AttributeImpl(const AttributeImpl &) = delete; + AttributeImpl &operator=(const AttributeImpl &) = delete; + virtual ~AttributeImpl(); bool isEnumAttribute() const { return KindID == EnumAttrEntry; } @@ -85,9 +90,6 @@ public: ID.AddString(Kind); if (!Values.empty()) ID.AddString(Values); } - - // FIXME: Remove this! - static uint64_t getAttrMask(Attribute::AttrKind Val); }; //===----------------------------------------------------------------------===// @@ -168,12 +170,9 @@ private: return getTrailingObjects<IndexAttrPair>() + Slot; } - // AttributesSet is uniqued, these should not be publicly available. - void operator=(const AttributeSetImpl &) = delete; - AttributeSetImpl(const AttributeSetImpl &) = delete; public: AttributeSetImpl(LLVMContext &C, - ArrayRef<std::pair<unsigned, AttributeSetNode *> > Slots) + ArrayRef<std::pair<unsigned, AttributeSetNode *>> Slots) : Context(C), NumSlots(Slots.size()), AvailableFunctionAttrs(0) { static_assert(Attribute::EndAttrKinds <= sizeof(AvailableFunctionAttrs) * CHAR_BIT, @@ -206,6 +205,10 @@ public: } } + // AttributesSetImpt is uniqued, these should not be available. + AttributeSetImpl(const AttributeSetImpl &) = delete; + AttributeSetImpl &operator=(const AttributeSetImpl &) = delete; + void operator delete(void *p) { ::operator delete(p); } /// \brief Get the context that created this AttributeSetImpl. @@ -251,19 +254,16 @@ public: Profile(ID, makeArrayRef(getNode(0), getNumSlots())); } static void Profile(FoldingSetNodeID &ID, - ArrayRef<std::pair<unsigned, AttributeSetNode*> > Nodes) { - for (unsigned i = 0, e = Nodes.size(); i != e; ++i) { - ID.AddInteger(Nodes[i].first); - ID.AddPointer(Nodes[i].second); + ArrayRef<std::pair<unsigned, AttributeSetNode*>> Nodes) { + for (const auto &Node : Nodes) { + ID.AddInteger(Node.first); + ID.AddPointer(Node.second); } } - // FIXME: This atrocity is temporary. - uint64_t Raw(unsigned Index) const; - void dump() const; }; -} // end llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_LIB_IR_ATTRIBUTEIMPL_H diff --git a/contrib/llvm/lib/IR/AttributeSetNode.h b/contrib/llvm/lib/IR/AttributeSetNode.h index fab1ed51e4d6..23ce3713c20b 100644 --- a/contrib/llvm/lib/IR/AttributeSetNode.h +++ b/contrib/llvm/lib/IR/AttributeSetNode.h @@ -15,10 +15,17 @@ #ifndef LLVM_IR_ATTRIBUTESETNODE_H #define LLVM_IR_ATTRIBUTESETNODE_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" #include "llvm/IR/Attributes.h" #include "llvm/Support/TrailingObjects.h" +#include <algorithm> #include <climits> +#include <cstdint> +#include <string> +#include <utility> namespace llvm { @@ -49,10 +56,11 @@ class AttributeSetNode final } } - // AttributesSetNode is uniqued, these should not be publicly available. - void operator=(const AttributeSetNode &) = delete; - AttributeSetNode(const AttributeSetNode &) = delete; public: + // AttributesSetNode is uniqued, these should not be available. + AttributeSetNode(const AttributeSetNode &) = delete; + AttributeSetNode &operator=(const AttributeSetNode &) = delete; + void operator delete(void *p) { ::operator delete(p); } static AttributeSetNode *get(LLVMContext &C, ArrayRef<Attribute> Attrs); @@ -88,11 +96,11 @@ public: Profile(ID, makeArrayRef(begin(), end())); } static void Profile(FoldingSetNodeID &ID, ArrayRef<Attribute> AttrList) { - for (unsigned I = 0, E = AttrList.size(); I != E; ++I) - AttrList[I].Profile(ID); + for (const auto &Attr : AttrList) + Attr.Profile(ID); } }; -} // end llvm namespace +} // end namespace llvm -#endif +#endif // LLVM_IR_ATTRIBUTESETNODE_H diff --git a/contrib/llvm/lib/IR/Attributes.cpp b/contrib/llvm/lib/IR/Attributes.cpp index d774c1ae9dfb..1ec53cf1e1d6 100644 --- a/contrib/llvm/lib/IR/Attributes.cpp +++ b/contrib/llvm/lib/IR/Attributes.cpp @@ -38,7 +38,7 @@ using namespace llvm; // // In order to do this, we need to reserve one value of the second (optional) // allocsize argument to signify "not present." -LLVM_CONSTEXPR static unsigned AllocSizeNumElemsNotPresent = -1; +static const unsigned AllocSizeNumElemsNotPresent = -1; static uint64_t packAllocSizeArgs(unsigned ElemSizeArg, const Optional<unsigned> &NumElemsArg) { @@ -381,10 +381,18 @@ std::string Attribute::getAsString(bool InAttrGrp) const { std::string Result; Result += (Twine('"') + getKindAsString() + Twine('"')).str(); - StringRef Val = pImpl->getValueAsString(); - if (Val.empty()) return Result; - - Result += ("=\"" + Val + Twine('"')).str(); + std::string AttrVal = pImpl->getValueAsString(); + if (AttrVal.empty()) return Result; + + // Since some attribute strings contain special characters that cannot be + // printable, those have to be escaped to make the attribute value printable + // as is. e.g. "\01__gnu_mcount_nc" + { + raw_string_ostream OS(Result); + OS << "=\""; + PrintEscapedString(AttrVal, OS); + OS << "\""; + } return Result; } @@ -464,78 +472,6 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const { return getKindAsString() < AI.getKindAsString(); } -uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) { - // FIXME: Remove this. - switch (Val) { - case Attribute::EndAttrKinds: - llvm_unreachable("Synthetic enumerators which should never get here"); - - case Attribute::None: return 0; - case Attribute::ZExt: return 1 << 0; - case Attribute::SExt: return 1 << 1; - case Attribute::NoReturn: return 1 << 2; - case Attribute::InReg: return 1 << 3; - case Attribute::StructRet: return 1 << 4; - case Attribute::NoUnwind: return 1 << 5; - case Attribute::NoAlias: return 1 << 6; - case Attribute::ByVal: return 1 << 7; - case Attribute::Nest: return 1 << 8; - case Attribute::ReadNone: return 1 << 9; - case Attribute::ReadOnly: return 1 << 10; - case Attribute::NoInline: return 1 << 11; - case Attribute::AlwaysInline: return 1 << 12; - case Attribute::OptimizeForSize: return 1 << 13; - case Attribute::StackProtect: return 1 << 14; - case Attribute::StackProtectReq: return 1 << 15; - case Attribute::Alignment: return 31 << 16; - case Attribute::NoCapture: return 1 << 21; - case Attribute::NoRedZone: return 1 << 22; - case Attribute::NoImplicitFloat: return 1 << 23; - case Attribute::Naked: return 1 << 24; - case Attribute::InlineHint: return 1 << 25; - case Attribute::StackAlignment: return 7 << 26; - case Attribute::ReturnsTwice: return 1 << 29; - case Attribute::UWTable: return 1 << 30; - case Attribute::NonLazyBind: return 1U << 31; - case Attribute::SanitizeAddress: return 1ULL << 32; - case Attribute::MinSize: return 1ULL << 33; - case Attribute::NoDuplicate: return 1ULL << 34; - case Attribute::StackProtectStrong: return 1ULL << 35; - case Attribute::SanitizeThread: return 1ULL << 36; - case Attribute::SanitizeMemory: return 1ULL << 37; - case Attribute::NoBuiltin: return 1ULL << 38; - case Attribute::Returned: return 1ULL << 39; - case Attribute::Cold: return 1ULL << 40; - case Attribute::Builtin: return 1ULL << 41; - case Attribute::OptimizeNone: return 1ULL << 42; - case Attribute::InAlloca: return 1ULL << 43; - case Attribute::NonNull: return 1ULL << 44; - case Attribute::JumpTable: return 1ULL << 45; - case Attribute::Convergent: return 1ULL << 46; - case Attribute::SafeStack: return 1ULL << 47; - case Attribute::NoRecurse: return 1ULL << 48; - case Attribute::InaccessibleMemOnly: return 1ULL << 49; - case Attribute::InaccessibleMemOrArgMemOnly: return 1ULL << 50; - case Attribute::SwiftSelf: return 1ULL << 51; - case Attribute::SwiftError: return 1ULL << 52; - case Attribute::WriteOnly: return 1ULL << 53; - case Attribute::Dereferenceable: - llvm_unreachable("dereferenceable attribute not supported in raw format"); - break; - case Attribute::DereferenceableOrNull: - llvm_unreachable("dereferenceable_or_null attribute not supported in raw " - "format"); - break; - case Attribute::ArgMemOnly: - llvm_unreachable("argmemonly attribute not supported in raw format"); - break; - case Attribute::AllocSize: - llvm_unreachable("allocsize not supported in raw format"); - break; - } - llvm_unreachable("Unsupported attribute type"); -} - //===----------------------------------------------------------------------===// // AttributeSetNode Definition //===----------------------------------------------------------------------===// @@ -645,39 +581,6 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const { // AttributeSetImpl Definition //===----------------------------------------------------------------------===// -uint64_t AttributeSetImpl::Raw(unsigned Index) const { - for (unsigned I = 0, E = getNumSlots(); I != E; ++I) { - if (getSlotIndex(I) != Index) continue; - const AttributeSetNode *ASN = getSlotNode(I); - uint64_t Mask = 0; - - for (AttributeSetNode::iterator II = ASN->begin(), - IE = ASN->end(); II != IE; ++II) { - Attribute Attr = *II; - - // This cannot handle string attributes. - if (Attr.isStringAttribute()) continue; - - Attribute::AttrKind Kind = Attr.getKindAsEnum(); - - if (Kind == Attribute::Alignment) - Mask |= (Log2_32(ASN->getAlignment()) + 1) << 16; - else if (Kind == Attribute::StackAlignment) - Mask |= (Log2_32(ASN->getStackAlignment()) + 1) << 26; - else if (Kind == Attribute::Dereferenceable) - llvm_unreachable("dereferenceable not supported in bit mask"); - else if (Kind == Attribute::AllocSize) - llvm_unreachable("allocsize not supported in bit mask"); - else - Mask |= AttributeImpl::getAttrMask(Kind); - } - - return Mask; - } - - return 0; -} - LLVM_DUMP_METHOD void AttributeSetImpl::dump() const { AttributeSet(const_cast<AttributeSetImpl *>(this)).dump(); } @@ -721,10 +624,11 @@ AttributeSet AttributeSet::get(LLVMContext &C, const std::pair<unsigned, Attribute> &RHS) { return LHS.first < RHS.first; }) && "Misordered Attributes list!"); - assert(std::none_of(Attrs.begin(), Attrs.end(), - [](const std::pair<unsigned, Attribute> &Pair) { - return Pair.second.hasAttribute(Attribute::None); - }) && "Pointless attribute!"); + assert(none_of(Attrs, + [](const std::pair<unsigned, Attribute> &Pair) { + return Pair.second.hasAttribute(Attribute::None); + }) && + "Pointless attribute!"); // Create a vector if (unsigned, AttributeSetNode*) pairs from the attributes // list. @@ -738,8 +642,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, ++I; } - AttrPairVec.push_back(std::make_pair(Index, - AttributeSetNode::get(C, AttrVec))); + AttrPairVec.emplace_back(Index, AttributeSetNode::get(C, AttrVec)); } return getImpl(C, AttrPairVec); @@ -791,13 +694,12 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, default: Attr = Attribute::get(C, Kind); } - Attrs.push_back(std::make_pair(Index, Attr)); + Attrs.emplace_back(Index, Attr); } // Add target-dependent (string) attributes. for (const auto &TDA : B.td_attrs()) - Attrs.push_back( - std::make_pair(Index, Attribute::get(C, TDA.first, TDA.second))); + Attrs.emplace_back(Index, Attribute::get(C, TDA.first, TDA.second)); return get(C, Attrs); } @@ -806,7 +708,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, ArrayRef<Attribute::AttrKind> Kinds) { SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; for (Attribute::AttrKind K : Kinds) - Attrs.push_back(std::make_pair(Index, Attribute::get(C, K))); + Attrs.emplace_back(Index, Attribute::get(C, K)); return get(C, Attrs); } @@ -814,7 +716,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, ArrayRef<StringRef> Kinds) { SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; for (StringRef K : Kinds) - Attrs.push_back(std::make_pair(Index, Attribute::get(C, K))); + Attrs.emplace_back(Index, Attribute::get(C, K)); return get(C, Attrs); } @@ -1108,6 +1010,10 @@ bool AttributeSet::hasFnAttribute(Attribute::AttrKind Kind) const { return pImpl && pImpl->hasFnAttribute(Kind); } +bool AttributeSet::hasFnAttribute(StringRef Kind) const { + return hasAttribute(AttributeSet::FunctionIndex, Kind); +} + bool AttributeSet::hasAttrSomewhere(Attribute::AttrKind Attr, unsigned *Index) const { if (!pImpl) return false; @@ -1158,7 +1064,7 @@ uint64_t AttributeSet::getDereferenceableOrNullBytes(unsigned Index) const { std::pair<unsigned, Optional<unsigned>> AttributeSet::getAllocSizeArgs(unsigned Index) const { AttributeSetNode *ASN = getAttributes(Index); - return ASN ? ASN->getAllocSizeArgs() : std::make_pair(0, 0); + return ASN ? ASN->getAllocSizeArgs() : std::make_pair(0u, Optional<unsigned>(0u)); } std::string AttributeSet::getAsString(unsigned Index, bool InAttrGrp) const { @@ -1209,11 +1115,6 @@ AttributeSet AttributeSet::getSlotAttributes(unsigned Slot) const { return pImpl->getSlotAttributes(Slot); } -uint64_t AttributeSet::Raw(unsigned Index) const { - // FIXME: Remove this. - return pImpl ? pImpl->Raw(Index) : 0; -} - LLVM_DUMP_METHOD void AttributeSet::dump() const { dbgs() << "PAL[\n"; @@ -1514,30 +1415,6 @@ bool AttrBuilder::operator==(const AttrBuilder &B) { DerefBytes == B.DerefBytes; } -AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) { - // FIXME: Remove this in 4.0. - if (!Val) return *this; - - for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; - I = Attribute::AttrKind(I + 1)) { - if (I == Attribute::Dereferenceable || - I == Attribute::DereferenceableOrNull || - I == Attribute::ArgMemOnly || - I == Attribute::AllocSize) - continue; - if (uint64_t A = (Val & AttributeImpl::getAttrMask(I))) { - Attrs[I] = true; - - if (I == Attribute::Alignment) - Alignment = 1ULL << ((A >> 16) - 1); - else if (I == Attribute::StackAlignment) - StackAlignment = 1ULL << ((A >> 26)-1); - } - } - - return *this; -} - //===----------------------------------------------------------------------===// // AttributeFuncs Function Defintions //===----------------------------------------------------------------------===// diff --git a/contrib/llvm/lib/IR/AutoUpgrade.cpp b/contrib/llvm/lib/IR/AutoUpgrade.cpp index 2e4a2f89e2c7..2d9d0f95efa5 100644 --- a/contrib/llvm/lib/IR/AutoUpgrade.cpp +++ b/contrib/llvm/lib/IR/AutoUpgrade.cpp @@ -31,6 +31,8 @@ #include <cstring> using namespace llvm; +static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } + // Upgrade the declarations of the SSE4.1 functions whose arguments have // changed their type from v4f32 to v2i64. static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, @@ -42,7 +44,7 @@ static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, return false; // Yes, it's old, replace it with new version. - F->setName(F->getName() + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), IID); return true; } @@ -58,7 +60,7 @@ static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, return false; // Move this function aside and map down. - F->setName(F->getName() + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), IID); return true; } @@ -135,25 +137,49 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { case 'c': { if (Name.startswith("ctlz.") && F->arg_size() == 1) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, F->arg_begin()->getType()); return true; } if (Name.startswith("cttz.") && F->arg_size() == 1) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, F->arg_begin()->getType()); return true; } break; } - + case 'i': { + if (Name.startswith("invariant.start")) { + auto Args = F->getFunctionType()->params(); + Type* ObjectPtr[1] = {Args[1]}; + if (F->getName() != + Intrinsic::getName(Intrinsic::invariant_start, ObjectPtr)) { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::invariant_start, ObjectPtr); + return true; + } + } + if (Name.startswith("invariant.end")) { + auto Args = F->getFunctionType()->params(); + Type* ObjectPtr[1] = {Args[2]}; + if (F->getName() != + Intrinsic::getName(Intrinsic::invariant_end, ObjectPtr)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::invariant_end, ObjectPtr); + return true; + } + } + break; + } case 'm': { if (Name.startswith("masked.load.")) { Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() }; if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::masked_load, Tys); @@ -164,7 +190,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { auto Args = F->getFunctionType()->params(); Type *Tys[] = { Args[0], Args[1] }; if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::masked_store, Tys); @@ -180,7 +206,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { if (F->arg_size() == 2 && Name.startswith("objectsize.")) { Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, Tys); return true; @@ -193,117 +219,172 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { NewFn = nullptr; return true; } + break; case 'x': { bool IsX86 = Name.startswith("x86."); if (IsX86) Name = Name.substr(4); + // All of the intrinsics matches below should be marked with which llvm + // version started autoupgrading them. At some point in the future we would + // like to use this information to remove upgrade code for some older + // intrinsics. It is currently undecided how we will determine that future + // point. if (IsX86 && - (Name.startswith("sse2.pcmpeq.") || - Name.startswith("sse2.pcmpgt.") || - Name.startswith("avx2.pcmpeq.") || - Name.startswith("avx2.pcmpgt.") || - Name.startswith("avx512.mask.pcmpeq.") || - Name.startswith("avx512.mask.pcmpgt.") || - Name == "sse41.pmaxsb" || - Name == "sse2.pmaxs.w" || - Name == "sse41.pmaxsd" || - Name == "sse2.pmaxu.b" || - Name == "sse41.pmaxuw" || - Name == "sse41.pmaxud" || - Name == "sse41.pminsb" || - Name == "sse2.pmins.w" || - Name == "sse41.pminsd" || - Name == "sse2.pminu.b" || - Name == "sse41.pminuw" || - Name == "sse41.pminud" || - Name.startswith("avx2.pmax") || - Name.startswith("avx2.pmin") || - Name.startswith("avx2.vbroadcast") || - Name.startswith("avx2.pbroadcast") || - Name.startswith("avx.vpermil.") || - Name.startswith("sse2.pshuf") || - Name.startswith("avx512.pbroadcast") || - Name.startswith("avx512.mask.broadcast.s") || - Name.startswith("avx512.mask.movddup") || - Name.startswith("avx512.mask.movshdup") || - Name.startswith("avx512.mask.movsldup") || - Name.startswith("avx512.mask.pshuf.d.") || - Name.startswith("avx512.mask.pshufl.w.") || - Name.startswith("avx512.mask.pshufh.w.") || - Name.startswith("avx512.mask.vpermil.p") || - Name.startswith("avx512.mask.perm.df.") || - Name.startswith("avx512.mask.perm.di.") || - Name.startswith("avx512.mask.punpckl") || - Name.startswith("avx512.mask.punpckh") || - Name.startswith("avx512.mask.unpckl.") || - Name.startswith("avx512.mask.unpckh.") || - Name.startswith("avx512.mask.pand.") || - Name.startswith("avx512.mask.pandn.") || - Name.startswith("avx512.mask.por.") || - Name.startswith("avx512.mask.pxor.") || - Name.startswith("sse41.pmovsx") || - Name.startswith("sse41.pmovzx") || - Name.startswith("avx2.pmovsx") || - Name.startswith("avx2.pmovzx") || - Name == "sse2.cvtdq2pd" || - Name == "sse2.cvtps2pd" || - Name == "avx.cvtdq2.pd.256" || - Name == "avx.cvt.ps2.pd.256" || - Name.startswith("avx.vinsertf128.") || - Name == "avx2.vinserti128" || - Name.startswith("avx.vextractf128.") || - Name == "avx2.vextracti128" || - Name.startswith("sse4a.movnt.") || - Name.startswith("avx.movnt.") || - Name.startswith("avx512.storent.") || - Name == "sse2.storel.dq" || - Name.startswith("sse.storeu.") || - Name.startswith("sse2.storeu.") || - Name.startswith("avx.storeu.") || - Name.startswith("avx512.mask.storeu.p") || - Name.startswith("avx512.mask.storeu.b.") || - Name.startswith("avx512.mask.storeu.w.") || - Name.startswith("avx512.mask.storeu.d.") || - Name.startswith("avx512.mask.storeu.q.") || - Name.startswith("avx512.mask.store.p") || - Name.startswith("avx512.mask.store.b.") || - Name.startswith("avx512.mask.store.w.") || - Name.startswith("avx512.mask.store.d.") || - Name.startswith("avx512.mask.store.q.") || - Name.startswith("avx512.mask.loadu.p") || - Name.startswith("avx512.mask.loadu.b.") || - Name.startswith("avx512.mask.loadu.w.") || - Name.startswith("avx512.mask.loadu.d.") || - Name.startswith("avx512.mask.loadu.q.") || - Name.startswith("avx512.mask.load.p") || - Name.startswith("avx512.mask.load.b.") || - Name.startswith("avx512.mask.load.w.") || - Name.startswith("avx512.mask.load.d.") || - Name.startswith("avx512.mask.load.q.") || - Name == "sse42.crc32.64.8" || - Name.startswith("avx.vbroadcast.s") || - Name.startswith("avx512.mask.palignr.") || - Name.startswith("sse2.psll.dq") || - Name.startswith("sse2.psrl.dq") || - Name.startswith("avx2.psll.dq") || - Name.startswith("avx2.psrl.dq") || - Name.startswith("avx512.psll.dq") || - Name.startswith("avx512.psrl.dq") || - Name == "sse41.pblendw" || - Name.startswith("sse41.blendp") || - Name.startswith("avx.blend.p") || - Name == "avx2.pblendw" || - Name.startswith("avx2.pblendd.") || - Name == "avx2.vbroadcasti128" || - Name == "xop.vpcmov" || - (Name.startswith("xop.vpcom") && F->arg_size() == 2))) { + (Name.startswith("sse2.pcmpeq.") || // Added in 3.1 + Name.startswith("sse2.pcmpgt.") || // Added in 3.1 + Name.startswith("avx2.pcmpeq.") || // Added in 3.1 + Name.startswith("avx2.pcmpgt.") || // Added in 3.1 + Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9 + Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9 + Name == "sse.add.ss" || // Added in 4.0 + Name == "sse2.add.sd" || // Added in 4.0 + Name == "sse.sub.ss" || // Added in 4.0 + Name == "sse2.sub.sd" || // Added in 4.0 + Name == "sse.mul.ss" || // Added in 4.0 + Name == "sse2.mul.sd" || // Added in 4.0 + Name == "sse.div.ss" || // Added in 4.0 + Name == "sse2.div.sd" || // Added in 4.0 + Name == "sse41.pmaxsb" || // Added in 3.9 + Name == "sse2.pmaxs.w" || // Added in 3.9 + Name == "sse41.pmaxsd" || // Added in 3.9 + Name == "sse2.pmaxu.b" || // Added in 3.9 + Name == "sse41.pmaxuw" || // Added in 3.9 + Name == "sse41.pmaxud" || // Added in 3.9 + Name == "sse41.pminsb" || // Added in 3.9 + Name == "sse2.pmins.w" || // Added in 3.9 + Name == "sse41.pminsd" || // Added in 3.9 + Name == "sse2.pminu.b" || // Added in 3.9 + Name == "sse41.pminuw" || // Added in 3.9 + Name == "sse41.pminud" || // Added in 3.9 + Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0 + Name.startswith("avx2.pmax") || // Added in 3.9 + Name.startswith("avx2.pmin") || // Added in 3.9 + Name.startswith("avx512.mask.pmax") || // Added in 4.0 + Name.startswith("avx512.mask.pmin") || // Added in 4.0 + Name.startswith("avx2.vbroadcast") || // Added in 3.8 + Name.startswith("avx2.pbroadcast") || // Added in 3.8 + Name.startswith("avx.vpermil.") || // Added in 3.1 + Name.startswith("sse2.pshuf") || // Added in 3.9 + Name.startswith("avx512.pbroadcast") || // Added in 3.9 + Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9 + Name.startswith("avx512.mask.movddup") || // Added in 3.9 + Name.startswith("avx512.mask.movshdup") || // Added in 3.9 + Name.startswith("avx512.mask.movsldup") || // Added in 3.9 + Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9 + Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9 + Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9 + Name.startswith("avx512.mask.shuf.p") || // Added in 4.0 + Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9 + Name.startswith("avx512.mask.perm.df.") || // Added in 3.9 + Name.startswith("avx512.mask.perm.di.") || // Added in 3.9 + Name.startswith("avx512.mask.punpckl") || // Added in 3.9 + Name.startswith("avx512.mask.punpckh") || // Added in 3.9 + Name.startswith("avx512.mask.unpckl.") || // Added in 3.9 + Name.startswith("avx512.mask.unpckh.") || // Added in 3.9 + Name.startswith("avx512.mask.pand.") || // Added in 3.9 + Name.startswith("avx512.mask.pandn.") || // Added in 3.9 + Name.startswith("avx512.mask.por.") || // Added in 3.9 + Name.startswith("avx512.mask.pxor.") || // Added in 3.9 + Name.startswith("avx512.mask.and.") || // Added in 3.9 + Name.startswith("avx512.mask.andn.") || // Added in 3.9 + Name.startswith("avx512.mask.or.") || // Added in 3.9 + Name.startswith("avx512.mask.xor.") || // Added in 3.9 + Name.startswith("avx512.mask.padd.") || // Added in 4.0 + Name.startswith("avx512.mask.psub.") || // Added in 4.0 + Name.startswith("avx512.mask.pmull.") || // Added in 4.0 + Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0 + Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0 + Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0 + Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0 + Name == "avx512.mask.add.pd.128" || // Added in 4.0 + Name == "avx512.mask.add.pd.256" || // Added in 4.0 + Name == "avx512.mask.add.ps.128" || // Added in 4.0 + Name == "avx512.mask.add.ps.256" || // Added in 4.0 + Name == "avx512.mask.div.pd.128" || // Added in 4.0 + Name == "avx512.mask.div.pd.256" || // Added in 4.0 + Name == "avx512.mask.div.ps.128" || // Added in 4.0 + Name == "avx512.mask.div.ps.256" || // Added in 4.0 + Name == "avx512.mask.mul.pd.128" || // Added in 4.0 + Name == "avx512.mask.mul.pd.256" || // Added in 4.0 + Name == "avx512.mask.mul.ps.128" || // Added in 4.0 + Name == "avx512.mask.mul.ps.256" || // Added in 4.0 + Name == "avx512.mask.sub.pd.128" || // Added in 4.0 + Name == "avx512.mask.sub.pd.256" || // Added in 4.0 + Name == "avx512.mask.sub.ps.128" || // Added in 4.0 + Name == "avx512.mask.sub.ps.256" || // Added in 4.0 + Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0 + Name.startswith("avx512.mask.psll.d") || // Added in 4.0 + Name.startswith("avx512.mask.psll.q") || // Added in 4.0 + Name.startswith("avx512.mask.psll.w") || // Added in 4.0 + Name.startswith("avx512.mask.psra.d") || // Added in 4.0 + Name.startswith("avx512.mask.psra.q") || // Added in 4.0 + Name.startswith("avx512.mask.psra.w") || // Added in 4.0 + Name.startswith("avx512.mask.psrl.d") || // Added in 4.0 + Name.startswith("avx512.mask.psrl.q") || // Added in 4.0 + Name.startswith("avx512.mask.psrl.w") || // Added in 4.0 + Name.startswith("avx512.mask.pslli") || // Added in 4.0 + Name.startswith("avx512.mask.psrai") || // Added in 4.0 + Name.startswith("avx512.mask.psrli") || // Added in 4.0 + Name.startswith("avx512.mask.psllv") || // Added in 4.0 + Name.startswith("avx512.mask.psrav") || // Added in 4.0 + Name.startswith("avx512.mask.psrlv") || // Added in 4.0 + Name.startswith("sse41.pmovsx") || // Added in 3.8 + Name.startswith("sse41.pmovzx") || // Added in 3.9 + Name.startswith("avx2.pmovsx") || // Added in 3.9 + Name.startswith("avx2.pmovzx") || // Added in 3.9 + Name.startswith("avx512.mask.pmovsx") || // Added in 4.0 + Name.startswith("avx512.mask.pmovzx") || // Added in 4.0 + Name == "sse2.cvtdq2pd" || // Added in 3.9 + Name == "sse2.cvtps2pd" || // Added in 3.9 + Name == "avx.cvtdq2.pd.256" || // Added in 3.9 + Name == "avx.cvt.ps2.pd.256" || // Added in 3.9 + Name.startswith("avx.vinsertf128.") || // Added in 3.7 + Name == "avx2.vinserti128" || // Added in 3.7 + Name.startswith("avx.vextractf128.") || // Added in 3.7 + Name == "avx2.vextracti128" || // Added in 3.7 + Name.startswith("sse4a.movnt.") || // Added in 3.9 + Name.startswith("avx.movnt.") || // Added in 3.2 + Name.startswith("avx512.storent.") || // Added in 3.9 + Name == "sse2.storel.dq" || // Added in 3.9 + Name.startswith("sse.storeu.") || // Added in 3.9 + Name.startswith("sse2.storeu.") || // Added in 3.9 + Name.startswith("avx.storeu.") || // Added in 3.9 + Name.startswith("avx512.mask.storeu.") || // Added in 3.9 + Name.startswith("avx512.mask.store.p") || // Added in 3.9 + Name.startswith("avx512.mask.store.b.") || // Added in 3.9 + Name.startswith("avx512.mask.store.w.") || // Added in 3.9 + Name.startswith("avx512.mask.store.d.") || // Added in 3.9 + Name.startswith("avx512.mask.store.q.") || // Added in 3.9 + Name.startswith("avx512.mask.loadu.") || // Added in 3.9 + Name.startswith("avx512.mask.load.") || // Added in 3.9 + Name == "sse42.crc32.64.8" || // Added in 3.4 + Name.startswith("avx.vbroadcast.s") || // Added in 3.5 + Name.startswith("avx512.mask.palignr.") || // Added in 3.9 + Name.startswith("avx512.mask.valign.") || // Added in 4.0 + Name.startswith("sse2.psll.dq") || // Added in 3.7 + Name.startswith("sse2.psrl.dq") || // Added in 3.7 + Name.startswith("avx2.psll.dq") || // Added in 3.7 + Name.startswith("avx2.psrl.dq") || // Added in 3.7 + Name.startswith("avx512.psll.dq") || // Added in 3.9 + Name.startswith("avx512.psrl.dq") || // Added in 3.9 + Name == "sse41.pblendw" || // Added in 3.7 + Name.startswith("sse41.blendp") || // Added in 3.7 + Name.startswith("avx.blend.p") || // Added in 3.7 + Name == "avx2.pblendw" || // Added in 3.7 + Name.startswith("avx2.pblendd.") || // Added in 3.7 + Name.startswith("avx.vbroadcastf128") || // Added in 4.0 + Name == "avx2.vbroadcasti128" || // Added in 3.7 + Name == "xop.vpcmov" || // Added in 3.8 + Name.startswith("avx512.mask.move.s") || // Added in 4.0 + (Name.startswith("xop.vpcom") && // Added in 3.2 + F->arg_size() == 2))) { NewFn = nullptr; return true; } // SSE4.1 ptest functions may have an old signature. - if (IsX86 && Name.startswith("sse41.ptest")) { + if (IsX86 && Name.startswith("sse41.ptest")) { // Added in 3.2 if (Name.substr(11) == "c") return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); if (Name.substr(11) == "z") @@ -313,67 +394,44 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } // Several blend and other instructions with masks used the wrong number of // bits. - if (IsX86 && Name == "sse41.insertps") + if (IsX86 && Name == "sse41.insertps") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, NewFn); - if (IsX86 && Name == "sse41.dppd") + if (IsX86 && Name == "sse41.dppd") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, NewFn); - if (IsX86 && Name == "sse41.dpps") + if (IsX86 && Name == "sse41.dpps") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, NewFn); - if (IsX86 && Name == "sse41.mpsadbw") + if (IsX86 && Name == "sse41.mpsadbw") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, NewFn); - if (IsX86 && Name == "avx.dp.ps.256") + if (IsX86 && Name == "avx.dp.ps.256") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, NewFn); - if (IsX86 && Name == "avx2.mpsadbw") + if (IsX86 && Name == "avx2.mpsadbw") // Added in 3.6 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, NewFn); - // frcz.ss/sd may need to have an argument dropped + // frcz.ss/sd may need to have an argument dropped. Added in 3.2 if (IsX86 && Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_xop_vfrcz_ss); return true; } if (IsX86 && Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) { - F->setName(Name + ".old"); + rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_xop_vfrcz_sd); return true; } - if (IsX86 && (Name.startswith("avx512.mask.pslli.") || - Name.startswith("avx512.mask.psrai.") || - Name.startswith("avx512.mask.psrli."))) { - Intrinsic::ID ShiftID; - if (Name.slice(12, 16) == "psll") - ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psll_di_512 - : Intrinsic::x86_avx512_mask_psll_qi_512; - else if (Name.slice(12, 16) == "psra") - ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psra_di_512 - : Intrinsic::x86_avx512_mask_psra_qi_512; - else - ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psrl_di_512 - : Intrinsic::x86_avx512_mask_psrl_qi_512; - F->setName("llvm.x86." + Name + ".old"); - NewFn = Intrinsic::getDeclaration(F->getParent(), ShiftID); - return true; - } - // Fix the FMA4 intrinsics to remove the 4 - if (IsX86 && Name.startswith("fma4.")) { - F->setName("llvm.x86.fma" + Name.substr(5)); - NewFn = F; - return true; - } // Upgrade any XOP PERMIL2 index operand still using a float/double vector. - if (IsX86 && Name.startswith("xop.vpermil2")) { + if (IsX86 && Name.startswith("xop.vpermil2")) { // Added in 3.9 auto Params = F->getFunctionType()->params(); auto Idx = Params[2]; if (Idx->getScalarType()->isFloatingPointTy()) { - F->setName("llvm.x86." + Name + ".old"); + rename(F); unsigned IdxSize = Idx->getPrimitiveSizeInBits(); unsigned EltSize = Idx->getScalarSizeInBits(); Intrinsic::ID Permil2ID; @@ -517,13 +575,23 @@ static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, return Builder.CreateSelect(Mask, Op0, Op1); } -static Value *UpgradeX86PALIGNRIntrinsics(IRBuilder<> &Builder, - Value *Op0, Value *Op1, Value *Shift, - Value *Passthru, Value *Mask) { +// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics. +// PALIGNR handles large immediates by shifting while VALIGN masks the immediate +// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes. +static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, + Value *Op1, Value *Shift, + Value *Passthru, Value *Mask, + bool IsVALIGN) { unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); unsigned NumElts = Op0->getType()->getVectorNumElements(); - assert(NumElts % 16 == 0); + assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"); + assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"); + assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!"); + + // Mask the immediate for VALIGN. + if (IsVALIGN) + ShiftVal &= (NumElts - 1); // If palignr is shifting the pair of vectors more than the size of two // lanes, emit zero. @@ -540,10 +608,10 @@ static Value *UpgradeX86PALIGNRIntrinsics(IRBuilder<> &Builder, uint32_t Indices[64]; // 256-bit palignr operates on 128-bit lanes so we need to handle that - for (unsigned l = 0; l != NumElts; l += 16) { + for (unsigned l = 0; l < NumElts; l += 16) { for (unsigned i = 0; i != 16; ++i) { unsigned Idx = ShiftVal + i; - if (Idx >= 16) + if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN. Idx += NumElts - 16; // End of lane, switch operand. Indices[l + i] = Idx + l; } @@ -601,7 +669,12 @@ static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI, Value *Op0 = CI.getArgOperand(0); Value *Op1 = CI.getArgOperand(1); Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1); - return Builder.CreateSelect(Cmp, Op0, Op1); + Value *Res = Builder.CreateSelect(Cmp, Op0, Op1); + + if (CI.getNumArgOperands() == 4) + Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); + + return Res; } static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, @@ -629,6 +702,30 @@ static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, std::max(NumElts, 8U))); } +// Replace a masked intrinsic with an older unmasked intrinsic. +static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI, + Intrinsic::ID IID) { + Function *F = CI.getCalledFunction(); + Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID); + Value *Rep = Builder.CreateCall(Intrin, + { CI.getArgOperand(0), CI.getArgOperand(1) }); + return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); +} + +static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { + Value* A = CI.getArgOperand(0); + Value* B = CI.getArgOperand(1); + Value* Src = CI.getArgOperand(2); + Value* Mask = CI.getArgOperand(3); + + Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1)); + Value* Cmp = Builder.CreateIsNotNull(AndNode); + Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0); + Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0); + Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2); + return Builder.CreateInsertElement(A, Select, (uint64_t)0); +} + /// Upgrade a call to an old intrinsic. All argument and return casting must be /// provided to seamlessly integrate with existing context. void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { @@ -650,67 +747,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (IsX86) Name = Name.substr(4); - Value *Rep; - // Upgrade packed integer vector compare intrinsics to compare instructions. - if (IsX86 && (Name.startswith("sse2.pcmpeq.") || - Name.startswith("avx2.pcmpeq."))) { - Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), - "pcmpeq"); - Rep = Builder.CreateSExt(Rep, CI->getType(), ""); - } else if (IsX86 && (Name.startswith("sse2.pcmpgt.") || - Name.startswith("avx2.pcmpgt."))) { - Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), - "pcmpgt"); - Rep = Builder.CreateSExt(Rep, CI->getType(), ""); - } else if (IsX86 && Name.startswith("avx512.mask.pcmpeq.")) { - Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_EQ); - } else if (IsX86 && Name.startswith("avx512.mask.pcmpgt.")) { - Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_SGT); - } else if (IsX86 && (Name == "sse41.pmaxsb" || - Name == "sse2.pmaxs.w" || - Name == "sse41.pmaxsd" || - Name.startswith("avx2.pmaxs"))) { - Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); - } else if (IsX86 && (Name == "sse2.pmaxu.b" || - Name == "sse41.pmaxuw" || - Name == "sse41.pmaxud" || - Name.startswith("avx2.pmaxu"))) { - Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); - } else if (IsX86 && (Name == "sse41.pminsb" || - Name == "sse2.pmins.w" || - Name == "sse41.pminsd" || - Name.startswith("avx2.pmins"))) { - Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); - } else if (IsX86 && (Name == "sse2.pminu.b" || - Name == "sse41.pminuw" || - Name == "sse41.pminud" || - Name.startswith("avx2.pminu"))) { - Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); - } else if (IsX86 && (Name == "sse2.cvtdq2pd" || - Name == "sse2.cvtps2pd" || - Name == "avx.cvtdq2.pd.256" || - Name == "avx.cvt.ps2.pd.256")) { - // Lossless i32/float to double conversion. - // Extract the bottom elements if necessary and convert to double vector. - Value *Src = CI->getArgOperand(0); - VectorType *SrcTy = cast<VectorType>(Src->getType()); - VectorType *DstTy = cast<VectorType>(CI->getType()); - Rep = CI->getArgOperand(0); - - unsigned NumDstElts = DstTy->getNumElements(); - if (NumDstElts < SrcTy->getNumElements()) { - assert(NumDstElts == 2 && "Unexpected vector size"); - uint32_t ShuffleMask[2] = { 0, 1 }; - Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy), - ShuffleMask); - } - - bool Int2Double = (StringRef::npos != Name.find("cvtdq2")); - if (Int2Double) - Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd"); - else - Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); - } else if (IsX86 && Name.startswith("sse4a.movnt.")) { + if (IsX86 && Name.startswith("sse4a.movnt.")) { Module *M = F->getParent(); SmallVector<Metadata *, 1> Elts; Elts.push_back( @@ -734,8 +771,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && (Name.startswith("avx.movnt.") || - Name.startswith("avx512.storent."))) { + } + + if (IsX86 && (Name.startswith("avx.movnt.") || + Name.startswith("avx512.storent."))) { Module *M = F->getParent(); SmallVector<Metadata *, 1> Elts; Elts.push_back( @@ -757,7 +796,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && Name == "sse2.storel.dq") { + } + + if (IsX86 && Name == "sse2.storel.dq") { Value *Arg0 = CI->getArgOperand(0); Value *Arg1 = CI->getArgOperand(1); @@ -772,9 +813,11 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && (Name.startswith("sse.storeu.") || - Name.startswith("sse2.storeu.") || - Name.startswith("avx.storeu."))) { + } + + if (IsX86 && (Name.startswith("sse.storeu.") || + Name.startswith("sse2.storeu.") || + Name.startswith("avx.storeu."))) { Value *Arg0 = CI->getArgOperand(0); Value *Arg1 = CI->getArgOperand(1); @@ -786,41 +829,140 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && (Name.startswith("avx512.mask.storeu.p") || - Name.startswith("avx512.mask.storeu.b.") || - Name.startswith("avx512.mask.storeu.w.") || - Name.startswith("avx512.mask.storeu.d.") || - Name.startswith("avx512.mask.storeu.q."))) { + } + + if (IsX86 && (Name.startswith("avx512.mask.storeu."))) { UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), /*Aligned*/false); // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && (Name.startswith("avx512.mask.store.p") || - Name.startswith("avx512.mask.store.b.") || - Name.startswith("avx512.mask.store.w.") || - Name.startswith("avx512.mask.store.d.") || - Name.startswith("avx512.mask.store.q."))) { + } + + if (IsX86 && (Name.startswith("avx512.mask.store."))) { UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), /*Aligned*/true); // Remove intrinsic. CI->eraseFromParent(); return; - } else if (IsX86 && (Name.startswith("avx512.mask.loadu.p") || - Name.startswith("avx512.mask.loadu.b.") || - Name.startswith("avx512.mask.loadu.w.") || - Name.startswith("avx512.mask.loadu.d.") || - Name.startswith("avx512.mask.loadu.q."))) { + } + + Value *Rep; + // Upgrade packed integer vector compare intrinsics to compare instructions. + if (IsX86 && (Name.startswith("sse2.pcmpeq.") || + Name.startswith("avx2.pcmpeq."))) { + Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), + "pcmpeq"); + Rep = Builder.CreateSExt(Rep, CI->getType(), ""); + } else if (IsX86 && (Name.startswith("sse2.pcmpgt.") || + Name.startswith("avx2.pcmpgt."))) { + Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), + "pcmpgt"); + Rep = Builder.CreateSExt(Rep, CI->getType(), ""); + } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) { + Type *I32Ty = Type::getInt32Ty(C); + Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), + ConstantInt::get(I32Ty, 0)); + Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), + ConstantInt::get(I32Ty, 0)); + Rep = Builder.CreateInsertElement(CI->getArgOperand(0), + Builder.CreateFAdd(Elt0, Elt1), + ConstantInt::get(I32Ty, 0)); + } else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) { + Type *I32Ty = Type::getInt32Ty(C); + Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), + ConstantInt::get(I32Ty, 0)); + Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), + ConstantInt::get(I32Ty, 0)); + Rep = Builder.CreateInsertElement(CI->getArgOperand(0), + Builder.CreateFSub(Elt0, Elt1), + ConstantInt::get(I32Ty, 0)); + } else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) { + Type *I32Ty = Type::getInt32Ty(C); + Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), + ConstantInt::get(I32Ty, 0)); + Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), + ConstantInt::get(I32Ty, 0)); + Rep = Builder.CreateInsertElement(CI->getArgOperand(0), + Builder.CreateFMul(Elt0, Elt1), + ConstantInt::get(I32Ty, 0)); + } else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) { + Type *I32Ty = Type::getInt32Ty(C); + Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), + ConstantInt::get(I32Ty, 0)); + Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), + ConstantInt::get(I32Ty, 0)); + Rep = Builder.CreateInsertElement(CI->getArgOperand(0), + Builder.CreateFDiv(Elt0, Elt1), + ConstantInt::get(I32Ty, 0)); + } else if (IsX86 && Name.startswith("avx512.mask.pcmpeq.")) { + Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_EQ); + } else if (IsX86 && Name.startswith("avx512.mask.pcmpgt.")) { + Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_SGT); + } else if (IsX86 && (Name == "sse41.pmaxsb" || + Name == "sse2.pmaxs.w" || + Name == "sse41.pmaxsd" || + Name.startswith("avx2.pmaxs") || + Name.startswith("avx512.mask.pmaxs"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); + } else if (IsX86 && (Name == "sse2.pmaxu.b" || + Name == "sse41.pmaxuw" || + Name == "sse41.pmaxud" || + Name.startswith("avx2.pmaxu") || + Name.startswith("avx512.mask.pmaxu"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); + } else if (IsX86 && (Name == "sse41.pminsb" || + Name == "sse2.pmins.w" || + Name == "sse41.pminsd" || + Name.startswith("avx2.pmins") || + Name.startswith("avx512.mask.pmins"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); + } else if (IsX86 && (Name == "sse2.pminu.b" || + Name == "sse41.pminuw" || + Name == "sse41.pminud" || + Name.startswith("avx2.pminu") || + Name.startswith("avx512.mask.pminu"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); + } else if (IsX86 && (Name == "sse2.cvtdq2pd" || + Name == "sse2.cvtps2pd" || + Name == "avx.cvtdq2.pd.256" || + Name == "avx.cvt.ps2.pd.256" || + Name.startswith("avx512.mask.cvtdq2pd.") || + Name.startswith("avx512.mask.cvtudq2pd."))) { + // Lossless i32/float to double conversion. + // Extract the bottom elements if necessary and convert to double vector. + Value *Src = CI->getArgOperand(0); + VectorType *SrcTy = cast<VectorType>(Src->getType()); + VectorType *DstTy = cast<VectorType>(CI->getType()); + Rep = CI->getArgOperand(0); + + unsigned NumDstElts = DstTy->getNumElements(); + if (NumDstElts < SrcTy->getNumElements()) { + assert(NumDstElts == 2 && "Unexpected vector size"); + uint32_t ShuffleMask[2] = { 0, 1 }; + Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy), + ShuffleMask); + } + + bool SInt2Double = (StringRef::npos != Name.find("cvtdq2")); + bool UInt2Double = (StringRef::npos != Name.find("cvtudq2")); + if (SInt2Double) + Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd"); + else if (UInt2Double) + Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd"); + else + Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); + + if (CI->getNumArgOperands() == 3) + Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, + CI->getArgOperand(1)); + } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) { Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), /*Aligned*/false); - } else if (IsX86 && (Name.startswith("avx512.mask.load.p") || - Name.startswith("avx512.mask.load.b.") || - Name.startswith("avx512.mask.load.w.") || - Name.startswith("avx512.mask.load.d.") || - Name.startswith("avx512.mask.load.q."))) { + } else if (IsX86 && (Name.startswith("avx512.mask.load."))) { Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),CI->getArgOperand(2), /*Aligned*/true); @@ -886,7 +1028,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); Rep = Builder.CreateZExt(Rep, CI->getType(), ""); - } else if (IsX86 && Name.startswith("avx.vbroadcast")) { + } else if (IsX86 && Name.startswith("avx.vbroadcast.s")) { // Replace broadcasts with a series of insertelements. Type *VecTy = CI->getType(); Type *EltTy = VecTy->getVectorElementType(); @@ -902,7 +1044,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { } else if (IsX86 && (Name.startswith("sse41.pmovsx") || Name.startswith("sse41.pmovzx") || Name.startswith("avx2.pmovsx") || - Name.startswith("avx2.pmovzx"))) { + Name.startswith("avx2.pmovzx") || + Name.startswith("avx512.mask.pmovsx") || + Name.startswith("avx512.mask.pmovzx"))) { VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType()); VectorType *DstTy = cast<VectorType>(CI->getType()); unsigned NumDstElts = DstTy->getNumElements(); @@ -918,15 +1062,25 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { bool DoSext = (StringRef::npos != Name.find("pmovsx")); Rep = DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy); - } else if (IsX86 && Name == "avx2.vbroadcasti128") { - // Replace vbroadcasts with a vector shuffle. - Type *VT = VectorType::get(Type::getInt64Ty(C), 2); + // If there are 3 arguments, it's a masked intrinsic so we need a select. + if (CI->getNumArgOperands() == 3) + Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, + CI->getArgOperand(1)); + } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") || + Name == "avx2.vbroadcasti128")) { + // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle. + Type *EltTy = CI->getType()->getVectorElementType(); + unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits(); + Type *VT = VectorType::get(EltTy, NumSrcElts); Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), PointerType::getUnqual(VT)); - Value *Load = Builder.CreateLoad(VT, Op); - uint32_t Idxs[4] = { 0, 1, 0, 1 }; - Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), - Idxs); + Value *Load = Builder.CreateAlignedLoad(Op, 1); + if (NumSrcElts == 2) + Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), + { 0, 1, 0, 1 }); + else + Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), + { 0, 1, 2, 3, 0, 1, 2, 3 }); } else if (IsX86 && (Name.startswith("avx2.pbroadcast") || Name.startswith("avx2.vbroadcast") || Name.startswith("avx512.pbroadcast") || @@ -942,11 +1096,19 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1)); } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) { - Rep = UpgradeX86PALIGNRIntrinsics(Builder, CI->getArgOperand(0), - CI->getArgOperand(1), - CI->getArgOperand(2), - CI->getArgOperand(3), - CI->getArgOperand(4)); + Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), + CI->getArgOperand(1), + CI->getArgOperand(2), + CI->getArgOperand(3), + CI->getArgOperand(4), + false); + } else if (IsX86 && Name.startswith("avx512.mask.valign.")) { + Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), + CI->getArgOperand(1), + CI->getArgOperand(2), + CI->getArgOperand(3), + CI->getArgOperand(4), + true); } else if (IsX86 && (Name == "sse2.psll.dq" || Name == "avx2.psll.dq")) { // 128/256-bit shift left specified in bits. @@ -1123,6 +1285,31 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (CI->getNumArgOperands() == 4) Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) { + Value *Op0 = CI->getArgOperand(0); + Value *Op1 = CI->getArgOperand(1); + unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); + unsigned NumElts = CI->getType()->getVectorNumElements(); + + unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); + unsigned HalfLaneElts = NumLaneElts / 2; + + SmallVector<uint32_t, 16> Idxs(NumElts); + for (unsigned i = 0; i != NumElts; ++i) { + // Base index is the starting element of the lane. + Idxs[i] = i - (i % NumLaneElts); + // If we are half way through the lane switch to the other source. + if ((i % NumLaneElts) >= HalfLaneElts) + Idxs[i] += NumElts; + // Now select the specific element. By adding HalfLaneElts bits from + // the immediate. Wrapping around the immediate every 8-bits. + Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1); + } + + Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); + + Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, + CI->getArgOperand(3)); } else if (IsX86 && (Name.startswith("avx512.mask.movddup") || Name.startswith("avx512.mask.movshdup") || Name.startswith("avx512.mask.movsldup"))) { @@ -1194,6 +1381,333 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1)); Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.and.")) { + VectorType *FTy = cast<VectorType>(CI->getType()); + VectorType *ITy = VectorType::getInteger(FTy); + Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), + Builder.CreateBitCast(CI->getArgOperand(1), ITy)); + Rep = Builder.CreateBitCast(Rep, FTy); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.andn.")) { + VectorType *FTy = cast<VectorType>(CI->getType()); + VectorType *ITy = VectorType::getInteger(FTy); + Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); + Rep = Builder.CreateAnd(Rep, + Builder.CreateBitCast(CI->getArgOperand(1), ITy)); + Rep = Builder.CreateBitCast(Rep, FTy); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.or.")) { + VectorType *FTy = cast<VectorType>(CI->getType()); + VectorType *ITy = VectorType::getInteger(FTy); + Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), + Builder.CreateBitCast(CI->getArgOperand(1), ITy)); + Rep = Builder.CreateBitCast(Rep, FTy); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.xor.")) { + VectorType *FTy = cast<VectorType>(CI->getType()); + VectorType *ITy = VectorType::getInteger(FTy); + Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), + Builder.CreateBitCast(CI->getArgOperand(1), ITy)); + Rep = Builder.CreateBitCast(Rep, FTy); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.padd.")) { + Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.psub.")) { + Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) { + Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) { + Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.div.p")) { + Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) { + Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) { + Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) { + VectorType *VecTy = cast<VectorType>(CI->getType()); + Intrinsic::ID IID; + if (VecTy->getPrimitiveSizeInBits() == 128) + IID = Intrinsic::x86_ssse3_pshuf_b_128; + else if (VecTy->getPrimitiveSizeInBits() == 256) + IID = Intrinsic::x86_avx2_pshuf_b; + else if (VecTy->getPrimitiveSizeInBits() == 512) + IID = Intrinsic::x86_avx512_pshuf_b_512; + else + llvm_unreachable("Unexpected intrinsic"); + + Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), + { CI->getArgOperand(0), CI->getArgOperand(1) }); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name.startswith("avx512.mask.pmul.dq.") || + Name.startswith("avx512.mask.pmulu.dq."))) { + bool IsUnsigned = Name[16] == 'u'; + VectorType *VecTy = cast<VectorType>(CI->getType()); + Intrinsic::ID IID; + if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128) + IID = Intrinsic::x86_sse41_pmuldq; + else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256) + IID = Intrinsic::x86_avx2_pmul_dq; + else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512) + IID = Intrinsic::x86_avx512_pmul_dq_512; + else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128) + IID = Intrinsic::x86_sse2_pmulu_dq; + else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256) + IID = Intrinsic::x86_avx2_pmulu_dq; + else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512) + IID = Intrinsic::x86_avx512_pmulu_dq_512; + else + llvm_unreachable("Unexpected intrinsic"); + + Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), + { CI->getArgOperand(0), CI->getArgOperand(1) }); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.psll")) { + bool IsImmediate = Name[16] == 'i' || + (Name.size() > 18 && Name[18] == 'i'); + bool IsVariable = Name[16] == 'v'; + char Size = Name[16] == '.' ? Name[17] : + Name[17] == '.' ? Name[18] : + Name[18] == '.' ? Name[19] : + Name[20]; + + Intrinsic::ID IID; + if (IsVariable && Name[17] != '.') { + if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di + IID = Intrinsic::x86_avx2_psllv_q; + else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di + IID = Intrinsic::x86_avx2_psllv_q_256; + else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si + IID = Intrinsic::x86_avx2_psllv_d; + else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si + IID = Intrinsic::x86_avx2_psllv_d_256; + else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi + IID = Intrinsic::x86_avx512_psllv_w_128; + else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi + IID = Intrinsic::x86_avx512_psllv_w_256; + else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi + IID = Intrinsic::x86_avx512_psllv_w_512; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".128")) { + if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128 + IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d + : Intrinsic::x86_sse2_psll_d; + else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128 + IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q + : Intrinsic::x86_sse2_psll_q; + else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128 + IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w + : Intrinsic::x86_sse2_psll_w; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".256")) { + if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256 + IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d + : Intrinsic::x86_avx2_psll_d; + else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256 + IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q + : Intrinsic::x86_avx2_psll_q; + else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256 + IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w + : Intrinsic::x86_avx2_psll_w; + else + llvm_unreachable("Unexpected size"); + } else { + if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512 + IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 : + IsVariable ? Intrinsic::x86_avx512_psllv_d_512 : + Intrinsic::x86_avx512_psll_d_512; + else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512 + IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 : + IsVariable ? Intrinsic::x86_avx512_psllv_q_512 : + Intrinsic::x86_avx512_psll_q_512; + else if (Size == 'w') // psll.wi.512, pslli.w, psll.w + IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 + : Intrinsic::x86_avx512_psll_w_512; + else + llvm_unreachable("Unexpected size"); + } + + Rep = UpgradeX86MaskedShift(Builder, *CI, IID); + } else if (IsX86 && Name.startswith("avx512.mask.psrl")) { + bool IsImmediate = Name[16] == 'i' || + (Name.size() > 18 && Name[18] == 'i'); + bool IsVariable = Name[16] == 'v'; + char Size = Name[16] == '.' ? Name[17] : + Name[17] == '.' ? Name[18] : + Name[18] == '.' ? Name[19] : + Name[20]; + + Intrinsic::ID IID; + if (IsVariable && Name[17] != '.') { + if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di + IID = Intrinsic::x86_avx2_psrlv_q; + else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di + IID = Intrinsic::x86_avx2_psrlv_q_256; + else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si + IID = Intrinsic::x86_avx2_psrlv_d; + else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si + IID = Intrinsic::x86_avx2_psrlv_d_256; + else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi + IID = Intrinsic::x86_avx512_psrlv_w_128; + else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi + IID = Intrinsic::x86_avx512_psrlv_w_256; + else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi + IID = Intrinsic::x86_avx512_psrlv_w_512; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".128")) { + if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128 + IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d + : Intrinsic::x86_sse2_psrl_d; + else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128 + IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q + : Intrinsic::x86_sse2_psrl_q; + else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128 + IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w + : Intrinsic::x86_sse2_psrl_w; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".256")) { + if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256 + IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d + : Intrinsic::x86_avx2_psrl_d; + else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256 + IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q + : Intrinsic::x86_avx2_psrl_q; + else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256 + IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w + : Intrinsic::x86_avx2_psrl_w; + else + llvm_unreachable("Unexpected size"); + } else { + if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512 + IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 : + IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 : + Intrinsic::x86_avx512_psrl_d_512; + else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512 + IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 : + IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 : + Intrinsic::x86_avx512_psrl_q_512; + else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w) + IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 + : Intrinsic::x86_avx512_psrl_w_512; + else + llvm_unreachable("Unexpected size"); + } + + Rep = UpgradeX86MaskedShift(Builder, *CI, IID); + } else if (IsX86 && Name.startswith("avx512.mask.psra")) { + bool IsImmediate = Name[16] == 'i' || + (Name.size() > 18 && Name[18] == 'i'); + bool IsVariable = Name[16] == 'v'; + char Size = Name[16] == '.' ? Name[17] : + Name[17] == '.' ? Name[18] : + Name[18] == '.' ? Name[19] : + Name[20]; + + Intrinsic::ID IID; + if (IsVariable && Name[17] != '.') { + if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si + IID = Intrinsic::x86_avx2_psrav_d; + else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si + IID = Intrinsic::x86_avx2_psrav_d_256; + else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi + IID = Intrinsic::x86_avx512_psrav_w_128; + else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi + IID = Intrinsic::x86_avx512_psrav_w_256; + else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi + IID = Intrinsic::x86_avx512_psrav_w_512; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".128")) { + if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128 + IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d + : Intrinsic::x86_sse2_psra_d; + else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128 + IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 : + IsVariable ? Intrinsic::x86_avx512_psrav_q_128 : + Intrinsic::x86_avx512_psra_q_128; + else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128 + IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w + : Intrinsic::x86_sse2_psra_w; + else + llvm_unreachable("Unexpected size"); + } else if (Name.endswith(".256")) { + if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256 + IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d + : Intrinsic::x86_avx2_psra_d; + else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256 + IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 : + IsVariable ? Intrinsic::x86_avx512_psrav_q_256 : + Intrinsic::x86_avx512_psra_q_256; + else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256 + IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w + : Intrinsic::x86_avx2_psra_w; + else + llvm_unreachable("Unexpected size"); + } else { + if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512 + IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 : + IsVariable ? Intrinsic::x86_avx512_psrav_d_512 : + Intrinsic::x86_avx512_psra_d_512; + else if (Size == 'q') // psra.qi.512, psrai.q, psra.q + IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 : + IsVariable ? Intrinsic::x86_avx512_psrav_q_512 : + Intrinsic::x86_avx512_psra_q_512; + else if (Size == 'w') // psra.wi.512, psrai.w, psra.w + IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 + : Intrinsic::x86_avx512_psra_w_512; + else + llvm_unreachable("Unexpected size"); + } + + Rep = UpgradeX86MaskedShift(Builder, *CI, IID); + } else if (IsX86 && Name.startswith("avx512.mask.move.s")) { + Rep = upgradeMaskedMove(Builder, *CI); + } else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) { + Intrinsic::ID IID; + if (Name.endswith("ps.128")) + IID = Intrinsic::x86_avx_vpermilvar_ps; + else if (Name.endswith("pd.128")) + IID = Intrinsic::x86_avx_vpermilvar_pd; + else if (Name.endswith("ps.256")) + IID = Intrinsic::x86_avx_vpermilvar_ps_256; + else if (Name.endswith("pd.256")) + IID = Intrinsic::x86_avx_vpermilvar_pd_256; + else if (Name.endswith("ps.512")) + IID = Intrinsic::x86_avx512_vpermilvar_ps_512; + else if (Name.endswith("pd.512")) + IID = Intrinsic::x86_avx512_vpermilvar_pd_512; + else + llvm_unreachable("Unexpected vpermilvar intrinsic"); + + Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID); + Rep = Builder.CreateCall(Intrin, + { CI->getArgOperand(0), CI->getArgOperand(1) }); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); } else { llvm_unreachable("Unknown function for CallInst upgrade."); } @@ -1212,12 +1726,6 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { default: llvm_unreachable("Unknown function for CallInst upgrade."); - case Intrinsic::x86_avx512_mask_psll_di_512: - case Intrinsic::x86_avx512_mask_psra_di_512: - case Intrinsic::x86_avx512_mask_psrl_di_512: - case Intrinsic::x86_avx512_mask_psll_qi_512: - case Intrinsic::x86_avx512_mask_psra_qi_512: - case Intrinsic::x86_avx512_mask_psrl_qi_512: case Intrinsic::arm_neon_vld1: case Intrinsic::arm_neon_vld2: case Intrinsic::arm_neon_vld3: @@ -1332,6 +1840,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { return; } + case Intrinsic::invariant_start: + case Intrinsic::invariant_end: case Intrinsic::masked_load: case Intrinsic::masked_store: { SmallVector<Value *, 4> Args(CI->arg_operands().begin(), @@ -1361,28 +1871,26 @@ void llvm::UpgradeCallsToIntrinsic(Function *F) { } } -void llvm::UpgradeInstWithTBAATag(Instruction *I) { - MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa); - assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag"); +MDNode *llvm::UpgradeTBAANode(MDNode &MD) { // Check if the tag uses struct-path aware TBAA format. - if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3) - return; + if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3) + return &MD; - if (MD->getNumOperands() == 3) { - Metadata *Elts[] = {MD->getOperand(0), MD->getOperand(1)}; - MDNode *ScalarType = MDNode::get(I->getContext(), Elts); + auto &Context = MD.getContext(); + if (MD.getNumOperands() == 3) { + Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)}; + MDNode *ScalarType = MDNode::get(Context, Elts); // Create a MDNode <ScalarType, ScalarType, offset 0, const> Metadata *Elts2[] = {ScalarType, ScalarType, - ConstantAsMetadata::get(Constant::getNullValue( - Type::getInt64Ty(I->getContext()))), - MD->getOperand(2)}; - I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2)); - } else { - // Create a MDNode <MD, MD, offset 0> - Metadata *Elts[] = {MD, MD, ConstantAsMetadata::get(Constant::getNullValue( - Type::getInt64Ty(I->getContext())))}; - I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts)); + ConstantAsMetadata::get( + Constant::getNullValue(Type::getInt64Ty(Context))), + MD.getOperand(2)}; + return MDNode::get(Context, Elts2); } + // Create a MDNode <MD, MD, offset 0> + Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue( + Type::getInt64Ty(Context)))}; + return MDNode::get(Context, Elts); } Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, @@ -1462,11 +1970,11 @@ bool llvm::UpgradeModuleFlags(Module &M) { } // "Objective-C Class Properties" is recently added for Objective-C. We // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module - // flag of value 0, so we can correclty report error when trying to link - // an ObjC bitcode without this module flag with an ObjC bitcode with this - // module flag. + // flag of value 0, so we can correclty downgrade this flag when trying to + // link an ObjC bitcode without this module flag with an ObjC bitcode with + // this module flag. if (HasObjCFlag && !HasClassProperties) { - M.addModuleFlag(llvm::Module::Error, "Objective-C Class Properties", + M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties", (uint32_t)0); return true; } @@ -1524,7 +2032,7 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) { if (!T) return &N; - if (!llvm::any_of(T->operands(), isOldLoopArgument)) + if (none_of(T->operands(), isOldLoopArgument)) return &N; SmallVector<Metadata *, 8> Ops; diff --git a/contrib/llvm/lib/IR/BasicBlock.cpp b/contrib/llvm/lib/IR/BasicBlock.cpp index 4640b4f9d413..19e784923658 100644 --- a/contrib/llvm/lib/IR/BasicBlock.cpp +++ b/contrib/llvm/lib/IR/BasicBlock.cpp @@ -26,7 +26,7 @@ using namespace llvm; ValueSymbolTable *BasicBlock::getValueSymbolTable() { if (Function *F = getParent()) - return &F->getValueSymbolTable(); + return F->getValueSymbolTable(); return nullptr; } diff --git a/contrib/llvm/lib/IR/ConstantFold.cpp b/contrib/llvm/lib/IR/ConstantFold.cpp index c06a99c05363..098ff90a0a95 100644 --- a/contrib/llvm/lib/IR/ConstantFold.cpp +++ b/contrib/llvm/lib/IR/ConstantFold.cpp @@ -120,7 +120,6 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { IdxList.push_back(Zero); } else if (SequentialType *STy = dyn_cast<SequentialType>(ElTy)) { - if (ElTy->isPointerTy()) break; // Can't index into pointers! ElTy = STy->getElementType(); IdxList.push_back(Zero); } else { @@ -545,7 +544,10 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, } else if (CE->getOpcode() == Instruction::GetElementPtr && // Do not fold addrspacecast (gep 0, .., 0). It might make the // addrspacecast uncanonicalized. - opc != Instruction::AddrSpaceCast) { + opc != Instruction::AddrSpaceCast && + // Do not fold bitcast (gep) with inrange index, as this loses + // information. + !cast<GEPOperator>(CE)->getInRangeIndex().hasValue()) { // If all of the indexes in the GEP are null values, there is no pointer // adjustment going on. We might as well cast the source pointer. bool isAllNull = true; @@ -588,13 +590,13 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, if (ConstantFP *FPC = dyn_cast<ConstantFP>(V)) { bool ignored; APFloat Val = FPC->getValueAPF(); - Val.convert(DestTy->isHalfTy() ? APFloat::IEEEhalf : - DestTy->isFloatTy() ? APFloat::IEEEsingle : - DestTy->isDoubleTy() ? APFloat::IEEEdouble : - DestTy->isX86_FP80Ty() ? APFloat::x87DoubleExtended : - DestTy->isFP128Ty() ? APFloat::IEEEquad : - DestTy->isPPC_FP128Ty() ? APFloat::PPCDoubleDouble : - APFloat::Bogus, + Val.convert(DestTy->isHalfTy() ? APFloat::IEEEhalf() : + DestTy->isFloatTy() ? APFloat::IEEEsingle() : + DestTy->isDoubleTy() ? APFloat::IEEEdouble() : + DestTy->isX86_FP80Ty() ? APFloat::x87DoubleExtended() : + DestTy->isFP128Ty() ? APFloat::IEEEquad() : + DestTy->isPPC_FP128Ty() ? APFloat::PPCDoubleDouble() : + APFloat::Bogus(), APFloat::rmNearestTiesToEven, &ignored); return ConstantFP::get(V->getContext(), Val); } @@ -889,10 +891,8 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg, unsigned NumElts; if (StructType *ST = dyn_cast<StructType>(Agg->getType())) NumElts = ST->getNumElements(); - else if (ArrayType *AT = dyn_cast<ArrayType>(Agg->getType())) - NumElts = AT->getNumElements(); else - NumElts = Agg->getType()->getVectorNumElements(); + NumElts = cast<SequentialType>(Agg->getType())->getNumElements(); SmallVector<Constant*, 32> Result; for (unsigned i = 0; i != NumElts; ++i) { @@ -925,7 +925,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, // Handle undef ^ undef -> 0 special case. This is a common // idiom (misuse). return Constant::getNullValue(C1->getType()); - // Fallthrough + LLVM_FALLTHROUGH; case Instruction::Add: case Instruction::Sub: return UndefValue::get(C1->getType()); @@ -2016,22 +2016,8 @@ static bool isInBoundsIndices(ArrayRef<IndexTy> Idxs) { } /// Test whether a given ConstantInt is in-range for a SequentialType. -static bool isIndexInRangeOfSequentialType(SequentialType *STy, - const ConstantInt *CI) { - // And indices are valid when indexing along a pointer - if (isa<PointerType>(STy)) - return true; - - uint64_t NumElements = 0; - // Determine the number of elements in our sequential type. - if (auto *ATy = dyn_cast<ArrayType>(STy)) - NumElements = ATy->getNumElements(); - else if (auto *VTy = dyn_cast<VectorType>(STy)) - NumElements = VTy->getNumElements(); - - assert((isa<ArrayType>(STy) || NumElements > 0) && - "didn't expect non-array type to have zero elements!"); - +static bool isIndexInRangeOfArrayType(uint64_t NumElements, + const ConstantInt *CI) { // We cannot bounds check the index if it doesn't fit in an int64_t. if (CI->getValue().getActiveBits() > 64) return false; @@ -2046,22 +2032,18 @@ static bool isIndexInRangeOfSequentialType(SequentialType *STy, return true; } -template<typename IndexTy> -static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, - bool inBounds, - ArrayRef<IndexTy> Idxs) { +Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, + bool InBounds, + Optional<unsigned> InRangeIndex, + ArrayRef<Value *> Idxs) { if (Idxs.empty()) return C; Constant *Idx0 = cast<Constant>(Idxs[0]); if ((Idxs.size() == 1 && Idx0->isNullValue())) return C; if (isa<UndefValue>(C)) { - PointerType *PtrTy = cast<PointerType>(C->getType()->getScalarType()); - Type *Ty = GetElementPtrInst::getIndexedType(PointeeTy, Idxs); - assert(Ty && "Invalid indices for GEP!"); - Type *GEPTy = PointerType::get(Ty, PtrTy->getAddressSpace()); - if (VectorType *VT = dyn_cast<VectorType>(C->getType())) - GEPTy = VectorType::get(GEPTy, VT->getNumElements()); + Type *GEPTy = GetElementPtrInst::getGEPReturnType( + C, makeArrayRef((Value * const *)Idxs.data(), Idxs.size())); return UndefValue::get(GEPTy); } @@ -2090,10 +2072,10 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, // getelementptr instructions into a single instruction. // if (CE->getOpcode() == Instruction::GetElementPtr) { - Type *LastTy = nullptr; + gep_type_iterator LastI = gep_type_end(CE); for (gep_type_iterator I = gep_type_begin(CE), E = gep_type_end(CE); I != E; ++I) - LastTy = *I; + LastI = I; // We cannot combine indices if doing so would take us outside of an // array or vector. Doing otherwise could trick us if we evaluated such a @@ -2116,9 +2098,11 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, bool PerformFold = false; if (Idx0->isNullValue()) PerformFold = true; - else if (SequentialType *STy = dyn_cast_or_null<SequentialType>(LastTy)) + else if (LastI.isSequential()) if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx0)) - PerformFold = isIndexInRangeOfSequentialType(STy, CI); + PerformFold = + !LastI.isBoundedSequential() || + isIndexInRangeOfArrayType(LastI.getSequentialNumElements(), CI); if (PerformFold) { SmallVector<Value*, 16> NewIndices; @@ -2150,9 +2134,18 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, NewIndices.push_back(Combined); NewIndices.append(Idxs.begin() + 1, Idxs.end()); + + // The combined GEP normally inherits its index inrange attribute from + // the inner GEP, but if the inner GEP's last index was adjusted by the + // outer GEP, any inbounds attribute on that index is invalidated. + Optional<unsigned> IRIndex = cast<GEPOperator>(CE)->getInRangeIndex(); + if (IRIndex && *IRIndex == CE->getNumOperands() - 2 && !Idx0->isNullValue()) + IRIndex = None; + return ConstantExpr::getGetElementPtr( cast<GEPOperator>(CE)->getSourceElementType(), CE->getOperand(0), - NewIndices, inBounds && cast<GEPOperator>(CE)->isInBounds()); + NewIndices, InBounds && cast<GEPOperator>(CE)->isInBounds(), + IRIndex); } } @@ -2177,8 +2170,9 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, if (SrcArrayTy && DstArrayTy && SrcArrayTy->getElementType() == DstArrayTy->getElementType() && SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace()) - return ConstantExpr::getGetElementPtr( - SrcArrayTy, (Constant *)CE->getOperand(0), Idxs, inBounds); + return ConstantExpr::getGetElementPtr(SrcArrayTy, + (Constant *)CE->getOperand(0), + Idxs, InBounds, InRangeIndex); } } } @@ -2198,25 +2192,26 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, Unknown = true; continue; } + if (InRangeIndex && i == *InRangeIndex + 1) { + // If an index is marked inrange, we cannot apply this canonicalization to + // the following index, as that will cause the inrange index to point to + // the wrong element. + continue; + } if (isa<StructType>(Ty)) { // The verify makes sure that GEPs into a struct are in range. continue; } auto *STy = cast<SequentialType>(Ty); - if (isa<PointerType>(STy)) { - // We don't know if it's in range or not. - Unknown = true; - continue; - } if (isa<VectorType>(STy)) { // There can be awkward padding in after a non-power of two vector. Unknown = true; continue; } - if (isIndexInRangeOfSequentialType(STy, CI)) + if (isIndexInRangeOfArrayType(STy->getNumElements(), CI)) // It's in range, skip to the next index. continue; - if (!isa<SequentialType>(Prev)) { + if (isa<StructType>(Prev)) { // It's out of range, but the prior dimension is a struct // so we can't do anything about it. Unknown = true; @@ -2260,27 +2255,17 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, if (!NewIdxs.empty()) { for (unsigned i = 0, e = Idxs.size(); i != e; ++i) if (!NewIdxs[i]) NewIdxs[i] = cast<Constant>(Idxs[i]); - return ConstantExpr::getGetElementPtr(PointeeTy, C, NewIdxs, inBounds); + return ConstantExpr::getGetElementPtr(PointeeTy, C, NewIdxs, InBounds, + InRangeIndex); } // If all indices are known integers and normalized, we can do a simple // check for the "inbounds" property. - if (!Unknown && !inBounds) + if (!Unknown && !InBounds) if (auto *GV = dyn_cast<GlobalVariable>(C)) if (!GV->hasExternalWeakLinkage() && isInBoundsIndices(Idxs)) - return ConstantExpr::getInBoundsGetElementPtr(PointeeTy, C, Idxs); + return ConstantExpr::getGetElementPtr(PointeeTy, C, Idxs, + /*InBounds=*/true, InRangeIndex); return nullptr; } - -Constant *llvm::ConstantFoldGetElementPtr(Type *Ty, Constant *C, - bool inBounds, - ArrayRef<Constant *> Idxs) { - return ConstantFoldGetElementPtrImpl(Ty, C, inBounds, Idxs); -} - -Constant *llvm::ConstantFoldGetElementPtr(Type *Ty, Constant *C, - bool inBounds, - ArrayRef<Value *> Idxs) { - return ConstantFoldGetElementPtrImpl(Ty, C, inBounds, Idxs); -} diff --git a/contrib/llvm/lib/IR/ConstantFold.h b/contrib/llvm/lib/IR/ConstantFold.h index 9b0a937c84df..2d8de1132b96 100644 --- a/contrib/llvm/lib/IR/ConstantFold.h +++ b/contrib/llvm/lib/IR/ConstantFold.h @@ -19,6 +19,8 @@ #ifndef LLVM_LIB_IR_CONSTANTFOLD_H #define LLVM_LIB_IR_CONSTANTFOLD_H +#include "llvm/ADT/Optional.h" + namespace llvm { template <typename T> class ArrayRef; class Value; @@ -46,9 +48,8 @@ template <typename T> class ArrayRef; Constant *V2); Constant *ConstantFoldCompareInstruction(unsigned short predicate, Constant *C1, Constant *C2); - Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds, - ArrayRef<Constant *> Idxs); - Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds, + Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool InBounds, + Optional<unsigned> InRangeIndex, ArrayRef<Value *> Idxs); } // End llvm namespace diff --git a/contrib/llvm/lib/IR/ConstantRange.cpp b/contrib/llvm/lib/IR/ConstantRange.cpp index 0f5c7128f3d0..a85ad465317c 100644 --- a/contrib/llvm/lib/IR/ConstantRange.cpp +++ b/contrib/llvm/lib/IR/ConstantRange.cpp @@ -147,6 +147,14 @@ bool ConstantRange::getEquivalentICmp(CmpInst::Predicate &Pred, Pred = isEmptySet() ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE; RHS = APInt(getBitWidth(), 0); Success = true; + } else if (auto *OnlyElt = getSingleElement()) { + Pred = CmpInst::ICMP_EQ; + RHS = *OnlyElt; + Success = true; + } else if (auto *OnlyMissingElt = getSingleMissingElement()) { + Pred = CmpInst::ICMP_NE; + RHS = *OnlyMissingElt; + Success = true; } else if (getLower().isMinSignedValue() || getLower().isMinValue()) { Pred = getLower().isMinSignedValue() ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT; @@ -526,6 +534,49 @@ ConstantRange ConstantRange::unionWith(const ConstantRange &CR) const { return ConstantRange(L, U); } +ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp, + uint32_t ResultBitWidth) const { + switch (CastOp) { + default: + llvm_unreachable("unsupported cast type"); + case Instruction::Trunc: + return truncate(ResultBitWidth); + case Instruction::SExt: + return signExtend(ResultBitWidth); + case Instruction::ZExt: + return zeroExtend(ResultBitWidth); + case Instruction::BitCast: + return *this; + case Instruction::FPToUI: + case Instruction::FPToSI: + if (getBitWidth() == ResultBitWidth) + return *this; + else + return ConstantRange(getBitWidth(), /*isFullSet=*/true); + case Instruction::UIToFP: { + // TODO: use input range if available + auto BW = getBitWidth(); + APInt Min = APInt::getMinValue(BW).zextOrSelf(ResultBitWidth); + APInt Max = APInt::getMaxValue(BW).zextOrSelf(ResultBitWidth); + return ConstantRange(Min, Max); + } + case Instruction::SIToFP: { + // TODO: use input range if available + auto BW = getBitWidth(); + APInt SMin = APInt::getSignedMinValue(BW).sextOrSelf(ResultBitWidth); + APInt SMax = APInt::getSignedMaxValue(BW).sextOrSelf(ResultBitWidth); + return ConstantRange(SMin, SMax); + } + case Instruction::FPTrunc: + case Instruction::FPExt: + case Instruction::IntToPtr: + case Instruction::PtrToInt: + case Instruction::AddrSpaceCast: + // Conservatively return full set. + return ConstantRange(getBitWidth(), /*isFullSet=*/true); + }; +} + /// zeroExtend - Return a new range in the specified integer type, which must /// be strictly larger than the current type. The returned range will /// correspond to the possible range of values as if the source range had been @@ -645,6 +696,42 @@ ConstantRange ConstantRange::sextOrTrunc(uint32_t DstTySize) const { return *this; } +ConstantRange ConstantRange::binaryOp(Instruction::BinaryOps BinOp, + const ConstantRange &Other) const { + assert(BinOp >= Instruction::BinaryOpsBegin && + BinOp < Instruction::BinaryOpsEnd && "Binary operators only!"); + + switch (BinOp) { + case Instruction::Add: + return add(Other); + case Instruction::Sub: + return sub(Other); + case Instruction::Mul: + return multiply(Other); + case Instruction::UDiv: + return udiv(Other); + case Instruction::Shl: + return shl(Other); + case Instruction::LShr: + return lshr(Other); + case Instruction::And: + return binaryAnd(Other); + case Instruction::Or: + return binaryOr(Other); + // Note: floating point operations applied to abstract ranges are just + // ideal integer operations with a lossy representation + case Instruction::FAdd: + return add(Other); + case Instruction::FSub: + return sub(Other); + case Instruction::FMul: + return multiply(Other); + default: + // Conservatively return full set. + return ConstantRange(getBitWidth(), /*isFullSet=*/true); + } +} + ConstantRange ConstantRange::add(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) @@ -666,6 +753,19 @@ ConstantRange::add(const ConstantRange &Other) const { return X; } +ConstantRange ConstantRange::addWithNoSignedWrap(const APInt &Other) const { + // Calculate the subset of this range such that "X + Other" is + // guaranteed not to wrap (overflow) for all X in this subset. + // makeGuaranteedNoWrapRegion will produce an exact NSW range since we are + // passing a single element range. + auto NSWRange = ConstantRange::makeGuaranteedNoWrapRegion(BinaryOperator::Add, + ConstantRange(Other), + OverflowingBinaryOperator::NoSignedWrap); + auto NSWConstrainedRange = intersectWith(NSWRange); + + return NSWConstrainedRange.add(ConstantRange(Other)); +} + ConstantRange ConstantRange::sub(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) @@ -901,3 +1001,25 @@ void ConstantRange::print(raw_ostream &OS) const { LLVM_DUMP_METHOD void ConstantRange::dump() const { print(dbgs()); } + +ConstantRange llvm::getConstantRangeFromMetadata(const MDNode &Ranges) { + const unsigned NumRanges = Ranges.getNumOperands() / 2; + assert(NumRanges >= 1 && "Must have at least one range!"); + assert(Ranges.getNumOperands() % 2 == 0 && "Must be a sequence of pairs"); + + auto *FirstLow = mdconst::extract<ConstantInt>(Ranges.getOperand(0)); + auto *FirstHigh = mdconst::extract<ConstantInt>(Ranges.getOperand(1)); + + ConstantRange CR(FirstLow->getValue(), FirstHigh->getValue()); + + for (unsigned i = 1; i < NumRanges; ++i) { + auto *Low = mdconst::extract<ConstantInt>(Ranges.getOperand(2 * i + 0)); + auto *High = mdconst::extract<ConstantInt>(Ranges.getOperand(2 * i + 1)); + + // Note: unionWith will potentially create a range that contains values not + // contained in any of the original N ranges. + CR = CR.unionWith(ConstantRange(Low->getValue(), High->getValue())); + } + + return CR; +} diff --git a/contrib/llvm/lib/IR/Constants.cpp b/contrib/llvm/lib/IR/Constants.cpp index d8d55b472f32..533b9245277f 100644 --- a/contrib/llvm/lib/IR/Constants.cpp +++ b/contrib/llvm/lib/IR/Constants.cpp @@ -198,22 +198,22 @@ Constant *Constant::getNullValue(Type *Ty) { return ConstantInt::get(Ty, 0); case Type::HalfTyID: return ConstantFP::get(Ty->getContext(), - APFloat::getZero(APFloat::IEEEhalf)); + APFloat::getZero(APFloat::IEEEhalf())); case Type::FloatTyID: return ConstantFP::get(Ty->getContext(), - APFloat::getZero(APFloat::IEEEsingle)); + APFloat::getZero(APFloat::IEEEsingle())); case Type::DoubleTyID: return ConstantFP::get(Ty->getContext(), - APFloat::getZero(APFloat::IEEEdouble)); + APFloat::getZero(APFloat::IEEEdouble())); case Type::X86_FP80TyID: return ConstantFP::get(Ty->getContext(), - APFloat::getZero(APFloat::x87DoubleExtended)); + APFloat::getZero(APFloat::x87DoubleExtended())); case Type::FP128TyID: return ConstantFP::get(Ty->getContext(), - APFloat::getZero(APFloat::IEEEquad)); + APFloat::getZero(APFloat::IEEEquad())); case Type::PPC_FP128TyID: return ConstantFP::get(Ty->getContext(), - APFloat(APFloat::PPCDoubleDouble, + APFloat(APFloat::PPCDoubleDouble(), APInt::getNullValue(128))); case Type::PointerTyID: return ConstantPointerNull::get(cast<PointerType>(Ty)); @@ -347,10 +347,8 @@ static bool canTrapImpl(const Constant *C, return false; case Instruction::UDiv: case Instruction::SDiv: - case Instruction::FDiv: case Instruction::URem: case Instruction::SRem: - case Instruction::FRem: // Div and rem can trap if the RHS is not known to be non-zero. if (!isa<ConstantInt>(CE->getOperand(1)) ||CE->getOperand(1)->isNullValue()) return true; @@ -547,14 +545,14 @@ Constant *ConstantInt::getFalse(Type *Ty) { ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) { // get an existing value or the insertion position LLVMContextImpl *pImpl = Context.pImpl; - ConstantInt *&Slot = pImpl->IntConstants[V]; + std::unique_ptr<ConstantInt> &Slot = pImpl->IntConstants[V]; if (!Slot) { // Get the corresponding integer type for the bit width of the value. IntegerType *ITy = IntegerType::get(Context, V.getBitWidth()); - Slot = new ConstantInt(ITy, V); + Slot.reset(new ConstantInt(ITy, V)); } assert(Slot->getType() == IntegerType::get(Context, V.getBitWidth())); - return Slot; + return Slot.get(); } Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) { @@ -606,18 +604,18 @@ void ConstantInt::destroyConstantImpl() { static const fltSemantics *TypeToFloatSemantics(Type *Ty) { if (Ty->isHalfTy()) - return &APFloat::IEEEhalf; + return &APFloat::IEEEhalf(); if (Ty->isFloatTy()) - return &APFloat::IEEEsingle; + return &APFloat::IEEEsingle(); if (Ty->isDoubleTy()) - return &APFloat::IEEEdouble; + return &APFloat::IEEEdouble(); if (Ty->isX86_FP80Ty()) - return &APFloat::x87DoubleExtended; + return &APFloat::x87DoubleExtended(); else if (Ty->isFP128Ty()) - return &APFloat::IEEEquad; + return &APFloat::IEEEquad(); assert(Ty->isPPC_FP128Ty() && "Unknown FP format"); - return &APFloat::PPCDoubleDouble; + return &APFloat::PPCDoubleDouble(); } void ConstantFP::anchor() { } @@ -687,29 +685,29 @@ Constant *ConstantFP::getZeroValueForNegation(Type *Ty) { ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) { LLVMContextImpl* pImpl = Context.pImpl; - ConstantFP *&Slot = pImpl->FPConstants[V]; + std::unique_ptr<ConstantFP> &Slot = pImpl->FPConstants[V]; if (!Slot) { Type *Ty; - if (&V.getSemantics() == &APFloat::IEEEhalf) + if (&V.getSemantics() == &APFloat::IEEEhalf()) Ty = Type::getHalfTy(Context); - else if (&V.getSemantics() == &APFloat::IEEEsingle) + else if (&V.getSemantics() == &APFloat::IEEEsingle()) Ty = Type::getFloatTy(Context); - else if (&V.getSemantics() == &APFloat::IEEEdouble) + else if (&V.getSemantics() == &APFloat::IEEEdouble()) Ty = Type::getDoubleTy(Context); - else if (&V.getSemantics() == &APFloat::x87DoubleExtended) + else if (&V.getSemantics() == &APFloat::x87DoubleExtended()) Ty = Type::getX86_FP80Ty(Context); - else if (&V.getSemantics() == &APFloat::IEEEquad) + else if (&V.getSemantics() == &APFloat::IEEEquad()) Ty = Type::getFP128Ty(Context); else { - assert(&V.getSemantics() == &APFloat::PPCDoubleDouble && + assert(&V.getSemantics() == &APFloat::PPCDoubleDouble() && "Unknown FP format"); Ty = Type::getPPC_FP128Ty(Context); } - Slot = new ConstantFP(Ty, V); + Slot.reset(new ConstantFP(Ty, V)); } - return Slot; + return Slot.get(); } Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) { @@ -796,10 +794,8 @@ UndefValue *UndefValue::getElementValue(unsigned Idx) const { unsigned UndefValue::getNumElements() const { Type *Ty = getType(); - if (auto *AT = dyn_cast<ArrayType>(Ty)) - return AT->getNumElements(); - if (auto *VT = dyn_cast<VectorType>(Ty)) - return VT->getNumElements(); + if (auto *ST = dyn_cast<SequentialType>(Ty)) + return ST->getNumElements(); return Ty->getStructNumElements(); } @@ -1075,19 +1071,16 @@ bool ConstantExpr::isGEPWithNoNotionalOverIndexing() const { gep_type_iterator GEPI = gep_type_begin(this), E = gep_type_end(this); User::const_op_iterator OI = std::next(this->op_begin()); - // Skip the first index, as it has no static limit. - ++GEPI; - ++OI; - - // The remaining indices must be compile-time known integers within the - // bounds of the corresponding notional static array types. + // The remaining indices may be compile-time known integers within the bounds + // of the corresponding notional static array types. for (; GEPI != E; ++GEPI, ++OI) { - ConstantInt *CI = dyn_cast<ConstantInt>(*OI); - if (!CI) return false; - if (ArrayType *ATy = dyn_cast<ArrayType>(*GEPI)) - if (CI->getValue().getActiveBits() > 64 || - CI->getZExtValue() >= ATy->getNumElements()) - return false; + if (isa<UndefValue>(*OI)) + continue; + auto *CI = dyn_cast<ConstantInt>(*OI); + if (!CI || (GEPI.isBoundedSequential() && + (CI->getValue().getActiveBits() > 64 || + CI->getZExtValue() >= GEPI.getSequentialNumElements()))) + return false; } // All the indices checked out. @@ -1169,7 +1162,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty, assert(SrcTy || (Ops[0]->getType() == getOperand(0)->getType())); return ConstantExpr::getGetElementPtr( SrcTy ? SrcTy : GEPO->getSourceElementType(), Ops[0], Ops.slice(1), - GEPO->isInBounds(), OnlyIfReducedTy); + GEPO->isInBounds(), GEPO->getInRangeIndex(), OnlyIfReducedTy); } case Instruction::ICmp: case Instruction::FCmp: @@ -1217,40 +1210,40 @@ bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) { // FIXME rounding mode needs to be more flexible case Type::HalfTyID: { - if (&Val2.getSemantics() == &APFloat::IEEEhalf) + if (&Val2.getSemantics() == &APFloat::IEEEhalf()) return true; - Val2.convert(APFloat::IEEEhalf, APFloat::rmNearestTiesToEven, &losesInfo); + Val2.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &losesInfo); return !losesInfo; } case Type::FloatTyID: { - if (&Val2.getSemantics() == &APFloat::IEEEsingle) + if (&Val2.getSemantics() == &APFloat::IEEEsingle()) return true; - Val2.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven, &losesInfo); + Val2.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, &losesInfo); return !losesInfo; } case Type::DoubleTyID: { - if (&Val2.getSemantics() == &APFloat::IEEEhalf || - &Val2.getSemantics() == &APFloat::IEEEsingle || - &Val2.getSemantics() == &APFloat::IEEEdouble) + if (&Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::IEEEsingle() || + &Val2.getSemantics() == &APFloat::IEEEdouble()) return true; - Val2.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, &losesInfo); + Val2.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, &losesInfo); return !losesInfo; } case Type::X86_FP80TyID: - return &Val2.getSemantics() == &APFloat::IEEEhalf || - &Val2.getSemantics() == &APFloat::IEEEsingle || - &Val2.getSemantics() == &APFloat::IEEEdouble || - &Val2.getSemantics() == &APFloat::x87DoubleExtended; + return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::IEEEsingle() || + &Val2.getSemantics() == &APFloat::IEEEdouble() || + &Val2.getSemantics() == &APFloat::x87DoubleExtended(); case Type::FP128TyID: - return &Val2.getSemantics() == &APFloat::IEEEhalf || - &Val2.getSemantics() == &APFloat::IEEEsingle || - &Val2.getSemantics() == &APFloat::IEEEdouble || - &Val2.getSemantics() == &APFloat::IEEEquad; + return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::IEEEsingle() || + &Val2.getSemantics() == &APFloat::IEEEdouble() || + &Val2.getSemantics() == &APFloat::IEEEquad(); case Type::PPC_FP128TyID: - return &Val2.getSemantics() == &APFloat::IEEEhalf || - &Val2.getSemantics() == &APFloat::IEEEsingle || - &Val2.getSemantics() == &APFloat::IEEEdouble || - &Val2.getSemantics() == &APFloat::PPCDoubleDouble; + return &Val2.getSemantics() == &APFloat::IEEEhalf() || + &Val2.getSemantics() == &APFloat::IEEEsingle() || + &Val2.getSemantics() == &APFloat::IEEEdouble() || + &Val2.getSemantics() == &APFloat::PPCDoubleDouble(); } } @@ -1261,12 +1254,13 @@ bool ConstantFP::isValueValidForType(Type *Ty, const APFloat& Val) { ConstantAggregateZero *ConstantAggregateZero::get(Type *Ty) { assert((Ty->isStructTy() || Ty->isArrayTy() || Ty->isVectorTy()) && "Cannot create an aggregate zero of non-aggregate type!"); - - ConstantAggregateZero *&Entry = Ty->getContext().pImpl->CAZConstants[Ty]; + + std::unique_ptr<ConstantAggregateZero> &Entry = + Ty->getContext().pImpl->CAZConstants[Ty]; if (!Entry) - Entry = new ConstantAggregateZero(Ty); + Entry.reset(new ConstantAggregateZero(Ty)); - return Entry; + return Entry.get(); } /// Remove the constant from the constant table. @@ -1327,11 +1321,12 @@ const APInt &Constant::getUniqueInteger() const { // ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) { - ConstantPointerNull *&Entry = Ty->getContext().pImpl->CPNConstants[Ty]; + std::unique_ptr<ConstantPointerNull> &Entry = + Ty->getContext().pImpl->CPNConstants[Ty]; if (!Entry) - Entry = new ConstantPointerNull(Ty); + Entry.reset(new ConstantPointerNull(Ty)); - return Entry; + return Entry.get(); } /// Remove the constant from the constant table. @@ -1340,11 +1335,11 @@ void ConstantPointerNull::destroyConstantImpl() { } UndefValue *UndefValue::get(Type *Ty) { - UndefValue *&Entry = Ty->getContext().pImpl->UVConstants[Ty]; + std::unique_ptr<UndefValue> &Entry = Ty->getContext().pImpl->UVConstants[Ty]; if (!Entry) - Entry = new UndefValue(Ty); + Entry.reset(new UndefValue(Ty)); - return Entry; + return Entry.get(); } /// Remove the constant from the constant table. @@ -1893,6 +1888,7 @@ Constant *ConstantExpr::getSelect(Constant *C, Constant *V1, Constant *V2, Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, ArrayRef<Value *> Idxs, bool InBounds, + Optional<unsigned> InRangeIndex, Type *OnlyIfReducedTy) { if (!Ty) Ty = cast<PointerType>(C->getType()->getScalarType())->getElementType(); @@ -1901,7 +1897,8 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, Ty == cast<PointerType>(C->getType()->getScalarType())->getContainedType(0u)); - if (Constant *FC = ConstantFoldGetElementPtr(Ty, C, InBounds, Idxs)) + if (Constant *FC = + ConstantFoldGetElementPtr(Ty, C, InBounds, InRangeIndex, Idxs)) return FC; // Fold a few common cases. // Get the result type of the getelementptr! @@ -1937,9 +1934,12 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, Idx = ConstantVector::getSplat(NumVecElts, Idx); ArgVec.push_back(Idx); } + + unsigned SubClassOptionalData = InBounds ? GEPOperator::IsInBounds : 0; + if (InRangeIndex && *InRangeIndex < 63) + SubClassOptionalData |= (*InRangeIndex + 1) << 1; const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0, - InBounds ? GEPOperator::IsInBounds : 0, None, - Ty); + SubClassOptionalData, None, Ty); LLVMContextImpl *pImpl = C->getContext().pImpl; return pImpl->ExprConstants.getOrCreate(ReqTy, Key); @@ -2610,15 +2610,15 @@ APFloat ConstantDataSequential::getElementAsAPFloat(unsigned Elt) const { llvm_unreachable("Accessor can only be used when element is float/double!"); case Type::HalfTyID: { auto EltVal = *reinterpret_cast<const uint16_t *>(EltPtr); - return APFloat(APFloat::IEEEhalf, APInt(16, EltVal)); + return APFloat(APFloat::IEEEhalf(), APInt(16, EltVal)); } case Type::FloatTyID: { auto EltVal = *reinterpret_cast<const uint32_t *>(EltPtr); - return APFloat(APFloat::IEEEsingle, APInt(32, EltVal)); + return APFloat(APFloat::IEEEsingle(), APInt(32, EltVal)); } case Type::DoubleTyID: { auto EltVal = *reinterpret_cast<const uint64_t *>(EltPtr); - return APFloat(APFloat::IEEEdouble, APInt(64, EltVal)); + return APFloat(APFloat::IEEEdouble(), APInt(64, EltVal)); } } } diff --git a/contrib/llvm/lib/IR/ConstantsContext.h b/contrib/llvm/lib/IR/ConstantsContext.h index 7db87edff010..eda751d8af4a 100644 --- a/contrib/llvm/lib/IR/ConstantsContext.h +++ b/contrib/llvm/lib/IR/ConstantsContext.h @@ -1,4 +1,4 @@ -//===-- ConstantsContext.h - Constants-related Context Interals -----------===// +//===-- ConstantsContext.h - Constants-related Context Interals -*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,14 +15,26 @@ #ifndef LLVM_LIB_IR_CONSTANTSCONTEXT_H #define LLVM_LIB_IR_CONSTANTSCONTEXT_H +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/None.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/InlineAsm.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Operator.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/OperandTraits.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <utility> #define DEBUG_TYPE "ir" @@ -32,16 +44,20 @@ namespace llvm { /// behind the scenes to implement unary constant exprs. class UnaryConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly one operand - void *operator new(size_t s) { - return User::operator new(s, 1); - } UnaryConstantExpr(unsigned Opcode, Constant *C, Type *Ty) : ConstantExpr(Ty, Opcode, &Op<0>(), 1) { Op<0>() = C; } + + // allocate space for exactly one operand + void *operator new(size_t s) { + return User::operator new(s, 1); + } + + void *operator new(size_t, unsigned) = delete; + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -49,12 +65,8 @@ public: /// behind the scenes to implement binary constant exprs. class BinaryConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); - } BinaryConstantExpr(unsigned Opcode, Constant *C1, Constant *C2, unsigned Flags) : ConstantExpr(C1->getType(), Opcode, &Op<0>(), 2) { @@ -62,6 +74,14 @@ public: Op<1>() = C2; SubclassOptionalData = Flags; } + + // allocate space for exactly two operands + void *operator new(size_t s) { + return User::operator new(s, 2); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -70,18 +90,22 @@ public: /// behind the scenes to implement select constant exprs. class SelectConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } SelectConstantExpr(Constant *C1, Constant *C2, Constant *C3) : ConstantExpr(C2->getType(), Instruction::Select, &Op<0>(), 3) { Op<0>() = C1; Op<1>() = C2; Op<2>() = C3; } + + // allocate space for exactly three operands + void *operator new(size_t s) { + return User::operator new(s, 3); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -91,18 +115,22 @@ public: /// extractelement constant exprs. class ExtractElementConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); - } ExtractElementConstantExpr(Constant *C1, Constant *C2) : ConstantExpr(cast<VectorType>(C1->getType())->getElementType(), Instruction::ExtractElement, &Op<0>(), 2) { Op<0>() = C1; Op<1>() = C2; } + + // allocate space for exactly two operands + void *operator new(size_t s) { + return User::operator new(s, 2); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -112,12 +140,8 @@ public: /// insertelement constant exprs. class InsertElementConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } InsertElementConstantExpr(Constant *C1, Constant *C2, Constant *C3) : ConstantExpr(C1->getType(), Instruction::InsertElement, &Op<0>(), 3) { @@ -125,6 +149,14 @@ public: Op<1>() = C2; Op<2>() = C3; } + + // allocate space for exactly three operands + void *operator new(size_t s) { + return User::operator new(s, 3); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -134,12 +166,8 @@ public: /// shufflevector constant exprs. class ShuffleVectorConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly three operands - void *operator new(size_t s) { - return User::operator new(s, 3); - } ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3) : ConstantExpr(VectorType::get( cast<VectorType>(C1->getType())->getElementType(), @@ -150,6 +178,14 @@ public: Op<1>() = C2; Op<2>() = C3; } + + // allocate space for exactly three operands + void *operator new(size_t s) { + return User::operator new(s, 3); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); }; @@ -159,12 +195,8 @@ public: /// extractvalue constant exprs. class ExtractValueConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly one operand - void *operator new(size_t s) { - return User::operator new(s, 1); - } ExtractValueConstantExpr(Constant *Agg, ArrayRef<unsigned> IdxList, Type *DestTy) : ConstantExpr(DestTy, Instruction::ExtractValue, &Op<0>(), 1), @@ -172,6 +204,13 @@ public: Op<0>() = Agg; } + // allocate space for exactly one operand + void *operator new(size_t s) { + return User::operator new(s, 1); + } + + void *operator new(size_t, unsigned) = delete; + /// Indices - These identify which value to extract. const SmallVector<unsigned, 4> Indices; @@ -191,12 +230,8 @@ public: /// insertvalue constant exprs. class InsertValueConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly one operand - void *operator new(size_t s) { - return User::operator new(s, 2); - } InsertValueConstantExpr(Constant *Agg, Constant *Val, ArrayRef<unsigned> IdxList, Type *DestTy) : ConstantExpr(DestTy, Instruction::InsertValue, &Op<0>(), 2), @@ -205,6 +240,13 @@ public: Op<1>() = Val; } + // allocate space for exactly one operand + void *operator new(size_t s) { + return User::operator new(s, 2); + } + + void *operator new(size_t, unsigned) = delete; + /// Indices - These identify the position for the insertion. const SmallVector<unsigned, 4> Indices; @@ -224,10 +266,12 @@ public: class GetElementPtrConstantExpr : public ConstantExpr { Type *SrcElementTy; Type *ResElementTy; - void anchor() override; + GetElementPtrConstantExpr(Type *SrcElementTy, Constant *C, ArrayRef<Constant *> IdxList, Type *DestTy); + void anchor() override; + public: static GetElementPtrConstantExpr *Create(Type *SrcElementTy, Constant *C, ArrayRef<Constant *> IdxList, @@ -237,8 +281,10 @@ public: Result->SubclassOptionalData = Flags; return Result; } + Type *getSourceElementType() const; Type *getResultElementType() const; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -255,12 +301,8 @@ public: // needed in order to store the predicate value for these instructions. class CompareConstantExpr : public ConstantExpr { void anchor() override; - void *operator new(size_t, unsigned) = delete; + public: - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); - } unsigned short predicate; CompareConstantExpr(Type *ty, Instruction::OtherOps opc, unsigned short pred, Constant* LHS, Constant* RHS) @@ -268,6 +310,14 @@ public: Op<0>() = LHS; Op<1>() = RHS; } + + // allocate space for exactly two operands + void *operator new(size_t s) { + return User::operator new(s, 2); + } + + void *operator new(size_t, unsigned) = delete; + /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -373,6 +423,7 @@ template <class ConstantClass> struct ConstantAggrKeyType { bool operator==(const ConstantAggrKeyType &X) const { return Operands == X.Operands; } + bool operator==(const ConstantClass *C) const { if (Operands.size() != C->getNumOperands()) return false; @@ -381,6 +432,7 @@ template <class ConstantClass> struct ConstantAggrKeyType { return false; return true; } + unsigned getHash() const { return hash_combine_range(Operands.begin(), Operands.end()); } @@ -416,6 +468,7 @@ struct InlineAsmKeyType { AsmString == X.AsmString && Constraints == X.Constraints && FTy == X.FTy; } + bool operator==(const InlineAsm *Asm) const { return HasSideEffects == Asm->hasSideEffects() && IsAlignStack == Asm->isAlignStack() && @@ -424,6 +477,7 @@ struct InlineAsmKeyType { Constraints == Asm->getConstraintString() && FTy == Asm->getFunctionType(); } + unsigned getHash() const { return hash_combine(AsmString, Constraints, HasSideEffects, IsAlignStack, AsmDialect, FTy); @@ -553,22 +607,28 @@ private: static inline ConstantClass *getEmptyKey() { return ConstantClassInfo::getEmptyKey(); } + static inline ConstantClass *getTombstoneKey() { return ConstantClassInfo::getTombstoneKey(); } + static unsigned getHashValue(const ConstantClass *CP) { SmallVector<Constant *, 32> Storage; return getHashValue(LookupKey(CP->getType(), ValType(CP, Storage))); } + static bool isEqual(const ConstantClass *LHS, const ConstantClass *RHS) { return LHS == RHS; } + static unsigned getHashValue(const LookupKey &Val) { return hash_combine(Val.first, Val.second.getHash()); } + static unsigned getHashValue(const LookupKeyHashed &Val) { return Val.first; } + static bool isEqual(const LookupKey &LHS, const ConstantClass *RHS) { if (RHS == getEmptyKey() || RHS == getTombstoneKey()) return false; @@ -576,6 +636,7 @@ private: return false; return LHS.second == RHS; } + static bool isEqual(const LookupKeyHashed &LHS, const ConstantClass *RHS) { return isEqual(LHS.second, RHS); } @@ -595,6 +656,7 @@ public: for (auto &I : Map) delete I; // Asserts that use_empty(). } + private: ConstantClass *create(TypeClass *Ty, ValType V, LookupKeyHashed &HashKey) { ConstantClass *Result = V.create(Ty); @@ -665,4 +727,4 @@ public: } // end namespace llvm -#endif +#endif // LLVM_LIB_IR_CONSTANTSCONTEXT_H diff --git a/contrib/llvm/lib/IR/Core.cpp b/contrib/llvm/lib/IR/Core.cpp index 3c4b0cf2f8ff..00bb476c0b3c 100644 --- a/contrib/llvm/lib/IR/Core.cpp +++ b/contrib/llvm/lib/IR/Core.cpp @@ -14,7 +14,7 @@ #include "llvm-c/Core.h" #include "llvm/ADT/StringSwitch.h" -#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/Attributes.h" #include "AttributeSetNode.h" #include "llvm/IR/CallSite.h" @@ -578,8 +578,11 @@ LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount) { return wrap(VectorType::get(unwrap(ElementType), ElementCount)); } -LLVMTypeRef LLVMGetElementType(LLVMTypeRef Ty) { - return wrap(unwrap<SequentialType>(Ty)->getElementType()); +LLVMTypeRef LLVMGetElementType(LLVMTypeRef WrappedTy) { + auto *Ty = unwrap<Type>(WrappedTy); + if (auto *PTy = dyn_cast<PointerType>(Ty)) + return wrap(PTy->getElementType()); + return wrap(cast<SequentialType>(Ty)->getElementType()); } unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy) { @@ -980,7 +983,7 @@ double LLVMConstRealGetDouble(LLVMValueRef ConstantVal, LLVMBool *LosesInfo) { bool APFLosesInfo; APFloat APF = cFP->getValueAPF(); - APF.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven, &APFLosesInfo); + APF.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, &APFLosesInfo); *LosesInfo = APFLosesInfo; return APF.convertToDouble(); } @@ -1178,6 +1181,12 @@ LLVMValueRef LLVMConstUDiv(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant) { unwrap<Constant>(RHSConstant))); } +LLVMValueRef LLVMConstExactUDiv(LLVMValueRef LHSConstant, + LLVMValueRef RHSConstant) { + return wrap(ConstantExpr::getExactUDiv(unwrap<Constant>(LHSConstant), + unwrap<Constant>(RHSConstant))); +} + LLVMValueRef LLVMConstSDiv(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant) { return wrap(ConstantExpr::getSDiv(unwrap<Constant>(LHSConstant), unwrap<Constant>(RHSConstant))); @@ -1829,17 +1838,6 @@ void LLVMSetGC(LLVMValueRef Fn, const char *GC) { F->clearGC(); } -void LLVMAddFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA) { - Function *Func = unwrap<Function>(Fn); - const AttributeSet PAL = Func->getAttributes(); - AttrBuilder B(PA); - const AttributeSet PALnew = - PAL.addAttributes(Func->getContext(), AttributeSet::FunctionIndex, - AttributeSet::get(Func->getContext(), - AttributeSet::FunctionIndex, B)); - Func->setAttributes(PALnew); -} - void LLVMAddAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, LLVMAttributeRef A) { unwrap<Function>(F)->addAttribute(Idx, unwrap(A)); @@ -1847,12 +1845,16 @@ void LLVMAddAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, unsigned LLVMGetAttributeCountAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx) { auto *ASN = AttributeSetNode::get(unwrap<Function>(F)->getAttributes(), Idx); + if (!ASN) + return 0; return ASN->getNumAttributes(); } void LLVMGetAttributesAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, LLVMAttributeRef *Attrs) { auto *ASN = AttributeSetNode::get(unwrap<Function>(F)->getAttributes(), Idx); + if (!ASN) + return; for (auto A: make_range(ASN->begin(), ASN->end())) *Attrs++ = wrap(A); } @@ -1892,23 +1894,6 @@ void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A, Func->addAttributes(Idx, Set); } -void LLVMRemoveFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA) { - Function *Func = unwrap<Function>(Fn); - const AttributeSet PAL = Func->getAttributes(); - AttrBuilder B(PA); - const AttributeSet PALnew = - PAL.removeAttributes(Func->getContext(), AttributeSet::FunctionIndex, - AttributeSet::get(Func->getContext(), - AttributeSet::FunctionIndex, B)); - Func->setAttributes(PALnew); -} - -LLVMAttribute LLVMGetFunctionAttr(LLVMValueRef Fn) { - Function *Func = unwrap<Function>(Fn); - const AttributeSet PAL = Func->getAttributes(); - return (LLVMAttribute)PAL.Raw(AttributeSet::FunctionIndex); -} - /*--.. Operations on parameters ............................................--*/ unsigned LLVMCountParams(LLVMValueRef FnRef) { @@ -1967,24 +1952,6 @@ LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg) { return wrap(&*--I); } -void LLVMAddAttribute(LLVMValueRef Arg, LLVMAttribute PA) { - Argument *A = unwrap<Argument>(Arg); - AttrBuilder B(PA); - A->addAttr(AttributeSet::get(A->getContext(), A->getArgNo() + 1, B)); -} - -void LLVMRemoveAttribute(LLVMValueRef Arg, LLVMAttribute PA) { - Argument *A = unwrap<Argument>(Arg); - AttrBuilder B(PA); - A->removeAttr(AttributeSet::get(A->getContext(), A->getArgNo() + 1, B)); -} - -LLVMAttribute LLVMGetAttribute(LLVMValueRef Arg) { - Argument *A = unwrap<Argument>(Arg); - return (LLVMAttribute)A->getParent()->getAttributes(). - Raw(A->getArgNo()+1); -} - void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned align) { Argument *A = unwrap<Argument>(Arg); AttrBuilder B; @@ -2193,26 +2160,6 @@ void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC) { .setCallingConv(static_cast<CallingConv::ID>(CC)); } -void LLVMAddInstrAttribute(LLVMValueRef Instr, unsigned index, - LLVMAttribute PA) { - CallSite Call = CallSite(unwrap<Instruction>(Instr)); - AttrBuilder B(PA); - Call.setAttributes( - Call.getAttributes().addAttributes(Call->getContext(), index, - AttributeSet::get(Call->getContext(), - index, B))); -} - -void LLVMRemoveInstrAttribute(LLVMValueRef Instr, unsigned index, - LLVMAttribute PA) { - CallSite Call = CallSite(unwrap<Instruction>(Instr)); - AttrBuilder B(PA); - Call.setAttributes(Call.getAttributes() - .removeAttributes(Call->getContext(), index, - AttributeSet::get(Call->getContext(), - index, B))); -} - void LLVMSetInstrParamAlignment(LLVMValueRef Instr, unsigned index, unsigned align) { CallSite Call = CallSite(unwrap<Instruction>(Instr)); @@ -2233,6 +2180,8 @@ unsigned LLVMGetCallSiteAttributeCount(LLVMValueRef C, LLVMAttributeIndex Idx) { auto CS = CallSite(unwrap<Instruction>(C)); auto *ASN = AttributeSetNode::get(CS.getAttributes(), Idx); + if (!ASN) + return 0; return ASN->getNumAttributes(); } @@ -2240,6 +2189,8 @@ void LLVMGetCallSiteAttributes(LLVMValueRef C, LLVMAttributeIndex Idx, LLVMAttributeRef *Attrs) { auto CS = CallSite(unwrap<Instruction>(C)); auto *ASN = AttributeSetNode::get(CS.getAttributes(), Idx); + if (!ASN) + return; for (auto A: make_range(ASN->begin(), ASN->end())) *Attrs++ = wrap(A); } @@ -2410,8 +2361,8 @@ LLVMBuilderRef LLVMCreateBuilder(void) { void LLVMPositionBuilder(LLVMBuilderRef Builder, LLVMBasicBlockRef Block, LLVMValueRef Instr) { BasicBlock *BB = unwrap(Block); - Instruction *I = Instr? unwrap<Instruction>(Instr) : (Instruction*) BB->end(); - unwrap(Builder)->SetInsertPoint(BB, I->getIterator()); + auto I = Instr ? unwrap<Instruction>(Instr)->getIterator() : BB->end(); + unwrap(Builder)->SetInsertPoint(BB, I); } void LLVMPositionBuilderBefore(LLVMBuilderRef Builder, LLVMValueRef Instr) { @@ -2624,6 +2575,11 @@ LLVMValueRef LLVMBuildUDiv(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, return wrap(unwrap(B)->CreateUDiv(unwrap(LHS), unwrap(RHS), Name)); } +LLVMValueRef LLVMBuildExactUDiv(LLVMBuilderRef B, LLVMValueRef LHS, + LLVMValueRef RHS, const char *Name) { + return wrap(unwrap(B)->CreateExactUDiv(unwrap(LHS), unwrap(RHS), Name)); +} + LLVMValueRef LLVMBuildSDiv(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *Name) { return wrap(unwrap(B)->CreateSDiv(unwrap(LHS), unwrap(RHS), Name)); diff --git a/contrib/llvm/lib/IR/DIBuilder.cpp b/contrib/llvm/lib/IR/DIBuilder.cpp index 01b47f386e1c..2ea572490b6d 100644 --- a/contrib/llvm/lib/IR/DIBuilder.cpp +++ b/contrib/llvm/lib/IR/DIBuilder.cpp @@ -109,21 +109,20 @@ static DIScope *getNonCompileUnitScope(DIScope *N) { } DICompileUnit *DIBuilder::createCompileUnit( - unsigned Lang, StringRef Filename, StringRef Directory, StringRef Producer, - bool isOptimized, StringRef Flags, unsigned RunTimeVer, StringRef SplitName, - DICompileUnit::DebugEmissionKind Kind, uint64_t DWOId) { + unsigned Lang, DIFile *File, StringRef Producer, bool isOptimized, + StringRef Flags, unsigned RunTimeVer, StringRef SplitName, + DICompileUnit::DebugEmissionKind Kind, uint64_t DWOId, + bool SplitDebugInlining) { assert(((Lang <= dwarf::DW_LANG_Fortran08 && Lang >= dwarf::DW_LANG_C89) || (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) && "Invalid Language tag"); - assert(!Filename.empty() && - "Unable to create compile unit without filename"); assert(!CUNode && "Can only make one compile unit per DIBuilder instance"); CUNode = DICompileUnit::getDistinct( - VMContext, Lang, DIFile::get(VMContext, Filename, Directory), Producer, - isOptimized, Flags, RunTimeVer, SplitName, Kind, nullptr, nullptr, - nullptr, nullptr, nullptr, DWOId); + VMContext, Lang, File, Producer, isOptimized, Flags, RunTimeVer, + SplitName, Kind, nullptr, nullptr, nullptr, nullptr, nullptr, DWOId, + SplitDebugInlining); // Create a named metadata so that it is easier to find cu in a module. NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.cu"); @@ -175,8 +174,9 @@ DIImportedEntity *DIBuilder::createImportedDeclaration(DIScope *Context, Context, Decl, Line, Name, AllImportedModules); } -DIFile *DIBuilder::createFile(StringRef Filename, StringRef Directory) { - return DIFile::get(VMContext, Filename, Directory); +DIFile *DIBuilder::createFile(StringRef Filename, StringRef Directory, + DIFile::ChecksumKind CSKind, StringRef Checksum) { + return DIFile::get(VMContext, Filename, Directory, CSKind, Checksum); } DIEnumerator *DIBuilder::createEnumerator(StringRef Name, int64_t Val) { @@ -194,33 +194,32 @@ DIBasicType *DIBuilder::createNullPtrType() { } DIBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, - uint64_t AlignInBits, unsigned Encoding) { assert(!Name.empty() && "Unable to create type without name"); return DIBasicType::get(VMContext, dwarf::DW_TAG_base_type, Name, SizeInBits, - AlignInBits, Encoding); + 0, Encoding); } DIDerivedType *DIBuilder::createQualifiedType(unsigned Tag, DIType *FromTy) { return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, FromTy, 0, - 0, 0, 0); + 0, 0, DINode::FlagZero); } DIDerivedType *DIBuilder::createPointerType(DIType *PointeeTy, uint64_t SizeInBits, - uint64_t AlignInBits, + uint32_t AlignInBits, StringRef Name) { // FIXME: Why is there a name here? return DIDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name, nullptr, 0, nullptr, PointeeTy, SizeInBits, - AlignInBits, 0, 0); + AlignInBits, 0, DINode::FlagZero); } DIDerivedType *DIBuilder::createMemberPointerType(DIType *PointeeTy, DIType *Base, uint64_t SizeInBits, - uint64_t AlignInBits, - unsigned Flags) { + uint32_t AlignInBits, + DINode::DIFlags Flags) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "", nullptr, 0, nullptr, PointeeTy, SizeInBits, AlignInBits, 0, Flags, Base); @@ -228,10 +227,10 @@ DIDerivedType *DIBuilder::createMemberPointerType(DIType *PointeeTy, DIDerivedType *DIBuilder::createReferenceType(unsigned Tag, DIType *RTy, uint64_t SizeInBits, - uint64_t AlignInBits) { + uint32_t AlignInBits) { assert(RTy && "Unable to create reference type"); return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, RTy, - SizeInBits, AlignInBits, 0, 0); + SizeInBits, AlignInBits, 0, DINode::FlagZero); } DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name, @@ -239,19 +238,19 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name, DIScope *Context) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo, getNonCompileUnitScope(Context), Ty, 0, 0, - 0, 0); + 0, DINode::FlagZero); } DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) { assert(Ty && "Invalid type!"); assert(FriendTy && "Invalid friend type!"); return DIDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, Ty, - FriendTy, 0, 0, 0, 0); + FriendTy, 0, 0, 0, DINode::FlagZero); } DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy, uint64_t BaseOffset, - unsigned Flags) { + DINode::DIFlags Flags) { assert(Ty && "Unable to create inheritance"); return DIDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr, 0, Ty, BaseTy, 0, 0, BaseOffset, Flags); @@ -260,9 +259,9 @@ DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy, DIDerivedType *DIBuilder::createMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, uint64_t SizeInBits, - uint64_t AlignInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, - unsigned Flags, DIType *Ty) { + DINode::DIFlags Flags, DIType *Ty) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, getNonCompileUnitScope(Scope), Ty, SizeInBits, AlignInBits, OffsetInBits, Flags); @@ -276,33 +275,33 @@ static ConstantAsMetadata *getConstantOrNull(Constant *C) { DIDerivedType *DIBuilder::createBitFieldMemberType( DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, - uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, - uint64_t StorageOffsetInBits, unsigned Flags, DIType *Ty) { + uint64_t SizeInBits, uint64_t OffsetInBits, uint64_t StorageOffsetInBits, + DINode::DIFlags Flags, DIType *Ty) { Flags |= DINode::FlagBitField; return DIDerivedType::get( VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - getNonCompileUnitScope(Scope), Ty, SizeInBits, AlignInBits, OffsetInBits, - Flags, ConstantAsMetadata::get(ConstantInt::get( - IntegerType::get(VMContext, 64), StorageOffsetInBits))); + getNonCompileUnitScope(Scope), Ty, SizeInBits, /* AlignInBits */ 0, + OffsetInBits, Flags, + ConstantAsMetadata::get(ConstantInt::get(IntegerType::get(VMContext, 64), + StorageOffsetInBits))); } -DIDerivedType *DIBuilder::createStaticMemberType(DIScope *Scope, StringRef Name, - DIFile *File, - unsigned LineNumber, - DIType *Ty, unsigned Flags, - llvm::Constant *Val) { +DIDerivedType * +DIBuilder::createStaticMemberType(DIScope *Scope, StringRef Name, DIFile *File, + unsigned LineNumber, DIType *Ty, + DINode::DIFlags Flags, llvm::Constant *Val, + uint32_t AlignInBits) { Flags |= DINode::FlagStaticMember; return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, - LineNumber, getNonCompileUnitScope(Scope), Ty, 0, 0, - 0, Flags, getConstantOrNull(Val)); + LineNumber, getNonCompileUnitScope(Scope), Ty, 0, + AlignInBits, 0, Flags, getConstantOrNull(Val)); } -DIDerivedType *DIBuilder::createObjCIVar(StringRef Name, DIFile *File, - unsigned LineNumber, - uint64_t SizeInBits, - uint64_t AlignInBits, - uint64_t OffsetInBits, unsigned Flags, - DIType *Ty, MDNode *PropertyNode) { +DIDerivedType * +DIBuilder::createObjCIVar(StringRef Name, DIFile *File, unsigned LineNumber, + uint64_t SizeInBits, uint32_t AlignInBits, + uint64_t OffsetInBits, DINode::DIFlags Flags, + DIType *Ty, MDNode *PropertyNode) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, getNonCompileUnitScope(File), Ty, SizeInBits, AlignInBits, OffsetInBits, Flags, @@ -358,8 +357,8 @@ DIBuilder::createTemplateParameterPack(DIScope *Context, StringRef Name, DICompositeType *DIBuilder::createClassType( DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber, - uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, - unsigned Flags, DIType *DerivedFrom, DINodeArray Elements, + uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, + DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, DIType *VTableHolder, MDNode *TemplateParams, StringRef UniqueIdentifier) { assert((!Context || isa<DIScope>(Context)) && "createClassType should be called with a valid Context"); @@ -375,7 +374,7 @@ DICompositeType *DIBuilder::createClassType( DICompositeType *DIBuilder::createStructType( DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber, - uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, + uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang, DIType *VTableHolder, StringRef UniqueIdentifier) { auto *R = DICompositeType::get( @@ -388,7 +387,7 @@ DICompositeType *DIBuilder::createStructType( DICompositeType *DIBuilder::createUnionType( DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, - uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, + uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DINodeArray Elements, unsigned RunTimeLang, StringRef UniqueIdentifier) { auto *R = DICompositeType::get( VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber, @@ -399,7 +398,8 @@ DICompositeType *DIBuilder::createUnionType( } DISubroutineType *DIBuilder::createSubroutineType(DITypeRefArray ParameterTypes, - unsigned Flags, unsigned CC) { + DINode::DIFlags Flags, + unsigned CC) { return DISubroutineType::get(VMContext, Flags, CC, ParameterTypes); } @@ -413,29 +413,29 @@ DICompositeType *DIBuilder::createExternalTypeRef(unsigned Tag, DIFile *File, DICompositeType *DIBuilder::createEnumerationType( DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, - uint64_t SizeInBits, uint64_t AlignInBits, DINodeArray Elements, + uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements, DIType *UnderlyingType, StringRef UniqueIdentifier) { auto *CTy = DICompositeType::get( VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber, getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0, - 0, Elements, 0, nullptr, nullptr, UniqueIdentifier); + DINode::FlagZero, Elements, 0, nullptr, nullptr, UniqueIdentifier); AllEnumTypes.push_back(CTy); trackIfUnresolved(CTy); return CTy; } -DICompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, - DIType *Ty, +DICompositeType *DIBuilder::createArrayType(uint64_t Size, + uint32_t AlignInBits, DIType *Ty, DINodeArray Subscripts) { auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty, Size, AlignInBits, 0, - 0, Subscripts, 0, nullptr); + DINode::FlagZero, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } DICompositeType *DIBuilder::createVectorType(uint64_t Size, - uint64_t AlignInBits, DIType *Ty, + uint32_t AlignInBits, DIType *Ty, DINodeArray Subscripts) { auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty, Size, AlignInBits, 0, @@ -445,7 +445,7 @@ DICompositeType *DIBuilder::createVectorType(uint64_t Size, } static DIType *createTypeWithFlags(LLVMContext &Context, DIType *Ty, - unsigned FlagsToSet) { + DINode::DIFlags FlagsToSet) { auto NewTy = Ty->clone(); NewTy->setFlags(NewTy->getFlags() | FlagsToSet); return MDNode::replaceWithUniqued(std::move(NewTy)); @@ -462,7 +462,7 @@ DIType *DIBuilder::createObjectPointerType(DIType *Ty) { // FIXME: Restrict this to the nodes where it's valid. if (Ty->isObjectPointer()) return Ty; - unsigned Flags = DINode::FlagObjectPointer | DINode::FlagArtificial; + DINode::DIFlags Flags = DINode::FlagObjectPointer | DINode::FlagArtificial; return createTypeWithFlags(VMContext, Ty, Flags); } @@ -479,7 +479,7 @@ DIBasicType *DIBuilder::createUnspecifiedParameter() { return nullptr; } DICompositeType * DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line, unsigned RuntimeLang, - uint64_t SizeInBits, uint64_t AlignInBits, + uint64_t SizeInBits, uint32_t AlignInBits, StringRef UniqueIdentifier) { // FIXME: Define in terms of createReplaceableForwardDecl() by calling // replaceWithUniqued(). @@ -493,8 +493,8 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope, DICompositeType *DIBuilder::createReplaceableCompositeType( unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line, - unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, - unsigned Flags, StringRef UniqueIdentifier) { + unsigned RuntimeLang, uint64_t SizeInBits, uint32_t AlignInBits, + DINode::DIFlags Flags, StringRef UniqueIdentifier) { auto *RetTy = DICompositeType::getTemporary( VMContext, Tag, Name, F, Line, getNonCompileUnitScope(Scope), nullptr, @@ -533,30 +533,31 @@ static void checkGlobalVariableScope(DIScope *Context) { #endif } -DIGlobalVariable *DIBuilder::createGlobalVariable( +DIGlobalVariableExpression *DIBuilder::createGlobalVariableExpression( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *F, - unsigned LineNumber, DIType *Ty, bool isLocalToUnit, Constant *Val, - MDNode *Decl) { + unsigned LineNumber, DIType *Ty, bool isLocalToUnit, DIExpression *Expr, + MDNode *Decl, uint32_t AlignInBits) { checkGlobalVariableScope(Context); - auto *N = DIGlobalVariable::getDistinct( + auto *GV = DIGlobalVariable::getDistinct( VMContext, cast_or_null<DIScope>(Context), Name, LinkageName, F, - LineNumber, Ty, isLocalToUnit, true, Val, - cast_or_null<DIDerivedType>(Decl)); + LineNumber, Ty, isLocalToUnit, true, cast_or_null<DIDerivedType>(Decl), + AlignInBits); + auto *N = DIGlobalVariableExpression::get(VMContext, GV, Expr); AllGVs.push_back(N); return N; } DIGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *F, - unsigned LineNumber, DIType *Ty, bool isLocalToUnit, Constant *Val, - MDNode *Decl) { + unsigned LineNumber, DIType *Ty, bool isLocalToUnit, MDNode *Decl, + uint32_t AlignInBits) { checkGlobalVariableScope(Context); return DIGlobalVariable::getTemporary( VMContext, cast_or_null<DIScope>(Context), Name, LinkageName, F, - LineNumber, Ty, isLocalToUnit, false, Val, - cast_or_null<DIDerivedType>(Decl)) + LineNumber, Ty, isLocalToUnit, false, + cast_or_null<DIDerivedType>(Decl), AlignInBits) .release(); } @@ -564,7 +565,8 @@ static DILocalVariable *createLocalVariable( LLVMContext &VMContext, DenseMap<MDNode *, SmallVector<TrackingMDNodeRef, 1>> &PreservedVariables, DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File, - unsigned LineNo, DIType *Ty, bool AlwaysPreserve, unsigned Flags) { + unsigned LineNo, DIType *Ty, bool AlwaysPreserve, DINode::DIFlags Flags, + uint32_t AlignInBits) { // FIXME: Why getNonCompileUnitScope()? // FIXME: Why is "!Context" okay here? // FIXME: Why doesn't this check for a subprogram or lexical block (AFAICT @@ -573,7 +575,7 @@ static DILocalVariable *createLocalVariable( auto *Node = DILocalVariable::get(VMContext, cast_or_null<DILocalScope>(Context), Name, - File, LineNo, Ty, ArgNo, Flags); + File, LineNo, Ty, ArgNo, Flags, AlignInBits); if (AlwaysPreserve) { // The optimizer may remove local variables. If there is an interest // to preserve variable info in such situation then stash it in a @@ -588,18 +590,20 @@ static DILocalVariable *createLocalVariable( DILocalVariable *DIBuilder::createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve, - unsigned Flags) { + DINode::DIFlags Flags, + uint32_t AlignInBits) { return createLocalVariable(VMContext, PreservedVariables, Scope, Name, /* ArgNo */ 0, File, LineNo, Ty, AlwaysPreserve, - Flags); + Flags, AlignInBits); } DILocalVariable *DIBuilder::createParameterVariable( DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File, - unsigned LineNo, DIType *Ty, bool AlwaysPreserve, unsigned Flags) { + unsigned LineNo, DIType *Ty, bool AlwaysPreserve, DINode::DIFlags Flags) { assert(ArgNo && "Expected non-zero argument number for parameter"); return createLocalVariable(VMContext, PreservedVariables, Scope, Name, ArgNo, - File, LineNo, Ty, AlwaysPreserve, Flags); + File, LineNo, Ty, AlwaysPreserve, Flags, + /* AlignInBits */0); } DIExpression *DIBuilder::createExpression(ArrayRef<uint64_t> Addr) { @@ -612,9 +616,9 @@ DIExpression *DIBuilder::createExpression(ArrayRef<int64_t> Signed) { return createExpression(Addr); } -DIExpression *DIBuilder::createBitPieceExpression(unsigned OffsetInBytes, +DIExpression *DIBuilder::createFragmentExpression(unsigned OffsetInBytes, unsigned SizeInBytes) { - uint64_t Addr[] = {dwarf::DW_OP_bit_piece, OffsetInBytes, SizeInBytes}; + uint64_t Addr[] = {dwarf::DW_OP_LLVM_fragment, OffsetInBytes, SizeInBytes}; return DIExpression::get(VMContext, Addr); } @@ -628,8 +632,8 @@ static DISubprogram *getSubprogram(bool IsDistinct, Ts &&... Args) { DISubprogram *DIBuilder::createFunction( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, - bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, - DITemplateParameterArray TParams, DISubprogram *Decl) { + bool isDefinition, unsigned ScopeLine, DINode::DIFlags Flags, + bool isOptimized, DITemplateParameterArray TParams, DISubprogram *Decl) { auto *Node = getSubprogram( /* IsDistinct = */ isDefinition, VMContext, getNonCompileUnitScope(Context), Name, LinkageName, File, LineNo, Ty, @@ -646,8 +650,8 @@ DISubprogram *DIBuilder::createFunction( DISubprogram *DIBuilder::createTempFunctionFwdDecl( DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, - bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, - DITemplateParameterArray TParams, DISubprogram *Decl) { + bool isDefinition, unsigned ScopeLine, DINode::DIFlags Flags, + bool isOptimized, DITemplateParameterArray TParams, DISubprogram *Decl) { return DISubprogram::getTemporary( VMContext, getNonCompileUnitScope(Context), Name, LinkageName, File, LineNo, Ty, isLocalToUnit, isDefinition, ScopeLine, nullptr, @@ -656,13 +660,14 @@ DISubprogram *DIBuilder::createTempFunctionFwdDecl( .release(); } -DISubprogram * -DIBuilder::createMethod(DIScope *Context, StringRef Name, StringRef LinkageName, - DIFile *F, unsigned LineNo, DISubroutineType *Ty, - bool isLocalToUnit, bool isDefinition, unsigned VK, - unsigned VIndex, int ThisAdjustment, - DIType *VTableHolder, unsigned Flags, bool isOptimized, - DITemplateParameterArray TParams) { +DISubprogram *DIBuilder::createMethod(DIScope *Context, StringRef Name, + StringRef LinkageName, DIFile *F, + unsigned LineNo, DISubroutineType *Ty, + bool isLocalToUnit, bool isDefinition, + unsigned VK, unsigned VIndex, + int ThisAdjustment, DIType *VTableHolder, + DINode::DIFlags Flags, bool isOptimized, + DITemplateParameterArray TParams) { assert(getNonCompileUnitScope(Context) && "Methods should have both a Context and a context that isn't " "the compile unit."); @@ -680,9 +685,10 @@ DIBuilder::createMethod(DIScope *Context, StringRef Name, StringRef LinkageName, } DINamespace *DIBuilder::createNameSpace(DIScope *Scope, StringRef Name, - DIFile *File, unsigned LineNo) { + DIFile *File, unsigned LineNo, + bool ExportSymbols) { return DINamespace::get(VMContext, getNonCompileUnitScope(Scope), File, Name, - LineNo); + LineNo, ExportSymbols); } DIModule *DIBuilder::createModule(DIScope *Scope, StringRef Name, diff --git a/contrib/llvm/lib/IR/DataLayout.cpp b/contrib/llvm/lib/IR/DataLayout.cpp index 20a15fb8831f..d15a34c0b936 100644 --- a/contrib/llvm/lib/IR/DataLayout.cpp +++ b/contrib/llvm/lib/IR/DataLayout.cpp @@ -182,6 +182,7 @@ void DataLayout::reset(StringRef Desc) { BigEndian = false; StackNaturalAlign = 0; ManglingMode = MM_None; + NonIntegralAddressSpaces.clear(); // Default alignments for (const LayoutAlignElem &E : DefaultAlignments) { @@ -234,6 +235,19 @@ void DataLayout::parseSpecifier(StringRef Desc) { StringRef &Tok = Split.first; // Current token. StringRef &Rest = Split.second; // The rest of the string. + if (Tok == "ni") { + do { + Split = split(Rest, ':'); + Rest = Split.second; + unsigned AS = getInt(Split.first); + if (AS == 0) + report_fatal_error("Address space 0 can never be non-integral"); + NonIntegralAddressSpaces.push_back(AS); + } while (!Rest.empty()); + + continue; + } + char Specifier = Tok.front(); Tok = Tok.substr(1); @@ -492,10 +506,7 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, // with what clang and llvm-gcc do. unsigned Align = getTypeAllocSize(cast<VectorType>(Ty)->getElementType()); Align *= cast<VectorType>(Ty)->getNumElements(); - // If the alignment is not a power of 2, round up to the next power of 2. - // This happens for non-power-of-2 length vectors. - if (Align & (Align-1)) - Align = NextPowerOf2(Align); + Align = PowerOf2Ceil(Align); return Align; } } @@ -508,8 +519,7 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, // layout. if (BestMatchIdx == -1) { unsigned Align = getTypeStoreSize(Ty); - if (Align & (Align-1)) - Align = NextPowerOf2(Align); + Align = PowerOf2Ceil(Align); return Align; } @@ -727,15 +737,12 @@ int64_t DataLayout::getIndexedOffsetInType(Type *ElemTy, ArrayRef<Value *> Indices) const { int64_t Result = 0; - // We can use 0 as the address space as we don't need - // to get pointer types back from gep_type_iterator. - unsigned AS = 0; generic_gep_type_iterator<Value* const*> - GTI = gep_type_begin(ElemTy, AS, Indices), - GTE = gep_type_end(ElemTy, AS, Indices); + GTI = gep_type_begin(ElemTy, Indices), + GTE = gep_type_end(ElemTy, Indices); for (; GTI != GTE; ++GTI) { Value *Idx = GTI.getOperand(); - if (auto *STy = dyn_cast<StructType>(*GTI)) { + if (StructType *STy = GTI.getStructTypeOrNull()) { assert(Idx->getType()->isIntegerTy(32) && "Illegal struct idx"); unsigned FieldNo = cast<ConstantInt>(Idx)->getZExtValue(); diff --git a/contrib/llvm/lib/IR/DebugInfo.cpp b/contrib/llvm/lib/IR/DebugInfo.cpp index 1d3c8299255b..6b9bc689a446 100644 --- a/contrib/llvm/lib/IR/DebugInfo.cpp +++ b/contrib/llvm/lib/IR/DebugInfo.cpp @@ -53,11 +53,12 @@ void DebugInfoFinder::reset() { void DebugInfoFinder::processModule(const Module &M) { for (auto *CU : M.debug_compile_units()) { addCompileUnit(CU); - for (auto *DIG : CU->getGlobalVariables()) { - if (addGlobalVariable(DIG)) { - processScope(DIG->getScope()); - processType(DIG->getType().resolve()); - } + for (auto DIG : CU->getGlobalVariables()) { + if (!addGlobalVariable(DIG)) + continue; + auto *GV = DIG->getVariable(); + processScope(GV->getScope()); + processType(GV->getType().resolve()); } for (auto *ET : CU->getEnumTypes()) processType(ET); @@ -206,10 +207,7 @@ bool DebugInfoFinder::addCompileUnit(DICompileUnit *CU) { return true; } -bool DebugInfoFinder::addGlobalVariable(DIGlobalVariable *DIG) { - if (!DIG) - return false; - +bool DebugInfoFinder::addGlobalVariable(DIGlobalVariableExpression *DIG) { if (!NodesSeen.insert(DIG).second) return false; @@ -272,7 +270,11 @@ bool llvm::StripDebugInfo(Module &M) { NME = M.named_metadata_end(); NMI != NME;) { NamedMDNode *NMD = &*NMI; ++NMI; - if (NMD->getName().startswith("llvm.dbg.")) { + + // We're stripping debug info, and without them, coverage information + // doesn't quite make sense. + if (NMD->getName().startswith("llvm.dbg.") || + NMD->getName() == "llvm.gcov") { NMD->eraseFromParent(); Changed = true; } @@ -281,12 +283,314 @@ bool llvm::StripDebugInfo(Module &M) { for (Function &F : M) Changed |= stripDebugInfo(F); + for (auto &GV : M.globals()) { + SmallVector<MDNode *, 1> MDs; + GV.getMetadata(LLVMContext::MD_dbg, MDs); + if (!MDs.empty()) { + GV.eraseMetadata(LLVMContext::MD_dbg); + Changed = true; + } + } + if (GVMaterializer *Materializer = M.getMaterializer()) Materializer->setStripDebugInfo(); return Changed; } +namespace { + +/// Helper class to downgrade -g metadata to -gline-tables-only metadata. +class DebugTypeInfoRemoval { + DenseMap<Metadata *, Metadata *> Replacements; + +public: + /// The (void)() type. + MDNode *EmptySubroutineType; + +private: + /// Remember what linkage name we originally had before stripping. If we end + /// up making two subprograms identical who originally had different linkage + /// names, then we need to make one of them distinct, to avoid them getting + /// uniqued. Maps the new node to the old linkage name. + DenseMap<DISubprogram *, StringRef> NewToLinkageName; + + // TODO: Remember the distinct subprogram we created for a given linkage name, + // so that we can continue to unique whenever possible. Map <newly created + // node, old linkage name> to the first (possibly distinct) mdsubprogram + // created for that combination. This is not strictly needed for correctness, + // but can cut down on the number of MDNodes and let us diff cleanly with the + // output of -gline-tables-only. + +public: + DebugTypeInfoRemoval(LLVMContext &C) + : EmptySubroutineType(DISubroutineType::get(C, DINode::FlagZero, 0, + MDNode::get(C, {}))) {} + + Metadata *map(Metadata *M) { + if (!M) + return nullptr; + auto Replacement = Replacements.find(M); + if (Replacement != Replacements.end()) + return Replacement->second; + + return M; + } + MDNode *mapNode(Metadata *N) { return dyn_cast_or_null<MDNode>(map(N)); } + + /// Recursively remap N and all its referenced children. Does a DF post-order + /// traversal, so as to remap bottoms up. + void traverseAndRemap(MDNode *N) { traverse(N); } + +private: + // Create a new DISubprogram, to replace the one given. + DISubprogram *getReplacementSubprogram(DISubprogram *MDS) { + auto *FileAndScope = cast_or_null<DIFile>(map(MDS->getFile())); + StringRef LinkageName = MDS->getName().empty() ? MDS->getLinkageName() : ""; + DISubprogram *Declaration = nullptr; + auto *Type = cast_or_null<DISubroutineType>(map(MDS->getType())); + DITypeRef ContainingType(map(MDS->getContainingType())); + auto *Unit = cast_or_null<DICompileUnit>(map(MDS->getUnit())); + auto Variables = nullptr; + auto TemplateParams = nullptr; + + // Make a distinct DISubprogram, for situations that warrent it. + auto distinctMDSubprogram = [&]() { + return DISubprogram::getDistinct( + MDS->getContext(), FileAndScope, MDS->getName(), LinkageName, + FileAndScope, MDS->getLine(), Type, MDS->isLocalToUnit(), + MDS->isDefinition(), MDS->getScopeLine(), ContainingType, + MDS->getVirtuality(), MDS->getVirtualIndex(), + MDS->getThisAdjustment(), MDS->getFlags(), MDS->isOptimized(), Unit, + TemplateParams, Declaration, Variables); + }; + + if (MDS->isDistinct()) + return distinctMDSubprogram(); + + auto *NewMDS = DISubprogram::get( + MDS->getContext(), FileAndScope, MDS->getName(), LinkageName, + FileAndScope, MDS->getLine(), Type, MDS->isLocalToUnit(), + MDS->isDefinition(), MDS->getScopeLine(), ContainingType, + MDS->getVirtuality(), MDS->getVirtualIndex(), MDS->getThisAdjustment(), + MDS->getFlags(), MDS->isOptimized(), Unit, TemplateParams, Declaration, + Variables); + + StringRef OldLinkageName = MDS->getLinkageName(); + + // See if we need to make a distinct one. + auto OrigLinkage = NewToLinkageName.find(NewMDS); + if (OrigLinkage != NewToLinkageName.end()) { + if (OrigLinkage->second == OldLinkageName) + // We're good. + return NewMDS; + + // Otherwise, need to make a distinct one. + // TODO: Query the map to see if we already have one. + return distinctMDSubprogram(); + } + + NewToLinkageName.insert({NewMDS, MDS->getLinkageName()}); + return NewMDS; + } + + /// Create a new compile unit, to replace the one given + DICompileUnit *getReplacementCU(DICompileUnit *CU) { + // Drop skeleton CUs. + if (CU->getDWOId()) + return nullptr; + + auto *File = cast_or_null<DIFile>(map(CU->getFile())); + MDTuple *EnumTypes = nullptr; + MDTuple *RetainedTypes = nullptr; + MDTuple *GlobalVariables = nullptr; + MDTuple *ImportedEntities = nullptr; + return DICompileUnit::getDistinct( + CU->getContext(), CU->getSourceLanguage(), File, CU->getProducer(), + CU->isOptimized(), CU->getFlags(), CU->getRuntimeVersion(), + CU->getSplitDebugFilename(), DICompileUnit::LineTablesOnly, EnumTypes, + RetainedTypes, GlobalVariables, ImportedEntities, CU->getMacros(), + CU->getDWOId(), CU->getSplitDebugInlining()); + } + + DILocation *getReplacementMDLocation(DILocation *MLD) { + auto *Scope = map(MLD->getScope()); + auto *InlinedAt = map(MLD->getInlinedAt()); + if (MLD->isDistinct()) + return DILocation::getDistinct(MLD->getContext(), MLD->getLine(), + MLD->getColumn(), Scope, InlinedAt); + return DILocation::get(MLD->getContext(), MLD->getLine(), MLD->getColumn(), + Scope, InlinedAt); + } + + /// Create a new generic MDNode, to replace the one given + MDNode *getReplacementMDNode(MDNode *N) { + SmallVector<Metadata *, 8> Ops; + Ops.reserve(N->getNumOperands()); + for (auto &I : N->operands()) + if (I) + Ops.push_back(map(I)); + auto *Ret = MDNode::get(N->getContext(), Ops); + return Ret; + } + + /// Attempt to re-map N to a newly created node. + void remap(MDNode *N) { + if (Replacements.count(N)) + return; + + auto doRemap = [&](MDNode *N) -> MDNode * { + if (!N) + return nullptr; + if (auto *MDSub = dyn_cast<DISubprogram>(N)) { + remap(MDSub->getUnit()); + return getReplacementSubprogram(MDSub); + } + if (isa<DISubroutineType>(N)) + return EmptySubroutineType; + if (auto *CU = dyn_cast<DICompileUnit>(N)) + return getReplacementCU(CU); + if (isa<DIFile>(N)) + return N; + if (auto *MDLB = dyn_cast<DILexicalBlockBase>(N)) + // Remap to our referenced scope (recursively). + return mapNode(MDLB->getScope()); + if (auto *MLD = dyn_cast<DILocation>(N)) + return getReplacementMDLocation(MLD); + + // Otherwise, if we see these, just drop them now. Not strictly necessary, + // but this speeds things up a little. + if (isa<DINode>(N)) + return nullptr; + + return getReplacementMDNode(N); + }; + Replacements[N] = doRemap(N); + } + + /// Do the remapping traversal. + void traverse(MDNode *); +}; + +} // Anonymous namespace. + +void DebugTypeInfoRemoval::traverse(MDNode *N) { + if (!N || Replacements.count(N)) + return; + + // To avoid cycles, as well as for efficiency sake, we will sometimes prune + // parts of the graph. + auto prune = [](MDNode *Parent, MDNode *Child) { + if (auto *MDS = dyn_cast<DISubprogram>(Parent)) + return Child == MDS->getVariables().get(); + return false; + }; + + SmallVector<MDNode *, 16> ToVisit; + DenseSet<MDNode *> Opened; + + // Visit each node starting at N in post order, and map them. + ToVisit.push_back(N); + while (!ToVisit.empty()) { + auto *N = ToVisit.back(); + if (!Opened.insert(N).second) { + // Close it. + remap(N); + ToVisit.pop_back(); + continue; + } + for (auto &I : N->operands()) + if (auto *MDN = dyn_cast_or_null<MDNode>(I)) + if (!Opened.count(MDN) && !Replacements.count(MDN) && !prune(N, MDN) && + !isa<DICompileUnit>(MDN)) + ToVisit.push_back(MDN); + } +} + +bool llvm::stripNonLineTableDebugInfo(Module &M) { + bool Changed = false; + + // First off, delete the debug intrinsics. + auto RemoveUses = [&](StringRef Name) { + if (auto *DbgVal = M.getFunction(Name)) { + while (!DbgVal->use_empty()) + cast<Instruction>(DbgVal->user_back())->eraseFromParent(); + DbgVal->eraseFromParent(); + Changed = true; + } + }; + RemoveUses("llvm.dbg.declare"); + RemoveUses("llvm.dbg.value"); + + // Delete non-CU debug info named metadata nodes. + for (auto NMI = M.named_metadata_begin(), NME = M.named_metadata_end(); + NMI != NME;) { + NamedMDNode *NMD = &*NMI; + ++NMI; + // Specifically keep dbg.cu around. + if (NMD->getName() == "llvm.dbg.cu") + continue; + } + + // Drop all dbg attachments from global variables. + for (auto &GV : M.globals()) + GV.eraseMetadata(LLVMContext::MD_dbg); + + DebugTypeInfoRemoval Mapper(M.getContext()); + auto remap = [&](llvm::MDNode *Node) -> llvm::MDNode * { + if (!Node) + return nullptr; + Mapper.traverseAndRemap(Node); + auto *NewNode = Mapper.mapNode(Node); + Changed |= Node != NewNode; + Node = NewNode; + return NewNode; + }; + + // Rewrite the DebugLocs to be equivalent to what + // -gline-tables-only would have created. + for (auto &F : M) { + if (auto *SP = F.getSubprogram()) { + Mapper.traverseAndRemap(SP); + auto *NewSP = cast<DISubprogram>(Mapper.mapNode(SP)); + Changed |= SP != NewSP; + F.setSubprogram(NewSP); + } + for (auto &BB : F) { + for (auto &I : BB) { + if (I.getDebugLoc() == DebugLoc()) + continue; + + // Make a replacement. + auto &DL = I.getDebugLoc(); + auto *Scope = DL.getScope(); + MDNode *InlinedAt = DL.getInlinedAt(); + Scope = remap(Scope); + InlinedAt = remap(InlinedAt); + I.setDebugLoc( + DebugLoc::get(DL.getLine(), DL.getCol(), Scope, InlinedAt)); + } + } + } + + // Create a new llvm.dbg.cu, which is equivalent to the one + // -gline-tables-only would have created. + for (auto &NMD : M.getNamedMDList()) { + SmallVector<MDNode *, 8> Ops; + for (MDNode *Op : NMD.operands()) + Ops.push_back(remap(Op)); + + if (!Changed) + continue; + + NMD.clearOperands(); + for (auto *Op : Ops) + if (Op) + NMD.addOperand(Op); + } + return Changed; +} + unsigned llvm::getDebugMetadataVersionFromModule(const Module &M) { if (auto *Val = mdconst::dyn_extract_or_null<ConstantInt>( M.getModuleFlag("Debug Info Version"))) diff --git a/contrib/llvm/lib/IR/DebugInfoMetadata.cpp b/contrib/llvm/lib/IR/DebugInfoMetadata.cpp index c58e3685b3c9..8e21a907e15e 100644 --- a/contrib/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/contrib/llvm/lib/IR/DebugInfoMetadata.cpp @@ -65,29 +65,29 @@ DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line, Storage, Context.pImpl->DILocations); } -unsigned DINode::getFlag(StringRef Flag) { - return StringSwitch<unsigned>(Flag) +DINode::DIFlags DINode::getFlag(StringRef Flag) { + return StringSwitch<DIFlags>(Flag) #define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME) #include "llvm/IR/DebugInfoFlags.def" - .Default(0); + .Default(DINode::FlagZero); } -const char *DINode::getFlagString(unsigned Flag) { +StringRef DINode::getFlagString(DIFlags Flag) { switch (Flag) { - default: - return ""; #define HANDLE_DI_FLAG(ID, NAME) \ case Flag##NAME: \ return "DIFlag" #NAME; #include "llvm/IR/DebugInfoFlags.def" } + return ""; } -unsigned DINode::splitFlags(unsigned Flags, - SmallVectorImpl<unsigned> &SplitFlags) { - // Accessibility and member pointer flags need to be specially handled, since - // they're packed together. - if (unsigned A = Flags & FlagAccessibility) { +DINode::DIFlags DINode::splitFlags(DIFlags Flags, + SmallVectorImpl<DIFlags> &SplitFlags) { + // Flags that are packed together need to be specially handled, so + // that, for example, we emit "DIFlagPublic" and not + // "DIFlagPrivate | DIFlagProtected". + if (DIFlags A = Flags & FlagAccessibility) { if (A == FlagPrivate) SplitFlags.push_back(FlagPrivate); else if (A == FlagProtected) @@ -96,7 +96,7 @@ unsigned DINode::splitFlags(unsigned Flags, SplitFlags.push_back(FlagPublic); Flags &= ~A; } - if (unsigned R = Flags & FlagPtrToMemberRep) { + if (DIFlags R = Flags & FlagPtrToMemberRep) { if (R == FlagSingleInheritance) SplitFlags.push_back(FlagSingleInheritance); else if (R == FlagMultipleInheritance) @@ -105,14 +105,17 @@ unsigned DINode::splitFlags(unsigned Flags, SplitFlags.push_back(FlagVirtualInheritance); Flags &= ~R; } + if ((Flags & FlagIndirectVirtualBase) == FlagIndirectVirtualBase) { + Flags &= ~FlagIndirectVirtualBase; + SplitFlags.push_back(FlagIndirectVirtualBase); + } #define HANDLE_DI_FLAG(ID, NAME) \ - if (unsigned Bit = Flags & ID) { \ + if (DIFlags Bit = Flags & Flag##NAME) { \ SplitFlags.push_back(Bit); \ Flags &= ~Bit; \ } #include "llvm/IR/DebugInfoFlags.def" - return Flags; } @@ -229,7 +232,7 @@ DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, int64_t Value, DIBasicType *DIBasicType::getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, uint64_t SizeInBits, - uint64_t AlignInBits, unsigned Encoding, + uint32_t AlignInBits, unsigned Encoding, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DIBasicType, @@ -242,7 +245,7 @@ DIBasicType *DIBasicType::getImpl(LLVMContext &Context, unsigned Tag, DIDerivedType *DIDerivedType::getImpl( LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, + uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, Metadata *ExtraData, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DIDerivedType, @@ -257,7 +260,7 @@ DIDerivedType *DIDerivedType::getImpl( DICompositeType *DICompositeType::getImpl( LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, + uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, MDString *Identifier, StorageType Storage, bool ShouldCreate) { @@ -278,8 +281,8 @@ DICompositeType *DICompositeType::getImpl( DICompositeType *DICompositeType::buildODRType( LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, - uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, - unsigned Flags, Metadata *Elements, unsigned RuntimeLang, + uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, + DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams) { assert(!Identifier.getString().empty() && "Expected valid identifier"); if (!Context.isODRUniquingDebugTypes()) @@ -312,8 +315,8 @@ DICompositeType *DICompositeType::buildODRType( DICompositeType *DICompositeType::getODRType( LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, - uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, - unsigned Flags, Metadata *Elements, unsigned RuntimeLang, + uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, + DIFlags Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams) { assert(!Identifier.getString().empty() && "Expected valid identifier"); if (!Context.isODRUniquingDebugTypes()) @@ -335,9 +338,8 @@ DICompositeType *DICompositeType::getODRTypeIfExists(LLVMContext &Context, return Context.pImpl->DITypeMap->lookup(&Identifier); } -DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, - unsigned Flags, uint8_t CC, - Metadata *TypeArray, +DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, DIFlags Flags, + uint8_t CC, Metadata *TypeArray, StorageType Storage, bool ShouldCreate) { DEFINE_GETIMPL_LOOKUP(DISubroutineType, (Flags, CC, TypeArray)); @@ -345,14 +347,34 @@ DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, DEFINE_GETIMPL_STORE(DISubroutineType, (Flags, CC), Ops); } +static const char *ChecksumKindName[DIFile::CSK_Last + 1] = { + "CSK_None", + "CSK_MD5", + "CSK_SHA1" +}; + +DIFile::ChecksumKind DIFile::getChecksumKind(StringRef CSKindStr) { + return StringSwitch<DIFile::ChecksumKind>(CSKindStr) + .Case("CSK_MD5", DIFile::CSK_MD5) + .Case("CSK_SHA1", DIFile::CSK_SHA1) + .Default(DIFile::CSK_None); +} + +StringRef DIFile::getChecksumKindAsString() const { + assert(CSKind <= DIFile::CSK_Last && "Invalid checksum kind"); + return ChecksumKindName[CSKind]; +} + DIFile *DIFile::getImpl(LLVMContext &Context, MDString *Filename, - MDString *Directory, StorageType Storage, + MDString *Directory, DIFile::ChecksumKind CSKind, + MDString *Checksum, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Filename) && "Expected canonical MDString"); assert(isCanonical(Directory) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIFile, (Filename, Directory)); - Metadata *Ops[] = {Filename, Directory}; - DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIFile, Ops); + assert(isCanonical(Checksum) && "Expected canonical MDString"); + DEFINE_GETIMPL_LOOKUP(DIFile, (Filename, Directory, CSKind, Checksum)); + Metadata *Ops[] = {Filename, Directory, Checksum}; + DEFINE_GETIMPL_STORE(DIFile, (CSKind), Ops); } DICompileUnit *DICompileUnit::getImpl( @@ -361,7 +383,8 @@ DICompileUnit *DICompileUnit::getImpl( unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, - uint64_t DWOId, StorageType Storage, bool ShouldCreate) { + uint64_t DWOId, bool SplitDebugInlining, StorageType Storage, + bool ShouldCreate) { assert(Storage != Uniqued && "Cannot unique DICompileUnit"); assert(isCanonical(Producer) && "Expected canonical MDString"); assert(isCanonical(Flags) && "Expected canonical MDString"); @@ -371,9 +394,10 @@ DICompileUnit *DICompileUnit::getImpl( File, Producer, Flags, SplitDebugFilename, EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, Macros}; - return storeImpl(new (array_lengthof(Ops)) DICompileUnit( - Context, Storage, SourceLanguage, IsOptimized, - RuntimeVersion, EmissionKind, DWOId, Ops), + return storeImpl(new (array_lengthof(Ops)) + DICompileUnit(Context, Storage, SourceLanguage, + IsOptimized, RuntimeVersion, EmissionKind, + DWOId, SplitDebugInlining, Ops), Storage); } @@ -412,7 +436,7 @@ DISubprogram *DISubprogram::getImpl( MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, unsigned VirtualIndex, - int ThisAdjustment, unsigned Flags, bool IsOptimized, Metadata *Unit, + int ThisAdjustment, DIFlags Flags, bool IsOptimized, Metadata *Unit, Metadata *TemplateParams, Metadata *Declaration, Metadata *Variables, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); @@ -467,11 +491,12 @@ DILexicalBlockFile *DILexicalBlockFile::getImpl(LLVMContext &Context, DINamespace *DINamespace::getImpl(LLVMContext &Context, Metadata *Scope, Metadata *File, MDString *Name, unsigned Line, - StorageType Storage, bool ShouldCreate) { + bool ExportSymbols, StorageType Storage, + bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DINamespace, (Scope, File, Name, Line)); + DEFINE_GETIMPL_LOOKUP(DINamespace, (Scope, File, Name, Line, ExportSymbols)); Metadata *Ops[] = {File, Scope, Name}; - DEFINE_GETIMPL_STORE(DINamespace, (Line), Ops); + DEFINE_GETIMPL_STORE(DINamespace, (Line, ExportSymbols), Ops); } DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *Scope, @@ -509,25 +534,27 @@ DIGlobalVariable * DIGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, - Metadata *Variable, Metadata *StaticDataMemberDeclaration, - StorageType Storage, bool ShouldCreate) { + uint32_t AlignInBits, StorageType Storage, + bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); assert(isCanonical(LinkageName) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DIGlobalVariable, (Scope, Name, LinkageName, File, Line, Type, - IsLocalToUnit, IsDefinition, Variable, - StaticDataMemberDeclaration)); - Metadata *Ops[] = {Scope, Name, File, Type, - Name, LinkageName, Variable, StaticDataMemberDeclaration}; - DEFINE_GETIMPL_STORE(DIGlobalVariable, (Line, IsLocalToUnit, IsDefinition), + IsLocalToUnit, IsDefinition, + StaticDataMemberDeclaration, AlignInBits)); + Metadata *Ops[] = { + Scope, Name, File, Type, Name, LinkageName, StaticDataMemberDeclaration}; + DEFINE_GETIMPL_STORE(DIGlobalVariable, + (Line, IsLocalToUnit, IsDefinition, AlignInBits), Ops); } DILocalVariable *DILocalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, Metadata *File, unsigned Line, Metadata *Type, - unsigned Arg, unsigned Flags, + unsigned Arg, DIFlags Flags, + uint32_t AlignInBits, StorageType Storage, bool ShouldCreate) { // 64K ought to be enough for any frontend. @@ -536,9 +563,10 @@ DILocalVariable *DILocalVariable::getImpl(LLVMContext &Context, Metadata *Scope, assert(Scope && "Expected scope"); assert(isCanonical(Name) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DILocalVariable, - (Scope, Name, File, Line, Type, Arg, Flags)); + (Scope, Name, File, Line, Type, Arg, Flags, + AlignInBits)); Metadata *Ops[] = {Scope, Name, File, Type}; - DEFINE_GETIMPL_STORE(DILocalVariable, (Line, Arg, Flags), Ops); + DEFINE_GETIMPL_STORE(DILocalVariable, (Line, Arg, Flags, AlignInBits), Ops); } DIExpression *DIExpression::getImpl(LLVMContext &Context, @@ -550,8 +578,9 @@ DIExpression *DIExpression::getImpl(LLVMContext &Context, unsigned DIExpression::ExprOperand::getSize() const { switch (getOp()) { - case dwarf::DW_OP_bit_piece: + case dwarf::DW_OP_LLVM_fragment: return 3; + case dwarf::DW_OP_constu: case dwarf::DW_OP_plus: case dwarf::DW_OP_minus: return 2; @@ -570,9 +599,19 @@ bool DIExpression::isValid() const { switch (I->getOp()) { default: return false; - case dwarf::DW_OP_bit_piece: - // Piece expressions must be at the end. + case dwarf::DW_OP_LLVM_fragment: + // A fragment operator must appear at the end. return I->get() + I->getSize() == E->get(); + case dwarf::DW_OP_stack_value: { + // Must be the last one or followed by a DW_OP_LLVM_fragment. + if (I->get() + I->getSize() == E->get()) + break; + auto J = I; + if ((++J)->getOp() != dwarf::DW_OP_LLVM_fragment) + return false; + break; + } + case dwarf::DW_OP_constu: case dwarf::DW_OP_plus: case dwarf::DW_OP_minus: case dwarf::DW_OP_deref: @@ -582,22 +621,35 @@ bool DIExpression::isValid() const { return true; } -bool DIExpression::isBitPiece() const { - assert(isValid() && "Expected valid expression"); - if (unsigned N = getNumElements()) - if (N >= 3) - return getElement(N - 3) == dwarf::DW_OP_bit_piece; - return false; -} - -uint64_t DIExpression::getBitPieceOffset() const { - assert(isBitPiece() && "Expected bit piece"); - return getElement(getNumElements() - 2); +Optional<DIExpression::FragmentInfo> +DIExpression::getFragmentInfo(expr_op_iterator Start, expr_op_iterator End) { + for (auto I = Start; I != End; ++I) + if (I->getOp() == dwarf::DW_OP_LLVM_fragment) { + DIExpression::FragmentInfo Info = {I->getArg(1), I->getArg(0)}; + return Info; + } + return None; +} + +bool DIExpression::isConstant() const { + // Recognize DW_OP_constu C DW_OP_stack_value (DW_OP_LLVM_fragment Len Ofs)?. + if (getNumElements() != 3 && getNumElements() != 6) + return false; + if (getElement(0) != dwarf::DW_OP_constu || + getElement(2) != dwarf::DW_OP_stack_value) + return false; + if (getNumElements() == 6 && getElement(3) != dwarf::DW_OP_LLVM_fragment) + return false; + return true; } -uint64_t DIExpression::getBitPieceSize() const { - assert(isBitPiece() && "Expected bit piece"); - return getElement(getNumElements() - 1); +DIGlobalVariableExpression * +DIGlobalVariableExpression::getImpl(LLVMContext &Context, Metadata *Variable, + Metadata *Expression, StorageType Storage, + bool ShouldCreate) { + DEFINE_GETIMPL_LOOKUP(DIGlobalVariableExpression, (Variable, Expression)); + Metadata *Ops[] = {Variable, Expression}; + DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIGlobalVariableExpression, Ops); } DIObjCProperty *DIObjCProperty::getImpl( diff --git a/contrib/llvm/lib/IR/DiagnosticInfo.cpp b/contrib/llvm/lib/IR/DiagnosticInfo.cpp index ce67be328ab1..ea71fde26e0e 100644 --- a/contrib/llvm/lib/IR/DiagnosticInfo.cpp +++ b/contrib/llvm/lib/IR/DiagnosticInfo.cpp @@ -12,11 +12,12 @@ // Diagnostics reporting is still done as part of the LLVMContext. //===----------------------------------------------------------------------===// +#include "llvm/IR/DiagnosticInfo.h" #include "LLVMContextImpl.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" -#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" @@ -91,7 +92,7 @@ int llvm::getNextAvailablePluginDiagnosticKind() { return ++PluginKindID; } -const char *DiagnosticInfoOptimizationRemarkAnalysis::AlwaysPrint = ""; +const char *OptimizationRemarkAnalysis::AlwaysPrint = ""; DiagnosticInfoInlineAsm::DiagnosticInfoInlineAsm(const Instruction &I, const Twine &MsgStr, @@ -170,23 +171,110 @@ const std::string DiagnosticInfoWithDebugLocBase::getLocationStr() const { return (Filename + ":" + Twine(Line) + ":" + Twine(Column)).str(); } +DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, Value *V) + : Key(Key) { + if (auto *F = dyn_cast<Function>(V)) { + if (DISubprogram *SP = F->getSubprogram()) + DLoc = DebugLoc::get(SP->getScopeLine(), 0, SP); + } + else if (auto *I = dyn_cast<Instruction>(V)) + DLoc = I->getDebugLoc(); + + // Only include names that correspond to user variables. FIXME: we should use + // debug info if available to get the name of the user variable. + if (isa<llvm::Argument>(V) || isa<GlobalValue>(V)) + Val = GlobalValue::getRealLinkageName(V->getName()); + else if (isa<Constant>(V)) { + raw_string_ostream OS(Val); + V->printAsOperand(OS, /*PrintType=*/false); + } else if (auto *I = dyn_cast<Instruction>(V)) + Val = I->getOpcodeName(); +} + +DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, Type *T) + : Key(Key) { + raw_string_ostream OS(Val); + OS << *T; +} + +DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, int N) + : Key(Key), Val(itostr(N)) {} + +DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, unsigned N) + : Key(Key), Val(utostr(N)) {} + void DiagnosticInfoOptimizationBase::print(DiagnosticPrinter &DP) const { DP << getLocationStr() << ": " << getMsg(); if (Hotness) DP << " (hotness: " << *Hotness << ")"; } -bool DiagnosticInfoOptimizationRemark::isEnabled() const { +OptimizationRemark::OptimizationRemark(const char *PassName, + StringRef RemarkName, + const DebugLoc &DLoc, Value *CodeRegion) + : DiagnosticInfoOptimizationBase( + DK_OptimizationRemark, DS_Remark, PassName, RemarkName, + *cast<BasicBlock>(CodeRegion)->getParent(), DLoc, CodeRegion) {} + +OptimizationRemark::OptimizationRemark(const char *PassName, + StringRef RemarkName, Instruction *Inst) + : DiagnosticInfoOptimizationBase(DK_OptimizationRemark, DS_Remark, PassName, + RemarkName, + *Inst->getParent()->getParent(), + Inst->getDebugLoc(), Inst->getParent()) {} + +bool OptimizationRemark::isEnabled() const { return PassRemarksOptLoc.Pattern && PassRemarksOptLoc.Pattern->match(getPassName()); } -bool DiagnosticInfoOptimizationRemarkMissed::isEnabled() const { +OptimizationRemarkMissed::OptimizationRemarkMissed(const char *PassName, + StringRef RemarkName, + const DebugLoc &DLoc, + Value *CodeRegion) + : DiagnosticInfoOptimizationBase( + DK_OptimizationRemarkMissed, DS_Remark, PassName, RemarkName, + *cast<BasicBlock>(CodeRegion)->getParent(), DLoc, CodeRegion) {} + +OptimizationRemarkMissed::OptimizationRemarkMissed(const char *PassName, + StringRef RemarkName, + Instruction *Inst) + : DiagnosticInfoOptimizationBase(DK_OptimizationRemarkMissed, DS_Remark, + PassName, RemarkName, + *Inst->getParent()->getParent(), + Inst->getDebugLoc(), Inst->getParent()) {} + +bool OptimizationRemarkMissed::isEnabled() const { return PassRemarksMissedOptLoc.Pattern && PassRemarksMissedOptLoc.Pattern->match(getPassName()); } -bool DiagnosticInfoOptimizationRemarkAnalysis::isEnabled() const { +OptimizationRemarkAnalysis::OptimizationRemarkAnalysis(const char *PassName, + StringRef RemarkName, + const DebugLoc &DLoc, + Value *CodeRegion) + : DiagnosticInfoOptimizationBase( + DK_OptimizationRemarkAnalysis, DS_Remark, PassName, RemarkName, + *cast<BasicBlock>(CodeRegion)->getParent(), DLoc, CodeRegion) {} + +OptimizationRemarkAnalysis::OptimizationRemarkAnalysis(const char *PassName, + StringRef RemarkName, + Instruction *Inst) + : DiagnosticInfoOptimizationBase(DK_OptimizationRemarkAnalysis, DS_Remark, + PassName, RemarkName, + *Inst->getParent()->getParent(), + Inst->getDebugLoc(), Inst->getParent()) {} + +OptimizationRemarkAnalysis::OptimizationRemarkAnalysis(enum DiagnosticKind Kind, + const char *PassName, + StringRef RemarkName, + const DebugLoc &DLoc, + Value *CodeRegion) + : DiagnosticInfoOptimizationBase(Kind, DS_Remark, PassName, RemarkName, + *cast<BasicBlock>(CodeRegion)->getParent(), + DLoc, CodeRegion) {} + +bool OptimizationRemarkAnalysis::isEnabled() const { return shouldAlwaysPrint() || (PassRemarksAnalysisOptLoc.Pattern && PassRemarksAnalysisOptLoc.Pattern->match(getPassName())); @@ -199,14 +287,14 @@ void DiagnosticInfoMIRParser::print(DiagnosticPrinter &DP) const { void llvm::emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { - Ctx.diagnose(DiagnosticInfoOptimizationRemark(PassName, Fn, DLoc, Msg)); + Ctx.diagnose(OptimizationRemark(PassName, Fn, DLoc, Msg)); } void llvm::emitOptimizationRemarkMissed(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { - Ctx.diagnose(DiagnosticInfoOptimizationRemarkMissed(PassName, Fn, DLoc, Msg)); + Ctx.diagnose(OptimizationRemarkMissed(PassName, Fn, DLoc, Msg)); } void llvm::emitOptimizationRemarkAnalysis(LLVMContext &Ctx, @@ -214,8 +302,7 @@ void llvm::emitOptimizationRemarkAnalysis(LLVMContext &Ctx, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { - Ctx.diagnose( - DiagnosticInfoOptimizationRemarkAnalysis(PassName, Fn, DLoc, Msg)); + Ctx.diagnose(OptimizationRemarkAnalysis(PassName, Fn, DLoc, Msg)); } void llvm::emitOptimizationRemarkAnalysisFPCommute(LLVMContext &Ctx, @@ -223,8 +310,7 @@ void llvm::emitOptimizationRemarkAnalysisFPCommute(LLVMContext &Ctx, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { - Ctx.diagnose(DiagnosticInfoOptimizationRemarkAnalysisFPCommute(PassName, Fn, - DLoc, Msg)); + Ctx.diagnose(OptimizationRemarkAnalysisFPCommute(PassName, Fn, DLoc, Msg)); } void llvm::emitOptimizationRemarkAnalysisAliasing(LLVMContext &Ctx, @@ -232,8 +318,7 @@ void llvm::emitOptimizationRemarkAnalysisAliasing(LLVMContext &Ctx, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { - Ctx.diagnose(DiagnosticInfoOptimizationRemarkAnalysisAliasing(PassName, Fn, - DLoc, Msg)); + Ctx.diagnose(OptimizationRemarkAnalysisAliasing(PassName, Fn, DLoc, Msg)); } bool DiagnosticInfoOptimizationFailure::isEnabled() const { @@ -262,3 +347,42 @@ void llvm::emitLoopInterleaveWarning(LLVMContext &Ctx, const Function &Fn, Ctx.diagnose(DiagnosticInfoOptimizationFailure( Fn, DLoc, Twine("loop not interleaved: " + Msg))); } + +void DiagnosticInfoISelFallback::print(DiagnosticPrinter &DP) const { + DP << "Instruction selection used fallback path for " << getFunction(); +} + +DiagnosticInfoOptimizationBase &DiagnosticInfoOptimizationBase:: +operator<<(StringRef S) { + Args.emplace_back(S); + return *this; +} + +DiagnosticInfoOptimizationBase &DiagnosticInfoOptimizationBase:: +operator<<(Argument A) { + Args.push_back(std::move(A)); + return *this; +} + +DiagnosticInfoOptimizationBase &DiagnosticInfoOptimizationBase:: +operator<<(setIsVerbose V) { + IsVerbose = true; + return *this; +} + +DiagnosticInfoOptimizationBase &DiagnosticInfoOptimizationBase:: +operator<<(setExtraArgs EA) { + FirstExtraArgIndex = Args.size(); + return *this; +} + +std::string DiagnosticInfoOptimizationBase::getMsg() const { + std::string Str; + raw_string_ostream OS(Str); + for (const DiagnosticInfoOptimizationBase::Argument &Arg : + make_range(Args.begin(), FirstExtraArgIndex == -1 + ? Args.end() + : Args.begin() + FirstExtraArgIndex)) + OS << Arg.Val; + return OS.str(); +} diff --git a/contrib/llvm/lib/IR/Dominators.cpp b/contrib/llvm/lib/IR/Dominators.cpp index 57e3df76d023..1880807da7eb 100644 --- a/contrib/llvm/lib/IR/Dominators.cpp +++ b/contrib/llvm/lib/IR/Dominators.cpp @@ -64,9 +64,13 @@ template class llvm::DomTreeNodeBase<BasicBlock>; template class llvm::DominatorTreeBase<BasicBlock>; template void llvm::Calculate<Function, BasicBlock *>( - DominatorTreeBase<GraphTraits<BasicBlock *>::NodeType> &DT, Function &F); + DominatorTreeBase< + typename std::remove_pointer<GraphTraits<BasicBlock *>::NodeRef>::type> + &DT, + Function &F); template void llvm::Calculate<Function, Inverse<BasicBlock *>>( - DominatorTreeBase<GraphTraits<Inverse<BasicBlock *>>::NodeType> &DT, + DominatorTreeBase<typename std::remove_pointer< + GraphTraits<Inverse<BasicBlock *>>::NodeRef>::type> &DT, Function &F); // dominates - Return true if Def dominates a use in User. This performs @@ -301,13 +305,13 @@ void DominatorTree::verifyDomTree() const { //===----------------------------------------------------------------------===// DominatorTree DominatorTreeAnalysis::run(Function &F, - AnalysisManager<Function> &) { + FunctionAnalysisManager &) { DominatorTree DT; DT.recalculate(F); return DT; } -char DominatorTreeAnalysis::PassID; +AnalysisKey DominatorTreeAnalysis::Key; DominatorTreePrinterPass::DominatorTreePrinterPass(raw_ostream &OS) : OS(OS) {} diff --git a/contrib/llvm/lib/IR/Function.cpp b/contrib/llvm/lib/IR/Function.cpp index e1223d0d0337..05419aa3d2bb 100644 --- a/contrib/llvm/lib/IR/Function.cpp +++ b/contrib/llvm/lib/IR/Function.cpp @@ -26,10 +26,6 @@ #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/RWMutex.h" -#include "llvm/Support/StringPool.h" -#include "llvm/Support/Threading.h" using namespace llvm; // Explicit instantiations of SymbolTableListTraits since some of the methods @@ -262,7 +258,10 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name, assert(FunctionType::isValidReturnType(getReturnType()) && "invalid return type"); setGlobalObjectSubClassData(0); - SymTab = new ValueSymbolTable(); + + // We only need a symbol table for a function if the context keeps value names + if (!getContext().shouldDiscardValueNames()) + SymTab = make_unique<ValueSymbolTable>(); // If the function has arguments, mark them as lazily built. if (Ty->getNumParams()) @@ -271,6 +270,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name, if (ParentModule) ParentModule->getFunctionList().push_back(this); + HasLLVMReservedName = getName().startswith("llvm."); // Ensure intrinsics have the right parameter attributes. // Note, the IntID field will have been set in Value::setName if this function // name is a valid intrinsic ID. @@ -283,7 +283,6 @@ Function::~Function() { // Delete all of the method arguments and unlink from symbol table... ArgumentList.clear(); - delete SymTab; // Remove the function from the on-the-side GC table. clearGC(); @@ -332,10 +331,6 @@ bool Function::arg_empty() const { return getFunctionType()->getNumParams() == 0; } -void Function::setParent(Module *parent) { - Parent = parent; -} - // dropAllReferences() - This function causes all the subinstructions to "let // go" of all references that they are maintaining. This allows one to // 'delete' a whole class at a time, even though there may be circular @@ -488,9 +483,7 @@ static ArrayRef<const char *> findTargetSubtable(StringRef Name) { /// \brief This does the actual lookup of an intrinsic ID which /// matches the given function name. -static Intrinsic::ID lookupIntrinsicID(const ValueName *ValName) { - StringRef Name = ValName->getKey(); - +Intrinsic::ID Function::lookupIntrinsicID(StringRef Name) { ArrayRef<const char *> NameTable = findTargetSubtable(Name); int Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name); if (Idx == -1) @@ -508,12 +501,14 @@ static Intrinsic::ID lookupIntrinsicID(const ValueName *ValName) { } void Function::recalculateIntrinsicID() { - const ValueName *ValName = this->getValueName(); - if (!ValName || !isIntrinsic()) { + StringRef Name = getName(); + if (!Name.startswith("llvm.")) { + HasLLVMReservedName = false; IntID = Intrinsic::not_intrinsic; return; } - IntID = lookupIntrinsicID(ValName); + HasLLVMReservedName = true; + IntID = lookupIntrinsicID(Name); } /// Returns a stable mangling for the type specified for use in the name @@ -557,6 +552,13 @@ static std::string getMangledTypeStr(Type* Ty) { return Result; } +StringRef Intrinsic::getName(ID id) { + assert(id < num_intrinsics && "Invalid intrinsic ID!"); + assert(!isOverloaded(id) && + "This version of getName does not support overloading"); + return IntrinsicNameTable[id]; +} + std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) { assert(id < num_intrinsics && "Invalid intrinsic ID!"); std::string Result(IntrinsicNameTable[id]); @@ -608,10 +610,11 @@ enum IIT_Info { IIT_HALF_VEC_ARG = 30, IIT_SAME_VEC_WIDTH_ARG = 31, IIT_PTR_TO_ARG = 32, - IIT_VEC_OF_PTRS_TO_ELT = 33, - IIT_I128 = 34, - IIT_V512 = 35, - IIT_V1024 = 36 + IIT_PTR_TO_ELT = 33, + IIT_VEC_OF_PTRS_TO_ELT = 34, + IIT_I128 = 35, + IIT_V512 = 36, + IIT_V1024 = 37 }; @@ -745,6 +748,11 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, ArgInfo)); return; } + case IIT_PTR_TO_ELT: { + unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToElt, ArgInfo)); + return; + } case IIT_VEC_OF_PTRS_TO_ELT: { unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecOfPtrsToElt, @@ -754,9 +762,9 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, case IIT_EMPTYSTRUCT: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0)); return; - case IIT_STRUCT5: ++StructElts; // FALL THROUGH. - case IIT_STRUCT4: ++StructElts; // FALL THROUGH. - case IIT_STRUCT3: ++StructElts; // FALL THROUGH. + case IIT_STRUCT5: ++StructElts; LLVM_FALLTHROUGH; + case IIT_STRUCT4: ++StructElts; LLVM_FALLTHROUGH; + case IIT_STRUCT3: ++StructElts; LLVM_FALLTHROUGH; case IIT_STRUCT2: { OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct,StructElts)); @@ -871,6 +879,14 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, Type *Ty = Tys[D.getArgumentNumber()]; return PointerType::getUnqual(Ty); } + case IITDescriptor::PtrToElt: { + Type *Ty = Tys[D.getArgumentNumber()]; + VectorType *VTy = dyn_cast<VectorType>(Ty); + if (!VTy) + llvm_unreachable("Expected an argument of Vector Type"); + Type *EltTy = VTy->getVectorElementType(); + return PointerType::getUnqual(EltTy); + } case IITDescriptor::VecOfPtrsToElt: { Type *Ty = Tys[D.getArgumentNumber()]; VectorType *VTy = dyn_cast<VectorType>(Ty); @@ -1049,7 +1065,7 @@ bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> if (D.getArgumentNumber() >= ArgTys.size()) return true; VectorType * ReferenceType = - dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]); + dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]); VectorType *ThisArgType = dyn_cast<VectorType>(Ty); if (!ThisArgType || !ReferenceType || (ReferenceType->getVectorNumElements() != @@ -1065,6 +1081,16 @@ bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> PointerType *ThisArgType = dyn_cast<PointerType>(Ty); return (!ThisArgType || ThisArgType->getElementType() != ReferenceType); } + case IITDescriptor::PtrToElt: { + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + VectorType * ReferenceType = + dyn_cast<VectorType> (ArgTys[D.getArgumentNumber()]); + PointerType *ThisArgType = dyn_cast<PointerType>(Ty); + + return (!ThisArgType || !ReferenceType || + ThisArgType->getElementType() != ReferenceType->getElementType()); + } case IITDescriptor::VecOfPtrsToElt: { if (D.getArgumentNumber() >= ArgTys.size()) return true; @@ -1264,7 +1290,27 @@ Optional<uint64_t> Function::getEntryCount() const { if (MDString *MDS = dyn_cast<MDString>(MD->getOperand(0))) if (MDS->getString().equals("function_entry_count")) { ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(1)); - return CI->getValue().getZExtValue(); + uint64_t Count = CI->getValue().getZExtValue(); + if (Count == 0) + return None; + return Count; } return None; } + +void Function::setSectionPrefix(StringRef Prefix) { + MDBuilder MDB(getContext()); + setMetadata(LLVMContext::MD_section_prefix, + MDB.createFunctionSectionPrefix(Prefix)); +} + +Optional<StringRef> Function::getSectionPrefix() const { + if (MDNode *MD = getMetadata(LLVMContext::MD_section_prefix)) { + assert(dyn_cast<MDString>(MD->getOperand(0)) + ->getString() + .equals("function_section_prefix") && + "Metadata not match"); + return dyn_cast<MDString>(MD->getOperand(1))->getString(); + } + return None; +} diff --git a/contrib/llvm/lib/IR/GCOV.cpp b/contrib/llvm/lib/IR/GCOV.cpp index a9f7f45ee305..3bbcf781e5dd 100644 --- a/contrib/llvm/lib/IR/GCOV.cpp +++ b/contrib/llvm/lib/IR/GCOV.cpp @@ -17,7 +17,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Format.h" -#include "llvm/Support/MemoryObject.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> diff --git a/contrib/llvm/lib/IR/Globals.cpp b/contrib/llvm/lib/IR/Globals.cpp index 6715484518a9..31f89514151c 100644 --- a/contrib/llvm/lib/IR/Globals.cpp +++ b/contrib/llvm/lib/IR/Globals.cpp @@ -15,12 +15,14 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/ConstantRange.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" +#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" using namespace llvm; @@ -28,12 +30,19 @@ using namespace llvm; // GlobalValue Class //===----------------------------------------------------------------------===// +// GlobalValue should be a Constant, plus a type, a module, some flags, and an +// intrinsic ID. Add an assert to prevent people from accidentally growing +// GlobalValue while adding flags. +static_assert(sizeof(GlobalValue) == + sizeof(Constant) + 2 * sizeof(void *) + 2 * sizeof(unsigned), + "unexpected GlobalValue size growth"); + bool GlobalValue::isMaterializable() const { if (const Function *F = dyn_cast<Function>(this)) return F->isMaterializable(); return false; } -std::error_code GlobalValue::materialize() { +Error GlobalValue::materialize() { return getParent()->materialize(this); } @@ -213,6 +222,34 @@ bool GlobalValue::canIncreaseAlignment() const { return true; } +GlobalObject *GlobalValue::getBaseObject() { + if (auto *GO = dyn_cast<GlobalObject>(this)) + return GO; + if (auto *GA = dyn_cast<GlobalAlias>(this)) + return GA->getBaseObject(); + return nullptr; +} + +bool GlobalValue::isAbsoluteSymbolRef() const { + auto *GO = dyn_cast<GlobalObject>(this); + if (!GO) + return false; + + return GO->getMetadata(LLVMContext::MD_absolute_symbol); +} + +Optional<ConstantRange> GlobalValue::getAbsoluteSymbolRange() const { + auto *GO = dyn_cast<GlobalObject>(this); + if (!GO) + return None; + + MDNode *MD = GO->getMetadata(LLVMContext::MD_absolute_symbol); + if (!MD) + return None; + + return getConstantRangeFromMetadata(*MD); +} + //===----------------------------------------------------------------------===// // GlobalVariable Implementation //===----------------------------------------------------------------------===// @@ -257,10 +294,6 @@ GlobalVariable::GlobalVariable(Module &M, Type *Ty, bool constant, M.getGlobalList().push_back(this); } -void GlobalVariable::setParent(Module *parent) { - Parent = parent; -} - void GlobalVariable::removeFromParent() { getParent()->getGlobalList().remove(getIterator()); } @@ -359,10 +392,6 @@ GlobalAlias *GlobalAlias::create(const Twine &Name, GlobalValue *Aliasee) { return create(Aliasee->getLinkage(), Name, Aliasee); } -void GlobalAlias::setParent(Module *parent) { - Parent = parent; -} - void GlobalAlias::removeFromParent() { getParent()->getAliasList().remove(getIterator()); } @@ -396,10 +425,6 @@ GlobalIFunc *GlobalIFunc::create(Type *Ty, unsigned AddressSpace, return new GlobalIFunc(Ty, AddressSpace, Link, Name, Resolver, ParentModule); } -void GlobalIFunc::setParent(Module *parent) { - Parent = parent; -} - void GlobalIFunc::removeFromParent() { getParent()->getIFuncList().remove(getIterator()); } diff --git a/contrib/llvm/lib/IR/IRBuilder.cpp b/contrib/llvm/lib/IR/IRBuilder.cpp index 298331d51c88..d3e410d6d033 100644 --- a/contrib/llvm/lib/IR/IRBuilder.cpp +++ b/contrib/llvm/lib/IR/IRBuilder.cpp @@ -191,6 +191,26 @@ CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) { return createCallHelper(TheFn, Ops, this); } +CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { + + assert(isa<PointerType>(Ptr->getType()) && + "invariant.start only applies to pointers."); + Ptr = getCastedInt8PtrValue(Ptr); + if (!Size) + Size = getInt64(-1); + else + assert(Size->getType() == getInt64Ty() && + "invariant.start requires the size to be an i64"); + + Value *Ops[] = {Size, Ptr}; + // Fill in the single overloaded type: memory object type. + Type *ObjectPtr[1] = {Ptr->getType()}; + Module *M = BB->getParent()->getParent(); + Value *TheFn = + Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr); + return createCallHelper(TheFn, Ops, this); +} + CallInst *IRBuilderBase::CreateAssumption(Value *Cond) { assert(Cond->getType() == getInt1Ty() && "an assumption condition must be of type i1"); diff --git a/contrib/llvm/lib/IR/IRPrintingPasses.cpp b/contrib/llvm/lib/IR/IRPrintingPasses.cpp index 4d2f9b98911b..05e206cfd6cb 100644 --- a/contrib/llvm/lib/IR/IRPrintingPasses.cpp +++ b/contrib/llvm/lib/IR/IRPrintingPasses.cpp @@ -26,7 +26,7 @@ PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner, : OS(OS), Banner(Banner), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {} -PreservedAnalyses PrintModulePass::run(Module &M, AnalysisManager<Module> &) { +PreservedAnalyses PrintModulePass::run(Module &M, ModuleAnalysisManager &) { OS << Banner; if (llvm::isFunctionInPrintList("*")) M.print(OS, nullptr, ShouldPreserveUseListOrder); @@ -43,7 +43,7 @@ PrintFunctionPass::PrintFunctionPass(raw_ostream &OS, const std::string &Banner) : OS(OS), Banner(Banner) {} PreservedAnalyses PrintFunctionPass::run(Function &F, - AnalysisManager<Function> &) { + FunctionAnalysisManager &) { if (isFunctionInPrintList(F.getName())) OS << Banner << static_cast<Value &>(F); return PreservedAnalyses::all(); diff --git a/contrib/llvm/lib/IR/InlineAsm.cpp b/contrib/llvm/lib/IR/InlineAsm.cpp index d6cf8c543dbd..5a9118571040 100644 --- a/contrib/llvm/lib/IR/InlineAsm.cpp +++ b/contrib/llvm/lib/IR/InlineAsm.cpp @@ -265,7 +265,7 @@ bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) { break; } ++NumIndirect; - // FALLTHROUGH for Indirect Outputs. + LLVM_FALLTHROUGH; // We fall through for Indirect Outputs. case InlineAsm::isInput: if (NumClobbers) return false; // inputs before clobbers. ++NumInputs; diff --git a/contrib/llvm/lib/IR/Instruction.cpp b/contrib/llvm/lib/IR/Instruction.cpp index ed08f85c60b6..2fa03489081d 100644 --- a/contrib/llvm/lib/IR/Instruction.cpp +++ b/contrib/llvm/lib/IR/Instruction.cpp @@ -11,6 +11,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/DenseSet.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" @@ -92,8 +93,13 @@ void Instruction::insertAfter(Instruction *InsertPos) { /// Unlink this instruction from its current basic block and insert it into the /// basic block that MovePos lives in, right before MovePos. void Instruction::moveBefore(Instruction *MovePos) { - MovePos->getParent()->getInstList().splice( - MovePos->getIterator(), getParent()->getInstList(), getIterator()); + moveBefore(*MovePos->getParent(), MovePos->getIterator()); +} + +void Instruction::moveBefore(BasicBlock &BB, + SymbolTableList<Instruction>::iterator I) { + assert(I == BB.end() || I->getParent() == &BB); + BB.getInstList().splice(I, getParent()->getInstList(), getIterator()); } void Instruction::setHasNoUnsignedWrap(bool b) { @@ -120,47 +126,31 @@ bool Instruction::isExact() const { return cast<PossiblyExactOperator>(this)->isExact(); } -/// Set or clear the unsafe-algebra flag on this instruction, which must be an -/// operator which supports this flag. See LangRef.html for the meaning of this -/// flag. void Instruction::setHasUnsafeAlgebra(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasUnsafeAlgebra(B); } -/// Set or clear the NoNaNs flag on this instruction, which must be an operator -/// which supports this flag. See LangRef.html for the meaning of this flag. void Instruction::setHasNoNaNs(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasNoNaNs(B); } -/// Set or clear the no-infs flag on this instruction, which must be an operator -/// which supports this flag. See LangRef.html for the meaning of this flag. void Instruction::setHasNoInfs(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasNoInfs(B); } -/// Set or clear the no-signed-zeros flag on this instruction, which must be an -/// operator which supports this flag. See LangRef.html for the meaning of this -/// flag. void Instruction::setHasNoSignedZeros(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasNoSignedZeros(B); } -/// Set or clear the allow-reciprocal flag on this instruction, which must be an -/// operator which supports this flag. See LangRef.html for the meaning of this -/// flag. void Instruction::setHasAllowReciprocal(bool B) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setHasAllowReciprocal(B); } -/// Convenience function for setting all the fast-math flags on this -/// instruction, which must be an operator which supports these flags. See -/// LangRef.html for the meaning of these flats. void Instruction::setFastMathFlags(FastMathFlags FMF) { assert(isa<FPMathOperator>(this) && "setting fast-math flag on invalid op"); cast<FPMathOperator>(this)->setFastMathFlags(FMF); @@ -171,45 +161,36 @@ void Instruction::copyFastMathFlags(FastMathFlags FMF) { cast<FPMathOperator>(this)->copyFastMathFlags(FMF); } -/// Determine whether the unsafe-algebra flag is set. bool Instruction::hasUnsafeAlgebra() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->hasUnsafeAlgebra(); } -/// Determine whether the no-NaNs flag is set. bool Instruction::hasNoNaNs() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->hasNoNaNs(); } -/// Determine whether the no-infs flag is set. bool Instruction::hasNoInfs() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->hasNoInfs(); } -/// Determine whether the no-signed-zeros flag is set. bool Instruction::hasNoSignedZeros() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->hasNoSignedZeros(); } -/// Determine whether the allow-reciprocal flag is set. bool Instruction::hasAllowReciprocal() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->hasAllowReciprocal(); } -/// Convenience function for getting all the fast-math flags, which must be an -/// operator which supports these flags. See LangRef.html for the meaning of -/// these flags. FastMathFlags Instruction::getFastMathFlags() const { assert(isa<FPMathOperator>(this) && "getting fast-math flag on invalid op"); return cast<FPMathOperator>(this)->getFastMathFlags(); } -/// Copy I's fast-math flags void Instruction::copyFastMathFlags(const Instruction *I) { copyFastMathFlags(I->getFastMathFlags()); } @@ -343,7 +324,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) { } } -/// Return true if both instructions have the same special state This must be +/// Return true if both instructions have the same special state. This must be /// kept in sync with FunctionComparator::cmpOperations in /// lib/Transforms/IPO/MergeFunctions.cpp. static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, @@ -402,17 +383,11 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, return true; } -/// isIdenticalTo - Return true if the specified instruction is exactly -/// identical to the current one. This means that all operands match and any -/// extra information (e.g. load is volatile) agree. bool Instruction::isIdenticalTo(const Instruction *I) const { return isIdenticalToWhenDefined(I) && SubclassOptionalData == I->SubclassOptionalData; } -/// isIdenticalToWhenDefined - This is like isIdenticalTo, except that it -/// ignores the SubclassOptionalData flags, which specify conditions -/// under which the instruction's result is undefined. bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const { if (getOpcode() != I->getOpcode() || getNumOperands() != I->getNumOperands() || @@ -463,9 +438,6 @@ bool Instruction::isSameOperationAs(const Instruction *I, return haveSameSpecialState(this, I, IgnoreAlignment); } -/// isUsedOutsideOfBlock - Return true if there are any uses of I outside of the -/// specified block. Note that PHI nodes are considered to evaluate their -/// operands in the corresponding predecessor block. bool Instruction::isUsedOutsideOfBlock(const BasicBlock *BB) const { for (const Use &U : uses()) { // PHI nodes uses values in the corresponding predecessor block. For other @@ -484,8 +456,6 @@ bool Instruction::isUsedOutsideOfBlock(const BasicBlock *BB) const { return false; } -/// mayReadFromMemory - Return true if this instruction may read memory. -/// bool Instruction::mayReadFromMemory() const { switch (getOpcode()) { default: return false; @@ -506,8 +476,6 @@ bool Instruction::mayReadFromMemory() const { } } -/// mayWriteToMemory - Return true if this instruction may modify memory. -/// bool Instruction::mayWriteToMemory() const { switch (getOpcode()) { default: return false; @@ -553,7 +521,7 @@ bool Instruction::mayThrow() const { return isa<ResumeInst>(this); } -/// isAssociative - Return true if the instruction is associative: +/// Return true if the instruction is associative: /// /// Associative operators satisfy: x op (y op z) === (x op y) op z /// @@ -578,7 +546,7 @@ bool Instruction::isAssociative() const { } } -/// isCommutative - Return true if the instruction is commutative: +/// Return true if the instruction is commutative: /// /// Commutative operators satisfy: (x op y) === (y op x) /// @@ -600,7 +568,7 @@ bool Instruction::isCommutative(unsigned op) { } } -/// isIdempotent - Return true if the instruction is idempotent: +/// Return true if the instruction is idempotent: /// /// Idempotent operators satisfy: x op x === x /// @@ -610,7 +578,7 @@ bool Instruction::isIdempotent(unsigned Opcode) { return Opcode == And || Opcode == Or; } -/// isNilpotent - Return true if the instruction is nilpotent: +/// Return true if the instruction is nilpotent: /// /// Nilpotent operators satisfy: x op x === Id, /// @@ -627,6 +595,45 @@ Instruction *Instruction::cloneImpl() const { llvm_unreachable("Subclass of Instruction failed to implement cloneImpl"); } +void Instruction::swapProfMetadata() { + MDNode *ProfileData = getMetadata(LLVMContext::MD_prof); + if (!ProfileData || ProfileData->getNumOperands() != 3 || + !isa<MDString>(ProfileData->getOperand(0))) + return; + + MDString *MDName = cast<MDString>(ProfileData->getOperand(0)); + if (MDName->getString() != "branch_weights") + return; + + // The first operand is the name. Fetch them backwards and build a new one. + Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2), + ProfileData->getOperand(1)}; + setMetadata(LLVMContext::MD_prof, + MDNode::get(ProfileData->getContext(), Ops)); +} + +void Instruction::copyMetadata(const Instruction &SrcInst, + ArrayRef<unsigned> WL) { + if (!SrcInst.hasMetadata()) + return; + + DenseSet<unsigned> WLS; + for (unsigned M : WL) + WLS.insert(M); + + // Otherwise, enumerate and copy over metadata from the old instruction to the + // new one. + SmallVector<std::pair<unsigned, MDNode *>, 4> TheMDs; + SrcInst.getAllMetadataOtherThanDebugLoc(TheMDs); + for (const auto &MD : TheMDs) { + if (WL.empty() || WLS.count(MD.first)) + setMetadata(MD.first, MD.second); + } + if (WL.empty() || WLS.count(LLVMContext::MD_dbg)) + setDebugLoc(SrcInst.getDebugLoc()); + return; +} + Instruction *Instruction::clone() const { Instruction *New = nullptr; switch (getOpcode()) { @@ -641,16 +648,6 @@ Instruction *Instruction::clone() const { } New->SubclassOptionalData = SubclassOptionalData; - if (!hasMetadata()) - return New; - - // Otherwise, enumerate and copy over metadata from the old instruction to the - // new one. - SmallVector<std::pair<unsigned, MDNode *>, 4> TheMDs; - getAllMetadataOtherThanDebugLoc(TheMDs); - for (const auto &MD : TheMDs) - New->setMetadata(MD.first, MD.second); - - New->setDebugLoc(getDebugLoc()); + New->copyMetadata(*this); return New; } diff --git a/contrib/llvm/lib/IR/Instructions.cpp b/contrib/llvm/lib/IR/Instructions.cpp index b9c693ff19ad..b67926943429 100644 --- a/contrib/llvm/lib/IR/Instructions.cpp +++ b/contrib/llvm/lib/IR/Instructions.cpp @@ -350,12 +350,6 @@ void CallInst::addAttribute(unsigned i, Attribute::AttrKind Kind) { setAttributes(PAL); } -void CallInst::addAttribute(unsigned i, StringRef Kind, StringRef Value) { - AttributeSet PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, Kind, Value); - setAttributes(PAL); -} - void CallInst::addAttribute(unsigned i, Attribute Attr) { AttributeSet PAL = getAttributes(); PAL = PAL.addAttribute(getContext(), i, Attr); @@ -374,15 +368,6 @@ void CallInst::removeAttribute(unsigned i, StringRef Kind) { setAttributes(PAL); } -void CallInst::removeAttribute(unsigned i, Attribute Attr) { - AttributeSet PAL = getAttributes(); - AttrBuilder B(Attr); - LLVMContext &Context = getContext(); - PAL = PAL.removeAttributes(Context, i, - AttributeSet::get(Context, i, B)); - setAttributes(PAL); -} - void CallInst::addDereferenceableAttr(unsigned i, uint64_t Bytes) { AttributeSet PAL = getAttributes(); PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes); @@ -405,14 +390,6 @@ bool CallInst::paramHasAttr(unsigned i, Attribute::AttrKind Kind) const { return false; } -Attribute CallInst::getAttribute(unsigned i, Attribute::AttrKind Kind) const { - return getAttributes().getAttribute(i, Kind); -} - -Attribute CallInst::getAttribute(unsigned i, StringRef Kind) const { - return getAttributes().getAttribute(i, Kind); -} - bool CallInst::dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { // There are getNumOperands() - 1 data operands. The last operand is the @@ -766,23 +743,6 @@ void InvokeInst::removeAttribute(unsigned i, StringRef Kind) { setAttributes(PAL); } -void InvokeInst::removeAttribute(unsigned i, Attribute Attr) { - AttributeSet PAL = getAttributes(); - AttrBuilder B(Attr); - PAL = PAL.removeAttributes(getContext(), i, - AttributeSet::get(getContext(), i, B)); - setAttributes(PAL); -} - -Attribute InvokeInst::getAttribute(unsigned i, - Attribute::AttrKind Kind) const { - return getAttributes().getAttribute(i, Kind); -} - -Attribute InvokeInst::getAttribute(unsigned i, StringRef Kind) const { - return getAttributes().getAttribute(i, Kind); -} - void InvokeInst::addDereferenceableAttr(unsigned i, uint64_t Bytes) { AttributeSet PAL = getAttributes(); PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes); @@ -1209,15 +1169,7 @@ void BranchInst::swapSuccessors() { // Update profile metadata if present and it matches our structural // expectations. - MDNode *ProfileData = getMetadata(LLVMContext::MD_prof); - if (!ProfileData || ProfileData->getNumOperands() != 3) - return; - - // The first operand is the name. Fetch them backwards and build a new one. - Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2), - ProfileData->getOperand(1)}; - setMetadata(LLVMContext::MD_prof, - MDNode::get(ProfileData->getContext(), Ops)); + swapProfMetadata(); } BasicBlock *BranchInst::getSuccessorV(unsigned idx) const { @@ -1916,9 +1868,6 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, return false; } -/// getMaskValue - Return the index from the shuffle mask for the specified -/// output result. This is either -1 if the element is undef or a number less -/// than 2*numelements. int ShuffleVectorInst::getMaskValue(Constant *Mask, unsigned i) { assert(i < Mask->getType()->getVectorNumElements() && "Index out of range"); if (ConstantDataSequential *CDS =dyn_cast<ConstantDataSequential>(Mask)) @@ -1929,8 +1878,6 @@ int ShuffleVectorInst::getMaskValue(Constant *Mask, unsigned i) { return cast<ConstantInt>(C)->getZExtValue(); } -/// getShuffleMask - Return the full mask for this instruction, where each -/// element is the element number and undef's are returned as -1. void ShuffleVectorInst::getShuffleMask(Constant *Mask, SmallVectorImpl<int> &Result) { unsigned NumElts = Mask->getType()->getVectorNumElements(); @@ -2289,11 +2236,10 @@ const Value *BinaryOperator::getNotArgument(const Value *BinOp) { } -// swapOperands - Exchange the two operands to this instruction. This -// instruction is safe to use on any binary instruction and does not -// modify the semantics of the instruction. If the instruction is -// order dependent (SetLT f.e.) the opcode is changed. -// +// Exchange the two operands to this instruction. This instruction is safe to +// use on any binary instruction and does not modify the semantics of the +// instruction. If the instruction is order-dependent (SetLT f.e.), the opcode +// is changed. bool BinaryOperator::swapOperands() { if (!isCommutative()) return true; // Can't commute operands @@ -2306,9 +2252,6 @@ bool BinaryOperator::swapOperands() { // FPMathOperator Class //===----------------------------------------------------------------------===// -/// getFPAccuracy - Get the maximum error permitted by this operation in ULPs. -/// An accuracy of 0.0 means that the operation should be performed with the -/// default precision. float FPMathOperator::getFPAccuracy() const { const MDNode *MD = cast<Instruction>(this)->getMetadata(LLVMContext::MD_fpmath); @@ -2603,7 +2546,8 @@ unsigned CastInst::isEliminableCastPair( case 14: // bitcast, addrspacecast -> addrspacecast if the element type of // bitcast's source is the same as that of addrspacecast's destination. - if (SrcTy->getPointerElementType() == DstTy->getPointerElementType()) + if (SrcTy->getScalarType()->getPointerElementType() == + DstTy->getScalarType()->getPointerElementType()) return Instruction::AddrSpaceCast; return 0; @@ -3465,6 +3409,38 @@ CmpInst::Predicate CmpInst::getInversePredicate(Predicate pred) { } } +StringRef CmpInst::getPredicateName(Predicate Pred) { + switch (Pred) { + default: return "unknown"; + case FCmpInst::FCMP_FALSE: return "false"; + case FCmpInst::FCMP_OEQ: return "oeq"; + case FCmpInst::FCMP_OGT: return "ogt"; + case FCmpInst::FCMP_OGE: return "oge"; + case FCmpInst::FCMP_OLT: return "olt"; + case FCmpInst::FCMP_OLE: return "ole"; + case FCmpInst::FCMP_ONE: return "one"; + case FCmpInst::FCMP_ORD: return "ord"; + case FCmpInst::FCMP_UNO: return "uno"; + case FCmpInst::FCMP_UEQ: return "ueq"; + case FCmpInst::FCMP_UGT: return "ugt"; + case FCmpInst::FCMP_UGE: return "uge"; + case FCmpInst::FCMP_ULT: return "ult"; + case FCmpInst::FCMP_ULE: return "ule"; + case FCmpInst::FCMP_UNE: return "une"; + case FCmpInst::FCMP_TRUE: return "true"; + case ICmpInst::ICMP_EQ: return "eq"; + case ICmpInst::ICMP_NE: return "ne"; + case ICmpInst::ICMP_SGT: return "sgt"; + case ICmpInst::ICMP_SGE: return "sge"; + case ICmpInst::ICMP_SLT: return "slt"; + case ICmpInst::ICMP_SLE: return "sle"; + case ICmpInst::ICMP_UGT: return "ugt"; + case ICmpInst::ICMP_UGE: return "uge"; + case ICmpInst::ICMP_ULT: return "ult"; + case ICmpInst::ICMP_ULE: return "ule"; + } +} + void ICmpInst::anchor() {} ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) { @@ -3493,69 +3469,6 @@ ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) { } } -/// Initialize a set of values that all satisfy the condition with C. -/// -ConstantRange -ICmpInst::makeConstantRange(Predicate pred, const APInt &C) { - APInt Lower(C); - APInt Upper(C); - uint32_t BitWidth = C.getBitWidth(); - switch (pred) { - default: llvm_unreachable("Invalid ICmp opcode to ConstantRange ctor!"); - case ICmpInst::ICMP_EQ: ++Upper; break; - case ICmpInst::ICMP_NE: ++Lower; break; - case ICmpInst::ICMP_ULT: - Lower = APInt::getMinValue(BitWidth); - // Check for an empty-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/false); - break; - case ICmpInst::ICMP_SLT: - Lower = APInt::getSignedMinValue(BitWidth); - // Check for an empty-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/false); - break; - case ICmpInst::ICMP_UGT: - ++Lower; Upper = APInt::getMinValue(BitWidth); // Min = Next(Max) - // Check for an empty-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/false); - break; - case ICmpInst::ICMP_SGT: - ++Lower; Upper = APInt::getSignedMinValue(BitWidth); // Min = Next(Max) - // Check for an empty-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/false); - break; - case ICmpInst::ICMP_ULE: - Lower = APInt::getMinValue(BitWidth); ++Upper; - // Check for a full-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/true); - break; - case ICmpInst::ICMP_SLE: - Lower = APInt::getSignedMinValue(BitWidth); ++Upper; - // Check for a full-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/true); - break; - case ICmpInst::ICMP_UGE: - Upper = APInt::getMinValue(BitWidth); // Min = Next(Max) - // Check for a full-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/true); - break; - case ICmpInst::ICMP_SGE: - Upper = APInt::getSignedMinValue(BitWidth); // Min = Next(Max) - // Check for a full-set condition. - if (Lower == Upper) - return ConstantRange(BitWidth, /*isFullSet=*/true); - break; - } - return ConstantRange(Lower, Upper); -} - CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) { switch (pred) { default: llvm_unreachable("Unknown cmp predicate!"); diff --git a/contrib/llvm/lib/IR/IntrinsicInst.cpp b/contrib/llvm/lib/IR/IntrinsicInst.cpp index 3f747117b728..240250662aec 100644 --- a/contrib/llvm/lib/IR/IntrinsicInst.cpp +++ b/contrib/llvm/lib/IR/IntrinsicInst.cpp @@ -25,6 +25,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -83,3 +84,12 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable, return LastLow - NameTable.begin(); return -1; } + +Value *InstrProfIncrementInst::getStep() const { + if (InstrProfIncrementInstStep::classof(this)) { + return const_cast<Value *>(getArgOperand(4)); + } + const Module *M = getModule(); + LLVMContext &Context = M->getContext(); + return ConstantInt::get(Type::getInt64Ty(Context), 1); +} diff --git a/contrib/llvm/lib/IR/LLVMContext.cpp b/contrib/llvm/lib/IR/LLVMContext.cpp index d27fcfb1b7a8..dd66f144f04f 100644 --- a/contrib/llvm/lib/IR/LLVMContext.cpp +++ b/contrib/llvm/lib/IR/LLVMContext.cpp @@ -35,108 +35,36 @@ using namespace llvm; LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { // Create the fixed metadata kinds. This is done in the same order as the // MD_* enum values so that they correspond. - - // Create the 'dbg' metadata kind. - unsigned DbgID = getMDKindID("dbg"); - assert(DbgID == MD_dbg && "dbg kind id drifted"); (void)DbgID; - - // Create the 'tbaa' metadata kind. - unsigned TBAAID = getMDKindID("tbaa"); - assert(TBAAID == MD_tbaa && "tbaa kind id drifted"); (void)TBAAID; - - // Create the 'prof' metadata kind. - unsigned ProfID = getMDKindID("prof"); - assert(ProfID == MD_prof && "prof kind id drifted"); (void)ProfID; - - // Create the 'fpmath' metadata kind. - unsigned FPAccuracyID = getMDKindID("fpmath"); - assert(FPAccuracyID == MD_fpmath && "fpmath kind id drifted"); - (void)FPAccuracyID; - - // Create the 'range' metadata kind. - unsigned RangeID = getMDKindID("range"); - assert(RangeID == MD_range && "range kind id drifted"); - (void)RangeID; - - // Create the 'tbaa.struct' metadata kind. - unsigned TBAAStructID = getMDKindID("tbaa.struct"); - assert(TBAAStructID == MD_tbaa_struct && "tbaa.struct kind id drifted"); - (void)TBAAStructID; - - // Create the 'invariant.load' metadata kind. - unsigned InvariantLdId = getMDKindID("invariant.load"); - assert(InvariantLdId == MD_invariant_load && "invariant.load kind id drifted"); - (void)InvariantLdId; - - // Create the 'alias.scope' metadata kind. - unsigned AliasScopeID = getMDKindID("alias.scope"); - assert(AliasScopeID == MD_alias_scope && "alias.scope kind id drifted"); - (void)AliasScopeID; - - // Create the 'noalias' metadata kind. - unsigned NoAliasID = getMDKindID("noalias"); - assert(NoAliasID == MD_noalias && "noalias kind id drifted"); - (void)NoAliasID; - - // Create the 'nontemporal' metadata kind. - unsigned NonTemporalID = getMDKindID("nontemporal"); - assert(NonTemporalID == MD_nontemporal && "nontemporal kind id drifted"); - (void)NonTemporalID; - - // Create the 'llvm.mem.parallel_loop_access' metadata kind. - unsigned MemParallelLoopAccessID = getMDKindID("llvm.mem.parallel_loop_access"); - assert(MemParallelLoopAccessID == MD_mem_parallel_loop_access && - "mem_parallel_loop_access kind id drifted"); - (void)MemParallelLoopAccessID; - - // Create the 'nonnull' metadata kind. - unsigned NonNullID = getMDKindID("nonnull"); - assert(NonNullID == MD_nonnull && "nonnull kind id drifted"); - (void)NonNullID; - - // Create the 'dereferenceable' metadata kind. - unsigned DereferenceableID = getMDKindID("dereferenceable"); - assert(DereferenceableID == MD_dereferenceable && - "dereferenceable kind id drifted"); - (void)DereferenceableID; - - // Create the 'dereferenceable_or_null' metadata kind. - unsigned DereferenceableOrNullID = getMDKindID("dereferenceable_or_null"); - assert(DereferenceableOrNullID == MD_dereferenceable_or_null && - "dereferenceable_or_null kind id drifted"); - (void)DereferenceableOrNullID; - - // Create the 'make.implicit' metadata kind. - unsigned MakeImplicitID = getMDKindID("make.implicit"); - assert(MakeImplicitID == MD_make_implicit && - "make.implicit kind id drifted"); - (void)MakeImplicitID; - - // Create the 'unpredictable' metadata kind. - unsigned UnpredictableID = getMDKindID("unpredictable"); - assert(UnpredictableID == MD_unpredictable && - "unpredictable kind id drifted"); - (void)UnpredictableID; - - // Create the 'invariant.group' metadata kind. - unsigned InvariantGroupId = getMDKindID("invariant.group"); - assert(InvariantGroupId == MD_invariant_group && - "invariant.group kind id drifted"); - (void)InvariantGroupId; - - // Create the 'align' metadata kind. - unsigned AlignID = getMDKindID("align"); - assert(AlignID == MD_align && "align kind id drifted"); - (void)AlignID; - - // Create the 'llvm.loop' metadata kind. - unsigned LoopID = getMDKindID("llvm.loop"); - assert(LoopID == MD_loop && "llvm.loop kind id drifted"); - (void)LoopID; - - unsigned TypeID = getMDKindID("type"); - assert(TypeID == MD_type && "type kind id drifted"); - (void)TypeID; + std::pair<unsigned, StringRef> MDKinds[] = { + {MD_dbg, "dbg"}, + {MD_tbaa, "tbaa"}, + {MD_prof, "prof"}, + {MD_fpmath, "fpmath"}, + {MD_range, "range"}, + {MD_tbaa_struct, "tbaa.struct"}, + {MD_invariant_load, "invariant.load"}, + {MD_alias_scope, "alias.scope"}, + {MD_noalias, "noalias"}, + {MD_nontemporal, "nontemporal"}, + {MD_mem_parallel_loop_access, "llvm.mem.parallel_loop_access"}, + {MD_nonnull, "nonnull"}, + {MD_dereferenceable, "dereferenceable"}, + {MD_dereferenceable_or_null, "dereferenceable_or_null"}, + {MD_make_implicit, "make.implicit"}, + {MD_unpredictable, "unpredictable"}, + {MD_invariant_group, "invariant.group"}, + {MD_align, "align"}, + {MD_loop, "llvm.loop"}, + {MD_type, "type"}, + {MD_section_prefix, "section_prefix"}, + {MD_absolute_symbol, "absolute_symbol"}, + }; + + for (auto &MDKind : MDKinds) { + unsigned ID = getMDKindID(MDKind.second); + assert(ID == MDKind.first && "metadata kind id drifted"); + (void)ID; + } auto *DeoptEntry = pImpl->getOrInsertBundleTag("deopt"); assert(DeoptEntry->second == LLVMContext::OB_deopt && @@ -203,6 +131,14 @@ bool LLVMContext::getDiagnosticHotnessRequested() const { return pImpl->DiagnosticHotnessRequested; } +yaml::Output *LLVMContext::getDiagnosticsOutputFile() { + return pImpl->DiagnosticsOutputFile.get(); +} + +void LLVMContext::setDiagnosticsOutputFile(std::unique_ptr<yaml::Output> F) { + pImpl->DiagnosticsOutputFile = std::move(F); +} + LLVMContext::DiagnosticHandlerTy LLVMContext::getDiagnosticHandler() const { return pImpl->DiagnosticHandler; } diff --git a/contrib/llvm/lib/IR/LLVMContextImpl.cpp b/contrib/llvm/lib/IR/LLVMContextImpl.cpp index b0b2c61bdf1f..c43356c53826 100644 --- a/contrib/llvm/lib/IR/LLVMContextImpl.cpp +++ b/contrib/llvm/lib/IR/LLVMContextImpl.cpp @@ -94,12 +94,13 @@ LLVMContextImpl::~LLVMContextImpl() { ArrayConstants.freeConstants(); StructConstants.freeConstants(); VectorConstants.freeConstants(); - DeleteContainerSeconds(CAZConstants); - DeleteContainerSeconds(CPNConstants); - DeleteContainerSeconds(UVConstants); InlineAsms.freeConstants(); - DeleteContainerSeconds(IntConstants); - DeleteContainerSeconds(FPConstants); + + CAZConstants.clear(); + CPNConstants.clear(); + UVConstants.clear(); + IntConstants.clear(); + FPConstants.clear(); for (auto &CDSConstant : CDSConstants) delete CDSConstant.second; diff --git a/contrib/llvm/lib/IR/LLVMContextImpl.h b/contrib/llvm/lib/IR/LLVMContextImpl.h index 7820e2ab958d..e9e30ef0656f 100644 --- a/contrib/llvm/lib/IR/LLVMContextImpl.h +++ b/contrib/llvm/lib/IR/LLVMContextImpl.h @@ -33,6 +33,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Dwarf.h" +#include "llvm/Support/YAMLTraits.h" #include <vector> namespace llvm { @@ -67,8 +68,8 @@ struct DenseMapAPIntKeyInfo { }; struct DenseMapAPFloatKeyInfo { - static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus, 1); } - static inline APFloat getTombstoneKey() { return APFloat(APFloat::Bogus, 2); } + static inline APFloat getEmptyKey() { return APFloat(APFloat::Bogus(), 1); } + static inline APFloat getTombstoneKey() { return APFloat(APFloat::Bogus(), 2); } static unsigned getHashValue(const APFloat &Key) { return static_cast<unsigned>(hash_value(Key)); } @@ -318,11 +319,11 @@ template <> struct MDNodeKeyImpl<DIBasicType> { unsigned Tag; MDString *Name; uint64_t SizeInBits; - uint64_t AlignInBits; + uint32_t AlignInBits; unsigned Encoding; MDNodeKeyImpl(unsigned Tag, MDString *Name, uint64_t SizeInBits, - uint64_t AlignInBits, unsigned Encoding) + uint32_t AlignInBits, unsigned Encoding) : Tag(Tag), Name(Name), SizeInBits(SizeInBits), AlignInBits(AlignInBits), Encoding(Encoding) {} MDNodeKeyImpl(const DIBasicType *N) @@ -348,23 +349,23 @@ template <> struct MDNodeKeyImpl<DIDerivedType> { Metadata *Scope; Metadata *BaseType; uint64_t SizeInBits; - uint64_t AlignInBits; uint64_t OffsetInBits; + uint32_t AlignInBits; unsigned Flags; Metadata *ExtraData; MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, + uint32_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *ExtraData) : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope), - BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits), - OffsetInBits(OffsetInBits), Flags(Flags), ExtraData(ExtraData) {} + BaseType(BaseType), SizeInBits(SizeInBits), OffsetInBits(OffsetInBits), + AlignInBits(AlignInBits), Flags(Flags), ExtraData(ExtraData) {} MDNodeKeyImpl(const DIDerivedType *N) : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Scope(N->getRawScope()), BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), - AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), + OffsetInBits(N->getOffsetInBits()), AlignInBits(N->getAlignInBits()), Flags(N->getFlags()), ExtraData(N->getRawExtraData()) {} bool isKeyOf(const DIDerivedType *RHS) const { @@ -429,8 +430,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> { Metadata *Scope; Metadata *BaseType; uint64_t SizeInBits; - uint64_t AlignInBits; uint64_t OffsetInBits; + uint32_t AlignInBits; unsigned Flags; Metadata *Elements; unsigned RuntimeLang; @@ -440,20 +441,20 @@ template <> struct MDNodeKeyImpl<DICompositeType> { MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, - uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, + uint32_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, MDString *Identifier) : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope), - BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits), - OffsetInBits(OffsetInBits), Flags(Flags), Elements(Elements), + BaseType(BaseType), SizeInBits(SizeInBits), OffsetInBits(OffsetInBits), + AlignInBits(AlignInBits), Flags(Flags), Elements(Elements), RuntimeLang(RuntimeLang), VTableHolder(VTableHolder), TemplateParams(TemplateParams), Identifier(Identifier) {} MDNodeKeyImpl(const DICompositeType *N) : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Scope(N->getRawScope()), BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), - AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), + OffsetInBits(N->getOffsetInBits()), AlignInBits(N->getAlignInBits()), Flags(N->getFlags()), Elements(N->getRawElements()), RuntimeLang(N->getRuntimeLang()), VTableHolder(N->getRawVTableHolder()), TemplateParams(N->getRawTemplateParams()), @@ -502,17 +503,26 @@ template <> struct MDNodeKeyImpl<DISubroutineType> { template <> struct MDNodeKeyImpl<DIFile> { MDString *Filename; MDString *Directory; + DIFile::ChecksumKind CSKind; + MDString *Checksum; - MDNodeKeyImpl(MDString *Filename, MDString *Directory) - : Filename(Filename), Directory(Directory) {} + MDNodeKeyImpl(MDString *Filename, MDString *Directory, + DIFile::ChecksumKind CSKind, MDString *Checksum) + : Filename(Filename), Directory(Directory), CSKind(CSKind), + Checksum(Checksum) {} MDNodeKeyImpl(const DIFile *N) - : Filename(N->getRawFilename()), Directory(N->getRawDirectory()) {} + : Filename(N->getRawFilename()), Directory(N->getRawDirectory()), + CSKind(N->getChecksumKind()), Checksum(N->getRawChecksum()) {} bool isKeyOf(const DIFile *RHS) const { return Filename == RHS->getRawFilename() && - Directory == RHS->getRawDirectory(); + Directory == RHS->getRawDirectory() && + CSKind == RHS->getChecksumKind() && + Checksum == RHS->getRawChecksum(); + } + unsigned getHashValue() const { + return hash_combine(Filename, Directory, CSKind, Checksum); } - unsigned getHashValue() const { return hash_combine(Filename, Directory); } }; template <> struct MDNodeKeyImpl<DISubprogram> { @@ -673,16 +683,20 @@ template <> struct MDNodeKeyImpl<DINamespace> { Metadata *File; MDString *Name; unsigned Line; + bool ExportSymbols; - MDNodeKeyImpl(Metadata *Scope, Metadata *File, MDString *Name, unsigned Line) - : Scope(Scope), File(File), Name(Name), Line(Line) {} + MDNodeKeyImpl(Metadata *Scope, Metadata *File, MDString *Name, unsigned Line, + bool ExportSymbols) + : Scope(Scope), File(File), Name(Name), Line(Line), + ExportSymbols(ExportSymbols) {} MDNodeKeyImpl(const DINamespace *N) : Scope(N->getRawScope()), File(N->getRawFile()), Name(N->getRawName()), - Line(N->getLine()) {} + Line(N->getLine()), ExportSymbols(N->getExportSymbols()) {} bool isKeyOf(const DINamespace *RHS) const { return Scope == RHS->getRawScope() && File == RHS->getRawFile() && - Name == RHS->getRawName() && Line == RHS->getLine(); + Name == RHS->getRawName() && Line == RHS->getLine() && + ExportSymbols == RHS->getExportSymbols(); } unsigned getHashValue() const { return hash_combine(Scope, File, Name, Line); @@ -758,24 +772,25 @@ template <> struct MDNodeKeyImpl<DIGlobalVariable> { Metadata *Type; bool IsLocalToUnit; bool IsDefinition; - Metadata *Variable; Metadata *StaticDataMemberDeclaration; + uint32_t AlignInBits; MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, - bool IsLocalToUnit, bool IsDefinition, Metadata *Variable, - Metadata *StaticDataMemberDeclaration) + bool IsLocalToUnit, bool IsDefinition, + Metadata *StaticDataMemberDeclaration, uint32_t AlignInBits) : Scope(Scope), Name(Name), LinkageName(LinkageName), File(File), Line(Line), Type(Type), IsLocalToUnit(IsLocalToUnit), - IsDefinition(IsDefinition), Variable(Variable), - StaticDataMemberDeclaration(StaticDataMemberDeclaration) {} + IsDefinition(IsDefinition), + StaticDataMemberDeclaration(StaticDataMemberDeclaration), + AlignInBits(AlignInBits) {} MDNodeKeyImpl(const DIGlobalVariable *N) : Scope(N->getRawScope()), Name(N->getRawName()), LinkageName(N->getRawLinkageName()), File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), IsLocalToUnit(N->isLocalToUnit()), IsDefinition(N->isDefinition()), - Variable(N->getRawVariable()), - StaticDataMemberDeclaration(N->getRawStaticDataMemberDeclaration()) {} + StaticDataMemberDeclaration(N->getRawStaticDataMemberDeclaration()), + AlignInBits(N->getAlignInBits()) {} bool isKeyOf(const DIGlobalVariable *RHS) const { return Scope == RHS->getRawScope() && Name == RHS->getRawName() && @@ -783,13 +798,20 @@ template <> struct MDNodeKeyImpl<DIGlobalVariable> { File == RHS->getRawFile() && Line == RHS->getLine() && Type == RHS->getRawType() && IsLocalToUnit == RHS->isLocalToUnit() && IsDefinition == RHS->isDefinition() && - Variable == RHS->getRawVariable() && StaticDataMemberDeclaration == - RHS->getRawStaticDataMemberDeclaration(); + RHS->getRawStaticDataMemberDeclaration() && + AlignInBits == RHS->getAlignInBits(); } unsigned getHashValue() const { + // We do not use AlignInBits in hashing function here on purpose: + // in most cases this param for local variable is zero (for function param + // it is always zero). This leads to lots of hash collisions and errors on + // cases with lots of similar variables. + // clang/test/CodeGen/debug-info-257-args.c is an example of this problem, + // generated IR is random for each run and test fails with Align included. + // TODO: make hashing work fine with such situations return hash_combine(Scope, Name, LinkageName, File, Line, Type, - IsLocalToUnit, IsDefinition, Variable, + IsLocalToUnit, IsDefinition, /* AlignInBits, */ StaticDataMemberDeclaration); } }; @@ -802,23 +824,32 @@ template <> struct MDNodeKeyImpl<DILocalVariable> { Metadata *Type; unsigned Arg; unsigned Flags; + uint32_t AlignInBits; MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line, - Metadata *Type, unsigned Arg, unsigned Flags) + Metadata *Type, unsigned Arg, unsigned Flags, + uint32_t AlignInBits) : Scope(Scope), Name(Name), File(File), Line(Line), Type(Type), Arg(Arg), - Flags(Flags) {} + Flags(Flags), AlignInBits(AlignInBits) {} MDNodeKeyImpl(const DILocalVariable *N) : Scope(N->getRawScope()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), Arg(N->getArg()), - Flags(N->getFlags()) {} + Flags(N->getFlags()), AlignInBits(N->getAlignInBits()) {} bool isKeyOf(const DILocalVariable *RHS) const { return Scope == RHS->getRawScope() && Name == RHS->getRawName() && File == RHS->getRawFile() && Line == RHS->getLine() && Type == RHS->getRawType() && Arg == RHS->getArg() && - Flags == RHS->getFlags(); + Flags == RHS->getFlags() && AlignInBits == RHS->getAlignInBits(); } unsigned getHashValue() const { + // We do not use AlignInBits in hashing function here on purpose: + // in most cases this param for local variable is zero (for function param + // it is always zero). This leads to lots of hash collisions and errors on + // cases with lots of similar variables. + // clang/test/CodeGen/debug-info-257-args.c is an example of this problem, + // generated IR is random for each run and test fails with Align included. + // TODO: make hashing work fine with such situations return hash_combine(Scope, Name, File, Line, Type, Arg, Flags); } }; @@ -837,6 +868,22 @@ template <> struct MDNodeKeyImpl<DIExpression> { } }; +template <> struct MDNodeKeyImpl<DIGlobalVariableExpression> { + Metadata *Variable; + Metadata *Expression; + + MDNodeKeyImpl(Metadata *Variable, Metadata *Expression) + : Variable(Variable), Expression(Expression) {} + MDNodeKeyImpl(const DIGlobalVariableExpression *N) + : Variable(N->getRawVariable()), Expression(N->getRawExpression()) {} + + bool isKeyOf(const DIGlobalVariableExpression *RHS) const { + return Variable == RHS->getRawVariable() && + Expression == RHS->getRawExpression(); + } + unsigned getHashValue() const { return hash_combine(Variable, Expression); } +}; + template <> struct MDNodeKeyImpl<DIObjCProperty> { MDString *Name; Metadata *File; @@ -928,7 +975,7 @@ template <> struct MDNodeKeyImpl<DIMacroFile> { bool isKeyOf(const DIMacroFile *RHS) const { return MIType == RHS->getMacinfoType() && Line == RHS->getLine() && - File == RHS->getRawFile() && File == RHS->getRawElements(); + File == RHS->getRawFile() && Elements == RHS->getRawElements(); } unsigned getHashValue() const { return hash_combine(MIType, Line, File, Elements); @@ -998,9 +1045,8 @@ public: /// /// Erases all attachments matching the \c shouldRemove predicate. template <class PredTy> void remove_if(PredTy shouldRemove) { - Attachments.erase( - std::remove_if(Attachments.begin(), Attachments.end(), shouldRemove), - Attachments.end()); + Attachments.erase(llvm::remove_if(Attachments, shouldRemove), + Attachments.end()); } }; @@ -1044,14 +1090,17 @@ public: void *DiagnosticContext; bool RespectDiagnosticFilters; bool DiagnosticHotnessRequested; + std::unique_ptr<yaml::Output> DiagnosticsOutputFile; LLVMContext::YieldCallbackTy YieldCallback; void *YieldOpaqueHandle; - typedef DenseMap<APInt, ConstantInt *, DenseMapAPIntKeyInfo> IntMapTy; + typedef DenseMap<APInt, std::unique_ptr<ConstantInt>, DenseMapAPIntKeyInfo> + IntMapTy; IntMapTy IntConstants; - typedef DenseMap<APFloat, ConstantFP *, DenseMapAPFloatKeyInfo> FPMapTy; + typedef DenseMap<APFloat, std::unique_ptr<ConstantFP>, DenseMapAPFloatKeyInfo> + FPMapTy; FPMapTy FPConstants; FoldingSet<AttributeImpl> AttrsSet; @@ -1077,7 +1126,7 @@ public: // them on context teardown. std::vector<MDNode *> DistinctMDNodes; - DenseMap<Type*, ConstantAggregateZero*> CAZConstants; + DenseMap<Type *, std::unique_ptr<ConstantAggregateZero>> CAZConstants; typedef ConstantUniqueMap<ConstantArray> ArrayConstantsTy; ArrayConstantsTy ArrayConstants; @@ -1087,11 +1136,11 @@ public: typedef ConstantUniqueMap<ConstantVector> VectorConstantsTy; VectorConstantsTy VectorConstants; - - DenseMap<PointerType*, ConstantPointerNull*> CPNConstants; - DenseMap<Type*, UndefValue*> UVConstants; - + DenseMap<PointerType *, std::unique_ptr<ConstantPointerNull>> CPNConstants; + + DenseMap<Type *, std::unique_ptr<UndefValue>> UVConstants; + StringMap<ConstantDataSequential*> CDSConstants; DenseMap<std::pair<const Function *, const BasicBlock *>, BlockAddress *> diff --git a/contrib/llvm/lib/IR/LegacyPassManager.cpp b/contrib/llvm/lib/IR/LegacyPassManager.cpp index 8f71d822d271..628a67bd639c 100644 --- a/contrib/llvm/lib/IR/LegacyPassManager.cpp +++ b/contrib/llvm/lib/IR/LegacyPassManager.cpp @@ -11,19 +11,19 @@ // //===----------------------------------------------------------------------===// - -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/LLVMContext.h" #include "llvm/IR/LegacyPassManagers.h" #include "llvm/IR/LegacyPassNameParser.h" #include "llvm/IR/Module.h" +#include "llvm/Support/Chrono.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" -#include "llvm/Support/TimeValue.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> @@ -56,8 +56,7 @@ PassDebugging("debug-pass", cl::Hidden, clEnumVal(Arguments , "print pass arguments to pass to 'opt'"), clEnumVal(Structure , "print pass structure before run()"), clEnumVal(Executions, "print pass name before it is executed"), - clEnumVal(Details , "print pass details when it is executed"), - clEnumValEnd)); + clEnumVal(Details , "print pass details when it is executed"))); namespace { typedef llvm::cl::list<const llvm::PassInfo *, bool, PassNameParser> @@ -193,9 +192,7 @@ public: PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } - const char *getPassName() const override { - return "BasicBlock Pass Manager"; - } + StringRef getPassName() const override { return "BasicBlock Pass Manager"; } // Print passes managed by this manager void dumpPassStructure(unsigned Offset) override { @@ -340,9 +337,7 @@ public: /// its runOnFunction() for function F. Pass* getOnTheFlyPass(Pass *MP, AnalysisID PI, Function &F) override; - const char *getPassName() const override { - return "Module Pass Manager"; - } + StringRef getPassName() const override { return "Module Pass Manager"; } PMDataManager *getAsPMDataManager() override { return this; } Pass *getAsPass() override { return this; } @@ -454,7 +449,7 @@ class TimingInfo { TimerGroup TG; public: // Use 'create' member to get this. - TimingInfo() : TG("... Pass execution timing report ...") {} + TimingInfo() : TG("pass", "... Pass execution timing report ...") {} // TimingDtor - Print out information about timing information ~TimingInfo() { @@ -477,8 +472,10 @@ public: sys::SmartScopedLock<true> Lock(*TimingInfoMutex); Timer *&T = TimingData[P]; - if (!T) - T = new Timer(P->getPassName(), TG); + if (!T) { + StringRef PassName = P->getPassName(); + T = new Timer(PassName, PassName, TG); + } return T; } }; @@ -538,12 +535,11 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) { // If AP is the last user of other passes then make P last user of // such passes. - for (DenseMap<Pass *, Pass *>::iterator LUI = LastUser.begin(), - LUE = LastUser.end(); LUI != LUE; ++LUI) { - if (LUI->second == AP) + for (auto LU : LastUser) { + if (LU.second == AP) // DenseMap iterator is not invalidated here because // this is just updating existing entries. - LastUser[LUI->first] = P; + LastUser[LU.first] = P; } } } @@ -684,7 +680,7 @@ void PMTopLevelManager::schedulePass(Pass *P) { if (PI && !PI->isAnalysis() && ShouldPrintBeforePass(PI)) { Pass *PP = P->createPrinterPass( - dbgs(), std::string("*** IR Dump Before ") + P->getPassName() + " ***"); + dbgs(), ("*** IR Dump Before " + P->getPassName() + " ***").str()); PP->assignPassManager(activeStack, getTopLevelPassManagerType()); } @@ -693,7 +689,7 @@ void PMTopLevelManager::schedulePass(Pass *P) { if (PI && !PI->isAnalysis() && ShouldPrintAfterPass(PI)) { Pass *PP = P->createPrinterPass( - dbgs(), std::string("*** IR Dump After ") + P->getPassName() + " ***"); + dbgs(), ("*** IR Dump After " + P->getPassName() + " ***").str()); PP->assignPassManager(activeStack, getTopLevelPassManagerType()); } } @@ -793,10 +789,9 @@ void PMTopLevelManager::initializeAllAnalysisInfo() { for (PMDataManager *IPM : IndirectPassManagers) IPM->initializeAnalysisInfo(); - for (DenseMap<Pass *, Pass *>::iterator DMI = LastUser.begin(), - DME = LastUser.end(); DMI != DME; ++DMI) { - SmallPtrSet<Pass *, 8> &L = InversedLastUser[DMI->second]; - L.insert(DMI->first); + for (auto LU : LastUser) { + SmallPtrSet<Pass *, 8> &L = InversedLastUser[LU.second]; + L.insert(LU.first); } } @@ -837,13 +832,9 @@ bool PMDataManager::preserveHigherLevelAnalysis(Pass *P) { return true; const AnalysisUsage::VectorType &PreservedSet = AnUsage->getPreservedSet(); - for (SmallVectorImpl<Pass *>::iterator I = HigherLevelAnalysis.begin(), - E = HigherLevelAnalysis.end(); I != E; ++I) { - Pass *P1 = *I; + for (Pass *P1 : HigherLevelAnalysis) { if (P1->getAsImmutablePass() == nullptr && - std::find(PreservedSet.begin(), PreservedSet.end(), - P1->getPassID()) == - PreservedSet.end()) + !is_contained(PreservedSet, P1->getPassID())) return false; } @@ -860,9 +851,7 @@ void PMDataManager::verifyPreservedAnalysis(Pass *P) { const AnalysisUsage::VectorType &PreservedSet = AnUsage->getPreservedSet(); // Verify preserved analysis - for (AnalysisUsage::VectorType::const_iterator I = PreservedSet.begin(), - E = PreservedSet.end(); I != E; ++I) { - AnalysisID AID = *I; + for (AnalysisID AID : PreservedSet) { if (Pass *AP = findAnalysisPass(AID, true)) { TimeRegion PassTimer(getPassTimer(AP)); AP->verifyAnalysis(); @@ -881,8 +870,7 @@ void PMDataManager::removeNotPreservedAnalysis(Pass *P) { E = AvailableAnalysis.end(); I != E; ) { DenseMap<AnalysisID, Pass*>::iterator Info = I++; if (Info->second->getAsImmutablePass() == nullptr && - std::find(PreservedSet.begin(), PreservedSet.end(), Info->first) == - PreservedSet.end()) { + !is_contained(PreservedSet, Info->first)) { // Remove this analysis if (PassDebugging >= Details) { Pass *S = Info->second; @@ -905,8 +893,7 @@ void PMDataManager::removeNotPreservedAnalysis(Pass *P) { E = InheritedAnalysis[Index]->end(); I != E; ) { DenseMap<AnalysisID, Pass *>::iterator Info = I++; if (Info->second->getAsImmutablePass() == nullptr && - std::find(PreservedSet.begin(), PreservedSet.end(), Info->first) == - PreservedSet.end()) { + !is_contained(PreservedSet, Info->first)) { // Remove this analysis if (PassDebugging >= Details) { Pass *S = Info->second; @@ -937,9 +924,8 @@ void PMDataManager::removeDeadPasses(Pass *P, StringRef Msg, dbgs() << " Free these instances\n"; } - for (SmallVectorImpl<Pass *>::iterator I = DeadPasses.begin(), - E = DeadPasses.end(); I != E; ++I) - freePass(*I, Msg, DBG_STR); + for (Pass *P : DeadPasses) + freePass(P, Msg, DBG_STR); } void PMDataManager::freePass(Pass *P, StringRef Msg, @@ -1145,7 +1131,7 @@ void PMDataManager::dumpPassInfo(Pass *P, enum PassDebuggingString S1, StringRef Msg) { if (PassDebugging < Executions) return; - dbgs() << "[" << sys::TimeValue::now().str() << "] " << (void *)this + dbgs() << "[" << std::chrono::system_clock::now() << "] " << (void *)this << std::string(getDepth() * 2 + 1, ' '); switch (S1) { case EXECUTION_MSG: @@ -1394,8 +1380,9 @@ void FunctionPassManager::add(Pass *P) { /// so, return true. /// bool FunctionPassManager::run(Function &F) { - if (std::error_code EC = F.materialize()) - report_fatal_error("Error reading bitcode file: " + EC.message()); + handleAllErrors(F.materialize(), [&](ErrorInfoBase &EIB) { + report_fatal_error("Error reading bitcode file: " + EIB.message()); + }); return FPM->run(F); } diff --git a/contrib/llvm/lib/IR/MDBuilder.cpp b/contrib/llvm/lib/IR/MDBuilder.cpp index a5a4cd06db09..f4bfd5992151 100644 --- a/contrib/llvm/lib/IR/MDBuilder.cpp +++ b/contrib/llvm/lib/IR/MDBuilder.cpp @@ -63,6 +63,12 @@ MDNode *MDBuilder::createFunctionEntryCount(uint64_t Count) { createConstant(ConstantInt::get(Int64Ty, Count))}); } +MDNode *MDBuilder::createFunctionSectionPrefix(StringRef Prefix) { + return MDNode::get(Context, + {createString("function_section_prefix"), + createString(Prefix)}); +} + MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) { assert(Lo.getBitWidth() == Hi.getBitWidth() && "Mismatched bitwidths!"); diff --git a/contrib/llvm/lib/IR/Mangler.cpp b/contrib/llvm/lib/IR/Mangler.cpp index ddf024df8ccc..41e11b3945e4 100644 --- a/contrib/llvm/lib/IR/Mangler.cpp +++ b/contrib/llvm/lib/IR/Mangler.cpp @@ -121,7 +121,7 @@ void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, // already. unsigned &ID = AnonGlobalIDs[GV]; if (ID == 0) - ID = NextAnonGlobalID++; + ID = AnonGlobalIDs.size(); // Must mangle the global into a unique ID. getNameWithPrefixImpl(OS, "__unnamed_" + Twine(ID), DL, PrefixTy); diff --git a/contrib/llvm/lib/IR/Metadata.cpp b/contrib/llvm/lib/IR/Metadata.cpp index f35c64b27b5b..1d1930459239 100644 --- a/contrib/llvm/lib/IR/Metadata.cpp +++ b/contrib/llvm/lib/IR/Metadata.cpp @@ -16,6 +16,7 @@ #include "MetadataImpl.h" #include "SymbolTableListTraitsImpl.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/ConstantRange.h" @@ -313,8 +314,7 @@ ValueAsMetadata *ValueAsMetadata::get(Value *V) { if (!Entry) { assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) && "Expected constant or function-local value"); - assert(!V->IsUsedByMD && - "Expected this to be the only metadata use"); + assert(!V->IsUsedByMD && "Expected this to be the only metadata use"); V->IsUsedByMD = true; if (auto *C = dyn_cast<Constant>(V)) Entry = new ConstantAsMetadata(C); @@ -359,14 +359,12 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) { auto &Store = Context.pImpl->ValuesAsMetadata; auto I = Store.find(From); if (I == Store.end()) { - assert(!From->IsUsedByMD && - "Expected From not to be used by metadata"); + assert(!From->IsUsedByMD && "Expected From not to be used by metadata"); return; } // Remove old entry from the map. - assert(From->IsUsedByMD && - "Expected From to be used by metadata"); + assert(From->IsUsedByMD && "Expected From to be used by metadata"); From->IsUsedByMD = false; ValueAsMetadata *MD = I->second; assert(MD && "Expected valid metadata"); @@ -403,8 +401,7 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) { } // Update MD in place (and update the map entry). - assert(!To->IsUsedByMD && - "Expected this to be the only metadata use"); + assert(!To->IsUsedByMD && "Expected this to be the only metadata use"); To->IsUsedByMD = true; MD->V = To; Entry = MD; @@ -416,7 +413,7 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) { MDString *MDString::get(LLVMContext &Context, StringRef Str) { auto &Store = Context.pImpl->MDStringCache; - auto I = Store.emplace_second(Str); + auto I = Store.try_emplace(Str); auto &MapEntry = I.first->getValue(); if (!I.second) return &MapEntry; @@ -437,7 +434,7 @@ StringRef MDString::getString() const { // prepended to them. #define HANDLE_MDNODE_LEAF(CLASS) \ static_assert( \ - llvm::AlignOf<uint64_t>::Alignment >= llvm::AlignOf<CLASS>::Alignment, \ + alignof(uint64_t) >= alignof(CLASS), \ "Alignment is insufficient after objects prepended to " #CLASS); #include "llvm/IR/Metadata.def" @@ -445,7 +442,7 @@ void *MDNode::operator new(size_t Size, unsigned NumOps) { size_t OpSize = NumOps * sizeof(MDOperand); // uint64_t is the most aligned type we need support (ensured by static_assert // above) - OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>()); + OpSize = alignTo(OpSize, alignof(uint64_t)); void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize; MDOperand *O = static_cast<MDOperand *>(Ptr); for (MDOperand *E = O - NumOps; O != E; --O) @@ -456,7 +453,7 @@ void *MDNode::operator new(size_t Size, unsigned NumOps) { void MDNode::operator delete(void *Mem) { MDNode *N = static_cast<MDNode *>(Mem); size_t OpSize = N->NumOperands * sizeof(MDOperand); - OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>()); + OpSize = alignTo(OpSize, alignof(uint64_t)); MDOperand *O = static_cast<MDOperand *>(Mem); for (MDOperand *E = O - N->NumOperands; O != E; --O) @@ -862,42 +859,32 @@ MDNode *MDNode::concatenate(MDNode *A, MDNode *B) { if (!B) return A; - SmallVector<Metadata *, 4> MDs; - MDs.reserve(A->getNumOperands() + B->getNumOperands()); - MDs.append(A->op_begin(), A->op_end()); - MDs.append(B->op_begin(), B->op_end()); + SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); + MDs.insert(B->op_begin(), B->op_end()); // FIXME: This preserves long-standing behaviour, but is it really the right // behaviour? Or was that an unintended side-effect of node uniquing? - return getOrSelfReference(A->getContext(), MDs); + return getOrSelfReference(A->getContext(), MDs.getArrayRef()); } MDNode *MDNode::intersect(MDNode *A, MDNode *B) { if (!A || !B) return nullptr; - SmallVector<Metadata *, 4> MDs; - for (Metadata *MD : A->operands()) - if (std::find(B->op_begin(), B->op_end(), MD) != B->op_end()) - MDs.push_back(MD); + SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); + SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); + MDs.remove_if([&](Metadata *MD) { return !is_contained(BSet, MD); }); // FIXME: This preserves long-standing behaviour, but is it really the right // behaviour? Or was that an unintended side-effect of node uniquing? - return getOrSelfReference(A->getContext(), MDs); + return getOrSelfReference(A->getContext(), MDs.getArrayRef()); } MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) { if (!A || !B) return nullptr; - SmallVector<Metadata *, 4> MDs(B->op_begin(), B->op_end()); - for (Metadata *MD : A->operands()) - if (std::find(B->op_begin(), B->op_end(), MD) == B->op_end()) - MDs.push_back(MD); - - // FIXME: This preserves long-standing behaviour, but is it really the right - // behaviour? Or was that an unintended side-effect of node uniquing? - return getOrSelfReference(A->getContext(), MDs); + return concatenate(A, B); } MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { @@ -1065,17 +1052,11 @@ void NamedMDNode::setOperand(unsigned I, MDNode *New) { getNMDOps(Operands)[I].reset(New); } -void NamedMDNode::eraseFromParent() { - getParent()->eraseNamedMetadata(this); -} +void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(this); } -void NamedMDNode::dropAllReferences() { - getNMDOps(Operands).clear(); -} +void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); } -StringRef NamedMDNode::getName() const { - return StringRef(Name); -} +StringRef NamedMDNode::getName() const { return StringRef(Name); } //===----------------------------------------------------------------------===// // Instruction Metadata method implementations. @@ -1173,14 +1154,13 @@ MDNode *Instruction::getMetadataImpl(StringRef Kind) const { } void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { - SmallSet<unsigned, 5> KnownSet; - KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); - if (!hasMetadataHashEntry()) return; // Nothing to remove! auto &InstructionMetadata = getContext().pImpl->InstructionMetadata; + SmallSet<unsigned, 4> KnownSet; + KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); if (KnownSet.empty()) { // Just drop our entry at the store. InstructionMetadata.erase(this); @@ -1209,7 +1189,7 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) { DbgLoc = DebugLoc(Node); return; } - + // Handle the case when we're adding/updating metadata on an instruction. if (Node) { auto &Info = getContext().pImpl->InstructionMetadata[this]; @@ -1226,7 +1206,7 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) { (getContext().pImpl->InstructionMetadata.count(this) > 0)) && "HasMetadata bit out of date!"); if (!hasMetadataHashEntry()) - return; // Nothing to remove! + return; // Nothing to remove! auto &Info = getContext().pImpl->InstructionMetadata[this]; // Handle removal of an existing value. @@ -1261,12 +1241,13 @@ MDNode *Instruction::getMetadataImpl(unsigned KindID) const { void Instruction::getAllMetadataImpl( SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { Result.clear(); - + // Handle 'dbg' as a special case since it is not stored in the hash table. if (DbgLoc) { Result.push_back( std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); - if (!hasMetadataHashEntry()) return; + if (!hasMetadataHashEntry()) + return; } assert(hasMetadataHashEntry() && @@ -1288,10 +1269,11 @@ void Instruction::getAllMetadataOtherThanDebugLocImpl( Info.getAll(Result); } -bool Instruction::extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) { - assert((getOpcode() == Instruction::Br || - getOpcode() == Instruction::Select) && - "Looking for branch weights on something besides branch or select"); +bool Instruction::extractProfMetadata(uint64_t &TrueVal, + uint64_t &FalseVal) const { + assert( + (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select) && + "Looking for branch weights on something besides branch or select"); auto *ProfileData = getMetadata(LLVMContext::MD_prof); if (!ProfileData || ProfileData->getNumOperands() != 3) @@ -1312,11 +1294,12 @@ bool Instruction::extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) { return true; } -bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) { +bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const { assert((getOpcode() == Instruction::Br || getOpcode() == Instruction::Select || getOpcode() == Instruction::Call || - getOpcode() == Instruction::Invoke) && + getOpcode() == Instruction::Invoke || + getOpcode() == Instruction::Switch) && "Looking for branch weights on something besides branch"); TotalVal = 0; @@ -1433,7 +1416,29 @@ void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { *MDNode::get(getContext(), {NewOffsetMD, TypeId})); continue; } - addMetadata(MD.first, *MD.second); + // If an offset adjustment was specified we need to modify the DIExpression + // to prepend the adjustment: + // !DIExpression(DW_OP_plus, Offset, [original expr]) + auto *Attachment = MD.second; + if (Offset != 0 && MD.first == LLVMContext::MD_dbg) { + DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Attachment); + DIExpression *E = nullptr; + if (!GV) { + auto *GVE = cast<DIGlobalVariableExpression>(Attachment); + GV = GVE->getVariable(); + E = GVE->getExpression(); + } + ArrayRef<uint64_t> OrigElements; + if (E) + OrigElements = E->getElements(); + std::vector<uint64_t> Elements(OrigElements.size() + 2); + Elements[0] = dwarf::DW_OP_plus; + Elements[1] = Offset; + std::copy(OrigElements.begin(), OrigElements.end(), Elements.begin() + 2); + E = DIExpression::get(getContext(), Elements); + Attachment = DIGlobalVariableExpression::get(getContext(), GV, E); + } + addMetadata(MD.first, *Attachment); } } @@ -1453,3 +1458,15 @@ void Function::setSubprogram(DISubprogram *SP) { DISubprogram *Function::getSubprogram() const { return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg)); } + +void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) { + addMetadata(LLVMContext::MD_dbg, *GV); +} + +void GlobalVariable::getDebugInfo( + SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const { + SmallVector<MDNode *, 1> MDs; + getMetadata(LLVMContext::MD_dbg, MDs); + for (MDNode *MD : MDs) + GVs.push_back(cast<DIGlobalVariableExpression>(MD)); +} diff --git a/contrib/llvm/lib/IR/Module.cpp b/contrib/llvm/lib/IR/Module.cpp index ae81b25b9517..1911f84340c6 100644 --- a/contrib/llvm/lib/IR/Module.cpp +++ b/contrib/llvm/lib/IR/Module.cpp @@ -25,6 +25,8 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/TypeFinder.h" #include "llvm/Support/Dwarf.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/RandomNumberGenerator.h" #include <algorithm> @@ -404,23 +406,23 @@ void Module::setMaterializer(GVMaterializer *GVM) { Materializer.reset(GVM); } -std::error_code Module::materialize(GlobalValue *GV) { +Error Module::materialize(GlobalValue *GV) { if (!Materializer) - return std::error_code(); + return Error::success(); return Materializer->materialize(GV); } -std::error_code Module::materializeAll() { +Error Module::materializeAll() { if (!Materializer) - return std::error_code(); + return Error::success(); std::unique_ptr<GVMaterializer> M = std::move(Materializer); return M->materializeModule(); } -std::error_code Module::materializeMetadata() { +Error Module::materializeMetadata() { if (!Materializer) - return std::error_code(); + return Error::success(); return Materializer->materializeMetadata(); } @@ -519,6 +521,10 @@ Metadata *Module::getProfileSummary() { return getModuleFlag("ProfileSummary"); } +void Module::setOwnedMemoryBuffer(std::unique_ptr<MemoryBuffer> MB) { + OwnedMemoryBuffer = std::move(MB); +} + GlobalVariable *llvm::collectUsedGlobalVariables( const Module &M, SmallPtrSetImpl<GlobalValue *> &Set, bool CompilerUsed) { const char *Name = CompilerUsed ? "llvm.compiler.used" : "llvm.used"; diff --git a/contrib/llvm/lib/IR/ModuleSummaryIndex.cpp b/contrib/llvm/lib/IR/ModuleSummaryIndex.cpp index 6107cf40a08d..9072f4bc7b12 100644 --- a/contrib/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/contrib/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -20,8 +20,17 @@ using namespace llvm; // per-module instances. void ModuleSummaryIndex::mergeFrom(std::unique_ptr<ModuleSummaryIndex> Other, uint64_t NextModuleId) { + if (Other->modulePaths().empty()) + return; + + assert(Other->modulePaths().size() == 1 && + "Can only merge from an single-module index at that time"); + + StringRef OtherModPath = Other->modulePaths().begin()->first(); + StringRef ModPath = addModulePath(OtherModPath, NextModuleId, + Other->getModuleHash(OtherModPath)) + ->first(); - StringRef ModPath; for (auto &OtherGlobalValSummaryLists : *Other) { GlobalValue::GUID ValueGUID = OtherGlobalValSummaryLists.first; GlobalValueSummaryList &List = OtherGlobalValSummaryLists.second; @@ -31,16 +40,6 @@ void ModuleSummaryIndex::mergeFrom(std::unique_ptr<ModuleSummaryIndex> Other, assert(List.size() == 1); std::unique_ptr<GlobalValueSummary> Summary = std::move(List.front()); - // Add the module path string ref for this module if we haven't already - // saved a reference to it. - if (ModPath.empty()) { - auto Path = Summary->modulePath(); - ModPath = addModulePath(Path, NextModuleId, Other->getModuleHash(Path)) - ->first(); - } else - assert(ModPath == Summary->modulePath() && - "Each module in the combined map should have a unique ID"); - // Note the module path string ref was copied above and is still owned by // the original per-module index. Reset it to the new module path // string reference owned by the combined index. diff --git a/contrib/llvm/lib/IR/Operator.cpp b/contrib/llvm/lib/IR/Operator.cpp index 8a94053a72c5..2fba24d99b30 100644 --- a/contrib/llvm/lib/IR/Operator.cpp +++ b/contrib/llvm/lib/IR/Operator.cpp @@ -33,7 +33,7 @@ bool GEPOperator::accumulateConstantOffset(const DataLayout &DL, continue; // Handle a struct index, which adds its field offset to the pointer. - if (StructType *STy = dyn_cast<StructType>(*GTI)) { + if (StructType *STy = GTI.getStructTypeOrNull()) { unsigned ElementIdx = OpC->getZExtValue(); const StructLayout *SL = DL.getStructLayout(STy); Offset += APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); diff --git a/contrib/llvm/lib/IR/Pass.cpp b/contrib/llvm/lib/IR/Pass.cpp index 69299fea7634..a42945ef3fff 100644 --- a/contrib/llvm/lib/IR/Pass.cpp +++ b/contrib/llvm/lib/IR/Pass.cpp @@ -64,7 +64,7 @@ void Pass::dumpPassStructure(unsigned Offset) { /// implemented in terms of the name that is registered by one of the /// Registration templates, but can be overloaded directly. /// -const char *Pass::getPassName() const { +StringRef Pass::getPassName() const { AnalysisID AID = getPassID(); const PassInfo *PI = PassRegistry::getPassRegistry()->getPassInfo(AID); if (PI) @@ -218,7 +218,7 @@ Pass *Pass::createPass(AnalysisID ID) { // RegisterAGBase implementation // -RegisterAGBase::RegisterAGBase(const char *Name, const void *InterfaceID, +RegisterAGBase::RegisterAGBase(StringRef Name, const void *InterfaceID, const void *PassID, bool isDefault) : PassInfo(Name, InterfaceID) { PassRegistry::getPassRegistry()->registerAnalysisGroup(InterfaceID, PassID, diff --git a/contrib/llvm/lib/IR/PassManager.cpp b/contrib/llvm/lib/IR/PassManager.cpp index 8563a4019a2b..8f68bb1daecf 100644 --- a/contrib/llvm/lib/IR/PassManager.cpp +++ b/contrib/llvm/lib/IR/PassManager.cpp @@ -13,12 +13,82 @@ using namespace llvm; -// Explicit template instantiations for core template typedefs. +// Explicit template instantiations and specialization defininitions for core +// template typedefs. namespace llvm { +template class AllAnalysesOn<Module>; +template class AllAnalysesOn<Function>; template class PassManager<Module>; template class PassManager<Function>; template class AnalysisManager<Module>; template class AnalysisManager<Function>; template class InnerAnalysisManagerProxy<FunctionAnalysisManager, Module>; template class OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>; + +template <> +bool FunctionAnalysisManagerModuleProxy::Result::invalidate( + Module &M, const PreservedAnalyses &PA, + ModuleAnalysisManager::Invalidator &Inv) { + // If literally everything is preserved, we're done. + if (PA.areAllPreserved()) + return false; // This is still a valid proxy. + + // If this proxy isn't marked as preserved, then even if the result remains + // valid, the key itself may no longer be valid, so we clear everything. + // + // Note that in order to preserve this proxy, a module pass must ensure that + // the FAM has been completely updated to handle the deletion of functions. + // Specifically, any FAM-cached results for those functions need to have been + // forcibly cleared. When preserved, this proxy will only invalidate results + // cached on functions *still in the module* at the end of the module pass. + auto PAC = PA.getChecker<FunctionAnalysisManagerModuleProxy>(); + if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) { + InnerAM->clear(); + return true; + } + + // Directly check if the relevant set is preserved. + bool AreFunctionAnalysesPreserved = + PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>(); + + // Now walk all the functions to see if any inner analysis invalidation is + // necessary. + for (Function &F : M) { + Optional<PreservedAnalyses> FunctionPA; + + // Check to see whether the preserved set needs to be pruned based on + // module-level analysis invalidation that triggers deferred invalidation + // registered with the outer analysis manager proxy for this function. + if (auto *OuterProxy = + InnerAM->getCachedResult<ModuleAnalysisManagerFunctionProxy>(F)) + for (const auto &OuterInvalidationPair : + OuterProxy->getOuterInvalidations()) { + AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; + const auto &InnerAnalysisIDs = OuterInvalidationPair.second; + if (Inv.invalidate(OuterAnalysisID, M, PA)) { + if (!FunctionPA) + FunctionPA = PA; + for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) + FunctionPA->abandon(InnerAnalysisID); + } + } + + // Check if we needed a custom PA set, and if so we'll need to run the + // inner invalidation. + if (FunctionPA) { + InnerAM->invalidate(F, *FunctionPA); + continue; + } + + // Otherwise we only need to do invalidation if the original PA set didn't + // preserve all function analyses. + if (!AreFunctionAnalysesPreserved) + InnerAM->invalidate(F, PA); + } + + // Return false to indicate that this result is still a valid proxy. + return false; } +} + +AnalysisSetKey PreservedAnalyses::AllAnalysesKey; diff --git a/contrib/llvm/lib/IR/PassRegistry.cpp b/contrib/llvm/lib/IR/PassRegistry.cpp index 09b17ba308dd..584dee2869c1 100644 --- a/contrib/llvm/lib/IR/PassRegistry.cpp +++ b/contrib/llvm/lib/IR/PassRegistry.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/PassRegistry.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/PassSupport.h" #include "llvm/Support/ManagedStatic.h" @@ -121,6 +122,6 @@ void PassRegistry::addRegistrationListener(PassRegistrationListener *L) { void PassRegistry::removeRegistrationListener(PassRegistrationListener *L) { sys::SmartScopedWriter<true> Guard(Lock); - auto I = std::find(Listeners.begin(), Listeners.end(), L); + auto I = find(Listeners, L); Listeners.erase(I); } diff --git a/contrib/llvm/lib/IR/SymbolTableListTraitsImpl.h b/contrib/llvm/lib/IR/SymbolTableListTraitsImpl.h index 50573d8d688a..6ddab6b4c69d 100644 --- a/contrib/llvm/lib/IR/SymbolTableListTraitsImpl.h +++ b/contrib/llvm/lib/IR/SymbolTableListTraitsImpl.h @@ -81,11 +81,10 @@ void SymbolTableListTraits<ValueSubClass>::removeNodeFromList( template <typename ValueSubClass> void SymbolTableListTraits<ValueSubClass>::transferNodesFromList( - SymbolTableListTraits &L2, ilist_iterator<ValueSubClass> first, - ilist_iterator<ValueSubClass> last) { + SymbolTableListTraits &L2, iterator first, iterator last) { // We only have to do work here if transferring instructions between BBs ItemParentClass *NewIP = getListOwner(), *OldIP = L2.getListOwner(); - if (NewIP == OldIP) return; // No work to do at all... + assert(NewIP != OldIP && "Expected different list owners"); // We only have to update symbol table entries if we are transferring the // instructions to a different symtab object... diff --git a/contrib/llvm/lib/IR/Type.cpp b/contrib/llvm/lib/IR/Type.cpp index 5c97a4ea0705..ca866738f882 100644 --- a/contrib/llvm/lib/IR/Type.cpp +++ b/contrib/llvm/lib/IR/Type.cpp @@ -296,9 +296,9 @@ FunctionType *FunctionType::get(Type *ReturnType, FunctionType *FT; if (I == pImpl->FunctionTypes.end()) { - FT = (FunctionType*) pImpl->TypeAllocator. - Allocate(sizeof(FunctionType) + sizeof(Type*) * (Params.size() + 1), - AlignOf<FunctionType>::Alignment); + FT = (FunctionType *)pImpl->TypeAllocator.Allocate( + sizeof(FunctionType) + sizeof(Type *) * (Params.size() + 1), + alignof(FunctionType)); new (FT) FunctionType(ReturnType, Params, isVarArg); pImpl->FunctionTypes.insert(FT); } else { @@ -601,9 +601,7 @@ bool CompositeType::indexValid(unsigned Idx) const { //===----------------------------------------------------------------------===// ArrayType::ArrayType(Type *ElType, uint64_t NumEl) - : SequentialType(ArrayTyID, ElType) { - NumElements = NumEl; -} + : SequentialType(ArrayTyID, ElType, NumEl) {} ArrayType *ArrayType::get(Type *ElementType, uint64_t NumElements) { assert(isValidElementType(ElementType) && "Invalid type for array element!"); @@ -628,9 +626,7 @@ bool ArrayType::isValidElementType(Type *ElemTy) { //===----------------------------------------------------------------------===// VectorType::VectorType(Type *ElType, unsigned NumEl) - : SequentialType(VectorTyID, ElType) { - NumElements = NumEl; -} + : SequentialType(VectorTyID, ElType, NumEl) {} VectorType *VectorType::get(Type *ElementType, unsigned NumElements) { assert(NumElements > 0 && "#Elements of a VectorType must be greater than 0"); @@ -673,13 +669,10 @@ PointerType *PointerType::get(Type *EltTy, unsigned AddressSpace) { PointerType::PointerType(Type *E, unsigned AddrSpace) - : SequentialType(PointerTyID, E) { -#ifndef NDEBUG - const unsigned oldNCT = NumContainedTys; -#endif + : Type(E->getContext(), PointerTyID), PointeeTy(E) { + ContainedTys = &PointeeTy; + NumContainedTys = 1; setSubclassData(AddrSpace); - // Check for miscompile. PR11652. - assert(oldNCT == NumContainedTys && "bitfield written out of bounds?"); } PointerType *Type::getPointerTo(unsigned addrs) const { diff --git a/contrib/llvm/lib/IR/User.cpp b/contrib/llvm/lib/IR/User.cpp index a75abe6938c9..497b4aa17643 100644 --- a/contrib/llvm/lib/IR/User.cpp +++ b/contrib/llvm/lib/IR/User.cpp @@ -43,10 +43,9 @@ void User::replaceUsesOfWith(Value *From, Value *To) { void User::allocHungoffUses(unsigned N, bool IsPhi) { assert(HasHungOffUses && "alloc must have hung off uses"); - static_assert(AlignOf<Use>::Alignment >= AlignOf<Use::UserRef>::Alignment, + static_assert(alignof(Use) >= alignof(Use::UserRef), "Alignment is insufficient for 'hung-off-uses' pieces"); - static_assert(AlignOf<Use::UserRef>::Alignment >= - AlignOf<BasicBlock *>::Alignment, + static_assert(alignof(Use::UserRef) >= alignof(BasicBlock *), "Alignment is insufficient for 'hung-off-uses' pieces"); // Allocate the array of Uses, followed by a pointer (with bottom bit set) to diff --git a/contrib/llvm/lib/IR/Value.cpp b/contrib/llvm/lib/IR/Value.cpp index 6ae37fa6f15a..91a999b58004 100644 --- a/contrib/llvm/lib/IR/Value.cpp +++ b/contrib/llvm/lib/IR/Value.cpp @@ -124,10 +124,10 @@ bool Value::isUsedInBasicBlock(const BasicBlock *BB) const { const_user_iterator UI = user_begin(), UE = user_end(); for (; BI != BE && UI != UE; ++BI, ++UI) { // Scan basic block: Check if this Value is used by the instruction at BI. - if (std::find(BI->op_begin(), BI->op_end(), this) != BI->op_end()) + if (is_contained(BI->operands(), this)) return true; // Scan use list: Check if the use at UI is in BB. - const Instruction *User = dyn_cast<Instruction>(*UI); + const auto *User = dyn_cast<Instruction>(*UI); if (User && User->getParent() == BB) return true; } @@ -143,16 +143,16 @@ static bool getSymTab(Value *V, ValueSymbolTable *&ST) { if (Instruction *I = dyn_cast<Instruction>(V)) { if (BasicBlock *P = I->getParent()) if (Function *PP = P->getParent()) - ST = &PP->getValueSymbolTable(); + ST = PP->getValueSymbolTable(); } else if (BasicBlock *BB = dyn_cast<BasicBlock>(V)) { if (Function *P = BB->getParent()) - ST = &P->getValueSymbolTable(); + ST = P->getValueSymbolTable(); } else if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) { if (Module *P = GV->getParent()) ST = &P->getValueSymbolTable(); } else if (Argument *A = dyn_cast<Argument>(V)) { if (Function *P = A->getParent()) - ST = &P->getValueSymbolTable(); + ST = P->getValueSymbolTable(); } else { assert(isa<Constant>(V) && "Unknown value type!"); return true; // no name is setable for this. @@ -367,7 +367,7 @@ static bool contains(Value *Expr, Value *V) { } #endif // NDEBUG -void Value::replaceAllUsesWith(Value *New) { +void Value::doRAUW(Value *New, bool NoMetadata) { assert(New && "Value::replaceAllUsesWith(<null>) is invalid!"); assert(!contains(New, this) && "this->replaceAllUsesWith(expr(this)) is NOT valid!"); @@ -377,7 +377,7 @@ void Value::replaceAllUsesWith(Value *New) { // Notify all ValueHandles (if present) that this value is going away. if (HasValueHandle) ValueHandleBase::ValueIsRAUWd(this, New); - if (isUsedByMetadata()) + if (!NoMetadata && isUsedByMetadata()) ValueAsMetadata::handleRAUW(this, New); while (!use_empty()) { @@ -398,6 +398,14 @@ void Value::replaceAllUsesWith(Value *New) { BB->replaceSuccessorsPhiUsesWith(cast<BasicBlock>(New)); } +void Value::replaceAllUsesWith(Value *New) { + doRAUW(New, false /* NoMetadata */); +} + +void Value::replaceNonMetadataUsesWith(Value *New) { + doRAUW(New, true /* NoMetadata */); +} + // Like replaceAllUsesWith except it does not handle constants or basic blocks. // This routine leaves uses within BB. void Value::replaceUsesOutsideBlock(Value *New, BasicBlock *BB) { @@ -449,7 +457,7 @@ static Value *stripPointerCastsAndOffsets(Value *V) { case PSK_InBoundsConstantIndices: if (!GEP->hasAllConstantIndices()) return V; - // fallthrough + LLVM_FALLTHROUGH; case PSK_InBounds: if (!GEP->isInBounds()) return V; @@ -664,6 +672,16 @@ void Value::reverseUseList() { Head->setPrev(&UseList); } +bool Value::isSwiftError() const { + auto *Arg = dyn_cast<Argument>(this); + if (Arg) + return Arg->hasSwiftErrorAttr(); + auto *Alloca = dyn_cast<AllocaInst>(this); + if (!Alloca) + return false; + return Alloca->isSwiftError(); +} + //===----------------------------------------------------------------------===// // ValueHandleBase Class //===----------------------------------------------------------------------===// @@ -848,7 +866,7 @@ void ValueHandleBase::ValueIsRAUWd(Value *Old, Value *New) { // virtual (or inline) interface to handle this though, so instead we make // the TrackingVH accessors guarantee that a client never sees this value. - // FALLTHROUGH + LLVM_FALLTHROUGH; case Weak: // Weak goes to the new value, which will unlink it from Old's list. Entry->operator=(New); diff --git a/contrib/llvm/lib/IR/ValueSymbolTable.cpp b/contrib/llvm/lib/IR/ValueSymbolTable.cpp index f6f1dd984e9f..8a6a320fc2d1 100644 --- a/contrib/llvm/lib/IR/ValueSymbolTable.cpp +++ b/contrib/llvm/lib/IR/ValueSymbolTable.cpp @@ -15,8 +15,13 @@ #include "llvm/ADT/SmallString.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Type.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <utility> + using namespace llvm; #define DEBUG_TYPE "valuesymtab" @@ -35,7 +40,7 @@ ValueSymbolTable::~ValueSymbolTable() { ValueName *ValueSymbolTable::makeUniqueName(Value *V, SmallString<256> &UniqueName) { unsigned BaseSize = UniqueName.size(); - while (1) { + while (true) { // Trim any suffix off and append the next number. UniqueName.resize(BaseSize); raw_svector_ostream S(UniqueName); @@ -94,7 +99,6 @@ ValueName *ValueSymbolTable::createValueName(StringRef Name, Value *V) { return makeUniqueName(V, UniqueName); } - // dump - print out the symbol table // LLVM_DUMP_METHOD void ValueSymbolTable::dump() const { diff --git a/contrib/llvm/lib/IR/ValueTypes.cpp b/contrib/llvm/lib/IR/ValueTypes.cpp index ff1e431c2e9a..2132e1659225 100644 --- a/contrib/llvm/lib/IR/ValueTypes.cpp +++ b/contrib/llvm/lib/IR/ValueTypes.cpp @@ -26,7 +26,7 @@ EVT EVT::changeExtendedTypeToInteger() const { EVT EVT::changeExtendedVectorElementTypeToInteger() const { LLVMContext &Context = LLVMTy->getContext(); - EVT IntTy = getIntegerVT(Context, getVectorElementType().getSizeInBits()); + EVT IntTy = getIntegerVT(Context, getScalarSizeInBits()); return getVectorVT(Context, IntTy, getVectorNumElements()); } diff --git a/contrib/llvm/lib/IR/Verifier.cpp b/contrib/llvm/lib/IR/Verifier.cpp index 682e934d4b0f..5855059a189c 100644 --- a/contrib/llvm/lib/IR/Verifier.cpp +++ b/contrib/llvm/lib/IR/Verifier.cpp @@ -45,48 +45,86 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/Verifier.h" +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/ilist.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" +#include "llvm/IR/Comdat.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InlineAsm.h" -#include "llvm/IR/InstIterator.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSlotTracker.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Statepoint.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Use.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" #include "llvm/Pass.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Dwarf.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> -#include <cstdarg> +#include <cassert> +#include <cstdint> +#include <memory> +#include <string> +#include <utility> + using namespace llvm; static cl::opt<bool> VerifyDebugInfo("verify-debug-info", cl::init(true)); -namespace { +namespace llvm { + struct VerifierSupport { raw_ostream *OS; - const Module *M = nullptr; - Optional<ModuleSlotTracker> MST; + const Module &M; + ModuleSlotTracker MST; + const DataLayout &DL; + LLVMContext &Context; /// Track the brokenness of the module while recursively visiting. bool Broken = false; @@ -95,16 +133,11 @@ struct VerifierSupport { /// Whether to treat broken debug info as an error. bool TreatBrokenDebugInfoAsError = true; - explicit VerifierSupport(raw_ostream *OS) : OS(OS) {} + explicit VerifierSupport(raw_ostream *OS, const Module &M) + : OS(OS), M(M), MST(&M), DL(M.getDataLayout()), Context(M.getContext()) {} private: - template <class NodeTy> void Write(const ilist_iterator<NodeTy> &I) { - Write(&*I); - } - void Write(const Module *M) { - if (!M) - return; *OS << "; ModuleID = '" << M->getModuleIdentifier() << "'\n"; } @@ -112,13 +145,14 @@ private: if (!V) return; if (isa<Instruction>(V)) { - V->print(*OS, *MST); + V->print(*OS, MST); *OS << '\n'; } else { - V->printAsOperand(*OS, true, *MST); + V->printAsOperand(*OS, true, MST); *OS << '\n'; } } + void Write(ImmutableCallSite CS) { Write(CS.getInstruction()); } @@ -126,7 +160,7 @@ private: void Write(const Metadata *MD) { if (!MD) return; - MD->print(*OS, *MST, M); + MD->print(*OS, MST, &M); *OS << '\n'; } @@ -137,7 +171,7 @@ private: void Write(const NamedMDNode *NMD) { if (!NMD) return; - NMD->print(*OS, *MST); + NMD->print(*OS, MST); *OS << '\n'; } @@ -153,6 +187,14 @@ private: *OS << *C; } + void Write(const APInt *AI) { + if (!AI) + return; + *OS << *AI << '\n'; + } + + void Write(const unsigned i) { *OS << i << '\n'; } + template <typename T> void Write(ArrayRef<T> Vs) { for (const T &V : Vs) Write(V); @@ -206,10 +248,13 @@ public: } }; +} // namespace llvm + +namespace { + class Verifier : public InstVisitor<Verifier>, VerifierSupport { friend class InstVisitor<Verifier>; - LLVMContext *Context; DominatorTree DT; /// \brief When verifying a basic block, keep track of all of the @@ -252,28 +297,23 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport { // constant expressions, we can arrive at a particular user many times. SmallPtrSet<const Value *, 32> GlobalValueVisited; - void checkAtomicMemAccessSize(const Module *M, Type *Ty, - const Instruction *I); + TBAAVerifier TBAAVerifyHelper; - void updateModule(const Module *NewM) { - if (M == NewM) - return; - MST.emplace(NewM); - M = NewM; - } + void checkAtomicMemAccessSize(Type *Ty, const Instruction *I); public: - explicit Verifier(raw_ostream *OS, bool ShouldTreatBrokenDebugInfoAsError) - : VerifierSupport(OS), Context(nullptr), LandingPadResultTy(nullptr), - SawFrameEscape(false) { + explicit Verifier(raw_ostream *OS, bool ShouldTreatBrokenDebugInfoAsError, + const Module &M) + : VerifierSupport(OS, M), LandingPadResultTy(nullptr), + SawFrameEscape(false), TBAAVerifyHelper(this) { TreatBrokenDebugInfoAsError = ShouldTreatBrokenDebugInfoAsError; } bool hasBrokenDebugInfo() const { return BrokenDebugInfo; } bool verify(const Function &F) { - updateModule(F.getParent()); - Context = &M->getContext(); + assert(F.getParent() == &M && + "An instance of this class only works with a specific module!"); // First ensure the function is well-enough formed to compute dominance // information, and directly compute a dominance tree. We don't rely on the @@ -291,7 +331,7 @@ public: if (OS) { *OS << "Basic Block in function '" << F.getName() << "' does not have terminator!\n"; - BB.printAsOperand(*OS, true, *MST); + BB.printAsOperand(*OS, true, MST); *OS << "\n"; } return false; @@ -309,9 +349,8 @@ public: return !Broken; } - bool verify(const Module &M) { - updateModule(&M); - Context = &M.getContext(); + /// Verify the module that this instance of \c Verifier was initialized with. + bool verify() { Broken = false; // Collect all declarations of the llvm.experimental.deoptimize intrinsic. @@ -364,8 +403,8 @@ private: SmallVectorImpl<const MDNode *> &Requirements); void visitFunction(const Function &F); void visitBasicBlock(BasicBlock &BB); - void visitRangeMetadata(Instruction& I, MDNode* Range, Type* Ty); - void visitDereferenceableMetadata(Instruction& I, MDNode* MD); + void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty); + void visitDereferenceableMetadata(Instruction &I, MDNode *MD); template <class Ty> bool isValidMetadataArray(const MDTuple &N); #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS) void visit##CLASS(const CLASS &N); @@ -428,6 +467,7 @@ private: void visitInsertValueInst(InsertValueInst &IVI); void visitEHPadPredecessors(Instruction &I); void visitLandingPadInst(LandingPadInst &LPI); + void visitResumeInst(ResumeInst &RI); void visitCatchPadInst(CatchPadInst &CPI); void visitCatchReturnInst(CatchReturnInst &CatchReturn); void visitCleanupPadInst(CleanupPadInst &CPI); @@ -456,7 +496,7 @@ private: void verifyFrameRecoverIndices(); void verifySiblingFuncletUnwinds(); - void verifyBitPieceExpression(const DbgInfoIntrinsic &I); + void verifyFragmentExpression(const DbgInfoIntrinsic &I); /// Module-level debug info verification... void verifyCompileUnits(); @@ -465,17 +505,17 @@ private: /// declarations share the same calling convention. void verifyDeoptimizeCallingConvs(); }; -} // End anonymous namespace + +} // end anonymous namespace /// We know that cond should be true, if not print an error message. #define Assert(C, ...) \ - do { if (!(C)) { CheckFailed(__VA_ARGS__); return; } } while (0) + do { if (!(C)) { CheckFailed(__VA_ARGS__); return; } } while (false) /// We know that a debug info condition should be true, if not print /// an error message. #define AssertDI(C, ...) \ - do { if (!(C)) { DebugInfoCheckFailed(__VA_ARGS__); return; } } while (0) - + do { if (!(C)) { DebugInfoCheckFailed(__VA_ARGS__); return; } } while (false) void Verifier::visit(Instruction &I) { for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) @@ -517,17 +557,17 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { forEachUser(&GV, GlobalValueVisited, [&](const Value *V) -> bool { if (const Instruction *I = dyn_cast<Instruction>(V)) { if (!I->getParent() || !I->getParent()->getParent()) - CheckFailed("Global is referenced by parentless instruction!", &GV, - M, I); - else if (I->getParent()->getParent()->getParent() != M) - CheckFailed("Global is referenced in a different module!", &GV, - M, I, I->getParent()->getParent(), + CheckFailed("Global is referenced by parentless instruction!", &GV, &M, + I); + else if (I->getParent()->getParent()->getParent() != &M) + CheckFailed("Global is referenced in a different module!", &GV, &M, I, + I->getParent()->getParent(), I->getParent()->getParent()->getParent()); return false; } else if (const Function *F = dyn_cast<Function>(V)) { - if (F->getParent() != M) - CheckFailed("Global is used by function in a different module", &GV, - M, F, F->getParent()); + if (F->getParent() != &M) + CheckFailed("Global is used by function in a different module", &GV, &M, + F, F->getParent()); return false; } return true; @@ -540,7 +580,6 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { "Global variable initializer type does not match global " "variable type!", &GV); - // If the global has common linkage, it must have a zero initializer and // cannot be constant. if (GV.hasCommonLinkage()) { @@ -561,7 +600,7 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { if (ArrayType *ATy = dyn_cast<ArrayType>(GV.getValueType())) { StructType *STy = dyn_cast<StructType>(ATy->getElementType()); PointerType *FuncPtrTy = - FunctionType::get(Type::getVoidTy(*Context), false)->getPointerTo(); + FunctionType::get(Type::getVoidTy(Context), false)->getPointerTo(); // FIXME: Reject the 2-field form in LLVM 4.0. Assert(STy && (STy->getNumElements() == 2 || STy->getNumElements() == 3) && @@ -606,6 +645,16 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { GV.hasAvailableExternallyLinkage(), "Global is marked as dllimport, but not external", &GV); + // Visit any debug info attachments. + SmallVector<MDNode *, 1> MDs; + GV.getMetadata(LLVMContext::MD_dbg, MDs); + for (auto *MD : MDs) { + if (auto *GVE = dyn_cast<DIGlobalVariableExpression>(MD)) + visitDIGlobalVariableExpression(*GVE); + else + AssertDI(false, "!dbg attachment of global variable must be a DIGlobalVariableExpression"); + } + if (!GV.hasInitializer()) { visitGlobalValue(GV); return; @@ -672,10 +721,15 @@ void Verifier::visitGlobalAlias(const GlobalAlias &GA) { } void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { + // There used to be various other llvm.dbg.* nodes, but we don't support + // upgrading them and we want to reserve the namespace for future uses. + if (NMD.getName().startswith("llvm.dbg.")) + AssertDI(NMD.getName() == "llvm.dbg.cu", + "unrecognized named metadata node in the llvm.dbg namespace", + &NMD); for (const MDNode *MD : NMD.operands()) { - if (NMD.getName() == "llvm.dbg.cu") { + if (NMD.getName() == "llvm.dbg.cu") AssertDI(MD && isa<DICompileUnit>(MD), "invalid compile unit", &NMD, MD); - } if (!MD) continue; @@ -769,7 +823,7 @@ static bool isScope(const Metadata *MD) { return !MD || isa<DIScope>(MD); } static bool isDINode(const Metadata *MD) { return !MD || isa<DINode>(MD); } template <class Ty> -bool isValidMetadataArrayImpl(const MDTuple &N, bool AllowNull) { +static bool isValidMetadataArrayImpl(const MDTuple &N, bool AllowNull) { for (Metadata *MD : N.operands()) { if (MD) { if (!isa<Ty>(MD)) @@ -782,13 +836,11 @@ bool isValidMetadataArrayImpl(const MDTuple &N, bool AllowNull) { return true; } -template <class Ty> -bool isValidMetadataArray(const MDTuple &N) { +template <class Ty> static bool isValidMetadataArray(const MDTuple &N) { return isValidMetadataArrayImpl<Ty>(N, /* AllowNull */ false); } -template <class Ty> -bool isValidMetadataNullArray(const MDTuple &N) { +template <class Ty> static bool isValidMetadataNullArray(const MDTuple &N) { return isValidMetadataArrayImpl<Ty>(N, /* AllowNull */ true); } @@ -835,6 +887,7 @@ void Verifier::visitDIDerivedType(const DIDerivedType &N) { N.getTag() == dwarf::DW_TAG_const_type || N.getTag() == dwarf::DW_TAG_volatile_type || N.getTag() == dwarf::DW_TAG_restrict_type || + N.getTag() == dwarf::DW_TAG_atomic_type || N.getTag() == dwarf::DW_TAG_member || N.getTag() == dwarf::DW_TAG_inheritance || N.getTag() == dwarf::DW_TAG_friend, @@ -908,6 +961,8 @@ void Verifier::visitDISubroutineType(const DISubroutineType &N) { void Verifier::visitDIFile(const DIFile &N) { AssertDI(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N); + AssertDI((N.getChecksumKind() != DIFile::CSK_None || + N.getChecksum().empty()), "invalid checksum kind", &N); } void Verifier::visitDICompileUnit(const DICompileUnit &N) { @@ -937,15 +992,15 @@ void Verifier::visitDICompileUnit(const DICompileUnit &N) { for (Metadata *Op : N.getRetainedTypes()->operands()) { AssertDI(Op && (isa<DIType>(Op) || (isa<DISubprogram>(Op) && - cast<DISubprogram>(Op)->isDefinition() == false)), + !cast<DISubprogram>(Op)->isDefinition())), "invalid retained type", &N, Op); } } if (auto *Array = N.getRawGlobalVariables()) { AssertDI(isa<MDTuple>(Array), "invalid global variable list", &N, Array); for (Metadata *Op : N.getGlobalVariables()->operands()) { - AssertDI(Op && isa<DIGlobalVariable>(Op), "invalid global variable ref", - &N, Op); + AssertDI(Op && (isa<DIGlobalVariableExpression>(Op)), + "invalid global variable ref", &N, Op); } } if (auto *Array = N.getRawImportedEntities()) { @@ -1088,12 +1143,6 @@ void Verifier::visitDIGlobalVariable(const DIGlobalVariable &N) { AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); AssertDI(!N.getName().empty(), "missing global variable name", &N); - if (auto *V = N.getRawVariable()) { - AssertDI(isa<ConstantAsMetadata>(V) && - !isa<Function>(cast<ConstantAsMetadata>(V)->getValue()), - "invalid global varaible ref", &N, V); - visitConstantExprsRecursively(cast<ConstantAsMetadata>(V)->getValue()); - } if (auto *Member = N.getRawStaticDataMemberDeclaration()) { AssertDI(isa<DIDerivedType>(Member), "invalid static data member declaration", &N, Member); @@ -1113,6 +1162,15 @@ void Verifier::visitDIExpression(const DIExpression &N) { AssertDI(N.isValid(), "invalid expression", &N); } +void Verifier::visitDIGlobalVariableExpression( + const DIGlobalVariableExpression &GVE) { + AssertDI(GVE.getVariable(), "missing variable"); + if (auto *Var = GVE.getVariable()) + visitDIGlobalVariable(*Var); + if (auto *Expr = GVE.getExpression()) + visitDIExpression(*Expr); +} + void Verifier::visitDIObjCProperty(const DIObjCProperty &N) { AssertDI(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); if (auto *T = N.getRawType()) @@ -1134,7 +1192,7 @@ void Verifier::visitDIImportedEntity(const DIImportedEntity &N) { void Verifier::visitComdat(const Comdat &C) { // The Module is invalid if the GlobalValue has private linkage. Entities // with private linkage don't have entries in the symbol table. - if (const GlobalValue *GV = M->getNamedValue(C.getName())) + if (const GlobalValue *GV = M.getNamedValue(C.getName())) Assert(!GV->hasPrivateLinkage(), "comdat global value has private linkage", GV); } @@ -1401,12 +1459,12 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, "'noinline and alwaysinline' are incompatible!", V); - Assert(!AttrBuilder(Attrs, Idx) - .overlaps(AttributeFuncs::typeIncompatible(Ty)), - "Wrong types for attribute: " + - AttributeSet::get(*Context, Idx, - AttributeFuncs::typeIncompatible(Ty)).getAsString(Idx), - V); + Assert( + !AttrBuilder(Attrs, Idx).overlaps(AttributeFuncs::typeIncompatible(Ty)), + "Wrong types for attribute: " + + AttributeSet::get(Context, Idx, AttributeFuncs::typeIncompatible(Ty)) + .getAsString(Idx), + V); if (PointerType *PTy = dyn_cast<PointerType>(Ty)) { SmallPtrSet<Type*, 4> Visited; @@ -1630,8 +1688,8 @@ void Verifier::visitConstantExprsRecursively(const Constant *EntryC) { if (const auto *GV = dyn_cast<GlobalValue>(C)) { // Global Values get visited separately, but we do need to make sure // that the global value is in the correct module - Assert(GV->getParent() == M, "Referencing global in another module!", - EntryC, M, GV, GV->getParent()); + Assert(GV->getParent() == &M, "Referencing global in another module!", + EntryC, &M, GV, GV->getParent()); continue; } @@ -1648,12 +1706,23 @@ void Verifier::visitConstantExprsRecursively(const Constant *EntryC) { } void Verifier::visitConstantExpr(const ConstantExpr *CE) { - if (CE->getOpcode() != Instruction::BitCast) - return; - - Assert(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0), - CE->getType()), - "Invalid bitcast", CE); + if (CE->getOpcode() == Instruction::BitCast) + Assert(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0), + CE->getType()), + "Invalid bitcast", CE); + + if (CE->getOpcode() == Instruction::IntToPtr || + CE->getOpcode() == Instruction::PtrToInt) { + auto *PtrTy = CE->getOpcode() == Instruction::IntToPtr + ? CE->getType() + : CE->getOperand(0)->getType(); + StringRef Msg = CE->getOpcode() == Instruction::IntToPtr + ? "inttoptr not supported for non-integral pointers" + : "ptrtoint not supported for non-integral pointers"; + Assert( + !DL.isNonIntegralPointerType(cast<PointerType>(PtrTy->getScalarType())), + Msg); + } } bool Verifier::verifyAttributeCount(AttributeSet Attrs, unsigned Params) { @@ -1783,7 +1852,7 @@ void Verifier::verifyStatepoint(ImmutableCallSite CS) { Assert(Call, "illegal use of statepoint token", &CI, U); if (!Call) continue; Assert(isa<GCRelocateInst>(Call) || isa<GCResultInst>(Call), - "gc.result or gc.relocate are the only value uses" + "gc.result or gc.relocate are the only value uses " "of a gc.statepoint", &CI, U); if (isa<GCResultInst>(Call)) { @@ -1880,7 +1949,7 @@ void Verifier::visitFunction(const Function &F) { FunctionType *FT = F.getFunctionType(); unsigned NumArgs = F.arg_size(); - Assert(Context == &F.getContext(), + Assert(&Context == &F.getContext(), "Function context does not match Module context!", &F); Assert(!F.hasCommonLinkage(), "Functions may not have common linkage", &F); @@ -2035,7 +2104,7 @@ void Verifier::visitFunction(const Function &F) { if (F.getIntrinsicID() && F.getParent()->isMaterialized()) { const User *U; if (F.hasAddressTaken(&U)) - Assert(0, "Invalid user of intrinsic instruction!", U); + Assert(false, "Invalid user of intrinsic instruction!", U); } Assert(!F.hasDLLImportStorageClass() || @@ -2078,9 +2147,9 @@ void Verifier::visitFunction(const Function &F) { continue; // FIXME: Once N is canonical, check "SP == &N". - Assert(SP->describes(&F), - "!dbg attachment points at wrong subprogram for function", N, &F, - &I, DL, Scope, SP); + AssertDI(SP->describes(&F), + "!dbg attachment points at wrong subprogram for function", N, &F, + &I, DL, Scope, SP); } } @@ -2218,7 +2287,7 @@ void Verifier::visitSelectInst(SelectInst &SI) { /// a pass, if any exist, it's an error. /// void Verifier::visitUserOp1(Instruction &I) { - Assert(0, "User-defined operators should not live outside of a pass!", &I); + Assert(false, "User-defined operators should not live outside of a pass!", &I); } void Verifier::visitTruncInst(TruncInst &I) { @@ -2409,6 +2478,11 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) { Assert(SrcTy->getScalarType()->isPointerTy(), "PtrToInt source must be pointer", &I); + + if (auto *PTy = dyn_cast<PointerType>(SrcTy->getScalarType())) + Assert(!DL.isNonIntegralPointerType(PTy), + "ptrtoint not supported for non-integral pointers"); + Assert(DestTy->getScalarType()->isIntegerTy(), "PtrToInt result must be integral", &I); Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "PtrToInt type mismatch", @@ -2433,6 +2507,11 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) { "IntToPtr source must be an integral", &I); Assert(DestTy->getScalarType()->isPointerTy(), "IntToPtr result must be a pointer", &I); + + if (auto *PTy = dyn_cast<PointerType>(DestTy->getScalarType())) + Assert(!DL.isNonIntegralPointerType(PTy), + "inttoptr not supported for non-integral pointers"); + Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "IntToPtr type mismatch", &I); if (SrcTy->isVectorTy()) { @@ -2541,15 +2620,20 @@ void Verifier::verifyCallSite(CallSite CS) { } // For each argument of the callsite, if it has the swifterror argument, - // make sure the underlying alloca has swifterror as well. + // make sure the underlying alloca/parameter it comes from has a swifterror as + // well. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) if (CS.paramHasAttr(i+1, Attribute::SwiftError)) { Value *SwiftErrorArg = CS.getArgument(i); - auto AI = dyn_cast<AllocaInst>(SwiftErrorArg->stripInBoundsOffsets()); - Assert(AI, "swifterror argument should come from alloca", AI, I); - if (AI) + if (auto AI = dyn_cast<AllocaInst>(SwiftErrorArg->stripInBoundsOffsets())) { Assert(AI->isSwiftError(), "swifterror argument for call has mismatched alloca", AI, I); + continue; + } + auto ArgI = dyn_cast<Argument>(SwiftErrorArg); + Assert(ArgI, "swifterror argument should come from an alloca or parameter", SwiftErrorArg, I); + Assert(ArgI->hasSwiftErrorAttr(), + "swifterror argument for call has mismatched parameter", ArgI, I); } if (FTy->isVarArg()) { @@ -2915,10 +2999,8 @@ static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); } -void Verifier::visitRangeMetadata(Instruction& I, - MDNode* Range, Type* Ty) { - assert(Range && - Range == I.getMetadata(LLVMContext::MD_range) && +void Verifier::visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty) { + assert(Range && Range == I.getMetadata(LLVMContext::MD_range) && "precondition violation"); unsigned NumOperands = Range->getNumOperands(); @@ -2965,9 +3047,8 @@ void Verifier::visitRangeMetadata(Instruction& I, } } -void Verifier::checkAtomicMemAccessSize(const Module *M, Type *Ty, - const Instruction *I) { - unsigned Size = M->getDataLayout().getTypeSizeInBits(Ty); +void Verifier::checkAtomicMemAccessSize(Type *Ty, const Instruction *I) { + unsigned Size = DL.getTypeSizeInBits(Ty); Assert(Size >= 8, "atomic memory access' size must be byte-sized", Ty, I); Assert(!(Size & (Size - 1)), "atomic memory access' operand must have a power-of-two size", Ty, I); @@ -2991,7 +3072,7 @@ void Verifier::visitLoadInst(LoadInst &LI) { "atomic load operand must have integer, pointer, or floating point " "type!", ElTy, &LI); - checkAtomicMemAccessSize(M, ElTy, &LI); + checkAtomicMemAccessSize(ElTy, &LI); } else { Assert(LI.getSynchScope() == CrossThread, "Non-atomic load cannot have SynchronizationScope specified", &LI); @@ -3020,7 +3101,7 @@ void Verifier::visitStoreInst(StoreInst &SI) { "atomic store operand must have integer, pointer, or floating point " "type!", ElTy, &SI); - checkAtomicMemAccessSize(M, ElTy, &SI); + checkAtomicMemAccessSize(ElTy, &SI); } else { Assert(SI.getSynchScope() == CrossThread, "Non-atomic store cannot have SynchronizationScope specified", &SI); @@ -3109,7 +3190,7 @@ void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) { Assert(ElTy->isIntegerTy() || ElTy->isPointerTy(), "cmpxchg operand must have integer or pointer type", ElTy, &CXI); - checkAtomicMemAccessSize(M, ElTy, &CXI); + checkAtomicMemAccessSize(ElTy, &CXI); Assert(ElTy == CXI.getOperand(1)->getType(), "Expected value type does not match pointer operand type!", &CXI, ElTy); @@ -3128,7 +3209,7 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) { Type *ElTy = PTy->getElementType(); Assert(ElTy->isIntegerTy(), "atomicrmw operand must have integer type!", &RMWI, ElTy); - checkAtomicMemAccessSize(M, ElTy, &RMWI); + checkAtomicMemAccessSize(ElTy, &RMWI); Assert(ElTy == RMWI.getOperand(1)->getType(), "Argument value type does not match pointer operand type!", &RMWI, ElTy); @@ -3288,6 +3369,21 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) { visitInstruction(LPI); } +void Verifier::visitResumeInst(ResumeInst &RI) { + Assert(RI.getFunction()->hasPersonalityFn(), + "ResumeInst needs to be in a function with a personality.", &RI); + + if (!LandingPadResultTy) + LandingPadResultTy = RI.getValue()->getType(); + else + Assert(LandingPadResultTy == RI.getValue()->getType(), + "The resume instruction should have a consistent result type " + "inside a function.", + &RI); + + visitTerminatorInst(RI); +} + void Verifier::visitCatchPadInst(CatchPadInst &CPI) { BasicBlock *BB = CPI.getParent(); @@ -3640,7 +3736,7 @@ void Verifier::visitInstruction(Instruction &I) { // Check to make sure that only first-class-values are operands to // instructions. if (!I.getOperand(i)->getType()->isFirstClassType()) { - Assert(0, "Instruction operands must be first-class values!", &I); + Assert(false, "Instruction operands must be first-class values!", &I); } if (Function *F = dyn_cast<Function>(I.getOperand(i))) { @@ -3653,14 +3749,16 @@ void Verifier::visitInstruction(Instruction &I) { Assert( !F->isIntrinsic() || isa<CallInst>(I) || F->getIntrinsicID() == Intrinsic::donothing || + F->getIntrinsicID() == Intrinsic::coro_resume || + F->getIntrinsicID() == Intrinsic::coro_destroy || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 || F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint, - "Cannot invoke an intrinsic other than donothing, patchpoint or " - "statepoint", + "Cannot invoke an intrinsic other than donothing, patchpoint, " + "statepoint, coro_resume or coro_destroy", &I); - Assert(F->getParent() == M, "Referencing function in another module!", - &I, M, F, F->getParent()); + Assert(F->getParent() == &M, "Referencing function in another module!", + &I, &M, F, F->getParent()); } else if (BasicBlock *OpBB = dyn_cast<BasicBlock>(I.getOperand(i))) { Assert(OpBB->getParent() == BB->getParent(), "Referring to a basic block in another function!", &I); @@ -3668,7 +3766,8 @@ void Verifier::visitInstruction(Instruction &I) { Assert(OpArg->getParent() == BB->getParent(), "Referring to an argument in another function!", &I); } else if (GlobalValue *GV = dyn_cast<GlobalValue>(I.getOperand(i))) { - Assert(GV->getParent() == M, "Referencing global in another module!", &I, M, GV, GV->getParent()); + Assert(GV->getParent() == &M, "Referencing global in another module!", &I, + &M, GV, GV->getParent()); } else if (isa<Instruction>(I.getOperand(i))) { verifyDominatesUse(I, i); } else if (isa<InlineAsm>(I.getOperand(i))) { @@ -3676,9 +3775,12 @@ void Verifier::visitInstruction(Instruction &I) { (i + 3 == e && isa<InvokeInst>(I)), "Cannot take the address of an inline asm!", &I); } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(I.getOperand(i))) { - if (CE->getType()->isPtrOrPtrVectorTy()) { + if (CE->getType()->isPtrOrPtrVectorTy() || + !DL.getNonIntegralAddressSpaces().empty()) { // If we have a ConstantExpr pointer, we need to see if it came from an - // illegal bitcast (inttoptr <constant int> ) + // illegal bitcast. If the datalayout string specifies non-integral + // address spaces then we also need to check for illegal ptrtoint and + // inttoptr expressions. visitConstantExprsRecursively(CE); } } @@ -3691,7 +3793,7 @@ void Verifier::visitInstruction(Instruction &I) { if (ConstantFP *CFP0 = mdconst::dyn_extract_or_null<ConstantFP>(MD->getOperand(0))) { const APFloat &Accuracy = CFP0->getValueAPF(); - Assert(&Accuracy.getSemantics() == &APFloat::IEEEsingle, + Assert(&Accuracy.getSemantics() == &APFloat::IEEEsingle(), "fpmath accuracy must have float type", &I); Assert(Accuracy.isFiniteNonZero() && !Accuracy.isNegative(), "fpmath accuracy not a positive number!", &I); @@ -3721,6 +3823,9 @@ void Verifier::visitInstruction(Instruction &I) { if (MDNode *MD = I.getMetadata(LLVMContext::MD_dereferenceable_or_null)) visitDereferenceableMetadata(I, MD); + if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa)) + TBAAVerifyHelper.visitTBAAMetadata(I, TBAA); + if (MDNode *AlignMD = I.getMetadata(LLVMContext::MD_align)) { Assert(I.getType()->isPointerTy(), "align applies only to pointer types", &I); @@ -3743,7 +3848,7 @@ void Verifier::visitInstruction(Instruction &I) { } if (auto *DII = dyn_cast<DbgInfoIntrinsic>(&I)) - verifyBitPieceExpression(*DII); + verifyFragmentExpression(*DII); InstsInThisBlock.insert(&I); } @@ -3803,6 +3908,20 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { switch (ID) { default: break; + case Intrinsic::coro_id: { + auto *InfoArg = CS.getArgOperand(3)->stripPointerCasts(); + if (isa<ConstantPointerNull>(InfoArg)) + break; + auto *GV = dyn_cast<GlobalVariable>(InfoArg); + Assert(GV && GV->isConstant() && GV->hasDefinitiveInitializer(), + "info argument of llvm.coro.begin must refer to an initialized " + "constant"); + Constant *Init = GV->getInitializer(); + Assert(isa<ConstantStruct>(Init) || isa<ConstantArray>(Init), + "info argument of llvm.coro.begin must refer to either a struct or " + "an array"); + break; + } case Intrinsic::ctlz: // llvm.ctlz case Intrinsic::cttz: // llvm.cttz Assert(isa<ConstantInt>(CS.getArgOperand(1)), @@ -3833,6 +3952,32 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { CS); break; } + case Intrinsic::memcpy_element_atomic: { + ConstantInt *ElementSizeCI = dyn_cast<ConstantInt>(CS.getArgOperand(3)); + Assert(ElementSizeCI, "element size of the element-wise atomic memory " + "intrinsic must be a constant int", + CS); + const APInt &ElementSizeVal = ElementSizeCI->getValue(); + Assert(ElementSizeVal.isPowerOf2(), + "element size of the element-wise atomic memory intrinsic " + "must be a power of 2", + CS); + + auto IsValidAlignment = [&](uint64_t Alignment) { + return isPowerOf2_64(Alignment) && ElementSizeVal.ule(Alignment); + }; + + uint64_t DstAlignment = CS.getParamAlignment(1), + SrcAlignment = CS.getParamAlignment(2); + + Assert(IsValidAlignment(DstAlignment), + "incorrect alignment of the destination argument", + CS); + Assert(IsValidAlignment(SrcAlignment), + "incorrect alignment of the source argument", + CS); + break; + } case Intrinsic::gcroot: case Intrinsic::gcwrite: case Intrinsic::gcread: @@ -4181,10 +4326,10 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { if (!VarSP || !LocSP) return; // Broken scope chains are checked elsewhere. - Assert(VarSP == LocSP, "mismatched subprogram between llvm.dbg." + Kind + - " variable and !dbg attachment", - &DII, BB, F, Var, Var->getScope()->getSubprogram(), Loc, - Loc->getScope()->getSubprogram()); + AssertDI(VarSP == LocSP, "mismatched subprogram between llvm.dbg." + Kind + + " variable and !dbg attachment", + &DII, BB, F, Var, Var->getScope()->getSubprogram(), Loc, + Loc->getScope()->getSubprogram()); } static uint64_t getVariableSize(const DILocalVariable &V) { @@ -4210,7 +4355,7 @@ static uint64_t getVariableSize(const DILocalVariable &V) { return 0; } -void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I) { +void Verifier::verifyFragmentExpression(const DbgInfoIntrinsic &I) { DILocalVariable *V; DIExpression *E; if (auto *DVI = dyn_cast<DbgValueInst>(&I)) { @@ -4227,7 +4372,8 @@ void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I) { return; // Nothing to do if this isn't a bit piece expression. - if (!E->isBitPiece()) + auto Fragment = E->getFragmentInfo(); + if (!Fragment) return; // The frontend helps out GDB by emitting the members of local anonymous @@ -4245,21 +4391,21 @@ void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I) { if (!VarSize) return; - unsigned PieceSize = E->getBitPieceSize(); - unsigned PieceOffset = E->getBitPieceOffset(); - Assert(PieceSize + PieceOffset <= VarSize, - "piece is larger than or outside of variable", &I, V, E); - Assert(PieceSize != VarSize, "piece covers entire variable", &I, V, E); + unsigned FragSize = Fragment->SizeInBits; + unsigned FragOffset = Fragment->OffsetInBits; + AssertDI(FragSize + FragOffset <= VarSize, + "fragment is larger than or outside of variable", &I, V, E); + AssertDI(FragSize != VarSize, "fragment covers entire variable", &I, V, E); } void Verifier::verifyCompileUnits() { - auto *CUs = M->getNamedMetadata("llvm.dbg.cu"); + auto *CUs = M.getNamedMetadata("llvm.dbg.cu"); SmallPtrSet<const Metadata *, 2> Listed; if (CUs) Listed.insert(CUs->op_begin(), CUs->op_end()); - Assert( - std::all_of(CUVisited.begin(), CUVisited.end(), - [&Listed](const Metadata *CU) { return Listed.count(CU); }), + AssertDI( + all_of(CUVisited, + [&Listed](const Metadata *CU) { return Listed.count(CU); }), "All DICompileUnits must be listed in llvm.dbg.cu"); CUVisited.clear(); } @@ -4285,7 +4431,7 @@ bool llvm::verifyFunction(const Function &f, raw_ostream *OS) { Function &F = const_cast<Function &>(f); // Don't use a raw_null_ostream. Printing IR is expensive. - Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/true); + Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/true, *f.getParent()); // Note that this function's return value is inverted from what you would // expect of a function called "verify". @@ -4295,13 +4441,13 @@ bool llvm::verifyFunction(const Function &f, raw_ostream *OS) { bool llvm::verifyModule(const Module &M, raw_ostream *OS, bool *BrokenDebugInfo) { // Don't use a raw_null_ostream. Printing IR is expensive. - Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/!BrokenDebugInfo); + Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/!BrokenDebugInfo, M); bool Broken = false; for (const Function &F : M) Broken |= !V.verify(F); - Broken |= !V.verify(M); + Broken |= !V.verify(); if (BrokenDebugInfo) *BrokenDebugInfo = V.hasBrokenDebugInfo(); // Note that this function's return value is inverted from what you would @@ -4310,26 +4456,30 @@ bool llvm::verifyModule(const Module &M, raw_ostream *OS, } namespace { + struct VerifierLegacyPass : public FunctionPass { static char ID; - Verifier V; + std::unique_ptr<Verifier> V; bool FatalErrors = true; - VerifierLegacyPass() - : FunctionPass(ID), - V(&dbgs(), /*ShouldTreatBrokenDebugInfoAsError=*/false) { + VerifierLegacyPass() : FunctionPass(ID) { initializeVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); } explicit VerifierLegacyPass(bool FatalErrors) : FunctionPass(ID), - V(&dbgs(), /*ShouldTreatBrokenDebugInfoAsError=*/false), FatalErrors(FatalErrors) { initializeVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); } + bool doInitialization(Module &M) override { + V = llvm::make_unique<Verifier>( + &dbgs(), /*ShouldTreatBrokenDebugInfoAsError=*/false, M); + return false; + } + bool runOnFunction(Function &F) override { - if (!V.verify(F) && FatalErrors) + if (!V->verify(F) && FatalErrors) report_fatal_error("Broken function found, compilation aborted!"); return false; @@ -4339,17 +4489,17 @@ struct VerifierLegacyPass : public FunctionPass { bool HasErrors = false; for (Function &F : M) if (F.isDeclaration()) - HasErrors |= !V.verify(F); + HasErrors |= !V->verify(F); - HasErrors |= !V.verify(M); + HasErrors |= !V->verify(); if (FatalErrors) { if (HasErrors) report_fatal_error("Broken module found, compilation aborted!"); - assert(!V.hasBrokenDebugInfo() && "Module contains invalid debug info"); + assert(!V->hasBrokenDebugInfo() && "Module contains invalid debug info"); } // Strip broken debug info. - if (V.hasBrokenDebugInfo()) { + if (V->hasBrokenDebugInfo()) { DiagnosticInfoIgnoringInvalidDebugMetadata DiagInvalid(M); M.getContext().diagnose(DiagInvalid); if (!StripDebugInfo(M)) @@ -4362,6 +4512,277 @@ struct VerifierLegacyPass : public FunctionPass { AU.setPreservesAll(); } }; + +} // end anonymous namespace + +/// Helper to issue failure from the TBAA verification +template <typename... Tys> void TBAAVerifier::CheckFailed(Tys &&... Args) { + if (Diagnostic) + return Diagnostic->CheckFailed(Args...); +} + +#define AssertTBAA(C, ...) \ + do { \ + if (!(C)) { \ + CheckFailed(__VA_ARGS__); \ + return false; \ + } \ + } while (false) + +/// Verify that \p BaseNode can be used as the "base type" in the struct-path +/// TBAA scheme. This means \p BaseNode is either a scalar node, or a +/// struct-type node describing an aggregate data structure (like a struct). +TBAAVerifier::TBAABaseNodeSummary +TBAAVerifier::verifyTBAABaseNode(Instruction &I, const MDNode *BaseNode) { + if (BaseNode->getNumOperands() < 2) { + CheckFailed("Base nodes must have at least two operands", &I, BaseNode); + return {true, ~0u}; + } + + auto Itr = TBAABaseNodes.find(BaseNode); + if (Itr != TBAABaseNodes.end()) + return Itr->second; + + auto Result = verifyTBAABaseNodeImpl(I, BaseNode); + auto InsertResult = TBAABaseNodes.insert({BaseNode, Result}); + (void)InsertResult; + assert(InsertResult.second && "We just checked!"); + return Result; +} + +TBAAVerifier::TBAABaseNodeSummary +TBAAVerifier::verifyTBAABaseNodeImpl(Instruction &I, const MDNode *BaseNode) { + const TBAAVerifier::TBAABaseNodeSummary InvalidNode = {true, ~0u}; + + if (BaseNode->getNumOperands() == 2) { + // Scalar nodes can only be accessed at offset 0. + return isValidScalarTBAANode(BaseNode) + ? TBAAVerifier::TBAABaseNodeSummary({false, 0}) + : InvalidNode; + } + + if (BaseNode->getNumOperands() % 2 != 1) { + CheckFailed("Struct tag nodes must have an odd number of operands!", + BaseNode); + return InvalidNode; + } + + if (!isa<MDString>(BaseNode->getOperand(0))) { + CheckFailed("Struct tag nodes have a string as their first operand", + BaseNode); + return InvalidNode; + } + + bool Failed = false; + + Optional<APInt> PrevOffset; + unsigned BitWidth = ~0u; + + // We've already checked that BaseNode is not a degenerate root node with one + // operand in \c verifyTBAABaseNode, so this loop should run at least once. + for (unsigned Idx = 1; Idx < BaseNode->getNumOperands(); Idx += 2) { + const MDOperand &FieldTy = BaseNode->getOperand(Idx); + const MDOperand &FieldOffset = BaseNode->getOperand(Idx + 1); + if (!isa<MDNode>(FieldTy)) { + CheckFailed("Incorrect field entry in struct type node!", &I, BaseNode); + Failed = true; + continue; + } + + auto *OffsetEntryCI = + mdconst::dyn_extract_or_null<ConstantInt>(FieldOffset); + if (!OffsetEntryCI) { + CheckFailed("Offset entries must be constants!", &I, BaseNode); + Failed = true; + continue; + } + + if (BitWidth == ~0u) + BitWidth = OffsetEntryCI->getBitWidth(); + + if (OffsetEntryCI->getBitWidth() != BitWidth) { + CheckFailed( + "Bitwidth between the offsets and struct type entries must match", &I, + BaseNode); + Failed = true; + continue; + } + + // NB! As far as I can tell, we generate a non-strictly increasing offset + // sequence only from structs that have zero size bit fields. When + // recursing into a contained struct in \c getFieldNodeFromTBAABaseNode we + // pick the field lexically the latest in struct type metadata node. This + // mirrors the actual behavior of the alias analysis implementation. + bool IsAscending = + !PrevOffset || PrevOffset->ule(OffsetEntryCI->getValue()); + + if (!IsAscending) { + CheckFailed("Offsets must be increasing!", &I, BaseNode); + Failed = true; + } + + PrevOffset = OffsetEntryCI->getValue(); + } + + return Failed ? InvalidNode + : TBAAVerifier::TBAABaseNodeSummary(false, BitWidth); +} + +static bool IsRootTBAANode(const MDNode *MD) { + return MD->getNumOperands() < 2; +} + +static bool IsScalarTBAANodeImpl(const MDNode *MD, + SmallPtrSetImpl<const MDNode *> &Visited) { + if (MD->getNumOperands() != 2 && MD->getNumOperands() != 3) + return false; + + if (!isa<MDString>(MD->getOperand(0))) + return false; + + if (MD->getNumOperands() == 3) { + auto *Offset = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2)); + if (!(Offset && Offset->isZero() && isa<MDString>(MD->getOperand(0)))) + return false; + } + + auto *Parent = dyn_cast_or_null<MDNode>(MD->getOperand(1)); + return Parent && Visited.insert(Parent).second && + (IsRootTBAANode(Parent) || IsScalarTBAANodeImpl(Parent, Visited)); +} + +bool TBAAVerifier::isValidScalarTBAANode(const MDNode *MD) { + auto ResultIt = TBAAScalarNodes.find(MD); + if (ResultIt != TBAAScalarNodes.end()) + return ResultIt->second; + + SmallPtrSet<const MDNode *, 4> Visited; + bool Result = IsScalarTBAANodeImpl(MD, Visited); + auto InsertResult = TBAAScalarNodes.insert({MD, Result}); + (void)InsertResult; + assert(InsertResult.second && "Just checked!"); + + return Result; +} + +/// Returns the field node at the offset \p Offset in \p BaseNode. Update \p +/// Offset in place to be the offset within the field node returned. +/// +/// We assume we've okayed \p BaseNode via \c verifyTBAABaseNode. +MDNode *TBAAVerifier::getFieldNodeFromTBAABaseNode(Instruction &I, + const MDNode *BaseNode, + APInt &Offset) { + assert(BaseNode->getNumOperands() >= 2 && "Invalid base node!"); + + // Scalar nodes have only one possible "field" -- their parent in the access + // hierarchy. Offset must be zero at this point, but our caller is supposed + // to Assert that. + if (BaseNode->getNumOperands() == 2) + return cast<MDNode>(BaseNode->getOperand(1)); + + for (unsigned Idx = 1; Idx < BaseNode->getNumOperands(); Idx += 2) { + auto *OffsetEntryCI = + mdconst::extract<ConstantInt>(BaseNode->getOperand(Idx + 1)); + if (OffsetEntryCI->getValue().ugt(Offset)) { + if (Idx == 1) { + CheckFailed("Could not find TBAA parent in struct type node", &I, + BaseNode, &Offset); + return nullptr; + } + + auto *PrevOffsetEntryCI = + mdconst::extract<ConstantInt>(BaseNode->getOperand(Idx - 1)); + Offset -= PrevOffsetEntryCI->getValue(); + return cast<MDNode>(BaseNode->getOperand(Idx - 2)); + } + } + + auto *LastOffsetEntryCI = mdconst::extract<ConstantInt>( + BaseNode->getOperand(BaseNode->getNumOperands() - 1)); + + Offset -= LastOffsetEntryCI->getValue(); + return cast<MDNode>(BaseNode->getOperand(BaseNode->getNumOperands() - 2)); +} + +bool TBAAVerifier::visitTBAAMetadata(Instruction &I, const MDNode *MD) { + AssertTBAA(isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) || + isa<VAArgInst>(I) || isa<AtomicRMWInst>(I) || + isa<AtomicCmpXchgInst>(I), + "TBAA is only for loads, stores and calls!", &I); + + bool IsStructPathTBAA = + isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3; + + AssertTBAA( + IsStructPathTBAA, + "Old-style TBAA is no longer allowed, use struct-path TBAA instead", &I); + + AssertTBAA(MD->getNumOperands() < 5, + "Struct tag metadata must have either 3 or 4 operands", &I, MD); + + MDNode *BaseNode = dyn_cast_or_null<MDNode>(MD->getOperand(0)); + MDNode *AccessType = dyn_cast_or_null<MDNode>(MD->getOperand(1)); + + if (MD->getNumOperands() == 4) { + auto *IsImmutableCI = + mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(3)); + AssertTBAA(IsImmutableCI, + "Immutability tag on struct tag metadata must be a constant", &I, + MD); + AssertTBAA( + IsImmutableCI->isZero() || IsImmutableCI->isOne(), + "Immutability part of the struct tag metadata must be either 0 or 1", + &I, MD); + } + + AssertTBAA(BaseNode && AccessType, + "Malformed struct tag metadata: base and access-type " + "should be non-null and point to Metadata nodes", + &I, MD, BaseNode, AccessType); + + AssertTBAA(isValidScalarTBAANode(AccessType), + "Access type node must be a valid scalar type", &I, MD, + AccessType); + + auto *OffsetCI = mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(2)); + AssertTBAA(OffsetCI, "Offset must be constant integer", &I, MD); + + APInt Offset = OffsetCI->getValue(); + bool SeenAccessTypeInPath = false; + + SmallPtrSet<MDNode *, 4> StructPath; + + for (/* empty */; BaseNode && !IsRootTBAANode(BaseNode); + BaseNode = getFieldNodeFromTBAABaseNode(I, BaseNode, Offset)) { + if (!StructPath.insert(BaseNode).second) { + CheckFailed("Cycle detected in struct path", &I, MD); + return false; + } + + bool Invalid; + unsigned BaseNodeBitWidth; + std::tie(Invalid, BaseNodeBitWidth) = verifyTBAABaseNode(I, BaseNode); + + // If the base node is invalid in itself, then we've already printed all the + // errors we wanted to print. + if (Invalid) + return false; + + SeenAccessTypeInPath |= BaseNode == AccessType; + + if (isValidScalarTBAANode(BaseNode) || BaseNode == AccessType) + AssertTBAA(Offset == 0, "Offset not zero at the point of scalar access", + &I, MD, &Offset); + + AssertTBAA(BaseNodeBitWidth == Offset.getBitWidth() || + (BaseNodeBitWidth == 0 && Offset == 0), + "Access bit-width not the same as description bit-width", &I, MD, + BaseNodeBitWidth, Offset.getBitWidth()); + } + + AssertTBAA(SeenAccessTypeInPath, "Did not see access type in access path!", + &I, MD); + return true; } char VerifierLegacyPass::ID = 0; @@ -4371,7 +4792,7 @@ FunctionPass *llvm::createVerifierPass(bool FatalErrors) { return new VerifierLegacyPass(FatalErrors); } -char VerifierAnalysis::PassID; +AnalysisKey VerifierAnalysis::Key; VerifierAnalysis::Result VerifierAnalysis::run(Module &M, ModuleAnalysisManager &) { Result Res; |