diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2023-07-26 19:03:47 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2023-07-26 19:04:23 +0000 |
commit | 7fa27ce4a07f19b07799a767fc29416f3b625afb (patch) | |
tree | 27825c83636c4de341eb09a74f49f5d38a15d165 /llvm/lib/IR | |
parent | e3b557809604d036af6e00c60f012c2025b59a5e (diff) | |
download | src-7fa27ce4a07f19b07799a767fc29416f3b625afb.tar.gz src-7fa27ce4a07f19b07799a767fc29416f3b625afb.zip |
Vendor import of llvm-project main llvmorg-17-init-19304-gd0b54bb50e51,vendor/llvm-project/llvmorg-17-init-19304-gd0b54bb50e51
the last commit before the upstream release/17.x branch was created.
Diffstat (limited to 'llvm/lib/IR')
45 files changed, 2805 insertions, 1455 deletions
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 6108ce09c289..be4a3ed79d88 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -329,8 +329,12 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::Swift: Out << "swiftcc"; break; case CallingConv::SwiftTail: Out << "swifttailcc"; break; case CallingConv::X86_INTR: Out << "x86_intrcc"; break; - case CallingConv::HHVM: Out << "hhvmcc"; break; - case CallingConv::HHVM_C: Out << "hhvm_ccc"; break; + case CallingConv::DUMMY_HHVM: + Out << "hhvmcc"; + break; + case CallingConv::DUMMY_HHVM_C: + Out << "hhvm_ccc"; + break; case CallingConv::AMDGPU_VS: Out << "amdgpu_vs"; break; case CallingConv::AMDGPU_LS: Out << "amdgpu_ls"; break; case CallingConv::AMDGPU_HS: Out << "amdgpu_hs"; break; @@ -338,6 +342,12 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::AMDGPU_GS: Out << "amdgpu_gs"; break; case CallingConv::AMDGPU_PS: Out << "amdgpu_ps"; break; case CallingConv::AMDGPU_CS: Out << "amdgpu_cs"; break; + case CallingConv::AMDGPU_CS_Chain: + Out << "amdgpu_cs_chain"; + break; + case CallingConv::AMDGPU_CS_ChainPreserve: + Out << "amdgpu_cs_chain_preserve"; + break; case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break; case CallingConv::AMDGPU_Gfx: Out << "amdgpu_gfx"; break; } @@ -421,8 +431,8 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) { bool FirstElt = true; if (all_of(Mask, [](int Elt) { return Elt == 0; })) { Out << "zeroinitializer"; - } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { - Out << "undef"; + } else if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) { + Out << "poison"; } else { Out << "<"; for (int Elt : Mask) { @@ -431,8 +441,8 @@ static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) { else Out << ", "; Out << "i32 "; - if (Elt == UndefMaskElem) - Out << "undef"; + if (Elt == PoisonMaskElem) + Out << "poison"; else Out << Elt; } @@ -585,16 +595,9 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) { } case Type::PointerTyID: { PointerType *PTy = cast<PointerType>(Ty); - if (PTy->isOpaque()) { - OS << "ptr"; - if (unsigned AddressSpace = PTy->getAddressSpace()) - OS << " addrspace(" << AddressSpace << ')'; - return; - } - print(PTy->getNonOpaquePointerElementType(), OS); + OS << "ptr"; if (unsigned AddressSpace = PTy->getAddressSpace()) OS << " addrspace(" << AddressSpace << ')'; - OS << '*'; return; } case Type::ArrayTyID: { @@ -1585,8 +1588,7 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, Out << CE->getOpcodeName(); WriteOptimizationInfo(Out, CE); if (CE->isCompare()) - Out << ' ' << CmpInst::getPredicateName( - static_cast<CmpInst::Predicate>(CE->getPredicate())); + Out << ' ' << static_cast<CmpInst::Predicate>(CE->getPredicate()); Out << " ("; std::optional<unsigned> InRangeOp; @@ -3207,10 +3209,7 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) { printTypeIdInfo(*TIdInfo); // The AllocationType identifiers capture the profiled context behavior - // reaching a specific static allocation site (possibly cloned). Thus - // "notcoldandcold" implies there are multiple contexts which reach this site, - // some of which are cold and some of which are not, and that need to - // disambiguate via cloning or other context identification. + // reaching a specific static allocation site (possibly cloned). auto AllocTypeName = [](uint8_t Type) -> const char * { switch (Type) { case (uint8_t)AllocationType::None: @@ -3219,8 +3218,8 @@ void AssemblyWriter::printFunctionSummary(const FunctionSummary *FS) { return "notcold"; case (uint8_t)AllocationType::Cold: return "cold"; - case (uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold: - return "notcoldandcold"; + case (uint8_t)AllocationType::Hot: + return "hot"; } llvm_unreachable("Unexpected alloc type"); }; @@ -4082,7 +4081,7 @@ void AssemblyWriter::printInstruction(const Instruction &I) { // Print out the compare instruction predicates if (const CmpInst *CI = dyn_cast<CmpInst>(&I)) - Out << ' ' << CmpInst::getPredicateName(CI->getPredicate()); + Out << ' ' << CI->getPredicate(); // Print out the atomicrmw operation if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&I)) diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h index 071c75e69377..78496786b0ae 100644 --- a/llvm/lib/IR/AttributeImpl.h +++ b/llvm/lib/IR/AttributeImpl.h @@ -266,6 +266,7 @@ public: UWTableKind getUWTableKind() const; AllocFnKind getAllocKind() const; MemoryEffects getMemoryEffects() const; + FPClassTest getNoFPClass() const; std::string getAsString(bool InAttrGrp) const; Type *getAttributeType(Attribute::AttrKind Kind) const; diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 8c989c464551..3d89d18e5822 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Config/llvm-config.h" +#include "llvm/IR/AttributeMask.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Type.h" @@ -216,6 +217,11 @@ Attribute Attribute::getWithMemoryEffects(LLVMContext &Context, return get(Context, Memory, ME.toIntValue()); } +Attribute Attribute::getWithNoFPClass(LLVMContext &Context, + FPClassTest ClassMask) { + return get(Context, NoFPClass, ClassMask); +} + Attribute Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const std::optional<unsigned> &NumElemsArg) { @@ -396,6 +402,12 @@ MemoryEffects Attribute::getMemoryEffects() const { return MemoryEffects::createFromIntValue(pImpl->getValueAsInt()); } +FPClassTest Attribute::getNoFPClass() const { + assert(hasAttribute(Attribute::NoFPClass) && + "Can only call getNoFPClass() on nofpclass attribute"); + return static_cast<FPClassTest>(pImpl->getValueAsInt()); +} + static const char *getModRefStr(ModRefInfo MR) { switch (MR) { case ModRefInfo::NoModRef: @@ -511,7 +523,7 @@ std::string Attribute::getAsString(bool InAttrGrp) const { // Print access kind for "other" as the default access kind. This way it // will apply to any new location kinds that get split out of "other". - ModRefInfo OtherMR = ME.getModRef(MemoryEffects::Other); + ModRefInfo OtherMR = ME.getModRef(IRMemLocation::Other); if (OtherMR != ModRefInfo::NoModRef || ME.getModRef() == OtherMR) { First = false; OS << getModRefStr(OtherMR); @@ -527,13 +539,13 @@ std::string Attribute::getAsString(bool InAttrGrp) const { First = false; switch (Loc) { - case MemoryEffects::ArgMem: + case IRMemLocation::ArgMem: OS << "argmem: "; break; - case MemoryEffects::InaccessibleMem: + case IRMemLocation::InaccessibleMem: OS << "inaccessiblemem: "; break; - case MemoryEffects::Other: + case IRMemLocation::Other: llvm_unreachable("This is represented as the default access kind"); } OS << getModRefStr(MR); @@ -543,6 +555,13 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return Result; } + if (hasAttribute(Attribute::NoFPClass)) { + std::string Result = "nofpclass"; + raw_string_ostream OS(Result); + OS << getNoFPClass(); + return Result; + } + // Convert target-dependent attributes to strings of the form: // // "kind" @@ -840,6 +859,10 @@ MemoryEffects AttributeSet::getMemoryEffects() const { return SetNode ? SetNode->getMemoryEffects() : MemoryEffects::unknown(); } +FPClassTest AttributeSet::getNoFPClass() const { + return SetNode ? SetNode->getNoFPClass() : fcNone; +} + std::string AttributeSet::getAsString(bool InAttrGrp) const { return SetNode ? SetNode->getAsString(InAttrGrp) : ""; } @@ -1024,6 +1047,12 @@ MemoryEffects AttributeSetNode::getMemoryEffects() const { return MemoryEffects::unknown(); } +FPClassTest AttributeSetNode::getNoFPClass() const { + if (auto A = findEnumAttribute(Attribute::NoFPClass)) + return A->getNoFPClass(); + return fcNone; +} + std::string AttributeSetNode::getAsString(bool InAttrGrp) const { std::string Str; for (iterator I = begin(), E = end(); I != E; ++I) { @@ -1560,6 +1589,14 @@ AttributeList::getParamDereferenceableOrNullBytes(unsigned Index) const { return getParamAttrs(Index).getDereferenceableOrNullBytes(); } +FPClassTest AttributeList::getRetNoFPClass() const { + return getRetAttrs().getNoFPClass(); +} + +FPClassTest AttributeList::getParamNoFPClass(unsigned Index) const { + return getParamAttrs(Index).getNoFPClass(); +} + UWTableKind AttributeList::getUWTableKind() const { return getFnAttrs().getUWTableKind(); } @@ -1803,6 +1840,13 @@ AttrBuilder &AttrBuilder::addMemoryAttr(MemoryEffects ME) { return addRawIntAttr(Attribute::Memory, ME.toIntValue()); } +AttrBuilder &AttrBuilder::addNoFPClassAttr(FPClassTest Mask) { + if (Mask == fcNone) + return *this; + + return addRawIntAttr(Attribute::NoFPClass, Mask); +} + AttrBuilder &AttrBuilder::addAllocKindAttr(AllocFnKind Kind) { return addRawIntAttr(Attribute::AllocKind, static_cast<uint64_t>(Kind)); } @@ -1885,6 +1929,16 @@ bool AttrBuilder::operator==(const AttrBuilder &B) const { // AttributeFuncs Function Defintions //===----------------------------------------------------------------------===// +/// Returns true if this is a type legal for the 'nofpclass' attribute. This +/// follows the same type rules as FPMathOperator. +/// +/// TODO: Consider relaxing to any FP type struct fields. +bool AttributeFuncs::isNoFPClassCompatibleType(Type *Ty) { + while (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) + Ty = ArrTy->getElementType(); + return Ty->isFPOrFPVectorTy(); +} + /// Which attributes cannot be applied to a type. AttributeMask AttributeFuncs::typeIncompatible(Type *Ty, AttributeSafetyKind ASK) { @@ -1926,6 +1980,11 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty, Incompatible.addAttribute(Attribute::Alignment); } + if (ASK & ASK_SAFE_TO_DROP) { + if (!isNoFPClassCompatibleType(Ty)) + Incompatible.addAttribute(Attribute::NoFPClass); + } + // Some attributes can apply to all "values" but there are no `void` values. if (Ty->isVoidTy()) { if (ASK & ASK_SAFE_TO_DROP) @@ -1943,6 +2002,41 @@ AttributeMask AttributeFuncs::getUBImplyingAttributes() { return AM; } +/// Callees with dynamic denormal modes are compatible with any caller mode. +static bool denormModeCompatible(DenormalMode CallerMode, + DenormalMode CalleeMode) { + if (CallerMode == CalleeMode || CalleeMode == DenormalMode::getDynamic()) + return true; + + // If they don't exactly match, it's OK if the mismatched component is + // dynamic. + if (CalleeMode.Input == CallerMode.Input && + CalleeMode.Output == DenormalMode::Dynamic) + return true; + + if (CalleeMode.Output == CallerMode.Output && + CalleeMode.Input == DenormalMode::Dynamic) + return true; + return false; +} + +static bool checkDenormMode(const Function &Caller, const Function &Callee) { + DenormalMode CallerMode = Caller.getDenormalModeRaw(); + DenormalMode CalleeMode = Callee.getDenormalModeRaw(); + + if (denormModeCompatible(CallerMode, CalleeMode)) { + DenormalMode CallerModeF32 = Caller.getDenormalModeF32Raw(); + DenormalMode CalleeModeF32 = Callee.getDenormalModeF32Raw(); + if (CallerModeF32 == DenormalMode::getInvalid()) + CallerModeF32 = CallerMode; + if (CalleeModeF32 == DenormalMode::getInvalid()) + CalleeModeF32 = CalleeMode; + return denormModeCompatible(CallerModeF32, CalleeModeF32); + } + + return false; +} + template<typename AttrClass> static bool isEqual(const Function &Caller, const Function &Callee) { return Caller.getFnAttribute(AttrClass::getKind()) == diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 7b9c55ff30a5..71b5722925a1 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -13,10 +13,13 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/AutoUpgrade.h" +#include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" -#include "llvm/ADT/Triple.h" +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/IR/AttributeMask.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" @@ -26,15 +29,26 @@ #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" #include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/IntrinsicsNVPTX.h" +#include "llvm/IR/IntrinsicsRISCV.h" +#include "llvm/IR/IntrinsicsWebAssembly.h" #include "llvm/IR/IntrinsicsX86.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Regex.h" +#include "llvm/TargetParser/Triple.h" #include <cstring> + using namespace llvm; +static cl::opt<bool> + DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", + cl::desc("Disable autoupgrade of debug info")); + static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have @@ -578,6 +592,71 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, return false; } +static Intrinsic::ID ShouldUpgradeNVPTXBF16Intrinsic(StringRef Name) { + return StringSwitch<Intrinsic::ID>(Name) + .Case("abs.bf16", Intrinsic::nvvm_abs_bf16) + .Case("abs.bf16x2", Intrinsic::nvvm_abs_bf16x2) + .Case("fma.rn.bf16", Intrinsic::nvvm_fma_rn_bf16) + .Case("fma.rn.bf16x2", Intrinsic::nvvm_fma_rn_bf16x2) + .Case("fma.rn.ftz_bf16", Intrinsic::nvvm_fma_rn_ftz_bf16) + .Case("fma.rn.ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2) + .Case("fma.rn.ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16) + .Case("fma.rn.ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2) + .Case("fma.rn.ftz_sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16) + .Case("fma.rn.ftz_sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2) + .Case("fma.rn.relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16) + .Case("fma.rn.relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2) + .Case("fma.rn.sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16) + .Case("fma.rn.sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2) + .Case("fmax.bf16", Intrinsic::nvvm_fmax_bf16) + .Case("fmax.bf16x2", Intrinsic::nvvm_fmax_bf16x2) + .Case("fmax.ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16) + .Case("fmax.ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2) + .Case("fmax.ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16) + .Case("fmax.ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2) + .Case("fmax.ftz.nan.xorsign.abs.bf16", + Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16) + .Case("fmax.ftz.nan.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2) + .Case("fmax.ftz.xorsign.abs.bf16", + Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16) + .Case("fmax.ftz.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2) + .Case("fmax.nan.bf16", Intrinsic::nvvm_fmax_nan_bf16) + .Case("fmax.nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2) + .Case("fmax.nan.xorsign.abs.bf16", + Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16) + .Case("fmax.nan.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2) + .Case("fmax.xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16) + .Case("fmax.xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2) + .Case("fmin.bf16", Intrinsic::nvvm_fmin_bf16) + .Case("fmin.bf16x2", Intrinsic::nvvm_fmin_bf16x2) + .Case("fmin.ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16) + .Case("fmin.ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2) + .Case("fmin.ftz.nan_bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16) + .Case("fmin.ftz.nan_bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2) + .Case("fmin.ftz.nan.xorsign.abs.bf16", + Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16) + .Case("fmin.ftz.nan.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2) + .Case("fmin.ftz.xorsign.abs.bf16", + Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16) + .Case("fmin.ftz.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2) + .Case("fmin.nan.bf16", Intrinsic::nvvm_fmin_nan_bf16) + .Case("fmin.nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2) + .Case("fmin.nan.xorsign.abs.bf16", + Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16) + .Case("fmin.nan.xorsign.abs.bf16x2", + Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2) + .Case("fmin.xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16) + .Case("fmin.xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2) + .Case("neg.bf16", Intrinsic::nvvm_neg_bf16) + .Case("neg.bf16x2", Intrinsic::nvvm_neg_bf16x2) + .Default(Intrinsic::not_intrinsic); +} + static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { assert(F && "Illegal to upgrade a non-existent Function."); @@ -802,10 +881,14 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" || Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" || Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" || + Name == + "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" || + Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" || Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" || Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" || + Name == + "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" || + Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" || Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" || Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" || Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" || @@ -814,16 +897,25 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1") return true; - if (Name == "amdgcn.alignbit") { + if (Name.startswith("amdgcn.")) + Name = Name.substr(7); // Strip off "amdgcn." + + if (Name == "alignbit") { // Target specific intrinsic became redundant NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::fshr, {F->getReturnType()}); return true; } + if (Name.startswith("atomic.inc") || Name.startswith("atomic.dec")) { + // This was replaced with atomicrmw uinc_wrap and udec_wrap, so there's no + // new declaration. + NewFn = nullptr; + return true; + } + break; } - case 'c': { if (Name.startswith("ctlz.") && F->arg_size() == 1) { rename(F); @@ -840,6 +932,11 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { break; } case 'd': { + if (Name == "dbg.addr") { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); + return true; + } if (Name == "dbg.value" && F->arg_size() == 4) { rename(F); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); @@ -1051,7 +1148,12 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { {F->getReturnType()}); return true; } - + IID = ShouldUpgradeNVPTXBF16Intrinsic(Name); + if (IID != Intrinsic::not_intrinsic && + !F->getReturnType()->getScalarType()->isBFloatTy()) { + NewFn = nullptr; + return true; + } // The following nvvm intrinsics correspond exactly to an LLVM idiom, but // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall. // @@ -1107,6 +1209,87 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } break; + case 'r': + if (Name == "riscv.aes32dsi" && + !F->getFunctionType()->getParamType(2)->isIntegerTy(32)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_aes32dsi); + return true; + } + if (Name == "riscv.aes32dsmi" && + !F->getFunctionType()->getParamType(2)->isIntegerTy(32)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_aes32dsmi); + return true; + } + if (Name == "riscv.aes32esi" && + !F->getFunctionType()->getParamType(2)->isIntegerTy(32)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_aes32esi); + return true; + } + if (Name == "riscv.aes32esmi" && + !F->getFunctionType()->getParamType(2)->isIntegerTy(32)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_aes32esmi); + return true; + } + if (Name.startswith("riscv.sm4ks") && + (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) || + F->getFunctionType()->getReturnType()->isIntegerTy(64))) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_sm4ks); + return true; + } + if (Name.startswith("riscv.sm4ed") && + (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) || + F->getFunctionType()->getReturnType()->isIntegerTy(64))) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_sm4ed); + return true; + } + if (Name.startswith("riscv.sha256sig0") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::riscv_sha256sig0); + return true; + } + if (Name.startswith("riscv.sha256sig1") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::riscv_sha256sig1); + return true; + } + if (Name.startswith("riscv.sha256sum0") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::riscv_sha256sum0); + return true; + } + if (Name.startswith("riscv.sha256sum1") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::riscv_sha256sum1); + return true; + } + if (Name.startswith("riscv.sm3p0") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_sm3p0); + return true; + } + if (Name.startswith("riscv.sm3p1") && + F->getFunctionType()->getReturnType()->isIntegerTy(64)) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::riscv_sm3p1); + return true; + } + break; + case 's': if (Name == "stackprotectorcheck") { NewFn = nullptr; @@ -1125,6 +1308,40 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { break; } + case 'w': + if (Name.startswith("wasm.fma.")) { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::wasm_relaxed_madd, F->getReturnType()); + return true; + } + if (Name.startswith("wasm.fms.")) { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::wasm_relaxed_nmadd, F->getReturnType()); + return true; + } + if (Name.startswith("wasm.laneselect.")) { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::wasm_relaxed_laneselect, + F->getReturnType()); + return true; + } + if (Name == "wasm.dot.i8x16.i7x16.signed") { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed); + return true; + } + if (Name == "wasm.dot.i8x16.i7x16.add.signed") { + rename(F); + NewFn = Intrinsic::getDeclaration( + F->getParent(), Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed); + return true; + } + break; + case 'x': if (UpgradeX86IntrinsicFunction(F, Name, NewFn)) return true; @@ -1994,10 +2211,14 @@ static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" || Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" || Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" || + Name == + "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" || + Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" || Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" || Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" || - Name == "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" || + Name == + "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" || + Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" || Name == "cde.vcx1q.predicated.v2i64.v4i1" || Name == "cde.vcx1qa.predicated.v2i64.v4i1" || Name == "cde.vcx2q.predicated.v2i64.v4i1" || @@ -2062,6 +2283,38 @@ static Value *UpgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, llvm_unreachable("Unknown function for ARM CallBase upgrade."); } +static Value *UpgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, + Function *F, IRBuilder<> &Builder) { + const bool IsInc = Name.startswith("atomic.inc."); + if (IsInc || Name.startswith("atomic.dec.")) { + if (CI->getNumOperands() != 6) // Malformed bitcode. + return nullptr; + + AtomicRMWInst::BinOp RMWOp = + IsInc ? AtomicRMWInst::UIncWrap : AtomicRMWInst::UDecWrap; + + Value *Ptr = CI->getArgOperand(0); + Value *Val = CI->getArgOperand(1); + ConstantInt *OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)); + ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4)); + + AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent; + if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue())) + Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue()); + if (Order == AtomicOrdering::NotAtomic || + Order == AtomicOrdering::Unordered) + Order = AtomicOrdering::SequentiallyConsistent; + + AtomicRMWInst *RMW = Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order); + + if (!VolatileArg || !VolatileArg->isZero()) + RMW->setVolatile(true); + return RMW; + } + + llvm_unreachable("Unknown function for AMDGPU intrinsic upgrade."); +} + /// Upgrade a call to an old intrinsic. All argument and return casting must be /// provided to seamlessly integrate with existing context. void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { @@ -2092,9 +2345,11 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { bool IsARM = Name.startswith("arm."); if (IsARM) Name = Name.substr(4); + bool IsAMDGCN = Name.startswith("amdgcn."); + if (IsAMDGCN) + Name = Name.substr(7); if (IsX86 && Name.startswith("sse4a.movnt.")) { - Module *M = F->getParent(); SmallVector<Metadata *, 1> Elts; Elts.push_back( ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); @@ -2112,7 +2367,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1)); - SI->setMetadata(M->getMDKindID("nontemporal"), Node); + SI->setMetadata(LLVMContext::MD_nontemporal, Node); // Remove intrinsic. CI->eraseFromParent(); @@ -2121,7 +2376,6 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { if (IsX86 && (Name.startswith("avx.movnt.") || Name.startswith("avx512.storent."))) { - Module *M = F->getParent(); SmallVector<Metadata *, 1> Elts; Elts.push_back( ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); @@ -2137,7 +2391,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { StoreInst *SI = Builder.CreateAlignedStore( Arg1, BC, Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); - SI->setMetadata(M->getMDKindID("nontemporal"), Node); + SI->setMetadata(LLVMContext::MD_nontemporal, Node); // Remove intrinsic. CI->eraseFromParent(); @@ -3465,7 +3719,6 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { } else if (IsX86 && Name.startswith("avx512.cvtmask2")) { Rep = UpgradeMaskToInt(Builder, *CI); } else if (IsX86 && Name.endswith(".movntdqa")) { - Module *M = F->getParent(); MDNode *Node = MDNode::get( C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); @@ -3477,7 +3730,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { LoadInst *LI = Builder.CreateAlignedLoad( CI->getType(), BC, Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)); - LI->setMetadata(M->getMDKindID("nontemporal"), Node); + LI->setMetadata(LLVMContext::MD_nontemporal, Node); Rep = LI; } else if (IsX86 && (Name.startswith("fma.vfmadd.") || Name.startswith("fma.vfmsub.") || @@ -3907,13 +4160,38 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { {Arg->getType()}), Arg, "ctpop"); Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc"); - } else if (IsNVVM && Name == "h2f") { - Rep = Builder.CreateCall(Intrinsic::getDeclaration( + } else if (IsNVVM) { + if (Name == "h2f") { + Rep = + Builder.CreateCall(Intrinsic::getDeclaration( F->getParent(), Intrinsic::convert_from_fp16, {Builder.getFloatTy()}), CI->getArgOperand(0), "h2f"); + } else { + Intrinsic::ID IID = ShouldUpgradeNVPTXBF16Intrinsic(Name); + if (IID != Intrinsic::not_intrinsic && + !F->getReturnType()->getScalarType()->isBFloatTy()) { + rename(F); + NewFn = Intrinsic::getDeclaration(F->getParent(), IID); + SmallVector<Value *, 2> Args; + for (size_t I = 0; I < NewFn->arg_size(); ++I) { + Value *Arg = CI->getArgOperand(I); + Type *OldType = Arg->getType(); + Type *NewType = NewFn->getArg(I)->getType(); + Args.push_back((OldType->isIntegerTy() && + NewType->getScalarType()->isBFloatTy()) + ? Builder.CreateBitCast(Arg, NewType) + : Arg); + } + Rep = Builder.CreateCall(NewFn, Args); + if (F->getReturnType()->isIntegerTy()) + Rep = Builder.CreateBitCast(Rep, F->getReturnType()); + } + } } else if (IsARM) { Rep = UpgradeARMIntrinsicCall(Name, CI, F, Builder); + } else if (IsAMDGCN) { + Rep = UpgradeAMDGCNIntrinsicCall(Name, CI, F, Builder); } else { llvm_unreachable("Unknown function for CallBase upgrade."); } @@ -4120,7 +4398,20 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); break; - case Intrinsic::dbg_value: + case Intrinsic::dbg_value: { + StringRef Name = F->getName(); + Name = Name.substr(5); // Strip llvm. + // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`. + if (Name.startswith("dbg.addr")) { + DIExpression *Expr = cast<DIExpression>( + cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata()); + Expr = DIExpression::append(Expr, dwarf::DW_OP_deref); + NewCall = + Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), + MetadataAsValue::get(C, Expr)}); + break; + } + // Upgrade from the old version that had an extra offset argument. assert(CI->arg_size() == 4); // Drop nonzero offsets instead of attempting to upgrade them. @@ -4133,6 +4424,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { } CI->eraseFromParent(); return; + } case Intrinsic::ptr_annotation: // Upgrade from versions that lacked the annotation attribute argument. @@ -4167,6 +4459,60 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) { CI->eraseFromParent(); return; + case Intrinsic::riscv_aes32dsi: + case Intrinsic::riscv_aes32dsmi: + case Intrinsic::riscv_aes32esi: + case Intrinsic::riscv_aes32esmi: + case Intrinsic::riscv_sm4ks: + case Intrinsic::riscv_sm4ed: { + // The last argument to these intrinsics used to be i8 and changed to i32. + // The type overload for sm4ks and sm4ed was removed. + Value *Arg2 = CI->getArgOperand(2); + if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64)) + return; + + Value *Arg0 = CI->getArgOperand(0); + Value *Arg1 = CI->getArgOperand(1); + if (CI->getType()->isIntegerTy(64)) { + Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty()); + Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty()); + } + + Arg2 = ConstantInt::get(Type::getInt32Ty(C), + cast<ConstantInt>(Arg2)->getZExtValue()); + + NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2}); + Value *Res = NewCall; + if (Res->getType() != CI->getType()) + Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); + NewCall->takeName(CI); + CI->replaceAllUsesWith(Res); + CI->eraseFromParent(); + return; + } + case Intrinsic::riscv_sha256sig0: + case Intrinsic::riscv_sha256sig1: + case Intrinsic::riscv_sha256sum0: + case Intrinsic::riscv_sha256sum1: + case Intrinsic::riscv_sm3p0: + case Intrinsic::riscv_sm3p1: { + // The last argument to these intrinsics used to be i8 and changed to i32. + // The type overload for sm4ks and sm4ed was removed. + if (!CI->getType()->isIntegerTy(64)) + return; + + Value *Arg = + Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty()); + + NewCall = Builder.CreateCall(NewFn, Arg); + Value *Res = + Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true); + NewCall->takeName(CI); + CI->replaceAllUsesWith(Res); + CI->eraseFromParent(); + return; + } + case Intrinsic::x86_xop_vfrcz_ss: case Intrinsic::x86_xop_vfrcz_sd: NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); @@ -4384,12 +4730,16 @@ void llvm::UpgradeCallsToIntrinsic(Function *F) { } MDNode *llvm::UpgradeTBAANode(MDNode &MD) { + const unsigned NumOperands = MD.getNumOperands(); + if (NumOperands == 0) + return &MD; // Invalid, punt to a verifier error. + // Check if the tag uses struct-path aware TBAA format. - if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3) + if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3) return &MD; auto &Context = MD.getContext(); - if (MD.getNumOperands() == 3) { + if (NumOperands == 3) { Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)}; MDNode *ScalarType = MDNode::get(Context, Elts); // Create a MDNode <ScalarType, ScalarType, offset 0, const> @@ -4450,6 +4800,9 @@ Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) { /// Check the debug info version number, if it is out-dated, drop the debug /// info. Return true if module is modified. bool llvm::UpgradeDebugInfo(Module &M) { + if (DisableAutoUpgradeDebugInfo) + return false; + unsigned Version = getDebugMetadataVersionFromModule(M); if (Version == DEBUG_METADATA_VERSION) { bool BrokenDebugInfo = false; @@ -4889,9 +5242,10 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) { std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { Triple T(TT); - // For AMDGPU we uprgrade older DataLayouts to include the default globals - // address space of 1. - if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) { + // The only data layout upgrades needed for pre-GCN are setting the address + // space of globals to 1. + if (T.isAMDGPU() && !T.isAMDGCN() && !DL.contains("-G") && + !DL.startswith("G")) { return DL.empty() ? std::string("G1") : (DL + "-G1").str(); } @@ -4904,6 +5258,31 @@ std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { } std::string Res = DL.str(); + // AMDGCN data layout upgrades. + if (T.isAMDGCN()) { + // Define address spaces for constants. + if (!DL.contains("-G") && !DL.starts_with("G")) + Res.append(Res.empty() ? "G1" : "-G1"); + + // Add missing non-integral declarations. + // This goes before adding new address spaces to prevent incoherent string + // values. + if (!DL.contains("-ni") && !DL.startswith("ni")) + Res.append("-ni:7:8"); + // Update ni:7 to ni:7:8. + if (DL.ends_with("ni:7")) + Res.append(":8"); + + // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer + // resources) An empty data layout has already been upgraded to G1 by now. + if (!DL.contains("-p7") && !DL.startswith("p7")) + Res.append("-p7:160:256:256:32"); + if (!DL.contains("-p8") && !DL.startswith("p8")) + Res.append("-p8:128:128"); + + return Res; + } + if (!T.isX86()) return Res; @@ -4958,7 +5337,6 @@ void llvm::UpgradeAttributes(AttrBuilder &B) { } void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) { - // clang.arc.attachedcall bundles are now required to have an operand. // If they don't, it's okay to drop them entirely: when there is an operand, // the "attachedcall" is meaningful and required, but without an operand, diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 63d363e2d082..14e1787c2b14 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -133,9 +133,8 @@ iplist<BasicBlock>::iterator BasicBlock::eraseFromParent() { return getParent()->getBasicBlockList().erase(getIterator()); } -void BasicBlock::moveBefore(BasicBlock *MovePos) { - MovePos->getParent()->splice(MovePos->getIterator(), getParent(), - getIterator()); +void BasicBlock::moveBefore(SymbolTableList<BasicBlock>::iterator MovePos) { + getParent()->splice(MovePos, getParent(), getIterator()); } void BasicBlock::moveAfter(BasicBlock *MovePos) { @@ -205,6 +204,15 @@ const CallInst *BasicBlock::getPostdominatingDeoptimizeCall() const { return BB->getTerminatingDeoptimizeCall(); } +const Instruction *BasicBlock::getFirstMayFaultInst() const { + if (InstList.empty()) + return nullptr; + for (const Instruction &I : *this) + if (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallBase>(I)) + return &I; + return nullptr; +} + const Instruction* BasicBlock::getFirstNonPHI() const { for (const Instruction &I : *this) if (!isa<PHINode>(I)) diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp index f84fe79b21be..4c3325063c09 100644 --- a/llvm/lib/IR/ConstantFold.cpp +++ b/llvm/lib/IR/ConstantFold.cpp @@ -111,29 +111,6 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { if (SrcTy == DestTy) return V; // no-op cast - // Check to see if we are casting a pointer to an aggregate to a pointer to - // the first element. If so, return the appropriate GEP instruction. - if (PointerType *PTy = dyn_cast<PointerType>(V->getType())) - if (PointerType *DPTy = dyn_cast<PointerType>(DestTy)) - if (PTy->getAddressSpace() == DPTy->getAddressSpace() && - !PTy->isOpaque() && !DPTy->isOpaque() && - PTy->getNonOpaquePointerElementType()->isSized()) { - SmallVector<Value*, 8> IdxList; - Value *Zero = - Constant::getNullValue(Type::getInt32Ty(DPTy->getContext())); - IdxList.push_back(Zero); - Type *ElTy = PTy->getNonOpaquePointerElementType(); - while (ElTy && ElTy != DPTy->getNonOpaquePointerElementType()) { - ElTy = GetElementPtrInst::getTypeAtIndex(ElTy, (uint64_t)0); - IdxList.push_back(Zero); - } - - if (ElTy == DPTy->getNonOpaquePointerElementType()) - // This GEP is inbounds because all indices are zero. - return ConstantExpr::getInBoundsGetElementPtr( - PTy->getNonOpaquePointerElementType(), V, IdxList); - } - // Handle casts from one vector constant to another. We know that the src // and dest type have the same size (otherwise its an illegal cast). if (VectorType *DestPTy = dyn_cast<VectorType>(DestTy)) { @@ -593,17 +570,6 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, if (isa<UndefValue>(V1) && NotPoison(V2)) return V2; if (isa<UndefValue>(V2) && NotPoison(V1)) return V1; - if (ConstantExpr *TrueVal = dyn_cast<ConstantExpr>(V1)) { - if (TrueVal->getOpcode() == Instruction::Select) - if (TrueVal->getOperand(0) == Cond) - return ConstantExpr::getSelect(Cond, TrueVal->getOperand(1), V2); - } - if (ConstantExpr *FalseVal = dyn_cast<ConstantExpr>(V2)) { - if (FalseVal->getOpcode() == Instruction::Select) - if (FalseVal->getOperand(0) == Cond) - return ConstantExpr::getSelect(Cond, V1, FalseVal->getOperand(2)); - } - return nullptr; } @@ -721,9 +687,9 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2, ElementCount::get(MaskNumElts, isa<ScalableVectorType>(V1VTy)); Type *EltTy = V1VTy->getElementType(); - // Undefined shuffle mask -> undefined value. - if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) { - return UndefValue::get(VectorType::get(EltTy, MaskEltCount)); + // Poison shuffle mask -> poison value. + if (all_of(Mask, [](int Elt) { return Elt == PoisonMaskElem; })) { + return PoisonValue::get(VectorType::get(EltTy, MaskEltCount)); } // If the mask is all zeros this is a splat, no need to go through all @@ -1053,7 +1019,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, isa<GlobalValue>(CE1->getOperand(0))) { GlobalValue *GV = cast<GlobalValue>(CE1->getOperand(0)); - MaybeAlign GVAlign; + Align GVAlign; // defaults to 1 if (Module *TheModule = GV->getParent()) { const DataLayout &DL = TheModule->getDataLayout(); @@ -1070,17 +1036,13 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, // appropriate defaults if (isa<Function>(GV) && !DL.getFunctionPtrAlign()) GVAlign = Align(4); - } else if (isa<Function>(GV)) { - // Without a datalayout we have to assume the worst case: that the - // function pointer isn't aligned at all. - GVAlign = std::nullopt; } else if (isa<GlobalVariable>(GV)) { - GVAlign = cast<GlobalVariable>(GV)->getAlign(); + GVAlign = cast<GlobalVariable>(GV)->getAlign().valueOrOne(); } - if (GVAlign && *GVAlign > 1) { + if (GVAlign > 1) { unsigned DstWidth = CI2->getType()->getBitWidth(); - unsigned SrcWidth = std::min(DstWidth, Log2(*GVAlign)); + unsigned SrcWidth = std::min(DstWidth, Log2(GVAlign)); APInt BitsNotSet(APInt::getLowBitsSet(DstWidth, SrcWidth)); // If checking bits we know are clear, return zero. @@ -1945,13 +1907,13 @@ static bool isInBoundsIndices(ArrayRef<IndexTy> Idxs) { static bool isIndexInRangeOfArrayType(uint64_t NumElements, const ConstantInt *CI) { // We cannot bounds check the index if it doesn't fit in an int64_t. - if (CI->getValue().getMinSignedBits() > 64) + if (CI->getValue().getSignificantBits() > 64) return false; // A negative index or an index past the end of our sequential type is // considered out-of-range. int64_t IndexVal = CI->getSExtValue(); - if (IndexVal < 0 || (NumElements > 0 && (uint64_t)IndexVal >= NumElements)) + if (IndexVal < 0 || (IndexVal != 0 && (uint64_t)IndexVal >= NumElements)) return false; // Otherwise, it is in-range. @@ -2038,7 +2000,7 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, if (Idxs.empty()) return C; Type *GEPTy = GetElementPtrInst::getGEPReturnType( - PointeeTy, C, ArrayRef((Value *const *)Idxs.data(), Idxs.size())); + C, ArrayRef((Value *const *)Idxs.data(), Idxs.size())); if (isa<PoisonValue>(C)) return PoisonValue::get(GEPTy); @@ -2048,11 +2010,6 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, return InBounds ? PoisonValue::get(GEPTy) : UndefValue::get(GEPTy); auto IsNoOp = [&]() { - // For non-opaque pointers having multiple indices will change the result - // type of the GEP. - if (!C->getType()->getScalarType()->isOpaquePointerTy() && Idxs.size() != 1) - return false; - // Avoid losing inrange information. if (InRangeIndex) return false; @@ -2101,41 +2058,11 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, } } - if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { + if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) if (auto *GEP = dyn_cast<GEPOperator>(CE)) if (Constant *C = foldGEPOfGEP(GEP, PointeeTy, InBounds, Idxs)) return C; - // Attempt to fold casts to the same type away. For example, folding: - // - // i32* getelementptr ([2 x i32]* bitcast ([3 x i32]* %X to [2 x i32]*), - // i64 0, i64 0) - // into: - // - // i32* getelementptr ([3 x i32]* %X, i64 0, i64 0) - // - // Don't fold if the cast is changing address spaces. - Constant *Idx0 = cast<Constant>(Idxs[0]); - if (CE->isCast() && Idxs.size() > 1 && Idx0->isNullValue()) { - PointerType *SrcPtrTy = - dyn_cast<PointerType>(CE->getOperand(0)->getType()); - PointerType *DstPtrTy = dyn_cast<PointerType>(CE->getType()); - if (SrcPtrTy && DstPtrTy && !SrcPtrTy->isOpaque() && - !DstPtrTy->isOpaque()) { - ArrayType *SrcArrayTy = - dyn_cast<ArrayType>(SrcPtrTy->getNonOpaquePointerElementType()); - ArrayType *DstArrayTy = - dyn_cast<ArrayType>(DstPtrTy->getNonOpaquePointerElementType()); - if (SrcArrayTy && DstArrayTy - && SrcArrayTy->getElementType() == DstArrayTy->getElementType() - && SrcPtrTy->getAddressSpace() == DstPtrTy->getAddressSpace()) - return ConstantExpr::getGetElementPtr(SrcArrayTy, - (Constant *)CE->getOperand(0), - Idxs, InBounds, InRangeIndex); - } - } - } - // Check to see if any array indices are not within the corresponding // notional array or vector bounds. If so, try to determine if they can be // factored out into preceding dimensions. @@ -2202,11 +2129,17 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, Unknown = true; continue; } + + // Determine the number of elements in our sequential type. + uint64_t NumElements = STy->getArrayNumElements(); + if (!NumElements) { + Unknown = true; + continue; + } + // It's out of range, but we can factor it into the prior // dimension. NewIdxs.resize(Idxs.size()); - // Determine the number of elements in our sequential type. - uint64_t NumElements = STy->getArrayNumElements(); // Expand the current index or the previous index to a vector from a scalar // if necessary. @@ -2280,7 +2213,8 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C, // check for the "inbounds" property. if (!Unknown && !InBounds) if (auto *GV = dyn_cast<GlobalVariable>(C)) - if (!GV->hasExternalWeakLinkage() && isInBoundsIndices(Idxs)) + if (!GV->hasExternalWeakLinkage() && GV->getValueType() == PointeeTy && + isInBoundsIndices(Idxs)) return ConstantExpr::getGetElementPtr(PointeeTy, C, Idxs, /*InBounds=*/true, InRangeIndex); diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 0dbccaa1a66a..e9344a8815c0 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -481,8 +481,8 @@ unsigned ConstantRange::getMinSignedBits() const { if (isEmptySet()) return 0; - return std::max(getSignedMin().getMinSignedBits(), - getSignedMax().getMinSignedBits()); + return std::max(getSignedMin().getSignificantBits(), + getSignedMax().getSignificantBits()); } ConstantRange ConstantRange::subtract(const APInt &Val) const { @@ -816,8 +816,7 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const { if (isUpperWrapped()) { // If Upper is greater than or equal to MaxValue(DstTy), it covers the whole // truncated range. - if (Upper.getActiveBits() > DstTySize || - Upper.countTrailingOnes() == DstTySize) + if (Upper.getActiveBits() > DstTySize || Upper.countr_one() == DstTySize) return getFull(DstTySize); Union = ConstantRange(APInt::getMaxValue(DstTySize),Upper.trunc(DstTySize)); @@ -945,6 +944,7 @@ bool ConstantRange::isIntrinsicSupported(Intrinsic::ID IntrinsicID) { case Intrinsic::smin: case Intrinsic::smax: case Intrinsic::abs: + case Intrinsic::ctlz: return true; default: return false; @@ -976,6 +976,12 @@ ConstantRange ConstantRange::intrinsic(Intrinsic::ID IntrinsicID, assert(IntMinIsPoison->getBitWidth() == 1 && "Must be boolean"); return Ops[0].abs(IntMinIsPoison->getBoolValue()); } + case Intrinsic::ctlz: { + const APInt *ZeroIsPoison = Ops[1].getSingleElement(); + assert(ZeroIsPoison && "Must be known (immarg)"); + assert(ZeroIsPoison->getBitWidth() == 1 && "Must be boolean"); + return Ops[0].ctlz(ZeroIsPoison->getBoolValue()); + } default: assert(!isIntrinsicSupported(IntrinsicID) && "Shouldn't be supported"); llvm_unreachable("Unsupported intrinsic"); @@ -1089,6 +1095,20 @@ ConstantRange::multiply(const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) return getEmpty(); + if (const APInt *C = getSingleElement()) { + if (C->isOne()) + return Other; + if (C->isAllOnes()) + return ConstantRange(APInt::getZero(getBitWidth())).sub(Other); + } + + if (const APInt *C = Other.getSingleElement()) { + if (C->isOne()) + return *this; + if (C->isAllOnes()) + return ConstantRange(APInt::getZero(getBitWidth())).sub(*this); + } + // Multiplication is signedness-independent. However different ranges can be // obtained depending on how the input ranges are treated. These different // ranges are all conservatively correct, but one might be better than the @@ -1448,7 +1468,7 @@ ConstantRange::shl(const ConstantRange &Other) const { if (RHS->uge(BW)) return getEmpty(); - unsigned EqualLeadingBits = (Min ^ Max).countLeadingZeros(); + unsigned EqualLeadingBits = (Min ^ Max).countl_zero(); if (RHS->ule(EqualLeadingBits)) return getNonEmpty(Min << *RHS, (Max << *RHS) + 1); @@ -1459,7 +1479,7 @@ ConstantRange::shl(const ConstantRange &Other) const { APInt OtherMax = Other.getUnsignedMax(); // There's overflow! - if (OtherMax.ugt(Max.countLeadingZeros())) + if (OtherMax.ugt(Max.countl_zero())) return getFull(); // FIXME: implement the other tricky cases @@ -1667,6 +1687,44 @@ ConstantRange ConstantRange::abs(bool IntMinIsPoison) const { APIntOps::umax(-SMin, SMax) + 1); } +ConstantRange ConstantRange::ctlz(bool ZeroIsPoison) const { + if (isEmptySet()) + return getEmpty(); + + APInt Zero = APInt::getZero(getBitWidth()); + if (ZeroIsPoison && contains(Zero)) { + // ZeroIsPoison is set, and zero is contained. We discern three cases, in + // which a zero can appear: + // 1) Lower is zero, handling cases of kind [0, 1), [0, 2), etc. + // 2) Upper is zero, wrapped set, handling cases of kind [3, 0], etc. + // 3) Zero contained in a wrapped set, e.g., [3, 2), [3, 1), etc. + + if (getLower().isZero()) { + if ((getUpper() - 1).isZero()) { + // We have in input interval of kind [0, 1). In this case we cannot + // really help but return empty-set. + return getEmpty(); + } + + // Compute the resulting range by excluding zero from Lower. + return ConstantRange( + APInt(getBitWidth(), (getUpper() - 1).countl_zero()), + APInt(getBitWidth(), (getLower() + 1).countl_zero() + 1)); + } else if ((getUpper() - 1).isZero()) { + // Compute the resulting range by excluding zero from Upper. + return ConstantRange(Zero, + APInt(getBitWidth(), getLower().countl_zero() + 1)); + } else { + return ConstantRange(Zero, APInt(getBitWidth(), getBitWidth())); + } + } + + // Zero is either safe or not in the range. The output range is composed by + // the result of countLeadingZero of the two extremes. + return getNonEmpty(APInt(getBitWidth(), getUnsignedMax().countl_zero()), + APInt(getBitWidth(), getUnsignedMin().countl_zero() + 1)); +} + ConstantRange::OverflowResult ConstantRange::unsignedAddMayOverflow( const ConstantRange &Other) const { if (isEmptySet() || Other.isEmptySet()) diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index a53671183f77..c69c7c095f78 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -547,8 +547,6 @@ void llvm::deleteConstant(Constant *C) { delete static_cast<CastConstantExpr *>(C); else if (isa<BinaryConstantExpr>(C)) delete static_cast<BinaryConstantExpr *>(C); - else if (isa<SelectConstantExpr>(C)) - delete static_cast<SelectConstantExpr *>(C); else if (isa<ExtractElementConstantExpr>(C)) delete static_cast<ExtractElementConstantExpr *>(C); else if (isa<InsertElementConstantExpr>(C)) @@ -874,7 +872,10 @@ Constant *ConstantInt::getBool(Type *Ty, bool V) { ConstantInt *ConstantInt::get(LLVMContext &Context, const APInt &V) { // get an existing value or the insertion position LLVMContextImpl *pImpl = Context.pImpl; - std::unique_ptr<ConstantInt> &Slot = pImpl->IntConstants[V]; + std::unique_ptr<ConstantInt> &Slot = + V.isZero() ? pImpl->IntZeroConstants[V.getBitWidth()] + : V.isOne() ? pImpl->IntOneConstants[V.getBitWidth()] + : pImpl->IntConstants[V]; if (!Slot) { // Get the corresponding integer type for the bit width of the value. IntegerType *ITy = IntegerType::get(Context, V.getBitWidth()); @@ -898,14 +899,6 @@ ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool isSigned) { return get(Ty->getContext(), APInt(Ty->getBitWidth(), V, isSigned)); } -ConstantInt *ConstantInt::getSigned(IntegerType *Ty, int64_t V) { - return get(Ty, V, true); -} - -Constant *ConstantInt::getSigned(Type *Ty, int64_t V) { - return get(Ty, V, true); -} - Constant *ConstantInt::get(Type *Ty, const APInt& V) { ConstantInt *C = get(Ty->getContext(), V); assert(C->getType() == Ty->getScalarType() && @@ -1016,13 +1009,6 @@ Constant *ConstantFP::getZero(Type *Ty, bool Negative) { return C; } -Constant *ConstantFP::getZeroValueForNegation(Type *Ty) { - if (Ty->isFPOrFPVectorTy()) - return getNegativeZero(Ty); - - return Constant::getNullValue(Ty); -} - // ConstantFP accessors. ConstantFP* ConstantFP::get(LLVMContext &Context, const APFloat& V) { @@ -1485,8 +1471,6 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty, case Instruction::BitCast: case Instruction::AddrSpaceCast: return ConstantExpr::getCast(getOpcode(), Ops[0], Ty, OnlyIfReduced); - case Instruction::Select: - return ConstantExpr::getSelect(Ops[0], Ops[1], Ops[2], OnlyIfReducedTy); case Instruction::InsertElement: return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2], OnlyIfReducedTy); @@ -2242,21 +2226,6 @@ Constant *ConstantExpr::getAddrSpaceCast(Constant *C, Type *DstTy, bool OnlyIfReduced) { assert(CastInst::castIsValid(Instruction::AddrSpaceCast, C, DstTy) && "Invalid constantexpr addrspacecast!"); - - // Canonicalize addrspacecasts between different pointer types by first - // bitcasting the pointer type and then converting the address space. - PointerType *SrcScalarTy = cast<PointerType>(C->getType()->getScalarType()); - PointerType *DstScalarTy = cast<PointerType>(DstTy->getScalarType()); - if (!SrcScalarTy->hasSameElementTypeAs(DstScalarTy)) { - Type *MidTy = PointerType::getWithSamePointeeType( - DstScalarTy, SrcScalarTy->getAddressSpace()); - if (VectorType *VT = dyn_cast<VectorType>(DstTy)) { - // Handle vectors of pointers. - MidTy = FixedVectorType::get(MidTy, - cast<FixedVectorType>(VT)->getNumElements()); - } - C = getBitCast(C, MidTy); - } return getFoldedCast(Instruction::AddrSpaceCast, C, DstTy, OnlyIfReduced); } @@ -2275,22 +2244,9 @@ Constant *ConstantExpr::get(unsigned Opcode, Constant *C1, Constant *C2, case Instruction::Add: case Instruction::Sub: case Instruction::Mul: - case Instruction::UDiv: - case Instruction::SDiv: - case Instruction::URem: - case Instruction::SRem: assert(C1->getType()->isIntOrIntVectorTy() && "Tried to create an integer operation on a non-integer type!"); break; - case Instruction::FAdd: - case Instruction::FSub: - case Instruction::FMul: - case Instruction::FDiv: - case Instruction::FRem: - assert(C1->getType()->isFPOrFPVectorTy() && - "Tried to create a floating-point operation on a " - "non-floating-point type!"); - break; case Instruction::And: case Instruction::Or: case Instruction::Xor: @@ -2398,24 +2354,6 @@ Constant *ConstantExpr::getAlignOf(Type* Ty) { Type::getInt64Ty(Ty->getContext())); } -Constant *ConstantExpr::getOffsetOf(StructType* STy, unsigned FieldNo) { - return getOffsetOf(STy, ConstantInt::get(Type::getInt32Ty(STy->getContext()), - FieldNo)); -} - -Constant *ConstantExpr::getOffsetOf(Type* Ty, Constant *FieldNo) { - // offsetof is implemented as: (i64) gep (Ty*)null, 0, FieldNo - // Note that a non-inbounds gep is used, as null isn't within any object. - Constant *GEPIdx[] = { - ConstantInt::get(Type::getInt64Ty(Ty->getContext()), 0), - FieldNo - }; - Constant *GEP = getGetElementPtr( - Ty, Constant::getNullValue(PointerType::getUnqual(Ty)), GEPIdx); - return getPtrToInt(GEP, - Type::getInt64Ty(Ty->getContext())); -} - Constant *ConstantExpr::getCompare(unsigned short Predicate, Constant *C1, Constant *C2, bool OnlyIfReduced) { assert(C1->getType() == C2->getType() && "Op types should be identical!"); @@ -2438,56 +2376,28 @@ Constant *ConstantExpr::getCompare(unsigned short Predicate, Constant *C1, } } -Constant *ConstantExpr::getSelect(Constant *C, Constant *V1, Constant *V2, - Type *OnlyIfReducedTy) { - assert(!SelectInst::areInvalidOperands(C, V1, V2)&&"Invalid select operands"); - - if (Constant *SC = ConstantFoldSelectInstruction(C, V1, V2)) - return SC; // Fold common cases - - if (OnlyIfReducedTy == V1->getType()) - return nullptr; - - Constant *ArgVec[] = { C, V1, V2 }; - ConstantExprKeyType Key(Instruction::Select, ArgVec); - - LLVMContextImpl *pImpl = C->getContext().pImpl; - return pImpl->ExprConstants.getOrCreate(V1->getType(), Key); -} - Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, ArrayRef<Value *> Idxs, bool InBounds, std::optional<unsigned> InRangeIndex, Type *OnlyIfReducedTy) { - PointerType *OrigPtrTy = cast<PointerType>(C->getType()->getScalarType()); assert(Ty && "Must specify element type"); - assert(OrigPtrTy->isOpaqueOrPointeeTypeMatches(Ty)); + assert(isSupportedGetElementPtr(Ty) && "Element type is unsupported!"); if (Constant *FC = ConstantFoldGetElementPtr(Ty, C, InBounds, InRangeIndex, Idxs)) return FC; // Fold a few common cases. + assert(GetElementPtrInst::getIndexedType(Ty, Idxs) && + "GEP indices invalid!");; + // Get the result type of the getelementptr! - Type *DestTy = GetElementPtrInst::getIndexedType(Ty, Idxs); - assert(DestTy && "GEP indices invalid!"); - unsigned AS = OrigPtrTy->getAddressSpace(); - Type *ReqTy = OrigPtrTy->isOpaque() - ? PointerType::get(OrigPtrTy->getContext(), AS) - : DestTy->getPointerTo(AS); + Type *ReqTy = GetElementPtrInst::getGEPReturnType(C, Idxs); + if (OnlyIfReducedTy == ReqTy) + return nullptr; auto EltCount = ElementCount::getFixed(0); - if (VectorType *VecTy = dyn_cast<VectorType>(C->getType())) + if (VectorType *VecTy = dyn_cast<VectorType>(ReqTy)) EltCount = VecTy->getElementCount(); - else - for (auto *Idx : Idxs) - if (VectorType *VecTy = dyn_cast<VectorType>(Idx->getType())) - EltCount = VecTy->getElementCount(); - - if (EltCount.isNonZero()) - ReqTy = VectorType::get(ReqTy, EltCount); - - if (OnlyIfReducedTy == ReqTy) - return nullptr; // Look up the constant in the table first to ensure uniqueness std::vector<Constant*> ArgVec; @@ -2644,8 +2554,7 @@ Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2, Constant *ConstantExpr::getNeg(Constant *C, bool HasNUW, bool HasNSW) { assert(C->getType()->isIntOrIntVectorTy() && "Cannot NEG a nonintegral value!"); - return getSub(ConstantFP::getZeroValueForNegation(C->getType()), - C, HasNUW, HasNSW); + return getSub(ConstantInt::get(C->getType(), 0), C, HasNUW, HasNSW); } Constant *ConstantExpr::getNot(Constant *C) { @@ -2687,11 +2596,6 @@ Constant *ConstantExpr::getXor(Constant *C1, Constant *C2) { return get(Instruction::Xor, C1, C2); } -Constant *ConstantExpr::getUMin(Constant *C1, Constant *C2) { - Constant *Cmp = ConstantExpr::getICmp(CmpInst::ICMP_ULT, C1, C2); - return getSelect(Cmp, C1, C2); -} - Constant *ConstantExpr::getShl(Constant *C1, Constant *C2, bool HasNUW, bool HasNSW) { unsigned Flags = (HasNUW ? OverflowingBinaryOperator::NoUnsignedWrap : 0) | @@ -3440,8 +3344,6 @@ Instruction *ConstantExpr::getAsInstruction(Instruction *InsertBefore) const { case Instruction::AddrSpaceCast: return CastInst::Create((Instruction::CastOps)getOpcode(), Ops[0], getType(), "", InsertBefore); - case Instruction::Select: - return SelectInst::Create(Ops[0], Ops[1], Ops[2], "", InsertBefore); case Instruction::InsertElement: return InsertElementInst::Create(Ops[0], Ops[1], Ops[2], "", InsertBefore); case Instruction::ExtractElement: diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index fbda443de7b2..6023216a5070 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -90,32 +90,6 @@ public: } }; -/// SelectConstantExpr - This class is private to Constants.cpp, and is used -/// behind the scenes to implement select constant exprs. -class SelectConstantExpr final : public ConstantExpr { -public: - SelectConstantExpr(Constant *C1, Constant *C2, Constant *C3) - : ConstantExpr(C2->getType(), Instruction::Select, &Op<0>(), 3) { - Op<0>() = C1; - Op<1>() = C2; - Op<2>() = C3; - } - - // allocate space for exactly three operands - void *operator new(size_t S) { return User::operator new(S, 3); } - void operator delete(void *Ptr) { User::operator delete(Ptr); } - - /// Transparently provide more efficient getOperand methods. - DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); - - static bool classof(const ConstantExpr *CE) { - return CE->getOpcode() == Instruction::Select; - } - static bool classof(const Value *V) { - return isa<ConstantExpr>(V) && classof(cast<ConstantExpr>(V)); - } -}; - /// ExtractElementConstantExpr - This class is private to /// Constants.cpp, and is used behind the scenes to implement /// extractelement constant exprs. @@ -280,11 +254,6 @@ struct OperandTraits<BinaryConstantExpr> DEFINE_TRANSPARENT_OPERAND_ACCESSORS(BinaryConstantExpr, Value) template <> -struct OperandTraits<SelectConstantExpr> - : public FixedNumOperandTraits<SelectConstantExpr, 3> {}; -DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectConstantExpr, Value) - -template <> struct OperandTraits<ExtractElementConstantExpr> : public FixedNumOperandTraits<ExtractElementConstantExpr, 2> {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementConstantExpr, Value) @@ -523,8 +492,6 @@ public: return new BinaryConstantExpr(Opcode, Ops[0], Ops[1], SubclassOptionalData); llvm_unreachable("Invalid ConstantExpr!"); - case Instruction::Select: - return new SelectConstantExpr(Ops[0], Ops[1], Ops[2]); case Instruction::ExtractElement: return new ExtractElementConstantExpr(Ops[0], Ops[1]); case Instruction::InsertElement: diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index ea7ee4f97f69..f7b6d54013de 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -53,10 +53,6 @@ void llvm::initializeCore(PassRegistry &Registry) { initializeVerifierLegacyPassPass(Registry); } -void LLVMInitializeCore(LLVMPassRegistryRef R) { - initializeCore(*unwrap(R)); -} - void LLVMShutdown() { llvm_shutdown(); } @@ -129,10 +125,6 @@ void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard) { unwrap(C)->setDiscardValueNames(Discard); } -void LLVMContextSetOpaquePointers(LLVMContextRef C, LLVMBool OpaquePointers) { - unwrap(C)->setOpaquePointers(OpaquePointers); -} - void LLVMContextDispose(LLVMContextRef C) { delete unwrap(C); } @@ -792,12 +784,16 @@ LLVMTypeRef LLVMArrayType(LLVMTypeRef ElementType, unsigned ElementCount) { return wrap(ArrayType::get(unwrap(ElementType), ElementCount)); } +LLVMTypeRef LLVMArrayType2(LLVMTypeRef ElementType, uint64_t ElementCount) { + return wrap(ArrayType::get(unwrap(ElementType), ElementCount)); +} + LLVMTypeRef LLVMPointerType(LLVMTypeRef ElementType, unsigned AddressSpace) { return wrap(PointerType::get(unwrap(ElementType), AddressSpace)); } LLVMBool LLVMPointerTypeIsOpaque(LLVMTypeRef Ty) { - return unwrap(Ty)->isOpaquePointerTy(); + return true; } LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount) { @@ -811,8 +807,6 @@ LLVMTypeRef LLVMScalableVectorType(LLVMTypeRef ElementType, LLVMTypeRef LLVMGetElementType(LLVMTypeRef WrappedTy) { auto *Ty = unwrap(WrappedTy); - if (auto *PTy = dyn_cast<PointerType>(Ty)) - return wrap(PTy->getNonOpaquePointerElementType()); if (auto *ATy = dyn_cast<ArrayType>(Ty)) return wrap(ATy->getElementType()); return wrap(cast<VectorType>(Ty)->getElementType()); @@ -826,6 +820,10 @@ unsigned LLVMGetArrayLength(LLVMTypeRef ArrayTy) { return unwrap<ArrayType>(ArrayTy)->getNumElements(); } +uint64_t LLVMGetArrayLength2(LLVMTypeRef ArrayTy) { + return unwrap<ArrayType>(ArrayTy)->getNumElements(); +} + unsigned LLVMGetPointerAddressSpace(LLVMTypeRef PointerTy) { return unwrap<PointerType>(PointerTy)->getAddressSpace(); } @@ -1013,6 +1011,13 @@ LLVMValueRef LLVMIsAMDNode(LLVMValueRef Val) { return nullptr; } +LLVMValueRef LLVMIsAValueAsMetadata(LLVMValueRef Val) { + if (auto *MD = dyn_cast_or_null<MetadataAsValue>(unwrap(Val))) + if (isa<ValueAsMetadata>(MD->getMetadata())) + return Val; + return nullptr; +} + LLVMValueRef LLVMIsAMDString(LLVMValueRef Val) { if (auto *MD = dyn_cast_or_null<MetadataAsValue>(unwrap(Val))) if (isa<MDString>(MD->getMetadata())) @@ -1272,6 +1277,13 @@ void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest) { Dest[i] = getMDNodeOperandImpl(Context, N, i); } +void LLVMReplaceMDNodeOperandWith(LLVMValueRef V, unsigned Index, + LLVMMetadataRef Replacement) { + auto *MD = cast<MetadataAsValue>(unwrap(V)); + auto *N = cast<MDNode>(MD->getMetadata()); + N->replaceOperandWith(Index, unwrap<Metadata>(Replacement)); +} + unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char *Name) { if (NamedMDNode *N = unwrap(M)->getNamedMetadata(Name)) { return N->getNumOperands(); @@ -1483,6 +1495,12 @@ LLVMValueRef LLVMConstArray(LLVMTypeRef ElementTy, return wrap(ConstantArray::get(ArrayType::get(unwrap(ElementTy), Length), V)); } +LLVMValueRef LLVMConstArray2(LLVMTypeRef ElementTy, LLVMValueRef *ConstantVals, + uint64_t Length) { + ArrayRef<Constant *> V(unwrap<Constant>(ConstantVals, Length), Length); + return wrap(ConstantArray::get(ArrayType::get(unwrap(ElementTy), Length), V)); +} + LLVMValueRef LLVMConstStructInContext(LLVMContextRef C, LLVMValueRef *ConstantVals, unsigned Count, LLVMBool Packed) { @@ -1777,14 +1795,6 @@ LLVMValueRef LLVMConstFPCast(LLVMValueRef ConstantVal, LLVMTypeRef ToType) { unwrap(ToType))); } -LLVMValueRef LLVMConstSelect(LLVMValueRef ConstantCondition, - LLVMValueRef ConstantIfTrue, - LLVMValueRef ConstantIfFalse) { - return wrap(ConstantExpr::getSelect(unwrap<Constant>(ConstantCondition), - unwrap<Constant>(ConstantIfTrue), - unwrap<Constant>(ConstantIfFalse))); -} - LLVMValueRef LLVMConstExtractElement(LLVMValueRef VectorConstant, LLVMValueRef IndexConstant) { return wrap(ConstantExpr::getExtractElement(unwrap<Constant>(VectorConstant), @@ -3434,6 +3444,36 @@ LLVMValueRef LLVMBuildNot(LLVMBuilderRef B, LLVMValueRef V, const char *Name) { return wrap(unwrap(B)->CreateNot(unwrap(V), Name)); } +LLVMBool LLVMGetNUW(LLVMValueRef ArithInst) { + Value *P = unwrap<Value>(ArithInst); + return cast<Instruction>(P)->hasNoUnsignedWrap(); +} + +void LLVMSetNUW(LLVMValueRef ArithInst, LLVMBool HasNUW) { + Value *P = unwrap<Value>(ArithInst); + cast<Instruction>(P)->setHasNoUnsignedWrap(HasNUW); +} + +LLVMBool LLVMGetNSW(LLVMValueRef ArithInst) { + Value *P = unwrap<Value>(ArithInst); + return cast<Instruction>(P)->hasNoSignedWrap(); +} + +void LLVMSetNSW(LLVMValueRef ArithInst, LLVMBool HasNSW) { + Value *P = unwrap<Value>(ArithInst); + cast<Instruction>(P)->setHasNoSignedWrap(HasNSW); +} + +LLVMBool LLVMGetExact(LLVMValueRef DivOrShrInst) { + Value *P = unwrap<Value>(DivOrShrInst); + return cast<Instruction>(P)->isExact(); +} + +void LLVMSetExact(LLVMValueRef DivOrShrInst, LLVMBool IsExact) { + Value *P = unwrap<Value>(DivOrShrInst); + cast<Instruction>(P)->setIsExact(IsExact); +} + /*--.. Memory ..............................................................--*/ LLVMValueRef LLVMBuildMalloc(LLVMBuilderRef B, LLVMTypeRef Ty, @@ -3939,7 +3979,7 @@ int LLVMGetMaskValue(LLVMValueRef SVInst, unsigned Elt) { return I->getMaskValue(Elt); } -int LLVMGetUndefMaskElem(void) { return UndefMaskElem; } +int LLVMGetUndefMaskElem(void) { return PoisonMaskElem; } LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) { Value *P = unwrap(AtomicInst); @@ -4057,12 +4097,6 @@ void LLVMDisposeMemoryBuffer(LLVMMemoryBufferRef MemBuf) { delete unwrap(MemBuf); } -/*===-- Pass Registry -----------------------------------------------------===*/ - -LLVMPassRegistryRef LLVMGetGlobalPassRegistry(void) { - return wrap(PassRegistry::getPassRegistry()); -} - /*===-- Pass Manager ------------------------------------------------------===*/ LLVMPassManagerRef LLVMCreatePassManager() { diff --git a/llvm/lib/IR/CycleInfo.cpp b/llvm/lib/IR/CycleInfo.cpp new file mode 100644 index 000000000000..a9b9129f24f0 --- /dev/null +++ b/llvm/lib/IR/CycleInfo.cpp @@ -0,0 +1,16 @@ +//===- CycleInfo.cpp - IR Cycle Info ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/CycleInfo.h" +#include "llvm/ADT/GenericCycleImpl.h" +#include "llvm/IR/CFG.h" + +using namespace llvm; + +template class llvm::GenericCycleInfo<SSAContext>; +template class llvm::GenericCycle<SSAContext>; diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp index 6c873c3c6644..1ce8c17f8a88 100644 --- a/llvm/lib/IR/DIBuilder.cpp +++ b/llvm/lib/IR/DIBuilder.cpp @@ -12,6 +12,8 @@ #include "llvm/IR/DIBuilder.h" #include "LLVMContextImpl.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/APSInt.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" @@ -23,14 +25,9 @@ using namespace llvm; using namespace llvm::dwarf; -static cl::opt<bool> - UseDbgAddr("use-dbg-addr", - llvm::cl::desc("Use llvm.dbg.addr for all local variables"), - cl::init(false), cl::Hidden); - DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU) : M(m), VMContext(M.getContext()), CUNode(CU), DeclareFn(nullptr), - ValueFn(nullptr), LabelFn(nullptr), AddrFn(nullptr), AssignFn(nullptr), + ValueFn(nullptr), LabelFn(nullptr), AssignFn(nullptr), AllowUnresolvedNodes(AllowUnresolvedNodes) { if (CUNode) { if (const auto &ETs = CUNode->getEnumTypes()) @@ -40,7 +37,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU) if (const auto &GVs = CUNode->getGlobalVariables()) AllGVs.assign(GVs.begin(), GVs.end()); if (const auto &IMs = CUNode->getImportedEntities()) - AllImportedModules.assign(IMs.begin(), IMs.end()); + ImportedModules.assign(IMs.begin(), IMs.end()); if (const auto &MNs = CUNode->getMacros()) AllMacrosPerParent.insert({nullptr, {MNs.begin(), MNs.end()}}); } @@ -57,23 +54,11 @@ void DIBuilder::trackIfUnresolved(MDNode *N) { } void DIBuilder::finalizeSubprogram(DISubprogram *SP) { - MDTuple *Temp = SP->getRetainedNodes().get(); - if (!Temp || !Temp->isTemporary()) - return; - - SmallVector<Metadata *, 16> RetainedNodes; - - auto PV = PreservedVariables.find(SP); - if (PV != PreservedVariables.end()) - RetainedNodes.append(PV->second.begin(), PV->second.end()); - - auto PL = PreservedLabels.find(SP); - if (PL != PreservedLabels.end()) - RetainedNodes.append(PL->second.begin(), PL->second.end()); - - DINodeArray Node = getOrCreateArray(RetainedNodes); - - TempMDTuple(Temp)->replaceAllUsesWith(Node.get()); + auto PN = SubprogramTrackedNodes.find(SP); + if (PN != SubprogramTrackedNodes.end()) + SP->replaceRetainedNodes( + MDTuple::get(VMContext, SmallVector<Metadata *, 16>(PN->second.begin(), + PN->second.end()))); } void DIBuilder::finalize() { @@ -101,8 +86,7 @@ void DIBuilder::finalize() { if (!RetainValues.empty()) CUNode->replaceRetainedTypes(MDTuple::get(VMContext, RetainValues)); - DISubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms); - for (auto *SP : SPs) + for (auto *SP : AllSubprograms) finalizeSubprogram(SP); for (auto *N : RetainValues) if (auto *SP = dyn_cast<DISubprogram>(N)) @@ -111,10 +95,10 @@ void DIBuilder::finalize() { if (!AllGVs.empty()) CUNode->replaceGlobalVariables(MDTuple::get(VMContext, AllGVs)); - if (!AllImportedModules.empty()) + if (!ImportedModules.empty()) CUNode->replaceImportedEntities(MDTuple::get( - VMContext, SmallVector<Metadata *, 16>(AllImportedModules.begin(), - AllImportedModules.end()))); + VMContext, SmallVector<Metadata *, 16>(ImportedModules.begin(), + ImportedModules.end()))); for (const auto &I : AllMacrosPerParent) { // DIMacroNode's with nullptr parent are DICompileUnit direct children. @@ -156,7 +140,7 @@ DICompileUnit *DIBuilder::createCompileUnit( DICompileUnit::DebugNameTableKind NameTableKind, bool RangesBaseAddress, StringRef SysRoot, StringRef SDK) { - assert(((Lang <= dwarf::DW_LANG_Ada2012 && Lang >= dwarf::DW_LANG_C89) || + assert(((Lang <= dwarf::DW_LANG_Mojo && Lang >= dwarf::DW_LANG_C89) || (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) && "Invalid Language tag"); @@ -178,7 +162,7 @@ static DIImportedEntity * createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope *Context, Metadata *NS, DIFile *File, unsigned Line, StringRef Name, DINodeArray Elements, - SmallVectorImpl<TrackingMDNodeRef> &AllImportedModules) { + SmallVectorImpl<TrackingMDNodeRef> &ImportedModules) { if (Line) assert(File && "Source location has line number but no file"); unsigned EntitiesCount = C.pImpl->DIImportedEntitys.size(); @@ -187,7 +171,7 @@ createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope *Context, if (EntitiesCount < C.pImpl->DIImportedEntitys.size()) // A new Imported Entity was just added to the context. // Add it to the Imported Modules list. - AllImportedModules.emplace_back(M); + ImportedModules.emplace_back(M); return M; } @@ -197,7 +181,7 @@ DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, DINodeArray Elements) { return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module, Context, NS, File, Line, StringRef(), Elements, - AllImportedModules); + getImportTrackingVector(Context)); } DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, @@ -206,7 +190,7 @@ DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, DINodeArray Elements) { return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module, Context, NS, File, Line, StringRef(), Elements, - AllImportedModules); + getImportTrackingVector(Context)); } DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, DIModule *M, @@ -214,7 +198,7 @@ DIImportedEntity *DIBuilder::createImportedModule(DIScope *Context, DIModule *M, DINodeArray Elements) { return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_module, Context, M, File, Line, StringRef(), Elements, - AllImportedModules); + getImportTrackingVector(Context)); } DIImportedEntity * @@ -225,7 +209,7 @@ DIBuilder::createImportedDeclaration(DIScope *Context, DINode *Decl, // types that have one. return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_declaration, Context, Decl, File, Line, Name, Elements, - AllImportedModules); + getImportTrackingVector(Context)); } DIFile *DIBuilder::createFile(StringRef Filename, StringRef Directory, @@ -588,14 +572,14 @@ DIBuilder::createArrayType(uint64_t Size, uint32_t AlignInBits, DIType *Ty, VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, nullptr, Ty, Size, AlignInBits, 0, DINode::FlagZero, Subscripts, 0, nullptr, nullptr, "", nullptr, - DL.is<DIExpression *>() ? (Metadata *)DL.get<DIExpression *>() - : (Metadata *)DL.get<DIVariable *>(), - AS.is<DIExpression *>() ? (Metadata *)AS.get<DIExpression *>() - : (Metadata *)AS.get<DIVariable *>(), - AL.is<DIExpression *>() ? (Metadata *)AL.get<DIExpression *>() - : (Metadata *)AL.get<DIVariable *>(), - RK.is<DIExpression *>() ? (Metadata *)RK.get<DIExpression *>() - : (Metadata *)RK.get<DIVariable *>()); + isa<DIExpression *>(DL) ? (Metadata *)cast<DIExpression *>(DL) + : (Metadata *)cast<DIVariable *>(DL), + isa<DIExpression *>(AS) ? (Metadata *)cast<DIExpression *>(AS) + : (Metadata *)cast<DIVariable *>(AS), + isa<DIExpression *>(AL) ? (Metadata *)cast<DIExpression *>(AL) + : (Metadata *)cast<DIVariable *>(AL), + isa<DIExpression *>(RK) ? (Metadata *)cast<DIExpression *>(RK) + : (Metadata *)cast<DIVariable *>(RK)); trackIfUnresolved(R); return R; } @@ -720,8 +704,8 @@ DIGenericSubrange *DIBuilder::getOrCreateGenericSubrange( DIGenericSubrange::BoundType CountNode, DIGenericSubrange::BoundType LB, DIGenericSubrange::BoundType UB, DIGenericSubrange::BoundType Stride) { auto ConvToMetadata = [&](DIGenericSubrange::BoundType Bound) -> Metadata * { - return Bound.is<DIExpression *>() ? (Metadata *)Bound.get<DIExpression *>() - : (Metadata *)Bound.get<DIVariable *>(); + return isa<DIExpression *>(Bound) ? (Metadata *)cast<DIExpression *>(Bound) + : (Metadata *)cast<DIVariable *>(Bound); }; return DIGenericSubrange::get(VMContext, ConvToMetadata(CountNode), ConvToMetadata(LB), ConvToMetadata(UB), @@ -772,26 +756,20 @@ DIGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl( static DILocalVariable *createLocalVariable( LLVMContext &VMContext, - DenseMap<MDNode *, SmallVector<TrackingMDNodeRef, 1>> &PreservedVariables, - DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File, + SmallVectorImpl<TrackingMDNodeRef> &PreservedNodes, + DIScope *Context, StringRef Name, unsigned ArgNo, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve, DINode::DIFlags Flags, uint32_t AlignInBits, DINodeArray Annotations = nullptr) { - // FIXME: Why getNonCompileUnitScope()? - // FIXME: Why is "!Context" okay here? // FIXME: Why doesn't this check for a subprogram or lexical block (AFAICT // the only valid scopes)? - DIScope *Context = getNonCompileUnitScope(Scope); - - auto *Node = DILocalVariable::get( - VMContext, cast_or_null<DILocalScope>(Context), Name, File, LineNo, Ty, - ArgNo, Flags, AlignInBits, Annotations); + auto *Scope = cast<DILocalScope>(Context); + auto *Node = DILocalVariable::get(VMContext, Scope, Name, File, LineNo, Ty, + ArgNo, Flags, AlignInBits, Annotations); if (AlwaysPreserve) { // The optimizer may remove local variables. If there is an interest // to preserve variable info in such situation then stash it in a // named mdnode. - DISubprogram *Fn = getDISubprogram(Scope); - assert(Fn && "Missing subprogram for local variable"); - PreservedVariables[Fn].emplace_back(Node); + PreservedNodes.emplace_back(Node); } return Node; } @@ -801,9 +779,11 @@ DILocalVariable *DIBuilder::createAutoVariable(DIScope *Scope, StringRef Name, DIType *Ty, bool AlwaysPreserve, DINode::DIFlags Flags, uint32_t AlignInBits) { - return createLocalVariable(VMContext, PreservedVariables, Scope, Name, - /* ArgNo */ 0, File, LineNo, Ty, AlwaysPreserve, - Flags, AlignInBits); + assert(Scope && isa<DILocalScope>(Scope) && + "Unexpected scope for a local variable."); + return createLocalVariable( + VMContext, getSubprogramNodesTrackingVector(Scope), Scope, Name, + /* ArgNo */ 0, File, LineNo, Ty, AlwaysPreserve, Flags, AlignInBits); } DILocalVariable *DIBuilder::createParameterVariable( @@ -811,25 +791,23 @@ DILocalVariable *DIBuilder::createParameterVariable( unsigned LineNo, DIType *Ty, bool AlwaysPreserve, DINode::DIFlags Flags, DINodeArray Annotations) { assert(ArgNo && "Expected non-zero argument number for parameter"); - return createLocalVariable(VMContext, PreservedVariables, Scope, Name, ArgNo, - File, LineNo, Ty, AlwaysPreserve, Flags, - /*AlignInBits=*/0, Annotations); + assert(Scope && isa<DILocalScope>(Scope) && + "Unexpected scope for a local variable."); + return createLocalVariable( + VMContext, getSubprogramNodesTrackingVector(Scope), Scope, Name, ArgNo, + File, LineNo, Ty, AlwaysPreserve, Flags, /*AlignInBits=*/0, Annotations); } -DILabel *DIBuilder::createLabel(DIScope *Scope, StringRef Name, DIFile *File, - unsigned LineNo, bool AlwaysPreserve) { - DIScope *Context = getNonCompileUnitScope(Scope); - - auto *Node = DILabel::get(VMContext, cast_or_null<DILocalScope>(Context), - Name, File, LineNo); +DILabel *DIBuilder::createLabel(DIScope *Context, StringRef Name, DIFile *File, + unsigned LineNo, bool AlwaysPreserve) { + auto *Scope = cast<DILocalScope>(Context); + auto *Node = DILabel::get(VMContext, Scope, Name, File, LineNo); if (AlwaysPreserve) { /// The optimizer may remove labels. If there is an interest /// to preserve label info in such situation then append it to /// the list of retained nodes of the DISubprogram. - DISubprogram *Fn = getDISubprogram(Scope); - assert(Fn && "Missing subprogram for label"); - PreservedLabels[Fn].emplace_back(Node); + getSubprogramNodesTrackingVector(Scope).emplace_back(Node); } return Node; } @@ -856,9 +834,8 @@ DISubprogram *DIBuilder::createFunction( auto *Node = getSubprogram( /*IsDistinct=*/IsDefinition, VMContext, getNonCompileUnitScope(Context), Name, LinkageName, File, LineNo, Ty, ScopeLine, nullptr, 0, 0, Flags, - SPFlags, IsDefinition ? CUNode : nullptr, TParams, Decl, - MDTuple::getTemporary(VMContext, std::nullopt).release(), ThrownTypes, - Annotations, TargetFuncName); + SPFlags, IsDefinition ? CUNode : nullptr, TParams, Decl, nullptr, + ThrownTypes, Annotations, TargetFuncName); if (IsDefinition) AllSubprograms.push_back(Node); @@ -1022,24 +999,6 @@ Instruction *DIBuilder::insertDbgValueIntrinsic(Value *V, return insertDbgValueIntrinsic(V, VarInfo, Expr, DL, InsertAtEnd, nullptr); } -Instruction *DIBuilder::insertDbgAddrIntrinsic(Value *V, - DILocalVariable *VarInfo, - DIExpression *Expr, - const DILocation *DL, - Instruction *InsertBefore) { - return insertDbgAddrIntrinsic( - V, VarInfo, Expr, DL, InsertBefore ? InsertBefore->getParent() : nullptr, - InsertBefore); -} - -Instruction *DIBuilder::insertDbgAddrIntrinsic(Value *V, - DILocalVariable *VarInfo, - DIExpression *Expr, - const DILocation *DL, - BasicBlock *InsertAtEnd) { - return insertDbgAddrIntrinsic(V, VarInfo, Expr, DL, InsertAtEnd, nullptr); -} - /// Initialize IRBuilder for inserting dbg.declare and dbg.value intrinsics. /// This abstracts over the various ways to specify an insert position. static void initIRBuilder(IRBuilder<> &Builder, const DILocation *DL, @@ -1057,8 +1016,7 @@ static Value *getDbgIntrinsicValueImpl(LLVMContext &VMContext, Value *V) { } static Function *getDeclareIntrin(Module &M) { - return Intrinsic::getDeclaration(&M, UseDbgAddr ? Intrinsic::dbg_addr - : Intrinsic::dbg_declare); + return Intrinsic::getDeclaration(&M, Intrinsic::dbg_declare); } Instruction *DIBuilder::insertDbgValueIntrinsic( @@ -1070,15 +1028,6 @@ Instruction *DIBuilder::insertDbgValueIntrinsic( InsertBefore); } -Instruction *DIBuilder::insertDbgAddrIntrinsic( - llvm::Value *Val, DILocalVariable *VarInfo, DIExpression *Expr, - const DILocation *DL, BasicBlock *InsertBB, Instruction *InsertBefore) { - if (!AddrFn) - AddrFn = Intrinsic::getDeclaration(&M, Intrinsic::dbg_addr); - return insertDbgIntrinsic(AddrFn, Val, VarInfo, Expr, DL, InsertBB, - InsertBefore); -} - Instruction *DIBuilder::insertDeclare(Value *Storage, DILocalVariable *VarInfo, DIExpression *Expr, const DILocation *DL, BasicBlock *InsertBB, diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp index 379c6d577b4e..53842b184ed6 100644 --- a/llvm/lib/IR/DataLayout.cpp +++ b/llvm/lib/IR/DataLayout.cpp @@ -18,7 +18,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Triple.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GetElementPtrTypeIterator.h" @@ -32,6 +31,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/MemAlloc.h" #include "llvm/Support/TypeSize.h" +#include "llvm/TargetParser/Triple.h" #include <algorithm> #include <cassert> #include <cstdint> @@ -45,21 +45,30 @@ using namespace llvm; // Support for StructLayout //===----------------------------------------------------------------------===// -StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { +StructLayout::StructLayout(StructType *ST, const DataLayout &DL) + : StructSize(TypeSize::Fixed(0)) { assert(!ST->isOpaque() && "Cannot get layout of opaque structs"); - StructSize = 0; IsPadded = false; NumElements = ST->getNumElements(); // Loop over each of the elements, placing them in memory. for (unsigned i = 0, e = NumElements; i != e; ++i) { Type *Ty = ST->getElementType(i); + if (i == 0 && Ty->isScalableTy()) + StructSize = TypeSize::Scalable(0); + const Align TyAlign = ST->isPacked() ? Align(1) : DL.getABITypeAlign(Ty); // Add padding if necessary to align the data element properly. - if (!isAligned(TyAlign, StructSize)) { + // Currently the only structure with scalable size will be the homogeneous + // scalable vector types. Homogeneous scalable vector types have members of + // the same data type so no alignment issue will happen. The condition here + // assumes so and needs to be adjusted if this assumption changes (e.g. we + // support structures with arbitrary scalable data type, or structure that + // contains both fixed size and scalable size data type members). + if (!StructSize.isScalable() && !isAligned(TyAlign, StructSize)) { IsPadded = true; - StructSize = alignTo(StructSize, TyAlign); + StructSize = TypeSize::Fixed(alignTo(StructSize, TyAlign)); } // Keep track of maximum alignment constraint. @@ -67,28 +76,39 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { getMemberOffsets()[i] = StructSize; // Consume space for this data item - StructSize += DL.getTypeAllocSize(Ty).getFixedValue(); + StructSize += DL.getTypeAllocSize(Ty); } // Add padding to the end of the struct so that it could be put in an array // and all array elements would be aligned correctly. - if (!isAligned(StructAlignment, StructSize)) { + if (!StructSize.isScalable() && !isAligned(StructAlignment, StructSize)) { IsPadded = true; - StructSize = alignTo(StructSize, StructAlignment); + StructSize = TypeSize::Fixed(alignTo(StructSize, StructAlignment)); } } /// getElementContainingOffset - Given a valid offset into the structure, /// return the structure index that contains it. -unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const { - ArrayRef<uint64_t> MemberOffsets = getMemberOffsets(); - auto SI = llvm::upper_bound(MemberOffsets, Offset); +unsigned StructLayout::getElementContainingOffset(uint64_t FixedOffset) const { + assert(!StructSize.isScalable() && + "Cannot get element at offset for structure containing scalable " + "vector types"); + TypeSize Offset = TypeSize::Fixed(FixedOffset); + ArrayRef<TypeSize> MemberOffsets = getMemberOffsets(); + + const auto *SI = + std::upper_bound(MemberOffsets.begin(), MemberOffsets.end(), Offset, + [](TypeSize LHS, TypeSize RHS) -> bool { + return TypeSize::isKnownLT(LHS, RHS); + }); assert(SI != MemberOffsets.begin() && "Offset not in structure type!"); --SI; - assert(*SI <= Offset && "upper_bound didn't work"); - assert((SI == MemberOffsets.begin() || *(SI - 1) <= Offset) && - (SI + 1 == MemberOffsets.end() || *(SI + 1) > Offset) && - "Upper bound didn't work!"); + assert(TypeSize::isKnownLE(*SI, Offset) && "upper_bound didn't work"); + assert( + (SI == MemberOffsets.begin() || TypeSize::isKnownLE(*(SI - 1), Offset)) && + (SI + 1 == MemberOffsets.end() || + TypeSize::isKnownGT(*(SI + 1), Offset)) && + "Upper bound didn't work!"); // Multiple fields can have the same offset if any of them are zero sized. // For example, in { i32, [0 x i32], i32 }, searching for offset 4 will stop @@ -102,23 +122,19 @@ unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const { // LayoutAlignElem, LayoutAlign support //===----------------------------------------------------------------------===// -LayoutAlignElem LayoutAlignElem::get(AlignTypeEnum align_type, Align abi_align, - Align pref_align, uint32_t bit_width) { - assert(abi_align <= pref_align && "Preferred alignment worse than ABI!"); +LayoutAlignElem LayoutAlignElem::get(Align ABIAlign, Align PrefAlign, + uint32_t BitWidth) { + assert(ABIAlign <= PrefAlign && "Preferred alignment worse than ABI!"); LayoutAlignElem retval; - retval.AlignType = align_type; - retval.ABIAlign = abi_align; - retval.PrefAlign = pref_align; - retval.TypeBitWidth = bit_width; + retval.ABIAlign = ABIAlign; + retval.PrefAlign = PrefAlign; + retval.TypeBitWidth = BitWidth; return retval; } -bool -LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const { - return (AlignType == rhs.AlignType - && ABIAlign == rhs.ABIAlign - && PrefAlign == rhs.PrefAlign - && TypeBitWidth == rhs.TypeBitWidth); +bool LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const { + return ABIAlign == rhs.ABIAlign && PrefAlign == rhs.PrefAlign && + TypeBitWidth == rhs.TypeBitWidth; } //===----------------------------------------------------------------------===// @@ -162,19 +178,18 @@ const char *DataLayout::getManglingComponent(const Triple &T) { return "-m:e"; } -static const LayoutAlignElem DefaultAlignments[] = { - {INTEGER_ALIGN, 1, Align(1), Align(1)}, // i1 - {INTEGER_ALIGN, 8, Align(1), Align(1)}, // i8 - {INTEGER_ALIGN, 16, Align(2), Align(2)}, // i16 - {INTEGER_ALIGN, 32, Align(4), Align(4)}, // i32 - {INTEGER_ALIGN, 64, Align(4), Align(8)}, // i64 - {FLOAT_ALIGN, 16, Align(2), Align(2)}, // half, bfloat - {FLOAT_ALIGN, 32, Align(4), Align(4)}, // float - {FLOAT_ALIGN, 64, Align(8), Align(8)}, // double - {FLOAT_ALIGN, 128, Align(16), Align(16)}, // ppcf128, quad, ... - {VECTOR_ALIGN, 64, Align(8), Align(8)}, // v2i32, v1i64, ... - {VECTOR_ALIGN, 128, Align(16), Align(16)}, // v16i8, v8i16, v4i32, ... - {AGGREGATE_ALIGN, 0, Align(1), Align(8)} // struct +static const std::pair<AlignTypeEnum, LayoutAlignElem> DefaultAlignments[] = { + {INTEGER_ALIGN, {1, Align(1), Align(1)}}, // i1 + {INTEGER_ALIGN, {8, Align(1), Align(1)}}, // i8 + {INTEGER_ALIGN, {16, Align(2), Align(2)}}, // i16 + {INTEGER_ALIGN, {32, Align(4), Align(4)}}, // i32 + {INTEGER_ALIGN, {64, Align(4), Align(8)}}, // i64 + {FLOAT_ALIGN, {16, Align(2), Align(2)}}, // half, bfloat + {FLOAT_ALIGN, {32, Align(4), Align(4)}}, // float + {FLOAT_ALIGN, {64, Align(8), Align(8)}}, // double + {FLOAT_ALIGN, {128, Align(16), Align(16)}}, // ppcf128, quad, ... + {VECTOR_ALIGN, {64, Align(8), Align(8)}}, // v2i32, v1i64, ... + {VECTOR_ALIGN, {128, Align(16), Align(16)}}, // v16i8, v8i16, v4i32, ... }; void DataLayout::reset(StringRef Desc) { @@ -190,11 +205,12 @@ void DataLayout::reset(StringRef Desc) { TheFunctionPtrAlignType = FunctionPtrAlignType::Independent; ManglingMode = MM_None; NonIntegralAddressSpaces.clear(); + StructAlignment = LayoutAlignElem::get(Align(1), Align(8), 0); // Default alignments - for (const LayoutAlignElem &E : DefaultAlignments) { - if (Error Err = setAlignment((AlignTypeEnum)E.AlignType, E.ABIAlign, - E.PrefAlign, E.TypeBitWidth)) + for (const auto &[Kind, Layout] : DefaultAlignments) { + if (Error Err = setAlignment(Kind, Layout.ABIAlign, Layout.PrefAlign, + Layout.TypeBitWidth)) return report_fatal_error(std::move(Err)); } if (Error Err = setPointerAlignmentInBits(0, Align(8), Align(8), 64, 64)) @@ -309,7 +325,7 @@ Error DataLayout::parseSpecifier(StringRef Desc) { if (Error Err = getInt(Tok, AddrSpace)) return Err; if (!isUInt<24>(AddrSpace)) - return reportError("Invalid address space, must be a 24bit integer"); + return reportError("Invalid address space, must be a 24-bit integer"); // Size. if (Rest.empty()) @@ -550,43 +566,63 @@ bool DataLayout::operator==(const DataLayout &Other) const { TheFunctionPtrAlignType == Other.TheFunctionPtrAlignType && ManglingMode == Other.ManglingMode && LegalIntWidths == Other.LegalIntWidths && - Alignments == Other.Alignments && Pointers == Other.Pointers; + IntAlignments == Other.IntAlignments && + FloatAlignments == Other.FloatAlignments && + VectorAlignments == Other.VectorAlignments && + StructAlignment == Other.StructAlignment && + Pointers == Other.Pointers; // Note: getStringRepresentation() might differs, it is not canonicalized return Ret; } -DataLayout::AlignmentsTy::iterator -DataLayout::findAlignmentLowerBound(AlignTypeEnum AlignType, - uint32_t BitWidth) { - auto Pair = std::make_pair((unsigned)AlignType, BitWidth); - return partition_point(Alignments, [=](const LayoutAlignElem &E) { - return std::make_pair(E.AlignType, E.TypeBitWidth) < Pair; +static SmallVectorImpl<LayoutAlignElem>::const_iterator +findAlignmentLowerBound(const SmallVectorImpl<LayoutAlignElem> &Alignments, + uint32_t BitWidth) { + return partition_point(Alignments, [BitWidth](const LayoutAlignElem &E) { + return E.TypeBitWidth < BitWidth; }); } -Error DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align, - Align pref_align, uint32_t bit_width) { +Error DataLayout::setAlignment(AlignTypeEnum AlignType, Align ABIAlign, + Align PrefAlign, uint32_t BitWidth) { // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as // uint16_t, it is unclear if there are requirements for alignment to be less // than 2^16 other than storage. In the meantime we leave the restriction as // an assert. See D67400 for context. - assert(Log2(abi_align) < 16 && Log2(pref_align) < 16 && "Alignment too big"); - if (!isUInt<24>(bit_width)) - return reportError("Invalid bit width, must be a 24bit integer"); - if (pref_align < abi_align) + assert(Log2(ABIAlign) < 16 && Log2(PrefAlign) < 16 && "Alignment too big"); + if (!isUInt<24>(BitWidth)) + return reportError("Invalid bit width, must be a 24-bit integer"); + if (PrefAlign < ABIAlign) return reportError( "Preferred alignment cannot be less than the ABI alignment"); - AlignmentsTy::iterator I = findAlignmentLowerBound(align_type, bit_width); - if (I != Alignments.end() && - I->AlignType == (unsigned)align_type && I->TypeBitWidth == bit_width) { + SmallVectorImpl<LayoutAlignElem> *Alignments; + switch (AlignType) { + case AGGREGATE_ALIGN: + StructAlignment.ABIAlign = ABIAlign; + StructAlignment.PrefAlign = PrefAlign; + return Error::success(); + case INTEGER_ALIGN: + Alignments = &IntAlignments; + break; + case FLOAT_ALIGN: + Alignments = &FloatAlignments; + break; + case VECTOR_ALIGN: + Alignments = &VectorAlignments; + break; + } + + auto I = partition_point(*Alignments, [BitWidth](const LayoutAlignElem &E) { + return E.TypeBitWidth < BitWidth; + }); + if (I != Alignments->end() && I->TypeBitWidth == BitWidth) { // Update the abi, preferred alignments. - I->ABIAlign = abi_align; - I->PrefAlign = pref_align; + I->ABIAlign = ABIAlign; + I->PrefAlign = PrefAlign; } else { // Insert before I to keep the vector sorted. - Alignments.insert(I, LayoutAlignElem::get(align_type, abi_align, - pref_align, bit_width)); + Alignments->insert(I, LayoutAlignElem::get(ABIAlign, PrefAlign, BitWidth)); } return Error::success(); } @@ -633,13 +669,12 @@ Error DataLayout::setPointerAlignmentInBits(uint32_t AddrSpace, Align ABIAlign, Align DataLayout::getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const { - auto I = findAlignmentLowerBound(INTEGER_ALIGN, BitWidth); + auto I = findAlignmentLowerBound(IntAlignments, BitWidth); // If we don't have an exact match, use alignment of next larger integer // type. If there is none, use alignment of largest integer type by going // back one element. - if (I == Alignments.end() || I->AlignType != INTEGER_ALIGN) + if (I == IntAlignments.end()) --I; - assert(I->AlignType == INTEGER_ALIGN && "Must be integer alignment"); return abi_or_pref ? I->ABIAlign : I->PrefAlign; } @@ -668,7 +703,9 @@ public: void DataLayout::clear() { LegalIntWidths.clear(); - Alignments.clear(); + IntAlignments.clear(); + FloatAlignments.clear(); + VectorAlignments.clear(); Pointers.clear(); delete static_cast<StructLayoutMap *>(LayoutMap); LayoutMap = nullptr; @@ -689,7 +726,7 @@ const StructLayout *DataLayout::getStructLayout(StructType *Ty) const { // Otherwise, create the struct layout. Because it is variable length, we // malloc it, then use placement new. StructLayout *L = (StructLayout *)safe_malloc( - StructLayout::totalSizeToAlloc<uint64_t>(Ty->getNumElements())); + StructLayout::totalSizeToAlloc<TypeSize>(Ty->getNumElements())); // Set SL before calling StructLayout's ctor. The ctor could cause other // entries to be added to TheMap, invalidating our reference. @@ -768,11 +805,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { // Get the layout annotation... which is lazily created on demand. const StructLayout *Layout = getStructLayout(cast<StructType>(Ty)); - const LayoutAlignElem &AggregateAlign = Alignments[0]; - assert(AggregateAlign.AlignType == AGGREGATE_ALIGN && - "Aggregate alignment must be first alignment entry"); const Align Align = - abi_or_pref ? AggregateAlign.ABIAlign : AggregateAlign.PrefAlign; + abi_or_pref ? StructAlignment.ABIAlign : StructAlignment.PrefAlign; return std::max(Align, Layout->getAlignment()); } case Type::IntegerTyID: @@ -787,9 +821,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { case Type::FP128TyID: case Type::X86_FP80TyID: { unsigned BitWidth = getTypeSizeInBits(Ty).getFixedValue(); - auto I = findAlignmentLowerBound(FLOAT_ALIGN, BitWidth); - if (I != Alignments.end() && I->AlignType == FLOAT_ALIGN && - I->TypeBitWidth == BitWidth) + auto I = findAlignmentLowerBound(FloatAlignments, BitWidth); + if (I != FloatAlignments.end() && I->TypeBitWidth == BitWidth) return abi_or_pref ? I->ABIAlign : I->PrefAlign; // If we still couldn't find a reasonable default alignment, fall back @@ -804,9 +837,8 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { case Type::FixedVectorTyID: case Type::ScalableVectorTyID: { unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinValue(); - auto I = findAlignmentLowerBound(VECTOR_ALIGN, BitWidth); - if (I != Alignments.end() && I->AlignType == VECTOR_ALIGN && - I->TypeBitWidth == BitWidth) + auto I = findAlignmentLowerBound(VectorAlignments, BitWidth); + if (I != VectorAlignments.end() && I->TypeBitWidth == BitWidth) return abi_or_pref ? I->ABIAlign : I->PrefAlign; // By default, use natural alignment for vector types. This is consistent @@ -828,11 +860,6 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const { } } -/// TODO: Remove this function once the transition to Align is over. -uint64_t DataLayout::getABITypeAlignment(Type *Ty) const { - return getABITypeAlign(Ty).value(); -} - Align DataLayout::getABITypeAlign(Type *Ty) const { return getAlignment(Ty, true); } @@ -873,6 +900,11 @@ unsigned DataLayout::getLargestLegalIntTypeSizeInBits() const { return Max != LegalIntWidths.end() ? *Max : 0; } +IntegerType *DataLayout::getIndexType(LLVMContext &C, + unsigned AddressSpace) const { + return IntegerType::get(C, getIndexSizeInBits(AddressSpace)); +} + Type *DataLayout::getIndexType(Type *Ty) const { assert(Ty->isPtrOrPtrVectorTy() && "Expected a pointer or pointer vector type."); diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 9198179674bd..48b5501c55ba 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -43,10 +43,7 @@ using namespace llvm; using namespace llvm::at; using namespace llvm::dwarf; -/// Finds all intrinsics declaring local variables as living in the memory that -/// 'V' points to. This may include a mix of dbg.declare and -/// dbg.addr intrinsics. -TinyPtrVector<DbgVariableIntrinsic *> llvm::FindDbgAddrUses(Value *V) { +TinyPtrVector<DbgDeclareInst *> llvm::FindDbgDeclareUses(Value *V) { // This function is hot. Check whether the value has any metadata to avoid a // DenseMap lookup. if (!V->isUsedByMetadata()) @@ -58,75 +55,54 @@ TinyPtrVector<DbgVariableIntrinsic *> llvm::FindDbgAddrUses(Value *V) { if (!MDV) return {}; - TinyPtrVector<DbgVariableIntrinsic *> Declares; + TinyPtrVector<DbgDeclareInst *> Declares; for (User *U : MDV->users()) { - if (auto *DII = dyn_cast<DbgVariableIntrinsic>(U)) - if (DII->isAddressOfVariable()) - Declares.push_back(DII); + if (auto *DDI = dyn_cast<DbgDeclareInst>(U)) + Declares.push_back(DDI); } return Declares; } -TinyPtrVector<DbgDeclareInst *> llvm::FindDbgDeclareUses(Value *V) { - TinyPtrVector<DbgDeclareInst *> DDIs; - for (DbgVariableIntrinsic *DVI : FindDbgAddrUses(V)) - if (auto *DDI = dyn_cast<DbgDeclareInst>(DVI)) - DDIs.push_back(DDI); - return DDIs; -} - -void llvm::findDbgValues(SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V) { +template <typename IntrinsicT> +static void findDbgIntrinsics(SmallVectorImpl<IntrinsicT *> &Result, Value *V) { // This function is hot. Check whether the value has any metadata to avoid a // DenseMap lookup. if (!V->isUsedByMetadata()) return; + + LLVMContext &Ctx = V->getContext(); // TODO: If this value appears multiple times in a DIArgList, we should still // only add the owning DbgValueInst once; use this set to track ArgListUsers. // This behaviour can be removed when we can automatically remove duplicates. - SmallPtrSet<DbgValueInst *, 4> EncounteredDbgValues; - if (auto *L = LocalAsMetadata::getIfExists(V)) { - if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L)) { + // V will also appear twice in a dbg.assign if its used in the both the value + // and address components. + SmallPtrSet<IntrinsicT *, 4> EncounteredIntrinsics; + + /// Append IntrinsicT users of MetadataAsValue(MD). + auto AppendUsers = [&Ctx, &EncounteredIntrinsics, &Result](Metadata *MD) { + if (auto *MDV = MetadataAsValue::getIfExists(Ctx, MD)) { for (User *U : MDV->users()) - if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(U)) - DbgValues.push_back(DVI); - } - for (Metadata *AL : L->getAllArgListUsers()) { - if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), AL)) { - for (User *U : MDV->users()) - if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(U)) - if (EncounteredDbgValues.insert(DVI).second) - DbgValues.push_back(DVI); - } + if (IntrinsicT *DVI = dyn_cast<IntrinsicT>(U)) + if (EncounteredIntrinsics.insert(DVI).second) + Result.push_back(DVI); } + }; + + if (auto *L = LocalAsMetadata::getIfExists(V)) { + AppendUsers(L); + for (Metadata *AL : L->getAllArgListUsers()) + AppendUsers(AL); } } +void llvm::findDbgValues(SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V) { + findDbgIntrinsics<DbgValueInst>(DbgValues, V); +} + void llvm::findDbgUsers(SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers, Value *V) { - // This function is hot. Check whether the value has any metadata to avoid a - // DenseMap lookup. - if (!V->isUsedByMetadata()) - return; - // TODO: If this value appears multiple times in a DIArgList, we should still - // only add the owning DbgValueInst once; use this set to track ArgListUsers. - // This behaviour can be removed when we can automatically remove duplicates. - SmallPtrSet<DbgVariableIntrinsic *, 4> EncounteredDbgValues; - if (auto *L = LocalAsMetadata::getIfExists(V)) { - if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L)) { - for (User *U : MDV->users()) - if (DbgVariableIntrinsic *DII = dyn_cast<DbgVariableIntrinsic>(U)) - DbgUsers.push_back(DII); - } - for (Metadata *AL : L->getAllArgListUsers()) { - if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), AL)) { - for (User *U : MDV->users()) - if (DbgVariableIntrinsic *DII = dyn_cast<DbgVariableIntrinsic>(U)) - if (EncounteredDbgValues.insert(DII).second) - DbgUsers.push_back(DII); - } - } - } + findDbgIntrinsics<DbgVariableIntrinsic>(DbgUsers, V); } DISubprogram *llvm::getDISubprogram(const MDNode *Scope) { @@ -410,16 +386,80 @@ static bool isDILocationReachable(SmallPtrSetImpl<Metadata *> &Visited, for (auto &OpIt : N->operands()) { Metadata *Op = OpIt.get(); if (isDILocationReachable(Visited, Reachable, Op)) { + // Don't return just yet as we want to visit all MD's children to + // initialize DILocationReachable in stripDebugLocFromLoopID Reachable.insert(N); - return true; } } - return false; + return Reachable.count(N); +} + +static bool isAllDILocation(SmallPtrSetImpl<Metadata *> &Visited, + SmallPtrSetImpl<Metadata *> &AllDILocation, + const SmallPtrSetImpl<Metadata *> &DIReachable, + Metadata *MD) { + MDNode *N = dyn_cast_or_null<MDNode>(MD); + if (!N) + return false; + if (isa<DILocation>(N) || AllDILocation.count(N)) + return true; + if (!DIReachable.count(N)) + return false; + if (!Visited.insert(N).second) + return false; + for (auto &OpIt : N->operands()) { + Metadata *Op = OpIt.get(); + if (Op == MD) + continue; + if (!isAllDILocation(Visited, AllDILocation, DIReachable, Op)) { + return false; + } + } + AllDILocation.insert(N); + return true; +} + +static Metadata * +stripLoopMDLoc(const SmallPtrSetImpl<Metadata *> &AllDILocation, + const SmallPtrSetImpl<Metadata *> &DIReachable, Metadata *MD) { + if (isa<DILocation>(MD) || AllDILocation.count(MD)) + return nullptr; + + if (!DIReachable.count(MD)) + return MD; + + MDNode *N = dyn_cast_or_null<MDNode>(MD); + if (!N) + return MD; + + SmallVector<Metadata *, 4> Args; + bool HasSelfRef = false; + for (unsigned i = 0; i < N->getNumOperands(); ++i) { + Metadata *A = N->getOperand(i); + if (!A) { + Args.push_back(nullptr); + } else if (A == MD) { + assert(i == 0 && "expected i==0 for self-reference"); + HasSelfRef = true; + Args.push_back(nullptr); + } else if (Metadata *NewArg = + stripLoopMDLoc(AllDILocation, DIReachable, A)) { + Args.push_back(NewArg); + } + } + if (Args.empty() || (HasSelfRef && Args.size() == 1)) + return nullptr; + + MDNode *NewMD = N->isDistinct() ? MDNode::getDistinct(N->getContext(), Args) + : MDNode::get(N->getContext(), Args); + if (HasSelfRef) + NewMD->replaceOperandWith(0, NewMD); + return NewMD; } static MDNode *stripDebugLocFromLoopID(MDNode *N) { assert(!N->operands().empty() && "Missing self reference?"); - SmallPtrSet<Metadata *, 8> Visited, DILocationReachable; + SmallPtrSet<Metadata *, 8> Visited, DILocationReachable, AllDILocation; // If we already visited N, there is nothing to do. if (!Visited.insert(N).second) return N; @@ -428,27 +468,27 @@ static MDNode *stripDebugLocFromLoopID(MDNode *N) { // MDNode. This loop also initializes DILocationReachable, later // needed by updateLoopMetadataDebugLocationsImpl; the use of // count_if avoids an early exit. - if (!std::count_if(N->op_begin() + 1, N->op_end(), + if (!llvm::count_if(llvm::drop_begin(N->operands()), [&Visited, &DILocationReachable](const MDOperand &Op) { return isDILocationReachable( Visited, DILocationReachable, Op.get()); })) return N; + Visited.clear(); // If there is only the debug location without any actual loop metadata, we // can remove the metadata. if (llvm::all_of(llvm::drop_begin(N->operands()), - [&Visited, &DILocationReachable](const MDOperand &Op) { - return isDILocationReachable(Visited, DILocationReachable, - Op.get()); + [&Visited, &AllDILocation, + &DILocationReachable](const MDOperand &Op) { + return isAllDILocation(Visited, AllDILocation, + DILocationReachable, Op.get()); })) return nullptr; return updateLoopMetadataDebugLocationsImpl( - N, [&DILocationReachable](Metadata *MD) -> Metadata * { - if (isa<DILocation>(MD) || DILocationReachable.count(MD)) - return nullptr; - return MD; + N, [&AllDILocation, &DILocationReachable](Metadata *MD) -> Metadata * { + return stripLoopMDLoc(AllDILocation, DILocationReachable, MD); }); } @@ -737,7 +777,6 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { Changed = true; } }; - RemoveUses("llvm.dbg.addr"); RemoveUses("llvm.dbg.declare"); RemoveUses("llvm.dbg.label"); RemoveUses("llvm.dbg.value"); @@ -806,7 +845,7 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { // Create a new llvm.dbg.cu, which is equivalent to the one // -gline-tables-only would have created. - for (auto &NMD : M.getNamedMDList()) { + for (auto &NMD : M.named_metadata()) { SmallVector<MDNode *, 8> Ops; for (MDNode *Op : NMD.operands()) Ops.push_back(remap(Op)); @@ -829,8 +868,7 @@ unsigned llvm::getDebugMetadataVersionFromModule(const Module &M) { return 0; } -void Instruction::applyMergedLocation(const DILocation *LocA, - const DILocation *LocB) { +void Instruction::applyMergedLocation(DILocation *LocA, DILocation *LocB) { setDebugLoc(DILocation::getMergedLocation(LocA, LocB)); } @@ -1444,8 +1482,12 @@ LLVMDIBuilderCreateArtificialType(LLVMDIBuilderRef Builder, return wrap(unwrap(Builder)->createArtificialType(unwrapDI<DIType>(Type))); } +uint16_t LLVMGetDINodeTag(LLVMMetadataRef MD) { + return unwrapDI<DINode>(MD)->getTag(); +} + const char *LLVMDITypeGetName(LLVMMetadataRef DType, size_t *Length) { - StringRef Str = unwrap<DIType>(DType)->getName(); + StringRef Str = unwrapDI<DIType>(DType)->getName(); *Length = Str.size(); return Str.data(); } @@ -1738,14 +1780,155 @@ void at::deleteAll(Function *F) { DAI->eraseFromParent(); } +bool at::calculateFragmentIntersect( + const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits, + uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DAI, + std::optional<DIExpression::FragmentInfo> &Result) { + // There are multiple offsets at play in this function, so let's break it + // down. Starting with how variables may be stored in allocas: + // + // 1 Simplest case: variable is alloca sized and starts at offset 0. + // 2 Variable is larger than the alloca: the alloca holds just a part of it. + // 3 Variable is smaller than the alloca: the alloca may hold multiple + // variables. + // + // Imagine we have a store to the entire alloca. In case (3) the store + // affects bits outside of the bounds of each variable. In case (2), where + // the alloca holds the Xth bit to the Yth bit of a variable, the + // zero-offset store doesn't represent an assignment at offset zero to the + // variable. It is an assignment to offset X. + // + // # Example 1 + // Obviously, not all stores are alloca-sized and have zero offset. Imagine + // the lower 32 bits of this store are dead and are going to be DSEd: + // + // store i64 %v, ptr %dest, !DIAssignID !1 + // dbg.assign(..., !DIExpression(fragment, 128, 32), !1, %dest, + // !DIExpression(DW_OP_plus_uconst, 4)) + // + // Goal: Given our dead bits at offset:0 size:32 for the store, determine the + // part of the variable, which fits in the fragment expressed by the + // dbg.assign, that has been killed, if any. + // + // calculateFragmentIntersect(..., SliceOffsetInBits=0, + // SliceSizeInBits=32, Dest=%dest, DAI=dbg.assign) + // + // Drawing the store (s) in memory followed by the shortened version ($), + // then the dbg.assign (d), with the fragment information on a seperate scale + // underneath: + // + // Memory + // offset + // from + // dest 0 63 + // | | + // s[######] - Original stores 64 bits to Dest. + // $----[##] - DSE says the lower 32 bits are dead, to be removed. + // d [##] - DAI's address-modifying expression adds 4 bytes to dest. + // Variable | | + // Fragment 128| + // Offsets 159 + // + // The answer is achieved in a few steps: + // 1. Add the fragment offset to the store offset: + // SliceOffsetInBits:0 + VarFrag.OffsetInBits:128 = 128 + // + // 2. Subtract the address-modifying expression offset plus difference + // between d.address and dest: + // 128 - (expression_offset:32 + (d.address - dest):0) = 96 + // + // 3. That offset along with the store size (32) represents the bits of the + // variable that'd be affected by the store. Call it SliceOfVariable. + // Intersect that with DAI's fragment info: + // SliceOfVariable ∩ DAI_fragment = none + // + // In this case: none of the dead bits of the store affect DAI. + // + // # Example 2 + // Similar example with the same goal. This time the upper 16 bits + // of the store are going to be DSE'd. + // + // store i64 %v, ptr %dest, !DIAssignID !1 + // dbg.assign(..., !DIExpression(fragment, 128, 32), !1, %dest, + // !DIExpression(DW_OP_plus_uconst, 4)) + // + // calculateFragmentIntersect(..., SliceOffsetInBits=48, + // SliceSizeInBits=16, Dest=%dest, DAI=dbg.assign) + // + // Memory + // offset + // from + // dest 0 63 + // | | + // s[######] - Original stores 64 bits to Dest. + // $[####]-- - DSE says the upper 16 bits are dead, to be removed. + // d [##] - DAI's address-modifying expression adds 4 bytes to dest. + // Variable | | + // Fragment 128| + // Offsets 159 + // + // Using the same steps in the first example: + // 1. SliceOffsetInBits:48 + VarFrag.OffsetInBits:128 = 176 + // 2. 176 - (expression_offset:32 + (d.address - dest):0) = 144 + // 3. SliceOfVariable offset = 144, size = 16: + // SliceOfVariable ∩ DAI_fragment = (offset: 144, size: 16) + // SliceOfVariable tells us the bits of the variable described by DAI that are + // affected by the DSE. + if (DAI->isKillAddress()) + return false; + + DIExpression::FragmentInfo VarFrag = DAI->getFragmentOrEntireVariable(); + if (VarFrag.SizeInBits == 0) + return false; // Variable size is unknown. + + // Calculate the difference between Dest and the dbg.assign address + + // address-modifying expression. + int64_t PointerOffsetInBits; + { + auto DestOffsetInBytes = DAI->getAddress()->getPointerOffsetFrom(Dest, DL); + if (!DestOffsetInBytes) + return false; // Can't calculate difference in addresses. + + int64_t ExprOffsetInBytes; + if (!DAI->getAddressExpression()->extractIfOffset(ExprOffsetInBytes)) + return false; + + int64_t PointerOffsetInBytes = *DestOffsetInBytes + ExprOffsetInBytes; + PointerOffsetInBits = PointerOffsetInBytes * 8; + } + + // Adjust the slice offset so that we go from describing the a slice + // of memory to a slice of the variable. + int64_t NewOffsetInBits = + SliceOffsetInBits + VarFrag.OffsetInBits - PointerOffsetInBits; + if (NewOffsetInBits < 0) + return false; // Fragment offsets can only be positive. + DIExpression::FragmentInfo SliceOfVariable(SliceSizeInBits, NewOffsetInBits); + // Intersect the variable slice with DAI's fragment to trim it down to size. + DIExpression::FragmentInfo TrimmedSliceOfVariable = + DIExpression::FragmentInfo::intersect(SliceOfVariable, VarFrag); + if (TrimmedSliceOfVariable == VarFrag) + Result = std::nullopt; + else + Result = TrimmedSliceOfVariable; + return true; +} + /// Collect constant properies (base, size, offset) of \p StoreDest. -/// Return std::nullopt if any properties are not constants. +/// Return std::nullopt if any properties are not constants or the +/// offset from the base pointer is negative. static std::optional<AssignmentInfo> getAssignmentInfoImpl(const DataLayout &DL, const Value *StoreDest, - uint64_t SizeInBits) { + TypeSize SizeInBits) { + if (SizeInBits.isScalable()) + return std::nullopt; APInt GEPOffset(DL.getIndexTypeSizeInBits(StoreDest->getType()), 0); const Value *Base = StoreDest->stripAndAccumulateConstantOffsets( DL, GEPOffset, /*AllowNonInbounds*/ true); + + if (GEPOffset.isNegative()) + return std::nullopt; + uint64_t OffsetInBytes = GEPOffset.getLimitedValue(); // Check for overflow. if (OffsetInBytes == UINT64_MAX) @@ -1764,22 +1947,22 @@ std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL, // We can't use a non-const size, bail. return std::nullopt; uint64_t SizeInBits = 8 * ConstLengthInBytes->getZExtValue(); - return getAssignmentInfoImpl(DL, StoreDest, SizeInBits); + return getAssignmentInfoImpl(DL, StoreDest, TypeSize::getFixed(SizeInBits)); } std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL, const StoreInst *SI) { - const Value *StoreDest = SI->getPointerOperand(); - uint64_t SizeInBits = DL.getTypeSizeInBits(SI->getValueOperand()->getType()); - return getAssignmentInfoImpl(DL, StoreDest, SizeInBits); + TypeSize SizeInBits = DL.getTypeSizeInBits(SI->getValueOperand()->getType()); + return getAssignmentInfoImpl(DL, SI->getPointerOperand(), SizeInBits); } std::optional<AssignmentInfo> at::getAssignmentInfo(const DataLayout &DL, const AllocaInst *AI) { - uint64_t SizeInBits = DL.getTypeSizeInBits(AI->getAllocatedType()); + TypeSize SizeInBits = DL.getTypeSizeInBits(AI->getAllocatedType()); return getAssignmentInfoImpl(DL, AI, SizeInBits); } +/// Returns nullptr if the assignment shouldn't be attributed to this variable. static CallInst *emitDbgAssign(AssignmentInfo Info, Value *Val, Value *Dest, Instruction &StoreLikeInst, const VarRecord &VarRec, DIBuilder &DIB) { @@ -1787,11 +1970,35 @@ static CallInst *emitDbgAssign(AssignmentInfo Info, Value *Val, Value *Dest, assert(ID && "Store instruction must have DIAssignID metadata"); (void)ID; + const uint64_t StoreStartBit = Info.OffsetInBits; + const uint64_t StoreEndBit = Info.OffsetInBits + Info.SizeInBits; + + uint64_t FragStartBit = StoreStartBit; + uint64_t FragEndBit = StoreEndBit; + + bool StoreToWholeVariable = Info.StoreToWholeAlloca; + if (auto Size = VarRec.Var->getSizeInBits()) { + // NOTE: trackAssignments doesn't understand base expressions yet, so all + // variables that reach here are guaranteed to start at offset 0 in the + // alloca. + const uint64_t VarStartBit = 0; + const uint64_t VarEndBit = *Size; + + // FIXME: trim FragStartBit when nonzero VarStartBit is supported. + FragEndBit = std::min(FragEndBit, VarEndBit); + + // Discard stores to bits outside this variable. + if (FragStartBit >= FragEndBit) + return nullptr; + + StoreToWholeVariable = FragStartBit <= VarStartBit && FragEndBit >= *Size; + } + DIExpression *Expr = DIExpression::get(StoreLikeInst.getContext(), std::nullopt); - if (!Info.StoreToWholeAlloca) { - auto R = DIExpression::createFragmentExpression(Expr, Info.OffsetInBits, - Info.SizeInBits); + if (!StoreToWholeVariable) { + auto R = DIExpression::createFragmentExpression(Expr, FragStartBit, + FragEndBit - FragStartBit); assert(R.has_value() && "failed to create fragment expression"); Expr = *R; } @@ -1889,13 +2096,19 @@ void at::trackAssignments(Function::iterator Start, Function::iterator End, auto *Assign = emitDbgAssign(*Info, ValueComponent, DestComponent, I, R, DIB); (void)Assign; - LLVM_DEBUG(errs() << " > INSERT: " << *Assign << "\n"); + LLVM_DEBUG(if (Assign) errs() << " > INSERT: " << *Assign << "\n"); } } } } -void AssignmentTrackingPass::runOnFunction(Function &F) { +bool AssignmentTrackingPass::runOnFunction(Function &F) { + // No value in assignment tracking without optimisations. + if (F.hasFnAttribute(Attribute::OptimizeNone)) + return /*Changed*/ false; + + bool Changed = false; + auto *DL = &F.getParent()->getDataLayout(); // Collect a map of {backing storage : dbg.declares} (currently "backing // storage" is limited to Allocas). We'll use this to find dbg.declares to // delete after running `trackAssignments`. @@ -1913,15 +2126,22 @@ void AssignmentTrackingPass::runOnFunction(Function &F) { // leave dbg.declares with non-empty expressions in place. if (DDI->getExpression()->getNumElements() != 0) continue; + if (!DDI->getAddress()) + continue; if (AllocaInst *Alloca = dyn_cast<AllocaInst>(DDI->getAddress()->stripPointerCasts())) { + // FIXME: Skip VLAs for now (let these variables use dbg.declares). + if (!Alloca->isStaticAlloca()) + continue; + // Similarly, skip scalable vectors (use dbg.declares instead). + if (auto Sz = Alloca->getAllocationSize(*DL); Sz && Sz->isScalable()) + continue; DbgDeclares[Alloca].insert(DDI); Vars[Alloca].insert(VarRecord(DDI)); } } } - auto DL = std::make_unique<DataLayout>(F.getParent()); // FIXME: Locals can be backed by caller allocas (sret, byval). // Note: trackAssignments doesn't respect dbg.declare's IR positions (as it // doesn't "understand" dbg.declares). However, this doesn't appear to break @@ -1940,16 +2160,22 @@ void AssignmentTrackingPass::runOnFunction(Function &F) { (void)Markers; for (DbgDeclareInst *DDI : P.second) { // Assert that the alloca that DDI uses is now linked to a dbg.assign - // describing the same variable (i.e. check that this dbg.declare - // has been replaced by a dbg.assign). + // describing the same variable (i.e. check that this dbg.declare has + // been replaced by a dbg.assign). Use DebugVariableAggregate to Discard + // the fragment part because trackAssignments may alter the + // fragment. e.g. if the alloca is smaller than the variable, then + // trackAssignments will create an alloca-sized fragment for the + // dbg.assign. assert(llvm::any_of(Markers, [DDI](DbgAssignIntrinsic *DAI) { - return DebugVariable(DAI) == DebugVariable(DDI); + return DebugVariableAggregate(DAI) == DebugVariableAggregate(DDI); })); // Delete DDI because the variable location is now tracked using // assignment tracking. DDI->eraseFromParent(); + Changed = true; } } + return Changed; } static const char *AssignmentTrackingModuleFlag = @@ -1972,7 +2198,8 @@ bool llvm::isAssignmentTrackingEnabled(const Module &M) { PreservedAnalyses AssignmentTrackingPass::run(Function &F, FunctionAnalysisManager &AM) { - runOnFunction(F); + if (!runOnFunction(F)) + return PreservedAnalyses::all(); // Record that this module uses assignment tracking. It doesn't matter that // some functons in the module may not use it - the debug info in those @@ -1988,8 +2215,12 @@ PreservedAnalyses AssignmentTrackingPass::run(Function &F, PreservedAnalyses AssignmentTrackingPass::run(Module &M, ModuleAnalysisManager &AM) { + bool Changed = false; for (auto &F : M) - runOnFunction(F); + Changed |= runOnFunction(F); + + if (!Changed) + return PreservedAnalyses::all(); // Record that this module uses assignment tracking. setAssignmentTrackingModuleFlag(M); diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp index fb9a7b882220..4933b6032688 100644 --- a/llvm/lib/IR/DebugInfoMetadata.cpp +++ b/llvm/lib/IR/DebugInfoMetadata.cpp @@ -13,6 +13,7 @@ #include "llvm/IR/DebugInfoMetadata.h" #include "LLVMContextImpl.h" #include "MetadataImpl.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -41,6 +42,10 @@ DebugVariable::DebugVariable(const DbgVariableIntrinsic *DII) Fragment(DII->getExpression()->getFragmentInfo()), InlinedAt(DII->getDebugLoc().getInlinedAt()) {} +DebugVariableAggregate::DebugVariableAggregate(const DbgVariableIntrinsic *DVI) + : DebugVariable(DVI->getVariable(), std::nullopt, + DVI->getDebugLoc()->getInlinedAt()) {} + DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line, unsigned Column, ArrayRef<Metadata *> MDs, bool ImplicitCode) @@ -90,14 +95,13 @@ DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line, Storage, Context.pImpl->DILocations); } -const DILocation * -DILocation::getMergedLocations(ArrayRef<const DILocation *> Locs) { +DILocation *DILocation::getMergedLocations(ArrayRef<DILocation *> Locs) { if (Locs.empty()) return nullptr; if (Locs.size() == 1) return Locs[0]; auto *Merged = Locs[0]; - for (const DILocation *L : llvm::drop_begin(Locs)) { + for (DILocation *L : llvm::drop_begin(Locs)) { Merged = getMergedLocation(Merged, L); if (Merged == nullptr) break; @@ -105,8 +109,7 @@ DILocation::getMergedLocations(ArrayRef<const DILocation *> Locs) { return Merged; } -const DILocation *DILocation::getMergedLocation(const DILocation *LocA, - const DILocation *LocB) { +DILocation *DILocation::getMergedLocation(DILocation *LocA, DILocation *LocB) { if (!LocA || !LocB) return nullptr; @@ -114,63 +117,122 @@ const DILocation *DILocation::getMergedLocation(const DILocation *LocA, return LocA; LLVMContext &C = LocA->getContext(); - SmallDenseMap<std::pair<DILocalScope *, DILocation *>, - std::pair<unsigned, unsigned>, 4> - Locations; - - DIScope *S = LocA->getScope(); - DILocation *L = LocA->getInlinedAt(); - unsigned Line = LocA->getLine(); - unsigned Col = LocA->getColumn(); - - // Walk from the current source locaiton until the file scope; - // then, do the same for the inlined-at locations. - auto AdvanceToParentLoc = [&S, &L, &Line, &Col]() { - S = S->getScope(); - if (!S && L) { - Line = L->getLine(); - Col = L->getColumn(); - S = L->getScope(); - L = L->getInlinedAt(); - } - }; - while (S) { - if (auto *LS = dyn_cast<DILocalScope>(S)) - Locations.try_emplace(std::make_pair(LS, L), std::make_pair(Line, Col)); - AdvanceToParentLoc(); + using LocVec = SmallVector<const DILocation *>; + LocVec ALocs; + LocVec BLocs; + SmallDenseMap<std::pair<const DISubprogram *, const DILocation *>, unsigned, + 4> + ALookup; + + // Walk through LocA and its inlined-at locations, populate them in ALocs and + // save the index for the subprogram and inlined-at pair, which we use to find + // a matching starting location in LocB's chain. + for (auto [L, I] = std::make_pair(LocA, 0U); L; L = L->getInlinedAt(), I++) { + ALocs.push_back(L); + auto Res = ALookup.try_emplace( + {L->getScope()->getSubprogram(), L->getInlinedAt()}, I); + assert(Res.second && "Multiple <SP, InlinedAt> pairs in a location chain?"); + (void)Res; } - // Walk the source locations of LocB until a match with LocA is found. - S = LocB->getScope(); - L = LocB->getInlinedAt(); - Line = LocB->getLine(); - Col = LocB->getColumn(); - while (S) { - if (auto *LS = dyn_cast<DILocalScope>(S)) { - auto MatchLoc = Locations.find(std::make_pair(LS, L)); - if (MatchLoc != Locations.end()) { - // If the lines match, keep the line, but set the column to '0' - // If the lines don't match, pick a "line 0" location but keep - // the current scope and inlined-at. - bool SameLine = Line == MatchLoc->second.first; - bool SameCol = Col == MatchLoc->second.second; - Line = SameLine ? Line : 0; - Col = SameLine && SameCol ? Col : 0; - break; - } - } - AdvanceToParentLoc(); + LocVec::reverse_iterator ARIt = ALocs.rend(); + LocVec::reverse_iterator BRIt = BLocs.rend(); + + // Populate BLocs and look for a matching starting location, the first + // location with the same subprogram and inlined-at location as in LocA's + // chain. Since the two locations have the same inlined-at location we do + // not need to look at those parts of the chains. + for (auto [L, I] = std::make_pair(LocB, 0U); L; L = L->getInlinedAt(), I++) { + BLocs.push_back(L); + + if (ARIt != ALocs.rend()) + // We have already found a matching starting location. + continue; + + auto IT = ALookup.find({L->getScope()->getSubprogram(), L->getInlinedAt()}); + if (IT == ALookup.end()) + continue; + + // The + 1 is to account for the &*rev_it = &(it - 1) relationship. + ARIt = LocVec::reverse_iterator(ALocs.begin() + IT->second + 1); + BRIt = LocVec::reverse_iterator(BLocs.begin() + I + 1); + + // If we have found a matching starting location we do not need to add more + // locations to BLocs, since we will only look at location pairs preceding + // the matching starting location, and adding more elements to BLocs could + // invalidate the iterator that we initialized here. + break; } - if (!S) { - // If the two locations are irreconsilable, pick any scope, - // and return a "line 0" location. - Line = Col = 0; - S = LocA->getScope(); + // Merge the two locations if possible, using the supplied + // inlined-at location for the created location. + auto MergeLocPair = [&C](const DILocation *L1, const DILocation *L2, + DILocation *InlinedAt) -> DILocation * { + if (L1 == L2) + return DILocation::get(C, L1->getLine(), L1->getColumn(), L1->getScope(), + InlinedAt); + + // If the locations originate from different subprograms we can't produce + // a common location. + if (L1->getScope()->getSubprogram() != L2->getScope()->getSubprogram()) + return nullptr; + + // Return the nearest common scope inside a subprogram. + auto GetNearestCommonScope = [](DIScope *S1, DIScope *S2) -> DIScope * { + SmallPtrSet<DIScope *, 8> Scopes; + for (; S1; S1 = S1->getScope()) { + Scopes.insert(S1); + if (isa<DISubprogram>(S1)) + break; + } + + for (; S2; S2 = S2->getScope()) { + if (Scopes.count(S2)) + return S2; + if (isa<DISubprogram>(S2)) + break; + } + + return nullptr; + }; + + auto Scope = GetNearestCommonScope(L1->getScope(), L2->getScope()); + assert(Scope && "No common scope in the same subprogram?"); + + bool SameLine = L1->getLine() == L2->getLine(); + bool SameCol = L1->getColumn() == L2->getColumn(); + unsigned Line = SameLine ? L1->getLine() : 0; + unsigned Col = SameLine && SameCol ? L1->getColumn() : 0; + + return DILocation::get(C, Line, Col, Scope, InlinedAt); + }; + + DILocation *Result = ARIt != ALocs.rend() ? (*ARIt)->getInlinedAt() : nullptr; + + // If we have found a common starting location, walk up the inlined-at chains + // and try to produce common locations. + for (; ARIt != ALocs.rend() && BRIt != BLocs.rend(); ++ARIt, ++BRIt) { + DILocation *Tmp = MergeLocPair(*ARIt, *BRIt, Result); + + if (!Tmp) + // We have walked up to a point in the chains where the two locations + // are irreconsilable. At this point Result contains the nearest common + // location in the inlined-at chains of LocA and LocB, so we break here. + break; + + Result = Tmp; } - return DILocation::get(C, Line, Col, S, L); + if (Result) + return Result; + + // We ended up with LocA and LocB as irreconsilable locations. Produce a + // location at 0:0 with one of the locations' scope. The function has + // historically picked A's scope, and a nullptr inlined-at location, so that + // behavior is mimicked here but I am not sure if this is always the correct + // way to handle this. + return DILocation::get(C, 0, 0, LocA->getScope(), nullptr); } std::optional<unsigned> @@ -908,6 +970,7 @@ DICompileUnit::getNameTableKind(StringRef Str) { return StringSwitch<std::optional<DebugNameTableKind>>(Str) .Case("Default", DebugNameTableKind::Default) .Case("GNU", DebugNameTableKind::GNU) + .Case("Apple", DebugNameTableKind::Apple) .Case("None", DebugNameTableKind::None) .Default(std::nullopt); } @@ -932,6 +995,8 @@ const char *DICompileUnit::nameTableKindString(DebugNameTableKind NTK) { return nullptr; case DebugNameTableKind::GNU: return "GNU"; + case DebugNameTableKind::Apple: + return "Apple"; case DebugNameTableKind::None: return "None"; } @@ -1285,6 +1350,9 @@ bool DIExpression::isEntryValue() const { bool DIExpression::startsWithDeref() const { return getNumElements() > 0 && getElement(0) == dwarf::DW_OP_deref; } +bool DIExpression::isDeref() const { + return getNumElements() == 1 && startsWithDeref(); +} DIAssignID *DIAssignID::getImpl(LLVMContext &Context, StorageType Storage, bool ShouldCreate) { @@ -1396,6 +1464,12 @@ bool DIExpression::isValid() const { case dwarf::DW_OP_push_object_address: case dwarf::DW_OP_over: case dwarf::DW_OP_consts: + case dwarf::DW_OP_eq: + case dwarf::DW_OP_ne: + case dwarf::DW_OP_gt: + case dwarf::DW_OP_ge: + case dwarf::DW_OP_lt: + case dwarf::DW_OP_le: break; } } @@ -1604,7 +1678,7 @@ bool DIExpression::hasAllLocationOps(unsigned N) const { if (ExprOp.getOp() == dwarf::DW_OP_LLVM_arg) SeenOps.insert(ExprOp.getArg(0)); for (uint64_t Idx = 0; Idx < N; ++Idx) - if (!is_contained(SeenOps, Idx)) + if (!SeenOps.contains(Idx)) return false; return true; } @@ -2026,7 +2100,7 @@ void DIArgList::handleChangedOperand(void *Ref, Metadata *New) { if (NewVM) VM = NewVM; else - VM = ValueAsMetadata::get(UndefValue::get(VM->getValue()->getType())); + VM = ValueAsMetadata::get(PoisonValue::get(VM->getValue()->getType())); } } if (Uniq) { diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp index fb238e2aac59..342c4cbbc39d 100644 --- a/llvm/lib/IR/DiagnosticInfo.cpp +++ b/llvm/lib/IR/DiagnosticInfo.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/Demangle/Demangle.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -440,7 +441,7 @@ void llvm::diagnoseDontCall(const CallInst &CI) { } void DiagnosticInfoDontCall::print(DiagnosticPrinter &DP) const { - DP << "call to " << getFunctionName() << " marked \"dontcall-"; + DP << "call to " << demangle(getFunctionName()) << " marked \"dontcall-"; if (getSeverity() == DiagnosticSeverity::DS_Error) DP << "error\""; else diff --git a/llvm/lib/IR/Dominators.cpp b/llvm/lib/IR/Dominators.cpp index 7c620c3a9331..24cc9f46ff79 100644 --- a/llvm/lib/IR/Dominators.cpp +++ b/llvm/lib/IR/Dominators.cpp @@ -194,13 +194,6 @@ bool DominatorTree::dominates(const Instruction *Def, return dominates(E, UseBB); } - // Callbr results are similarly only usable in the default destination. - if (const auto *CBI = dyn_cast<CallBrInst>(Def)) { - BasicBlock *NormalDest = CBI->getDefaultDest(); - BasicBlockEdge E(DefBB, NormalDest); - return dominates(E, UseBB); - } - return dominates(DefBB, UseBB); } @@ -311,13 +304,6 @@ bool DominatorTree::dominates(const Value *DefV, const Use &U) const { return dominates(E, U); } - // Callbr results are similarly only usable in the default destination. - if (const auto *CBI = dyn_cast<CallBrInst>(Def)) { - BasicBlock *NormalDest = CBI->getDefaultDest(); - BasicBlockEdge E(DefBB, NormalDest); - return dominates(E, U); - } - // If the def and use are in different blocks, do a simple CFG dominator // tree query. if (DefBB != UseBB) diff --git a/llvm/lib/IR/EHPersonalities.cpp b/llvm/lib/IR/EHPersonalities.cpp new file mode 100644 index 000000000000..afbb2bb8275d --- /dev/null +++ b/llvm/lib/IR/EHPersonalities.cpp @@ -0,0 +1,160 @@ +//===- EHPersonalities.cpp - Compute EH-related information ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/EHPersonalities.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Triple.h" +using namespace llvm; + +/// See if the given exception handling personality function is one that we +/// understand. If so, return a description of it; otherwise return Unknown. +EHPersonality llvm::classifyEHPersonality(const Value *Pers) { + const GlobalValue *F = + Pers ? dyn_cast<GlobalValue>(Pers->stripPointerCasts()) : nullptr; + if (!F || !F->getValueType() || !F->getValueType()->isFunctionTy()) + return EHPersonality::Unknown; + return StringSwitch<EHPersonality>(F->getName()) + .Case("__gnat_eh_personality", EHPersonality::GNU_Ada) + .Case("__gxx_personality_v0", EHPersonality::GNU_CXX) + .Case("__gxx_personality_seh0", EHPersonality::GNU_CXX) + .Case("__gxx_personality_sj0", EHPersonality::GNU_CXX_SjLj) + .Case("__gcc_personality_v0", EHPersonality::GNU_C) + .Case("__gcc_personality_seh0", EHPersonality::GNU_C) + .Case("__gcc_personality_sj0", EHPersonality::GNU_C_SjLj) + .Case("__objc_personality_v0", EHPersonality::GNU_ObjC) + .Case("_except_handler3", EHPersonality::MSVC_X86SEH) + .Case("_except_handler4", EHPersonality::MSVC_X86SEH) + .Case("__C_specific_handler", EHPersonality::MSVC_TableSEH) + .Case("__CxxFrameHandler3", EHPersonality::MSVC_CXX) + .Case("ProcessCLRException", EHPersonality::CoreCLR) + .Case("rust_eh_personality", EHPersonality::Rust) + .Case("__gxx_wasm_personality_v0", EHPersonality::Wasm_CXX) + .Case("__xlcxx_personality_v1", EHPersonality::XL_CXX) + .Default(EHPersonality::Unknown); +} + +StringRef llvm::getEHPersonalityName(EHPersonality Pers) { + switch (Pers) { + case EHPersonality::GNU_Ada: + return "__gnat_eh_personality"; + case EHPersonality::GNU_CXX: + return "__gxx_personality_v0"; + case EHPersonality::GNU_CXX_SjLj: + return "__gxx_personality_sj0"; + case EHPersonality::GNU_C: + return "__gcc_personality_v0"; + case EHPersonality::GNU_C_SjLj: + return "__gcc_personality_sj0"; + case EHPersonality::GNU_ObjC: + return "__objc_personality_v0"; + case EHPersonality::MSVC_X86SEH: + return "_except_handler3"; + case EHPersonality::MSVC_TableSEH: + return "__C_specific_handler"; + case EHPersonality::MSVC_CXX: + return "__CxxFrameHandler3"; + case EHPersonality::CoreCLR: + return "ProcessCLRException"; + case EHPersonality::Rust: + return "rust_eh_personality"; + case EHPersonality::Wasm_CXX: + return "__gxx_wasm_personality_v0"; + case EHPersonality::XL_CXX: + return "__xlcxx_personality_v1"; + case EHPersonality::Unknown: + llvm_unreachable("Unknown EHPersonality!"); + } + + llvm_unreachable("Invalid EHPersonality!"); +} + +EHPersonality llvm::getDefaultEHPersonality(const Triple &T) { + if (T.isPS5()) + return EHPersonality::GNU_CXX; + else + return EHPersonality::GNU_C; +} + +bool llvm::canSimplifyInvokeNoUnwind(const Function *F) { + EHPersonality Personality = classifyEHPersonality(F->getPersonalityFn()); + // We can't simplify any invokes to nounwind functions if the personality + // function wants to catch asynch exceptions. The nounwind attribute only + // implies that the function does not throw synchronous exceptions. + + // Cannot simplify CXX Personality under AsynchEH + const llvm::Module *M = (const llvm::Module *)F->getParent(); + bool EHa = M->getModuleFlag("eh-asynch"); + return !EHa && !isAsynchronousEHPersonality(Personality); +} + +DenseMap<BasicBlock *, ColorVector> llvm::colorEHFunclets(Function &F) { + SmallVector<std::pair<BasicBlock *, BasicBlock *>, 16> Worklist; + BasicBlock *EntryBlock = &F.getEntryBlock(); + DenseMap<BasicBlock *, ColorVector> BlockColors; + + // Build up the color map, which maps each block to its set of 'colors'. + // For any block B the "colors" of B are the set of funclets F (possibly + // including a root "funclet" representing the main function) such that + // F will need to directly contain B or a copy of B (where the term "directly + // contain" is used to distinguish from being "transitively contained" in + // a nested funclet). + // + // Note: Despite not being a funclet in the truest sense, a catchswitch is + // considered to belong to its own funclet for the purposes of coloring. + + DEBUG_WITH_TYPE("winehprepare-coloring", + dbgs() << "\nColoring funclets for " << F.getName() << "\n"); + + Worklist.push_back({EntryBlock, EntryBlock}); + + while (!Worklist.empty()) { + BasicBlock *Visiting; + BasicBlock *Color; + std::tie(Visiting, Color) = Worklist.pop_back_val(); + DEBUG_WITH_TYPE("winehprepare-coloring", + dbgs() << "Visiting " << Visiting->getName() << ", " + << Color->getName() << "\n"); + Instruction *VisitingHead = Visiting->getFirstNonPHI(); + if (VisitingHead->isEHPad()) { + // Mark this funclet head as a member of itself. + Color = Visiting; + } + // Note that this is a member of the given color. + ColorVector &Colors = BlockColors[Visiting]; + if (!is_contained(Colors, Color)) + Colors.push_back(Color); + else + continue; + + DEBUG_WITH_TYPE("winehprepare-coloring", + dbgs() << " Assigned color \'" << Color->getName() + << "\' to block \'" << Visiting->getName() + << "\'.\n"); + + BasicBlock *SuccColor = Color; + Instruction *Terminator = Visiting->getTerminator(); + if (auto *CatchRet = dyn_cast<CatchReturnInst>(Terminator)) { + Value *ParentPad = CatchRet->getCatchSwitchParentPad(); + if (isa<ConstantTokenNone>(ParentPad)) + SuccColor = EntryBlock; + else + SuccColor = cast<Instruction>(ParentPad)->getParent(); + } + + for (BasicBlock *Succ : successors(Visiting)) + Worklist.push_back({Succ, SuccColor}); + } + return BlockColors; +} diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 677db46124e4..27219e89dc5f 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -229,6 +229,10 @@ uint64_t Argument::getDereferenceableOrNullBytes() const { return getParent()->getParamDereferenceableOrNullBytes(getArgNo()); } +FPClassTest Argument::getNoFPClass() const { + return getParent()->getParamNoFPClass(getArgNo()); +} + bool Argument::hasNestAttr() const { if (!getType()->isPointerTy()) return false; return hasAttribute(Attribute::Nest); @@ -698,17 +702,30 @@ void Function::addDereferenceableOrNullParamAttr(unsigned ArgNo, DenormalMode Function::getDenormalMode(const fltSemantics &FPType) const { if (&FPType == &APFloat::IEEEsingle()) { - Attribute Attr = getFnAttribute("denormal-fp-math-f32"); - StringRef Val = Attr.getValueAsString(); - if (!Val.empty()) - return parseDenormalFPAttribute(Val); - + DenormalMode Mode = getDenormalModeF32Raw(); // If the f32 variant of the attribute isn't specified, try to use the // generic one. + if (Mode.isValid()) + return Mode; } + return getDenormalModeRaw(); +} + +DenormalMode Function::getDenormalModeRaw() const { Attribute Attr = getFnAttribute("denormal-fp-math"); - return parseDenormalFPAttribute(Attr.getValueAsString()); + StringRef Val = Attr.getValueAsString(); + return parseDenormalFPAttribute(Val); +} + +DenormalMode Function::getDenormalModeF32Raw() const { + Attribute Attr = getFnAttribute("denormal-fp-math-f32"); + if (Attr.isValid()) { + StringRef Val = Attr.getValueAsString(); + return parseDenormalFPAttribute(Val); + } + + return DenormalMode::getInvalid(); } const std::string &Function::getGC() const { @@ -900,11 +917,6 @@ static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) { std::string Result; if (PointerType *PTyp = dyn_cast<PointerType>(Ty)) { Result += "p" + utostr(PTyp->getAddressSpace()); - // Opaque pointer doesn't have pointee type information, so we just mangle - // address space for opaque pointer. - if (!PTyp->isOpaque()) - Result += getMangledTypeStr(PTyp->getNonOpaquePointerElementType(), - HasUnnamedType); } else if (ArrayType *ATyp = dyn_cast<ArrayType>(Ty)) { Result += "a" + utostr(ATyp->getNumElements()) + getMangledTypeStr(ATyp->getElementType(), HasUnnamedType); @@ -1019,70 +1031,11 @@ std::string Intrinsic::getNameNoUnnamedTypes(ID Id, ArrayRef<Type *> Tys) { /// IIT_Info - These are enumerators that describe the entries returned by the /// getIntrinsicInfoTableEntries function. /// -/// NOTE: This must be kept in synch with the copy in TblGen/IntrinsicEmitter! +/// Defined in Intrinsics.td. enum IIT_Info { - // Common values should be encoded with 0-15. - IIT_Done = 0, - IIT_I1 = 1, - IIT_I8 = 2, - IIT_I16 = 3, - IIT_I32 = 4, - IIT_I64 = 5, - IIT_F16 = 6, - IIT_F32 = 7, - IIT_F64 = 8, - IIT_V2 = 9, - IIT_V4 = 10, - IIT_V8 = 11, - IIT_V16 = 12, - IIT_V32 = 13, - IIT_PTR = 14, - IIT_ARG = 15, - - // Values from 16+ are only encodable with the inefficient encoding. - IIT_V64 = 16, - IIT_MMX = 17, - IIT_TOKEN = 18, - IIT_METADATA = 19, - IIT_EMPTYSTRUCT = 20, - IIT_STRUCT2 = 21, - IIT_STRUCT3 = 22, - IIT_STRUCT4 = 23, - IIT_STRUCT5 = 24, - IIT_EXTEND_ARG = 25, - IIT_TRUNC_ARG = 26, - IIT_ANYPTR = 27, - IIT_V1 = 28, - IIT_VARARG = 29, - IIT_HALF_VEC_ARG = 30, - IIT_SAME_VEC_WIDTH_ARG = 31, - IIT_PTR_TO_ARG = 32, - IIT_PTR_TO_ELT = 33, - IIT_VEC_OF_ANYPTRS_TO_ELT = 34, - IIT_I128 = 35, - IIT_V512 = 36, - IIT_V1024 = 37, - IIT_STRUCT6 = 38, - IIT_STRUCT7 = 39, - IIT_STRUCT8 = 40, - IIT_F128 = 41, - IIT_VEC_ELEMENT = 42, - IIT_SCALABLE_VEC = 43, - IIT_SUBDIVIDE2_ARG = 44, - IIT_SUBDIVIDE4_ARG = 45, - IIT_VEC_OF_BITCASTS_TO_INT = 46, - IIT_V128 = 47, - IIT_BF16 = 48, - IIT_STRUCT9 = 49, - IIT_V256 = 50, - IIT_AMX = 51, - IIT_PPCF128 = 52, - IIT_V3 = 53, - IIT_EXTERNREF = 54, - IIT_FUNCREF = 55, - IIT_ANYPTR_TO_ELT = 56, - IIT_I2 = 57, - IIT_I4 = 58, +#define GET_INTRINSIC_IITINFO +#include "llvm/IR/IntrinsicImpl.inc" +#undef GET_INTRINSIC_IITINFO }; static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, @@ -1141,6 +1094,9 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, case IIT_I4: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 4)); return; + case IIT_AARCH64_SVCOUNT: + OutputTable.push_back(IITDescriptor::get(IITDescriptor::AArch64Svcount, 0)); + return; case IIT_I8: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 8)); return; @@ -1206,22 +1162,17 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, return; case IIT_EXTERNREF: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 10)); - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0)); return; case IIT_FUNCREF: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 20)); - OutputTable.push_back(IITDescriptor::get(IITDescriptor::Integer, 8)); return; case IIT_PTR: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, 0)); - DecodeIITType(NextElt, Infos, Info, OutputTable); return; - case IIT_ANYPTR: { // [ANYPTR addrspace, subtype] + case IIT_ANYPTR: // [ANYPTR addrspace] OutputTable.push_back(IITDescriptor::get(IITDescriptor::Pointer, Infos[NextElt++])); - DecodeIITType(NextElt, Infos, Info, OutputTable); return; - } case IIT_ARG: { unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); OutputTable.push_back(IITDescriptor::get(IITDescriptor::Argument, ArgInfo)); @@ -1251,24 +1202,6 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, ArgInfo)); return; } - case IIT_PTR_TO_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToArgument, - ArgInfo)); - return; - } - case IIT_PTR_TO_ELT: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToElt, ArgInfo)); - return; - } - case IIT_ANYPTR_TO_ELT: { - unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::AnyPtrToElt, ArgNo, RefNo)); - return; - } case IIT_VEC_OF_ANYPTRS_TO_ELT: { unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); @@ -1382,6 +1315,8 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, case IITDescriptor::Double: return Type::getDoubleTy(Context); case IITDescriptor::Quad: return Type::getFP128Ty(Context); case IITDescriptor::PPCQuad: return Type::getPPC_FP128Ty(Context); + case IITDescriptor::AArch64Svcount: + return TargetExtType::get(Context, "aarch64.svcount"); case IITDescriptor::Integer: return IntegerType::get(Context, D.Integer_Width); @@ -1389,8 +1324,7 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, return VectorType::get(DecodeFixedType(Infos, Tys, Context), D.Vector_Width); case IITDescriptor::Pointer: - return PointerType::get(DecodeFixedType(Infos, Tys, Context), - D.Pointer_AddressSpace); + return PointerType::get(Context, D.Pointer_AddressSpace); case IITDescriptor::Struct: { SmallVector<Type *, 8> Elts; for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) @@ -1433,18 +1367,6 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, return VectorType::get(EltTy, VTy->getElementCount()); return EltTy; } - case IITDescriptor::PtrToArgument: { - Type *Ty = Tys[D.getArgumentNumber()]; - return PointerType::getUnqual(Ty); - } - case IITDescriptor::PtrToElt: { - Type *Ty = Tys[D.getArgumentNumber()]; - VectorType *VTy = dyn_cast<VectorType>(Ty); - if (!VTy) - llvm_unreachable("Expected an argument of Vector Type"); - Type *EltTy = VTy->getElementType(); - return PointerType::getUnqual(EltTy); - } case IITDescriptor::VecElementArgument: { Type *Ty = Tys[D.getArgumentNumber()]; if (VectorType *VTy = dyn_cast<VectorType>(Ty)) @@ -1460,9 +1382,6 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos, case IITDescriptor::VecOfAnyPtrsToElt: // Return the overloaded type (which determines the pointers address space) return Tys[D.getOverloadArgNumber()]; - case IITDescriptor::AnyPtrToElt: - // Return the overloaded type (which determines the pointers address space) - return Tys[D.getOverloadArgNumber()]; } llvm_unreachable("unhandled"); } @@ -1556,6 +1475,9 @@ static bool matchIntrinsicType( case IITDescriptor::Quad: return !Ty->isFP128Ty(); case IITDescriptor::PPCQuad: return !Ty->isPPC_FP128Ty(); case IITDescriptor::Integer: return !Ty->isIntegerTy(D.Integer_Width); + case IITDescriptor::AArch64Svcount: + return !isa<TargetExtType>(Ty) || + cast<TargetExtType>(Ty)->getName() != "aarch64.svcount"; case IITDescriptor::Vector: { VectorType *VT = dyn_cast<VectorType>(Ty); return !VT || VT->getElementCount() != D.Vector_Width || @@ -1564,33 +1486,7 @@ static bool matchIntrinsicType( } case IITDescriptor::Pointer: { PointerType *PT = dyn_cast<PointerType>(Ty); - if (!PT || PT->getAddressSpace() != D.Pointer_AddressSpace) - return true; - if (!PT->isOpaque()) { - /* Manually consume a pointer to empty struct descriptor, which is - * used for externref. We don't want to enforce that the struct is - * anonymous in this case. (This renders externref intrinsics - * non-unique, but this will go away with opaque pointers anyway.) */ - if (Infos.front().Kind == IITDescriptor::Struct && - Infos.front().Struct_NumElements == 0) { - Infos = Infos.slice(1); - return false; - } - return matchIntrinsicType(PT->getNonOpaquePointerElementType(), Infos, - ArgTys, DeferredChecks, IsDeferredCheck); - } - // Consume IIT descriptors relating to the pointer element type. - // FIXME: Intrinsic type matching of nested single value types or even - // aggregates doesn't work properly with opaque pointers but hopefully - // doesn't happen in practice. - while (Infos.front().Kind == IITDescriptor::Pointer || - Infos.front().Kind == IITDescriptor::Vector) - Infos = Infos.slice(1); - assert((Infos.front().Kind != IITDescriptor::Argument || - Infos.front().getArgumentKind() == IITDescriptor::AK_MatchType) && - "Unsupported polymorphic pointer type with opaque pointer"); - Infos = Infos.slice(1); - return false; + return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace; } case IITDescriptor::Struct: { @@ -1688,50 +1584,6 @@ static bool matchIntrinsicType( return matchIntrinsicType(EltTy, Infos, ArgTys, DeferredChecks, IsDeferredCheck); } - case IITDescriptor::PtrToArgument: { - if (D.getArgumentNumber() >= ArgTys.size()) - return IsDeferredCheck || DeferCheck(Ty); - Type * ReferenceType = ArgTys[D.getArgumentNumber()]; - PointerType *ThisArgType = dyn_cast<PointerType>(Ty); - return (!ThisArgType || - !ThisArgType->isOpaqueOrPointeeTypeMatches(ReferenceType)); - } - case IITDescriptor::PtrToElt: { - if (D.getArgumentNumber() >= ArgTys.size()) - return IsDeferredCheck || DeferCheck(Ty); - VectorType * ReferenceType = - dyn_cast<VectorType> (ArgTys[D.getArgumentNumber()]); - PointerType *ThisArgType = dyn_cast<PointerType>(Ty); - - if (!ThisArgType || !ReferenceType) - return true; - return !ThisArgType->isOpaqueOrPointeeTypeMatches( - ReferenceType->getElementType()); - } - case IITDescriptor::AnyPtrToElt: { - unsigned RefArgNumber = D.getRefArgNumber(); - if (RefArgNumber >= ArgTys.size()) { - if (IsDeferredCheck) - return true; - // If forward referencing, already add the pointer type and - // defer the checks for later. - ArgTys.push_back(Ty); - return DeferCheck(Ty); - } - - if (!IsDeferredCheck) { - assert(D.getOverloadArgNumber() == ArgTys.size() && - "Table consistency error"); - ArgTys.push_back(Ty); - } - - auto *ReferenceType = dyn_cast<VectorType>(ArgTys[RefArgNumber]); - auto *ThisArgType = dyn_cast<PointerType>(Ty); - if (!ThisArgType || !ReferenceType) - return true; - return !ThisArgType->isOpaqueOrPointeeTypeMatches( - ReferenceType->getElementType()); - } case IITDescriptor::VecOfAnyPtrsToElt: { unsigned RefArgNumber = D.getRefArgNumber(); if (RefArgNumber >= ArgTys.size()) { @@ -1757,12 +1609,7 @@ static bool matchIntrinsicType( if (!ThisArgVecTy || !ReferenceType || (ReferenceType->getElementCount() != ThisArgVecTy->getElementCount())) return true; - PointerType *ThisArgEltTy = - dyn_cast<PointerType>(ThisArgVecTy->getElementType()); - if (!ThisArgEltTy) - return true; - return !ThisArgEltTy->isOpaqueOrPointeeTypeMatches( - ReferenceType->getElementType()); + return !ThisArgVecTy->getElementType()->isPointerTy(); } case IITDescriptor::VecElementArgument: { if (D.getArgumentNumber() >= ArgTys.size()) diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index c208ab0f3d6b..7bd4503a689e 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -12,7 +12,6 @@ //===----------------------------------------------------------------------===// #include "LLVMContextImpl.h" -#include "llvm/ADT/Triple.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" @@ -22,6 +21,7 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/TargetParser/Triple.h" using namespace llvm; //===----------------------------------------------------------------------===// @@ -44,9 +44,7 @@ bool GlobalValue::isMaterializable() const { return F->isMaterializable(); return false; } -Error GlobalValue::materialize() { - return getParent()->materialize(this); -} +Error GlobalValue::materialize() { return getParent()->materialize(this); } /// Override destroyConstantImpl to make sure it doesn't get called on /// GlobalValue's because they shouldn't be treated like other constants. @@ -127,6 +125,16 @@ void GlobalObject::setAlignment(MaybeAlign Align) { assert(getAlign() == Align && "Alignment representation error!"); } +void GlobalObject::setAlignment(Align Align) { + assert(Align <= MaximumAlignment && + "Alignment is greater than MaximumAlignment!"); + unsigned AlignmentData = encode(Align); + unsigned OldData = getGlobalValueSubClassData(); + setGlobalValueSubClassData((OldData & ~AlignmentMask) | AlignmentData); + assert(getAlign() && *getAlign() == Align && + "Alignment representation error!"); +} + void GlobalObject::copyAttributesFrom(const GlobalObject *Src) { GlobalValue::copyAttributesFrom(Src); setAlignment(Src->getAlign()); @@ -428,35 +436,23 @@ GlobalVariable::GlobalVariable(Module &M, Type *Ty, bool constant, ThreadLocalMode TLMode, std::optional<unsigned> AddressSpace, bool isExternallyInitialized) - : GlobalObject(Ty, Value::GlobalVariableVal, - OperandTraits<GlobalVariable>::op_begin(this), - InitVal != nullptr, Link, Name, - AddressSpace - ? *AddressSpace - : M.getDataLayout().getDefaultGlobalsAddressSpace()), - isConstantGlobal(constant), - isExternallyInitializedConstant(isExternallyInitialized) { - assert(!Ty->isFunctionTy() && PointerType::isValidElementType(Ty) && - "invalid type for global variable"); - setThreadLocalMode(TLMode); - if (InitVal) { - assert(InitVal->getType() == Ty && - "Initializer should be the same type as the GlobalVariable!"); - Op<0>() = InitVal; - } - + : GlobalVariable(Ty, constant, Link, InitVal, Name, TLMode, + AddressSpace + ? *AddressSpace + : M.getDataLayout().getDefaultGlobalsAddressSpace(), + isExternallyInitialized) { if (Before) - Before->getParent()->getGlobalList().insert(Before->getIterator(), this); + Before->getParent()->insertGlobalVariable(Before->getIterator(), this); else - M.getGlobalList().push_back(this); + M.insertGlobalVariable(this); } void GlobalVariable::removeFromParent() { - getParent()->getGlobalList().remove(getIterator()); + getParent()->removeGlobalVariable(this); } void GlobalVariable::eraseFromParent() { - getParent()->getGlobalList().erase(getIterator()); + getParent()->eraseGlobalVariable(this); } void GlobalVariable::setInitializer(Constant *InitVal) { @@ -504,7 +500,7 @@ GlobalAlias::GlobalAlias(Type *Ty, unsigned AddressSpace, LinkageTypes Link, AddressSpace) { setAliasee(Aliasee); if (ParentModule) - ParentModule->getAliasList().push_back(this); + ParentModule->insertAlias(this); } GlobalAlias *GlobalAlias::create(Type *Ty, unsigned AddressSpace, @@ -535,13 +531,9 @@ GlobalAlias *GlobalAlias::create(const Twine &Name, GlobalValue *Aliasee) { return create(Aliasee->getLinkage(), Name, Aliasee); } -void GlobalAlias::removeFromParent() { - getParent()->getAliasList().remove(getIterator()); -} +void GlobalAlias::removeFromParent() { getParent()->removeAlias(this); } -void GlobalAlias::eraseFromParent() { - getParent()->getAliasList().erase(getIterator()); -} +void GlobalAlias::eraseFromParent() { getParent()->eraseAlias(this); } void GlobalAlias::setAliasee(Constant *Aliasee) { assert((!Aliasee || Aliasee->getType() == getType()) && @@ -565,7 +557,7 @@ GlobalIFunc::GlobalIFunc(Type *Ty, unsigned AddressSpace, LinkageTypes Link, AddressSpace) { setResolver(Resolver); if (ParentModule) - ParentModule->getIFuncList().push_back(this); + ParentModule->insertIFunc(this); } GlobalIFunc *GlobalIFunc::create(Type *Ty, unsigned AddressSpace, @@ -574,13 +566,9 @@ GlobalIFunc *GlobalIFunc::create(Type *Ty, unsigned AddressSpace, return new GlobalIFunc(Ty, AddressSpace, Link, Name, Resolver, ParentModule); } -void GlobalIFunc::removeFromParent() { - getParent()->getIFuncList().remove(getIterator()); -} +void GlobalIFunc::removeFromParent() { getParent()->removeIFunc(this); } -void GlobalIFunc::eraseFromParent() { - getParent()->getIFuncList().erase(getIterator()); -} +void GlobalIFunc::eraseFromParent() { getParent()->eraseIFunc(this); } const Function *GlobalIFunc::getResolverFunction() const { return dyn_cast<Function>(getResolver()->stripPointerCastsAndAliases()); diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index f871205843a7..094819dc39b5 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -60,15 +60,6 @@ Type *IRBuilderBase::getCurrentFunctionReturnType() const { return BB->getParent()->getReturnType(); } -Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) { - auto *PT = cast<PointerType>(Ptr->getType()); - if (PT->isOpaqueOrPointeeTypeMatches(getInt8Ty())) - return Ptr; - - // Otherwise, we need to insert a bitcast. - return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace())); -} - DebugLoc IRBuilderBase::getCurrentDebugLocation() const { for (auto &KV : MetadataToCopy) if (KV.first == LLVMContext::MD_dbg) @@ -102,9 +93,17 @@ Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) { Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()}); CallInst *CI = CreateCall(TheFn, {}, {}, Name); - return cast<ConstantInt>(Scaling)->getSExtValue() == 1 - ? CI - : CreateMul(CI, Scaling); + return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling); +} + +Value *IRBuilderBase::CreateElementCount(Type *DstType, ElementCount EC) { + Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue()); + return EC.isScalable() ? CreateVScale(MinEC) : MinEC; +} + +Value *IRBuilderBase::CreateTypeSize(Type *DstType, TypeSize Size) { + Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue()); + return Size.isScalable() ? CreateVScale(MinSize) : MinSize; } Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) { @@ -139,7 +138,6 @@ CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Ptr = getCastedInt8PtrValue(Ptr); Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)}; Type *Tys[] = { Ptr->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); @@ -168,7 +166,6 @@ CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, bool IsVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Dst = getCastedInt8PtrValue(Dst); Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)}; Type *Tys[] = {Dst->getType(), Size->getType()}; Module *M = BB->getParent()->getParent(); @@ -196,7 +193,6 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Ptr = getCastedInt8PtrValue(Ptr); Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)}; Type *Tys[] = {Ptr->getType(), Size->getType()}; Module *M = BB->getParent()->getParent(); @@ -224,9 +220,6 @@ CallInst *IRBuilderBase::CreateMemTransferInst( Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Dst = getCastedInt8PtrValue(Dst); - Src = getCastedInt8PtrValue(Src); - Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); @@ -261,9 +254,6 @@ CallInst *IRBuilderBase::CreateMemCpyInline( Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, Value *Size, bool IsVolatile, MDNode *TBAATag, MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Dst = getCastedInt8PtrValue(Dst); - Src = getCastedInt8PtrValue(Src); - Value *Ops[] = {Dst, Src, Size, getInt1(IsVolatile)}; Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; Function *F = BB->getParent(); @@ -303,9 +293,6 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( "Pointer alignment must be at least element size"); assert(SrcAlign >= ElementSize && "Pointer alignment must be at least element size"); - Dst = getCastedInt8PtrValue(Dst); - Src = getCastedInt8PtrValue(Src); - Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; Module *M = BB->getParent()->getParent(); @@ -341,9 +328,6 @@ CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { - Dst = getCastedInt8PtrValue(Dst); - Src = getCastedInt8PtrValue(Src); - Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); @@ -378,9 +362,6 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( "Pointer alignment must be at least element size"); assert(SrcAlign >= ElementSize && "Pointer alignment must be at least element size"); - Dst = getCastedInt8PtrValue(Dst); - Src = getCastedInt8PtrValue(Src); - Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; Module *M = BB->getParent()->getParent(); @@ -474,10 +455,17 @@ CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) { return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src); } +CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) { + return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src); +} + +CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) { + return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src); +} + CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) { assert(isa<PointerType>(Ptr->getType()) && "lifetime.start only applies to pointers."); - Ptr = getCastedInt8PtrValue(Ptr); if (!Size) Size = getInt64(-1); else @@ -493,7 +481,6 @@ CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) { CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) { assert(isa<PointerType>(Ptr->getType()) && "lifetime.end only applies to pointers."); - Ptr = getCastedInt8PtrValue(Ptr); if (!Size) Size = getInt64(-1); else @@ -510,7 +497,6 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { assert(isa<PointerType>(Ptr->getType()) && "invariant.start only applies to pointers."); - Ptr = getCastedInt8PtrValue(Ptr); if (!Size) Size = getInt64(-1); else @@ -590,7 +576,6 @@ CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, const Twine &Name) { auto *PtrTy = cast<PointerType>(Ptr->getType()); assert(Ty->isVectorTy() && "Type should be vector"); - assert(PtrTy->isOpaqueOrPointeeTypeMatches(Ty) && "Wrong element type"); assert(Mask && "Mask should not be all-ones (null)"); if (!PassThru) PassThru = PoisonValue::get(Ty); @@ -611,7 +596,6 @@ CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, auto *PtrTy = cast<PointerType>(Ptr->getType()); Type *DataTy = Val->getType(); assert(DataTy->isVectorTy() && "Val should be a vector"); - assert(PtrTy->isOpaqueOrPointeeTypeMatches(DataTy) && "Wrong element type"); assert(Mask && "Mask should not be all-ones (null)"); Type *OverloadedTypes[] = { DataTy, PtrTy }; Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask}; @@ -646,15 +630,10 @@ CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, auto *VecTy = cast<VectorType>(Ty); ElementCount NumElts = VecTy->getElementCount(); auto *PtrsTy = cast<VectorType>(Ptrs->getType()); - assert(cast<PointerType>(PtrsTy->getElementType()) - ->isOpaqueOrPointeeTypeMatches( - cast<VectorType>(Ty)->getElementType()) && - "Element type mismatch"); assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch"); if (!Mask) - Mask = Constant::getAllOnesValue( - VectorType::get(Type::getInt1Ty(Context), NumElts)); + Mask = getAllOnesMask(NumElts); if (!PassThru) PassThru = PoisonValue::get(Ty); @@ -681,16 +660,8 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, auto *DataTy = cast<VectorType>(Data->getType()); ElementCount NumElts = PtrsTy->getElementCount(); -#ifndef NDEBUG - auto *PtrTy = cast<PointerType>(PtrsTy->getElementType()); - assert(NumElts == DataTy->getElementCount() && - PtrTy->isOpaqueOrPointeeTypeMatches(DataTy->getElementType()) && - "Incompatible pointer and data types"); -#endif - if (!Mask) - Mask = Constant::getAllOnesValue( - VectorType::get(Type::getInt1Ty(Context), NumElts)); + Mask = getAllOnesMask(NumElts); Type *OverloadedTypes[] = {DataTy, PtrsTy}; Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask}; @@ -711,12 +682,7 @@ CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask, Value *PassThru, const Twine &Name) { - auto *PtrTy = cast<PointerType>(Ptr->getType()); assert(Ty->isVectorTy() && "Type should be vector"); - assert(PtrTy->isOpaqueOrPointeeTypeMatches( - cast<FixedVectorType>(Ty)->getElementType()) && - "Wrong element type"); - (void)PtrTy; assert(Mask && "Mask should not be all-ones (null)"); if (!PassThru) PassThru = PoisonValue::get(Ty); @@ -733,13 +699,8 @@ CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr, /// be accessed in memory CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask) { - auto *PtrTy = cast<PointerType>(Ptr->getType()); Type *DataTy = Val->getType(); assert(DataTy->isVectorTy() && "Val should be a vector"); - assert(PtrTy->isOpaqueOrPointeeTypeMatches( - cast<FixedVectorType>(DataTy)->getElementType()) && - "Wrong element type"); - (void)PtrTy; assert(Mask && "Mask should not be all-ones (null)"); Type *OverloadedTypes[] = {DataTy}; Value *Ops[] = {Val, Ptr, Mask}; @@ -1018,6 +979,23 @@ CallInst *IRBuilderBase::CreateConstrainedFPBinOp( return C; } +CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp( + Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, + const Twine &Name, MDNode *FPMathTag, + std::optional<fp::ExceptionBehavior> Except) { + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C = + CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name); + setConstrainedFPCallAttr(C); + setFPAttrs(C, FPMathTag, UseFMF); + return C; +} + Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, const Twine &Name, MDNode *FPMathTag) { if (Instruction::isBinaryOp(Opc)) { @@ -1143,9 +1121,6 @@ Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name) { assert(LHS->getType() == RHS->getType() && "Pointer subtraction operand types must match!"); - assert(cast<PointerType>(LHS->getType()) - ->isOpaqueOrPointeeTypeMatches(ElemTy) && - "Pointer type must match element type"); Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); Value *Difference = CreateSub(LHS_int, RHS_int); @@ -1156,50 +1131,34 @@ Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) { assert(isa<PointerType>(Ptr->getType()) && "launder.invariant.group only applies to pointers."); - // FIXME: we could potentially avoid casts to/from i8*. auto *PtrType = Ptr->getType(); - auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); - if (PtrType != Int8PtrTy) - Ptr = CreateBitCast(Ptr, Int8PtrTy); Module *M = BB->getParent()->getParent(); Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( - M, Intrinsic::launder_invariant_group, {Int8PtrTy}); + M, Intrinsic::launder_invariant_group, {PtrType}); - assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && + assert(FnLaunderInvariantGroup->getReturnType() == PtrType && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == - Int8PtrTy && + PtrType && "LaunderInvariantGroup should take and return the same type"); - CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); - - if (PtrType != Int8PtrTy) - return CreateBitCast(Fn, PtrType); - return Fn; + return CreateCall(FnLaunderInvariantGroup, {Ptr}); } Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { assert(isa<PointerType>(Ptr->getType()) && "strip.invariant.group only applies to pointers."); - // FIXME: we could potentially avoid casts to/from i8*. auto *PtrType = Ptr->getType(); - auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); - if (PtrType != Int8PtrTy) - Ptr = CreateBitCast(Ptr, Int8PtrTy); Module *M = BB->getParent()->getParent(); Function *FnStripInvariantGroup = Intrinsic::getDeclaration( - M, Intrinsic::strip_invariant_group, {Int8PtrTy}); + M, Intrinsic::strip_invariant_group, {PtrType}); - assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy && + assert(FnStripInvariantGroup->getReturnType() == PtrType && FnStripInvariantGroup->getFunctionType()->getParamType(0) == - Int8PtrTy && + PtrType && "StripInvariantGroup should take and return the same type"); - CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); - - if (PtrType != Int8PtrTy) - return CreateBitCast(Fn, PtrType); - return Fn; + return CreateCall(FnStripInvariantGroup, {Ptr}); } Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) { @@ -1295,16 +1254,13 @@ Value *IRBuilderBase::CreatePreserveArrayAccessIndex( auto *BaseType = Base->getType(); assert(isa<PointerType>(BaseType) && "Invalid Base ptr type for preserve.array.access.index."); - assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) && - "Pointer element type mismatch"); Value *LastIndexV = getInt32(LastIndex); Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); SmallVector<Value *, 4> IdxList(Dimension, Zero); IdxList.push_back(LastIndexV); - Type *ResultType = - GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); + Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList); Module *M = BB->getParent()->getParent(); Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( @@ -1346,13 +1302,11 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex( auto *BaseType = Base->getType(); assert(isa<PointerType>(BaseType) && "Invalid Base ptr type for preserve.struct.access.index."); - assert(cast<PointerType>(BaseType)->isOpaqueOrPointeeTypeMatches(ElTy) && - "Pointer element type mismatch"); Value *GEPIndex = getInt32(Index); Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); Type *ResultType = - GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); + GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex}); Module *M = BB->getParent()->getParent(); Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( @@ -1369,6 +1323,14 @@ Value *IRBuilderBase::CreatePreserveStructAccessIndex( return Fn; } +Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) { + ConstantInt *TestV = getInt32(Test); + Module *M = BB->getParent()->getParent(); + Function *FnIsFPClass = + Intrinsic::getDeclaration(M, Intrinsic::is_fpclass, {FPNum->getType()}); + return CreateCall(FnIsFPClass, {FPNum, TestV}); +} + CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL, Value *PtrValue, Value *AlignValue, diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index 9c88ca17ebde..0dcf0ac6a78a 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -12,6 +12,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/IR/AttributeMask.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -138,9 +139,10 @@ Instruction *Instruction::getInsertionPointAfterDef() { } else if (auto *II = dyn_cast<InvokeInst>(this)) { InsertBB = II->getNormalDest(); InsertPt = InsertBB->getFirstInsertionPt(); - } else if (auto *CB = dyn_cast<CallBrInst>(this)) { - InsertBB = CB->getDefaultDest(); - InsertPt = InsertBB->getFirstInsertionPt(); + } else if (isa<CallBrInst>(this)) { + // Def is available in multiple successors, there's no single dominating + // insertion point. + return nullptr; } else { assert(!isTerminator() && "Only invoke/callbr terminators return value"); InsertBB = getParent(); @@ -223,7 +225,7 @@ void Instruction::dropPoisonGeneratingMetadata() { eraseMetadata(LLVMContext::MD_align); } -void Instruction::dropUndefImplyingAttrsAndUnknownMetadata( +void Instruction::dropUBImplyingAttrsAndUnknownMetadata( ArrayRef<unsigned> KnownIDs) { dropUnknownNonDebugMetadata(KnownIDs); auto *CB = dyn_cast<CallBase>(this); @@ -242,6 +244,16 @@ void Instruction::dropUndefImplyingAttrsAndUnknownMetadata( CB->removeRetAttrs(UBImplyingAttributes); } +void Instruction::dropUBImplyingAttrsAndMetadata() { + // !annotation metadata does not impact semantics. + // !range, !nonnull and !align produce poison, so they are safe to speculate. + // !noundef and various AA metadata must be dropped, as it generally produces + // immediate undefined behavior. + unsigned KnownIDs[] = {LLVMContext::MD_annotation, LLVMContext::MD_range, + LLVMContext::MD_nonnull, LLVMContext::MD_align}; + dropUBImplyingAttrsAndUnknownMetadata(KnownIDs); +} + bool Instruction::isExact() const { return cast<PossiblyExactOperator>(this)->isExact(); } @@ -479,11 +491,11 @@ const char *Instruction::getOpcodeName(unsigned OpCode) { } } -/// Return true if both instructions have the same special state. This must be -/// kept in sync with FunctionComparator::cmpOperations in +/// This must be kept in sync with FunctionComparator::cmpOperations in /// lib/Transforms/IPO/MergeFunctions.cpp. -static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, - bool IgnoreAlignment = false) { +bool Instruction::hasSameSpecialState(const Instruction *I2, + bool IgnoreAlignment) const { + auto I1 = this; assert(I1->getOpcode() == I2->getOpcode() && "Can not compare special state of different instructions"); @@ -562,7 +574,7 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const { // If both instructions have no operands, they are identical. if (getNumOperands() == 0 && I->getNumOperands() == 0) - return haveSameSpecialState(this, I); + return this->hasSameSpecialState(I); // We have two instructions of identical opcode and #operands. Check to see // if all operands are the same. @@ -576,7 +588,7 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const { otherPHI->block_begin()); } - return haveSameSpecialState(this, I); + return this->hasSameSpecialState(I); } // Keep this in sync with FunctionComparator::cmpOperations in @@ -602,7 +614,7 @@ bool Instruction::isSameOperationAs(const Instruction *I, getOperand(i)->getType() != I->getOperand(i)->getType()) return false; - return haveSameSpecialState(this, I, IgnoreAlignment); + return this->hasSameSpecialState(I, IgnoreAlignment); } bool Instruction::isUsedOutsideOfBlock(const BasicBlock *BB) const { @@ -732,14 +744,89 @@ bool Instruction::isVolatile() const { } } -bool Instruction::mayThrow() const { - if (const CallInst *CI = dyn_cast<CallInst>(this)) - return !CI->doesNotThrow(); - if (const auto *CRI = dyn_cast<CleanupReturnInst>(this)) - return CRI->unwindsToCaller(); - if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(this)) - return CatchSwitch->unwindsToCaller(); - return isa<ResumeInst>(this); +Type *Instruction::getAccessType() const { + switch (getOpcode()) { + case Instruction::Store: + return cast<StoreInst>(this)->getValueOperand()->getType(); + case Instruction::Load: + case Instruction::AtomicRMW: + return getType(); + case Instruction::AtomicCmpXchg: + return cast<AtomicCmpXchgInst>(this)->getNewValOperand()->getType(); + case Instruction::Call: + case Instruction::Invoke: + if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(this)) { + switch (II->getIntrinsicID()) { + case Intrinsic::masked_load: + case Intrinsic::masked_gather: + case Intrinsic::masked_expandload: + case Intrinsic::vp_load: + case Intrinsic::vp_gather: + case Intrinsic::experimental_vp_strided_load: + return II->getType(); + case Intrinsic::masked_store: + case Intrinsic::masked_scatter: + case Intrinsic::masked_compressstore: + case Intrinsic::vp_store: + case Intrinsic::vp_scatter: + case Intrinsic::experimental_vp_strided_store: + return II->getOperand(0)->getType(); + default: + break; + } + } + } + + return nullptr; +} + +static bool canUnwindPastLandingPad(const LandingPadInst *LP, + bool IncludePhaseOneUnwind) { + // Because phase one unwinding skips cleanup landingpads, we effectively + // unwind past this frame, and callers need to have valid unwind info. + if (LP->isCleanup()) + return IncludePhaseOneUnwind; + + for (unsigned I = 0; I < LP->getNumClauses(); ++I) { + Constant *Clause = LP->getClause(I); + // catch ptr null catches all exceptions. + if (LP->isCatch(I) && isa<ConstantPointerNull>(Clause)) + return false; + // filter [0 x ptr] catches all exceptions. + if (LP->isFilter(I) && Clause->getType()->getArrayNumElements() == 0) + return false; + } + + // May catch only some subset of exceptions, in which case other exceptions + // will continue unwinding. + return true; +} + +bool Instruction::mayThrow(bool IncludePhaseOneUnwind) const { + switch (getOpcode()) { + case Instruction::Call: + return !cast<CallInst>(this)->doesNotThrow(); + case Instruction::CleanupRet: + return cast<CleanupReturnInst>(this)->unwindsToCaller(); + case Instruction::CatchSwitch: + return cast<CatchSwitchInst>(this)->unwindsToCaller(); + case Instruction::Resume: + return true; + case Instruction::Invoke: { + // Landingpads themselves don't unwind -- however, an invoke of a skipped + // landingpad may continue unwinding. + BasicBlock *UnwindDest = cast<InvokeInst>(this)->getUnwindDest(); + Instruction *Pad = UnwindDest->getFirstNonPHI(); + if (auto *LP = dyn_cast<LandingPadInst>(Pad)) + return canUnwindPastLandingPad(LP, IncludePhaseOneUnwind); + return false; + } + case Instruction::CleanupPad: + // Treat the same as cleanup landingpad. + return IncludePhaseOneUnwind; + default: + return false; + } } bool Instruction::mayHaveSideEffects() const { diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index 7c343a0ff00a..cb0ac0f8eae6 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -325,6 +325,22 @@ Intrinsic::ID CallBase::getIntrinsicID() const { return Intrinsic::not_intrinsic; } +FPClassTest CallBase::getRetNoFPClass() const { + FPClassTest Mask = Attrs.getRetNoFPClass(); + + if (const Function *F = getCalledFunction()) + Mask |= F->getAttributes().getRetNoFPClass(); + return Mask; +} + +FPClassTest CallBase::getParamNoFPClass(unsigned i) const { + FPClassTest Mask = Attrs.getParamNoFPClass(i); + + if (const Function *F = getCalledFunction()) + Mask |= F->getAttributes().getParamNoFPClass(i); + return Mask; +} + bool CallBase::isReturnNonNull() const { if (hasRetAttr(Attribute::NonNull)) return true; @@ -1577,7 +1593,6 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBef) : UnaryInstruction(Ty, Load, Ptr, InsertBef) { - assert(cast<PointerType>(Ptr->getType())->isOpaqueOrPointeeTypeMatches(Ty)); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -1589,7 +1604,6 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAE) : UnaryInstruction(Ty, Load, Ptr, InsertAE) { - assert(cast<PointerType>(Ptr->getType())->isOpaqueOrPointeeTypeMatches(Ty)); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -1605,9 +1619,6 @@ void StoreInst::AssertOK() { assert(getOperand(0) && getOperand(1) && "Both operands must be non-null!"); assert(getOperand(1)->getType()->isPointerTy() && "Ptr must have pointer type!"); - assert(cast<PointerType>(getOperand(1)->getType()) - ->isOpaqueOrPointeeTypeMatches(getOperand(0)->getType()) && - "Ptr must be a pointer to Val type!"); } StoreInst::StoreInst(Value *val, Value *addr, Instruction *InsertBefore) @@ -1687,12 +1698,6 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, "All operands must be non-null!"); assert(getOperand(0)->getType()->isPointerTy() && "Ptr must have pointer type!"); - assert(cast<PointerType>(getOperand(0)->getType()) - ->isOpaqueOrPointeeTypeMatches(getOperand(1)->getType()) && - "Ptr must be a pointer to Cmp type!"); - assert(cast<PointerType>(getOperand(0)->getType()) - ->isOpaqueOrPointeeTypeMatches(getOperand(2)->getType()) && - "Ptr must be a pointer to NewVal type!"); assert(getOperand(1)->getType() == getOperand(2)->getType() && "Cmp type and NewVal type must be same!"); } @@ -1745,9 +1750,6 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, "All operands must be non-null!"); assert(getOperand(0)->getType()->isPointerTy() && "Ptr must have pointer type!"); - assert(cast<PointerType>(getOperand(0)->getType()) - ->isOpaqueOrPointeeTypeMatches(getOperand(1)->getType()) && - "Ptr must be a pointer to Val type!"); assert(Ordering != AtomicOrdering::NotAtomic && "AtomicRMW instructions must be atomic!"); } @@ -2148,8 +2150,8 @@ void ShuffleVectorInst::commute() { SmallVector<int, 16> NewMask(NumMaskElts); for (int i = 0; i != NumMaskElts; ++i) { int MaskElt = getMaskValue(i); - if (MaskElt == UndefMaskElem) { - NewMask[i] = UndefMaskElem; + if (MaskElt == PoisonMaskElem) { + NewMask[i] = PoisonMaskElem; continue; } assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask"); @@ -2170,11 +2172,11 @@ bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2, int V1Size = cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue(); for (int Elem : Mask) - if (Elem != UndefMaskElem && Elem >= V1Size * 2) + if (Elem != PoisonMaskElem && Elem >= V1Size * 2) return false; if (isa<ScalableVectorType>(V1->getType())) - if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !all_equal(Mask)) + if ((Mask[0] != 0 && Mask[0] != PoisonMaskElem) || !all_equal(Mask)) return false; return true; @@ -2273,8 +2275,8 @@ Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask, } SmallVector<Constant *, 16> MaskConst; for (int Elem : Mask) { - if (Elem == UndefMaskElem) - MaskConst.push_back(UndefValue::get(Int32Ty)); + if (Elem == PoisonMaskElem) + MaskConst.push_back(PoisonValue::get(Int32Ty)); else MaskConst.push_back(ConstantInt::get(Int32Ty, Elem)); } @@ -2501,10 +2503,10 @@ bool ShuffleVectorInst::isInsertSubvectorMask(ArrayRef<int> Mask, // Determine lo/hi span ranges. // TODO: How should we handle undefs at the start of subvector insertions? - int Src0Lo = Src0Elts.countTrailingZeros(); - int Src1Lo = Src1Elts.countTrailingZeros(); - int Src0Hi = NumMaskElts - Src0Elts.countLeadingZeros(); - int Src1Hi = NumMaskElts - Src1Elts.countLeadingZeros(); + int Src0Lo = Src0Elts.countr_zero(); + int Src1Lo = Src1Elts.countr_zero(); + int Src0Hi = NumMaskElts - Src0Elts.countl_zero(); + int Src1Hi = NumMaskElts - Src1Elts.countl_zero(); // If src0 is in place, see if the src1 elements is inplace within its own // span. @@ -2611,7 +2613,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask, "Run out of mask?"); Mask = Mask.drop_front(ReplicationFactor); if (!all_of(CurrSubMask, [CurrElt](int MaskElt) { - return MaskElt == UndefMaskElem || MaskElt == CurrElt; + return MaskElt == PoisonMaskElem || MaskElt == CurrElt; })) return false; } @@ -2623,7 +2625,7 @@ static bool isReplicationMaskWithParams(ArrayRef<int> Mask, bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask, int &ReplicationFactor, int &VF) { // undef-less case is trivial. - if (!llvm::is_contained(Mask, UndefMaskElem)) { + if (!llvm::is_contained(Mask, PoisonMaskElem)) { ReplicationFactor = Mask.take_while([](int MaskElt) { return MaskElt == 0; }).size(); if (ReplicationFactor == 0 || Mask.size() % ReplicationFactor != 0) @@ -2641,7 +2643,7 @@ bool ShuffleVectorInst::isReplicationMask(ArrayRef<int> Mask, // Before doing that, let's perform basic correctness checking first. int Largest = -1; for (int MaskElt : Mask) { - if (MaskElt == UndefMaskElem) + if (MaskElt == PoisonMaskElem) continue; // Elements must be in non-decreasing order. if (MaskElt < Largest) @@ -2687,11 +2689,11 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(ArrayRef<int> Mask, int VF) { return false; for (unsigned K = 0, Sz = Mask.size(); K < Sz; K += VF) { ArrayRef<int> SubMask = Mask.slice(K, VF); - if (all_of(SubMask, [](int Idx) { return Idx == UndefMaskElem; })) + if (all_of(SubMask, [](int Idx) { return Idx == PoisonMaskElem; })) continue; SmallBitVector Used(VF, false); for_each(SubMask, [&Used, VF](int Idx) { - if (Idx != UndefMaskElem && Idx < VF) + if (Idx != PoisonMaskElem && Idx < VF) Used.set(Idx); }); if (!Used.all()) @@ -2712,6 +2714,98 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const { return isOneUseSingleSourceMask(ShuffleMask, VF); } +bool ShuffleVectorInst::isInterleave(unsigned Factor) { + FixedVectorType *OpTy = dyn_cast<FixedVectorType>(getOperand(0)->getType()); + // shuffle_vector can only interleave fixed length vectors - for scalable + // vectors, see the @llvm.experimental.vector.interleave2 intrinsic + if (!OpTy) + return false; + unsigned OpNumElts = OpTy->getNumElements(); + + return isInterleaveMask(ShuffleMask, Factor, OpNumElts * 2); +} + +bool ShuffleVectorInst::isInterleaveMask( + ArrayRef<int> Mask, unsigned Factor, unsigned NumInputElts, + SmallVectorImpl<unsigned> &StartIndexes) { + unsigned NumElts = Mask.size(); + if (NumElts % Factor) + return false; + + unsigned LaneLen = NumElts / Factor; + if (!isPowerOf2_32(LaneLen)) + return false; + + StartIndexes.resize(Factor); + + // Check whether each element matches the general interleaved rule. + // Ignore undef elements, as long as the defined elements match the rule. + // Outer loop processes all factors (x, y, z in the above example) + unsigned I = 0, J; + for (; I < Factor; I++) { + unsigned SavedLaneValue; + unsigned SavedNoUndefs = 0; + + // Inner loop processes consecutive accesses (x, x+1... in the example) + for (J = 0; J < LaneLen - 1; J++) { + // Lane computes x's position in the Mask + unsigned Lane = J * Factor + I; + unsigned NextLane = Lane + Factor; + int LaneValue = Mask[Lane]; + int NextLaneValue = Mask[NextLane]; + + // If both are defined, values must be sequential + if (LaneValue >= 0 && NextLaneValue >= 0 && + LaneValue + 1 != NextLaneValue) + break; + + // If the next value is undef, save the current one as reference + if (LaneValue >= 0 && NextLaneValue < 0) { + SavedLaneValue = LaneValue; + SavedNoUndefs = 1; + } + + // Undefs are allowed, but defined elements must still be consecutive: + // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, .... + // Verify this by storing the last non-undef followed by an undef + // Check that following non-undef masks are incremented with the + // corresponding distance. + if (SavedNoUndefs > 0 && LaneValue < 0) { + SavedNoUndefs++; + if (NextLaneValue >= 0 && + SavedLaneValue + SavedNoUndefs != (unsigned)NextLaneValue) + break; + } + } + + if (J < LaneLen - 1) + return false; + + int StartMask = 0; + if (Mask[I] >= 0) { + // Check that the start of the I range (J=0) is greater than 0 + StartMask = Mask[I]; + } else if (Mask[(LaneLen - 1) * Factor + I] >= 0) { + // StartMask defined by the last value in lane + StartMask = Mask[(LaneLen - 1) * Factor + I] - J; + } else if (SavedNoUndefs > 0) { + // StartMask defined by some non-zero value in the j loop + StartMask = SavedLaneValue - (LaneLen - 1 - SavedNoUndefs); + } + // else StartMask remains set to 0, i.e. all elements are undefs + + if (StartMask < 0) + return false; + // We must stay within the vectors; This case can happen with undefs. + if (StartMask + LaneLen > NumInputElts) + return false; + + StartIndexes[I] = StartMask; + } + + return true; +} + //===----------------------------------------------------------------------===// // InsertValueInst Class //===----------------------------------------------------------------------===// @@ -2965,42 +3059,42 @@ BinaryOperator *BinaryOperator::Create(BinaryOps Op, Value *S1, Value *S2, BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name, Instruction *InsertBefore) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); + Value *Zero = ConstantInt::get(Op->getType(), 0); return new BinaryOperator(Instruction::Sub, - zero, Op, + Zero, Op, Op->getType(), Name, InsertBefore); } BinaryOperator *BinaryOperator::CreateNeg(Value *Op, const Twine &Name, BasicBlock *InsertAtEnd) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); + Value *Zero = ConstantInt::get(Op->getType(), 0); return new BinaryOperator(Instruction::Sub, - zero, Op, + Zero, Op, Op->getType(), Name, InsertAtEnd); } BinaryOperator *BinaryOperator::CreateNSWNeg(Value *Op, const Twine &Name, Instruction *InsertBefore) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return BinaryOperator::CreateNSWSub(zero, Op, Name, InsertBefore); + Value *Zero = ConstantInt::get(Op->getType(), 0); + return BinaryOperator::CreateNSWSub(Zero, Op, Name, InsertBefore); } BinaryOperator *BinaryOperator::CreateNSWNeg(Value *Op, const Twine &Name, BasicBlock *InsertAtEnd) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return BinaryOperator::CreateNSWSub(zero, Op, Name, InsertAtEnd); + Value *Zero = ConstantInt::get(Op->getType(), 0); + return BinaryOperator::CreateNSWSub(Zero, Op, Name, InsertAtEnd); } BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name, Instruction *InsertBefore) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return BinaryOperator::CreateNUWSub(zero, Op, Name, InsertBefore); + Value *Zero = ConstantInt::get(Op->getType(), 0); + return BinaryOperator::CreateNUWSub(Zero, Op, Name, InsertBefore); } BinaryOperator *BinaryOperator::CreateNUWNeg(Value *Op, const Twine &Name, BasicBlock *InsertAtEnd) { - Value *zero = ConstantFP::getZeroValueForNegation(Op->getType()); - return BinaryOperator::CreateNUWSub(zero, Op, Name, InsertAtEnd); + Value *Zero = ConstantInt::get(Op->getType(), 0); + return BinaryOperator::CreateNUWSub(Zero, Op, Name, InsertAtEnd); } BinaryOperator *BinaryOperator::CreateNot(Value *Op, const Twine &Name, @@ -3059,23 +3153,6 @@ bool CastInst::isIntegerCast() const { } } -bool CastInst::isLosslessCast() const { - // Only BitCast can be lossless, exit fast if we're not BitCast - if (getOpcode() != Instruction::BitCast) - return false; - - // Identity cast is always lossless - Type *SrcTy = getOperand(0)->getType(); - Type *DstTy = getType(); - if (SrcTy == DstTy) - return true; - - // Pointer to pointer is always lossless. - if (SrcTy->isPointerTy()) - return DstTy->isPointerTy(); - return false; // Other types have no identity values -} - /// This function determines if the CastInst does not require any bits to be /// changed in order to effect the cast. Essentially, it identifies cases where /// no code gen is necessary for the cast, hence the name no-op cast. For @@ -3306,15 +3383,9 @@ unsigned CastInst::isEliminableCastPair( "Illegal addrspacecast, bitcast sequence!"); // Allowed, use first cast's opcode return firstOp; - case 14: { - // bitcast, addrspacecast -> addrspacecast if the element type of - // bitcast's source is the same as that of addrspacecast's destination. - PointerType *SrcPtrTy = cast<PointerType>(SrcTy->getScalarType()); - PointerType *DstPtrTy = cast<PointerType>(DstTy->getScalarType()); - if (SrcPtrTy->hasSameElementTypeAs(DstPtrTy)) - return Instruction::AddrSpaceCast; - return 0; - } + case 14: + // bitcast, addrspacecast -> addrspacecast + return Instruction::AddrSpaceCast; case 15: // FIXME: this state can be merged with (1), but the following assert // is useful to check the correcteness of the sequence due to semantic @@ -4138,6 +4209,11 @@ StringRef CmpInst::getPredicateName(Predicate Pred) { } } +raw_ostream &llvm::operator<<(raw_ostream &OS, CmpInst::Predicate Pred) { + OS << CmpInst::getPredicateName(Pred); + return OS; +} + ICmpInst::Predicate ICmpInst::getSignedPredicate(Predicate pred) { switch (pred) { default: llvm_unreachable("Unknown icmp predicate!"); diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp index b258e7bd3154..36d56699c64e 100644 --- a/llvm/lib/IR/IntrinsicInst.cpp +++ b/llvm/lib/IR/IntrinsicInst.cpp @@ -71,11 +71,9 @@ bool IntrinsicInst::mayLowerToFunctionCall(Intrinsic::ID IID) { /// intrinsics for variables. /// -iterator_range<DbgVariableIntrinsic::location_op_iterator> -DbgVariableIntrinsic::location_ops() const { - auto *MD = getRawLocation(); +iterator_range<location_op_iterator> RawLocationWrapper::location_ops() const { + Metadata *MD = getRawLocation(); assert(MD && "First operand of DbgVariableIntrinsic should be non-null."); - // If operand is ValueAsMetadata, return a range over just that operand. if (auto *VAM = dyn_cast<ValueAsMetadata>(MD)) { return {location_op_iterator(VAM), location_op_iterator(VAM + 1)}; @@ -89,8 +87,17 @@ DbgVariableIntrinsic::location_ops() const { location_op_iterator(static_cast<ValueAsMetadata *>(nullptr))}; } +iterator_range<location_op_iterator> +DbgVariableIntrinsic::location_ops() const { + return getWrappedLocation().location_ops(); +} + Value *DbgVariableIntrinsic::getVariableLocationOp(unsigned OpIdx) const { - auto *MD = getRawLocation(); + return getWrappedLocation().getVariableLocationOp(OpIdx); +} + +Value *RawLocationWrapper::getVariableLocationOp(unsigned OpIdx) const { + Metadata *MD = getRawLocation(); assert(MD && "First operand of DbgVariableIntrinsic should be non-null."); if (auto *AL = dyn_cast<DIArgList>(MD)) return AL->getArgs()[OpIdx]->getValue(); @@ -128,14 +135,14 @@ void DbgVariableIntrinsic::replaceVariableLocationOp(Value *OldValue, assert(NewValue && "Values must be non-null"); auto Locations = location_ops(); auto OldIt = find(Locations, OldValue); - assert((OldIt != Locations.end() || DbgAssignAddrReplaced) && - "OldValue must be a current location"); + if (OldIt == Locations.end()) { + assert(DbgAssignAddrReplaced && + "OldValue must be dbg.assign addr if unused in DIArgList"); + return; + } + + assert(OldIt != Locations.end() && "OldValue must be a current location"); if (!hasArgList()) { - // Additional check necessary to avoid unconditionally replacing this - // operand when a dbg.assign address is replaced (DbgAssignAddrReplaced is - // true). - if (OldValue != getVariableLocationOp(0)) - return; Value *NewOperand = isa<MetadataAsValue>(NewValue) ? NewValue : MetadataAsValue::get( @@ -206,8 +213,6 @@ void DbgAssignIntrinsic::setAssignId(DIAssignID *New) { } void DbgAssignIntrinsic::setAddress(Value *V) { - assert(V->getType()->isPointerTy() && - "Destination Component must be a pointer type"); setOperand(OpAddress, MetadataAsValue::get(getContext(), ValueAsMetadata::get(V))); } @@ -524,6 +529,20 @@ VPIntrinsic::getFunctionalOpcodeForVP(Intrinsic::ID ID) { return std::nullopt; } +// Equivalent non-predicated constrained intrinsic +std::optional<unsigned> +VPIntrinsic::getConstrainedIntrinsicIDForVP(Intrinsic::ID ID) { + switch (ID) { + default: + break; +#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID: +#define VP_PROPERTY_CONSTRAINEDFP(HASRND, HASEXCEPT, CID) return Intrinsic::CID; +#define END_REGISTER_VP_INTRINSIC(VPID) break; +#include "llvm/IR/VPIntrinsics.def" + } + return std::nullopt; +} + Intrinsic::ID VPIntrinsic::getForOpcode(unsigned IROPC) { switch (IROPC) { default: @@ -554,17 +573,11 @@ bool VPIntrinsic::canIgnoreVectorLengthParam() const { // Check whether "W == vscale * EC.getKnownMinValue()" if (EC.isScalable()) { - // Undig the DL - const auto *ParMod = this->getModule(); - if (!ParMod) - return false; - const auto &DL = ParMod->getDataLayout(); - // Compare vscale patterns uint64_t VScaleFactor; - if (match(VLParam, m_c_Mul(m_ConstantInt(VScaleFactor), m_VScale(DL)))) + if (match(VLParam, m_c_Mul(m_ConstantInt(VScaleFactor), m_VScale()))) return VScaleFactor >= EC.getKnownMinValue(); - return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale(DL)); + return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale()); } // standard SIMD operation diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 7911705776e3..8ddf51537ec1 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -92,6 +92,11 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { "kcfi operand bundle id drifted!"); (void)KCFIEntry; + auto *ConvergenceCtrlEntry = pImpl->getOrInsertBundleTag("convergencectrl"); + assert(ConvergenceCtrlEntry->second == LLVMContext::OB_convergencectrl && + "convergencectrl operand bundle id drifted!"); + (void)ConvergenceCtrlEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && @@ -369,9 +374,9 @@ std::unique_ptr<DiagnosticHandler> LLVMContext::getDiagnosticHandler() { } void LLVMContext::setOpaquePointers(bool Enable) const { - pImpl->setOpaquePointers(Enable); + assert(Enable && "Cannot disable opaque pointers"); } bool LLVMContext::supportsTypedPointers() const { - return !pImpl->getOpaquePointers(); + return false; } diff --git a/llvm/lib/IR/LLVMContextImpl.cpp b/llvm/lib/IR/LLVMContextImpl.cpp index 9acb1f654899..2076eeed9417 100644 --- a/llvm/lib/IR/LLVMContextImpl.cpp +++ b/llvm/lib/IR/LLVMContextImpl.cpp @@ -33,10 +33,6 @@ using namespace llvm; -static cl::opt<bool> - OpaquePointersCL("opaque-pointers", cl::desc("Use opaque pointers"), - cl::init(true)); - LLVMContextImpl::LLVMContextImpl(LLVMContext &C) : DiagHandler(std::make_unique<DiagnosticHandler>()), VoidTy(C, Type::VoidTyID), LabelTy(C, Type::LabelTyID), @@ -46,11 +42,7 @@ LLVMContextImpl::LLVMContextImpl(LLVMContext &C) X86_FP80Ty(C, Type::X86_FP80TyID), FP128Ty(C, Type::FP128TyID), PPC_FP128Ty(C, Type::PPC_FP128TyID), X86_MMXTy(C, Type::X86_MMXTyID), X86_AMXTy(C, Type::X86_AMXTyID), Int1Ty(C, 1), Int8Ty(C, 8), - Int16Ty(C, 16), Int32Ty(C, 32), Int64Ty(C, 64), Int128Ty(C, 128) { - if (OpaquePointersCL.getNumOccurrences()) { - OpaquePointers = OpaquePointersCL; - } -} + Int16Ty(C, 16), Int32Ty(C, 32), Int64Ty(C, 64), Int128Ty(C, 128) {} LLVMContextImpl::~LLVMContextImpl() { // NOTE: We need to delete the contents of OwnedModules, but Module's dtor @@ -116,6 +108,8 @@ LLVMContextImpl::~LLVMContextImpl() { CTNConstants.clear(); UVConstants.clear(); PVConstants.clear(); + IntZeroConstants.clear(); + IntOneConstants.clear(); IntConstants.clear(); FPConstants.clear(); CDSConstants.clear(); @@ -248,15 +242,3 @@ OptPassGate &LLVMContextImpl::getOptPassGate() const { void LLVMContextImpl::setOptPassGate(OptPassGate& OPG) { this->OPG = &OPG; } - -bool LLVMContextImpl::getOpaquePointers() { - if (LLVM_UNLIKELY(!OpaquePointers)) - OpaquePointers = OpaquePointersCL; - return *OpaquePointers; -} - -void LLVMContextImpl::setOpaquePointers(bool OP) { - assert((!OpaquePointers || *OpaquePointers == OP) && - "Cannot change opaque pointers mode once set"); - OpaquePointers = OP; -} diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index d2651a6ec72e..4cc3f8da6b75 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -513,11 +513,20 @@ template <> struct MDNodeKeyImpl<DIStringType> { bool isKeyOf(const DIStringType *RHS) const { return Tag == RHS->getTag() && Name == RHS->getRawName() && + StringLength == RHS->getRawStringLength() && + StringLengthExp == RHS->getRawStringLengthExp() && + StringLocationExp == RHS->getRawStringLocationExp() && SizeInBits == RHS->getSizeInBits() && AlignInBits == RHS->getAlignInBits() && Encoding == RHS->getEncoding(); } - unsigned getHashValue() const { return hash_combine(Tag, Name, Encoding); } + unsigned getHashValue() const { + // Intentionally computes the hash on a subset of the operands for + // performance reason. The subset has to be significant enough to avoid + // collision "most of the time". There is no correctness issue in case of + // collision because of the full check above. + return hash_combine(Tag, Name, StringLength, Encoding); + } }; template <> struct MDNodeKeyImpl<DIDerivedType> { @@ -1446,13 +1455,13 @@ public: DenseMap<const Value *, ValueName *> ValueNames; - using IntMapTy = - DenseMap<APInt, std::unique_ptr<ConstantInt>, DenseMapAPIntKeyInfo>; - IntMapTy IntConstants; + DenseMap<unsigned, std::unique_ptr<ConstantInt>> IntZeroConstants; + DenseMap<unsigned, std::unique_ptr<ConstantInt>> IntOneConstants; + DenseMap<APInt, std::unique_ptr<ConstantInt>, DenseMapAPIntKeyInfo> + IntConstants; - using FPMapTy = - DenseMap<APFloat, std::unique_ptr<ConstantFP>, DenseMapAPFloatKeyInfo>; - FPMapTy FPConstants; + DenseMap<APFloat, std::unique_ptr<ConstantFP>, DenseMapAPFloatKeyInfo> + FPConstants; FoldingSet<AttributeImpl> AttrsSet; FoldingSet<AttributeListImpl> AttrsLists; @@ -1535,8 +1544,9 @@ public: DenseMap<std::pair<Type *, uint64_t>, ArrayType *> ArrayTypes; DenseMap<std::pair<Type *, ElementCount>, VectorType *> VectorTypes; - DenseMap<Type *, PointerType *> PointerTypes; // Pointers in AddrSpace = 0 - DenseMap<std::pair<Type *, unsigned>, PointerType *> ASPointerTypes; + PointerType *AS0PointerType = nullptr; // AddrSpace = 0 + DenseMap<unsigned, PointerType *> PointerTypes; + DenseMap<std::pair<Type *, unsigned>, PointerType *> LegacyPointerTypes; DenseMap<std::pair<Type *, unsigned>, TypedPointerType *> ASTypedPointerTypes; /// ValueHandles - This map keeps track of all of the value handles that are @@ -1623,14 +1633,6 @@ public: /// The lifetime of the object must be guaranteed to extend as long as the /// LLVMContext is used by compilation. void setOptPassGate(OptPassGate &); - - // TODO: clean up the following after we no longer support non-opaque pointer - // types. - bool getOpaquePointers(); - void setOpaquePointers(bool OP); - -private: - std::optional<bool> OpaquePointers; }; } // end namespace llvm diff --git a/llvm/lib/IR/LLVMRemarkStreamer.cpp b/llvm/lib/IR/LLVMRemarkStreamer.cpp index 8fbc33328de8..71f8d4a4b1c7 100644 --- a/llvm/lib/IR/LLVMRemarkStreamer.cpp +++ b/llvm/lib/IR/LLVMRemarkStreamer.cpp @@ -96,8 +96,8 @@ Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks( LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional<uint64_t> RemarksHotnessThreshold) { - if (RemarksWithHotness) - Context.setDiagnosticsHotnessRequested(true); + if (RemarksWithHotness || RemarksHotnessThreshold.value_or(1)) + Context.setDiagnosticsHotnessRequested(true); Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold); @@ -143,7 +143,7 @@ Error llvm::setupLLVMOptimizationRemarks( LLVMContext &Context, raw_ostream &OS, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional<uint64_t> RemarksHotnessThreshold) { - if (RemarksWithHotness) + if (RemarksWithHotness || RemarksHotnessThreshold.value_or(1)) Context.setDiagnosticsHotnessRequested(true); Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold); diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp index ef3465177647..6c223d4ec381 100644 --- a/llvm/lib/IR/LegacyPassManager.cpp +++ b/llvm/lib/IR/LegacyPassManager.cpp @@ -1408,15 +1408,20 @@ bool FPPassManager::runOnFunction(Function &F) { FunctionSize = F.getInstructionCount(); } - llvm::TimeTraceScope FunctionScope("OptFunction", F.getName()); + // Store name outside of loop to avoid redundant calls. + const StringRef Name = F.getName(); + llvm::TimeTraceScope FunctionScope("OptFunction", Name); for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { FunctionPass *FP = getContainedPass(Index); bool LocalChanged = false; - llvm::TimeTraceScope PassScope("RunPass", FP->getPassName()); + // Call getPassName only when required. The call itself is fairly cheap, but + // still virtual and repeated calling adds unnecessary overhead. + llvm::TimeTraceScope PassScope( + "RunPass", [FP]() { return std::string(FP->getPassName()); }); - dumpPassInfo(FP, EXECUTION_MSG, ON_FUNCTION_MSG, F.getName()); + dumpPassInfo(FP, EXECUTION_MSG, ON_FUNCTION_MSG, Name); dumpRequiredSet(FP); initializeAnalysisImpl(FP); @@ -1455,7 +1460,7 @@ bool FPPassManager::runOnFunction(Function &F) { Changed |= LocalChanged; if (LocalChanged) - dumpPassInfo(FP, MODIFICATION_MSG, ON_FUNCTION_MSG, F.getName()); + dumpPassInfo(FP, MODIFICATION_MSG, ON_FUNCTION_MSG, Name); dumpPreservedSet(FP); dumpUsedSet(FP); @@ -1463,7 +1468,7 @@ bool FPPassManager::runOnFunction(Function &F) { if (LocalChanged) removeNotPreservedAnalysis(FP); recordAvailableAnalysis(FP); - removeDeadPasses(FP, F.getName(), ON_FUNCTION_MSG); + removeDeadPasses(FP, Name, ON_FUNCTION_MSG); } return Changed; diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp index 38ab1d3d1024..2490b3012bdc 100644 --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -336,12 +336,12 @@ MDNode *MDBuilder::createIrrLoopHeaderWeight(uint64_t Weight) { } MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, - Function *F) { + StringRef FName) { auto *Int64Ty = Type::getInt64Ty(Context); SmallVector<Metadata *, 3> Ops(3); Ops[0] = createConstant(ConstantInt::get(Int64Ty, GUID)); Ops[1] = createConstant(ConstantInt::get(Int64Ty, Hash)); - Ops[2] = createString(F->getName()); + Ops[2] = createString(FName); return MDNode::get(Context, Ops); } diff --git a/llvm/lib/IR/Mangler.cpp b/llvm/lib/IR/Mangler.cpp index 9011f5db6a40..8d9880ecba58 100644 --- a/llvm/lib/IR/Mangler.cpp +++ b/llvm/lib/IR/Mangler.cpp @@ -13,13 +13,13 @@ #include "llvm/IR/Mangler.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Triple.h" using namespace llvm; namespace { @@ -119,6 +119,7 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F, void Mangler::getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const { ManglerPrefixTy PrefixTy = Default; + assert(GV != nullptr && "Invalid Global Value"); if (GV->hasPrivateLinkage()) { if (CannotUsePrivateLabel) PrefixTy = LinkerPrivate; diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index bb757269e55f..c153ffb71a73 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -195,9 +195,9 @@ SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() { SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID; for (auto Pair : UseMap) { OwnerTy Owner = Pair.second.first; - if (!Owner.is<Metadata *>()) + if (!isa<Metadata *>(Owner)) continue; - Metadata *OwnerMD = Owner.get<Metadata *>(); + Metadata *OwnerMD = cast<Metadata *>(Owner); if (OwnerMD->getMetadataID() == Metadata::DIArgListKind) MDUsersWithID.push_back(&UseMap[Pair.first]); } @@ -206,7 +206,7 @@ SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() { }); SmallVector<Metadata *> MDUsers; for (auto *UserWithID : MDUsersWithID) - MDUsers.push_back(UserWithID->first.get<Metadata *>()); + MDUsers.push_back(cast<Metadata *>(UserWithID->first)); return MDUsers; } @@ -263,9 +263,9 @@ void ReplaceableMetadataImpl::SalvageDebugInfo(const Constant &C) { MetadataTracking::OwnerTy Owner = Pair.second.first; if (!Owner) continue; - if (!Owner.is<Metadata *>()) + if (!isa<Metadata *>(Owner)) continue; - auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); + auto *OwnerMD = dyn_cast_if_present<MDNode>(cast<Metadata *>(Owner)); if (!OwnerMD) continue; if (isa<DINode>(OwnerMD)) { @@ -282,7 +282,9 @@ void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { // Copy out uses since UseMap will get touched below. using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); - llvm::sort(Uses, llvm::less_second()); + llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { + return L.second.second < R.second.second; + }); for (const auto &Pair : Uses) { // Check that this Ref hasn't disappeared after RAUW (when updating a // previous Ref). @@ -301,13 +303,13 @@ void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { } // Check for MetadataAsValue. - if (Owner.is<MetadataAsValue *>()) { - Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD); + if (isa<MetadataAsValue *>(Owner)) { + cast<MetadataAsValue *>(Owner)->handleChangedMetadata(MD); continue; } // There's a Metadata owner -- dispatch. - Metadata *OwnerMD = Owner.get<Metadata *>(); + Metadata *OwnerMD = cast<Metadata *>(Owner); switch (OwnerMD->getMetadataID()) { #define HANDLE_METADATA_LEAF(CLASS) \ case Metadata::CLASS##Kind: \ @@ -341,11 +343,11 @@ void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { auto Owner = Pair.second.first; if (!Owner) continue; - if (Owner.is<MetadataAsValue *>()) + if (isa<MetadataAsValue *>(Owner)) continue; // Resolve MDNodes that point at this. - auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); + auto *OwnerMD = dyn_cast_if_present<MDNode>(cast<Metadata *>(Owner)); if (!OwnerMD) continue; if (OwnerMD->isResolved()) @@ -1072,6 +1074,70 @@ MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { return B; } +// Call instructions with branch weights are only used in SamplePGO as +// documented in +/// https://llvm.org/docs/BranchWeightMetadata.html#callinst). +MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B, + const Instruction *AInstr, + const Instruction *BInstr) { + assert(A && B && AInstr && BInstr && "Caller should guarantee"); + auto &Ctx = AInstr->getContext(); + MDBuilder MDHelper(Ctx); + + // LLVM IR verifier verifies !prof metadata has at least 2 operands. + assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 && + "!prof annotations should have no less than 2 operands"); + MDString *AMDS = dyn_cast<MDString>(A->getOperand(0)); + MDString *BMDS = dyn_cast<MDString>(B->getOperand(0)); + // LLVM IR verfier verifies first operand is MDString. + assert(AMDS != nullptr && BMDS != nullptr && + "first operand should be a non-null MDString"); + StringRef AProfName = AMDS->getString(); + StringRef BProfName = BMDS->getString(); + if (AProfName.equals("branch_weights") && + BProfName.equals("branch_weights")) { + ConstantInt *AInstrWeight = + mdconst::dyn_extract<ConstantInt>(A->getOperand(1)); + ConstantInt *BInstrWeight = + mdconst::dyn_extract<ConstantInt>(B->getOperand(1)); + assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier"); + return MDNode::get(Ctx, + {MDHelper.createString("branch_weights"), + MDHelper.createConstant(ConstantInt::get( + Type::getInt64Ty(Ctx), + SaturatingAdd(AInstrWeight->getZExtValue(), + BInstrWeight->getZExtValue())))}); + } + return nullptr; +} + +// Pass in both instructions and nodes. Instruction information (e.g., +// instruction type) helps interpret profiles and make implementation clearer. +MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B, + const Instruction *AInstr, + const Instruction *BInstr) { + if (!(A && B)) { + return A ? A : B; + } + + assert(AInstr->getMetadata(LLVMContext::MD_prof) == A && + "Caller should guarantee"); + assert(BInstr->getMetadata(LLVMContext::MD_prof) == B && + "Caller should guarantee"); + + const CallInst *ACall = dyn_cast<CallInst>(AInstr); + const CallInst *BCall = dyn_cast<CallInst>(BInstr); + + // Both ACall and BCall are direct callsites. + if (ACall && BCall && ACall->getCalledFunction() && + BCall->getCalledFunction()) + return mergeDirectCallProfMetadata(A, B, AInstr, BInstr); + + // The rest of the cases are not implemented but could be added + // when there are use cases. + return nullptr; +} + static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); } @@ -1475,23 +1541,54 @@ void Instruction::setMetadata(unsigned KindID, MDNode *Node) { Value::setMetadata(KindID, Node); } +void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) { + SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(), + Annotations.end()); + MDBuilder MDB(getContext()); + + auto *Existing = getMetadata(LLVMContext::MD_annotation); + SmallVector<Metadata *, 4> Names; + if (Existing) { + auto *Tuple = cast<MDTuple>(Existing); + for (auto &N : Tuple->operands()) { + if (isa<MDString>(N.get())) { + Names.push_back(N); + continue; + } + auto *MDAnnotationTuple = cast<MDTuple>(N); + if (any_of(MDAnnotationTuple->operands(), [&AnnotationsSet](auto &Op) { + return AnnotationsSet.contains(cast<MDString>(Op)->getString()); + })) + return; + Names.push_back(N); + } + } + + SmallVector<Metadata *> MDAnnotationStrings; + for (StringRef Annotation : Annotations) + MDAnnotationStrings.push_back(MDB.createString(Annotation)); + MDNode *InfoTuple = MDTuple::get(getContext(), MDAnnotationStrings); + Names.push_back(InfoTuple); + MDNode *MD = MDTuple::get(getContext(), Names); + setMetadata(LLVMContext::MD_annotation, MD); +} + void Instruction::addAnnotationMetadata(StringRef Name) { MDBuilder MDB(getContext()); auto *Existing = getMetadata(LLVMContext::MD_annotation); SmallVector<Metadata *, 4> Names; - bool AppendName = true; if (Existing) { auto *Tuple = cast<MDTuple>(Existing); for (auto &N : Tuple->operands()) { - if (cast<MDString>(N.get())->getString() == Name) - AppendName = false; + if (isa<MDString>(N.get()) && + cast<MDString>(N.get())->getString() == Name) + return; Names.push_back(N.get()); } } - if (AppendName) - Names.push_back(MDB.createString(Name)); + Names.push_back(MDB.createString(Name)); MDNode *MD = MDTuple::get(getContext(), Names); setMetadata(LLVMContext::MD_annotation, MD); } @@ -1517,6 +1614,11 @@ void Instruction::setAAMetadata(const AAMDNodes &N) { setMetadata(LLVMContext::MD_noalias, N.NoAlias); } +void Instruction::setNoSanitizeMetadata() { + setMetadata(llvm::LLVMContext::MD_nosanitize, + llvm::MDNode::get(getContext(), std::nullopt)); +} + MDNode *Instruction::getMetadataImpl(unsigned KindID) const { // Handle 'dbg' as a special case since it is not stored in the hash table. if (KindID == LLVMContext::MD_dbg) diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp index 49fadc9ed7e6..73354a8f36d2 100644 --- a/llvm/lib/IR/Module.cpp +++ b/llvm/lib/IR/Module.cpp @@ -262,7 +262,7 @@ NamedMDNode *Module::getOrInsertNamedMetadata(StringRef Name) { if (!NMD) { NMD = new NamedMDNode(Name); NMD->setParent(this); - NamedMDList.push_back(NMD); + insertNamedMDNode(NMD); } return NMD; } @@ -271,7 +271,7 @@ NamedMDNode *Module::getOrInsertNamedMetadata(StringRef Name) { /// delete it. void Module::eraseNamedMetadata(NamedMDNode *NMD) { NamedMDSymTab.erase(NMD->getName()); - NamedMDList.erase(NMD->getIterator()); + eraseNamedMDNode(NMD); } bool Module::isValidModFlagBehavior(Metadata *MD, ModFlagBehavior &MFB) { @@ -672,6 +672,18 @@ void Module::setRtLibUseGOT() { addModuleFlag(ModFlagBehavior::Max, "RtLibUseGOT", 1); } +bool Module::getDirectAccessExternalData() const { + auto *Val = cast_or_null<ConstantAsMetadata>( + getModuleFlag("direct-access-external-data")); + if (Val) + return cast<ConstantInt>(Val->getValue())->getZExtValue() > 0; + return getPICLevel() == PICLevel::NotPIC; +} + +void Module::setDirectAccessExternalData(bool Value) { + addModuleFlag(ModFlagBehavior::Max, "direct-access-external-data", Value); +} + UWTableKind Module::getUwtable() const { if (auto *Val = cast_or_null<ConstantAsMetadata>(getModuleFlag("uwtable"))) return UWTableKind(cast<ConstantInt>(Val->getValue())->getZExtValue()); @@ -746,6 +758,13 @@ unsigned Module::getOverrideStackAlignment() const { return 0; } +unsigned Module::getMaxTLSAlignment() const { + Metadata *MD = getModuleFlag("MaxTLSAlign"); + if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(MD)) + return CI->getZExtValue(); + return 0; +} + void Module::setOverrideStackAlignment(unsigned Align) { addModuleFlag(ModFlagBehavior::Error, "override-stack-alignment", Align); } diff --git a/llvm/lib/IR/ModuleSummaryIndex.cpp b/llvm/lib/IR/ModuleSummaryIndex.cpp index 2d1440756a95..15fe342969d6 100644 --- a/llvm/lib/IR/ModuleSummaryIndex.cpp +++ b/llvm/lib/IR/ModuleSummaryIndex.cpp @@ -107,11 +107,15 @@ uint64_t ModuleSummaryIndex::getFlags() const { Flags |= 0x40; if (withWholeProgramVisibility()) Flags |= 0x80; + if (withSupportsHotColdNew()) + Flags |= 0x100; + if (hasUnifiedLTO()) + Flags |= 0x200; return Flags; } void ModuleSummaryIndex::setFlags(uint64_t Flags) { - assert(Flags <= 0xff && "Unexpected bits in flag"); + assert(Flags <= 0x2ff && "Unexpected bits in flag"); // 1 bit: WithGlobalValueDeadStripping flag. // Set on combined index only. if (Flags & 0x1) @@ -145,6 +149,14 @@ void ModuleSummaryIndex::setFlags(uint64_t Flags) { // Set on combined index only. if (Flags & 0x80) setWithWholeProgramVisibility(); + // 1 bit: WithSupportsHotColdNew flag. + // Set on combined index only. + if (Flags & 0x100) + setWithSupportsHotColdNew(); + // 1 bit: WithUnifiedLTO flag. + // Set on combined index only. + if (Flags & 0x200) + setUnifiedLTO(); } // Collect for the given module the list of function it defines @@ -317,7 +329,7 @@ void ModuleSummaryIndex::propagateAttributes( } } -bool ModuleSummaryIndex::canImportGlobalVar(GlobalValueSummary *S, +bool ModuleSummaryIndex::canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const { auto HasRefsPreventingImport = [this](const GlobalVarSummary *GVS) { // We don't analyze GV references during attribute propagation, so diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp index ef850b8235b9..92b729c44d21 100644 --- a/llvm/lib/IR/PassManager.cpp +++ b/llvm/lib/IR/PassManager.cpp @@ -96,9 +96,9 @@ void ModuleToFunctionPassAdaptor::printPipeline( OS << "function"; if (EagerlyInvalidate) OS << "<eager-inv>"; - OS << "("; + OS << '('; Pass->printPipeline(OS, MapClassName2PassName); - OS << ")"; + OS << ')'; } PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M, @@ -122,13 +122,14 @@ PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M, continue; PreservedAnalyses PassPA = Pass->run(F, FAM); - PI.runAfterPass(*Pass, F, PassPA); // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); + PI.runAfterPass(*Pass, F, PassPA); + // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. PA.intersect(std::move(PassPA)); diff --git a/llvm/lib/IR/PseudoProbe.cpp b/llvm/lib/IR/PseudoProbe.cpp index f3802af26a61..df5f78c51182 100644 --- a/llvm/lib/IR/PseudoProbe.cpp +++ b/llvm/lib/IR/PseudoProbe.cpp @@ -22,12 +22,8 @@ using namespace llvm; namespace llvm { std::optional<PseudoProbe> -extractProbeFromDiscriminator(const Instruction &Inst) { - assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) && - "Only call instructions should have pseudo probe encodes as their " - "Dwarf discriminators"); - if (const DebugLoc &DLoc = Inst.getDebugLoc()) { - const DILocation *DIL = DLoc; +extractProbeFromDiscriminator(const DILocation *DIL) { + if (DIL) { auto Discriminator = DIL->getDiscriminator(); if (DILocation::isPseudoProbeDiscriminator(Discriminator)) { PseudoProbe Probe; @@ -40,12 +36,23 @@ extractProbeFromDiscriminator(const Instruction &Inst) { Probe.Factor = PseudoProbeDwarfDiscriminator::extractProbeFactor(Discriminator) / (float)PseudoProbeDwarfDiscriminator::FullDistributionFactor; + Probe.Discriminator = 0; return Probe; } } return std::nullopt; } +std::optional<PseudoProbe> +extractProbeFromDiscriminator(const Instruction &Inst) { + assert(isa<CallBase>(&Inst) && !isa<IntrinsicInst>(&Inst) && + "Only call instructions should have pseudo probe encodes as their " + "Dwarf discriminators"); + if (const DebugLoc &DLoc = Inst.getDebugLoc()) + return extractProbeFromDiscriminator(DLoc); + return std::nullopt; +} + std::optional<PseudoProbe> extractProbe(const Instruction &Inst) { if (const auto *II = dyn_cast<PseudoProbeInst>(&Inst)) { PseudoProbe Probe; @@ -54,6 +61,9 @@ std::optional<PseudoProbe> extractProbe(const Instruction &Inst) { Probe.Attr = II->getAttributes()->getZExtValue(); Probe.Factor = II->getFactor()->getZExtValue() / (float)PseudoProbeFullDistributionFactor; + Probe.Discriminator = 0; + if (const DebugLoc &DLoc = Inst.getDebugLoc()) + Probe.Discriminator = DLoc->getDiscriminator(); return Probe; } diff --git a/llvm/lib/IR/ReplaceConstant.cpp b/llvm/lib/IR/ReplaceConstant.cpp index 069da26e63b1..58aa040eb032 100644 --- a/llvm/lib/IR/ReplaceConstant.cpp +++ b/llvm/lib/IR/ReplaceConstant.cpp @@ -12,125 +12,91 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/ReplaceConstant.h" -#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SetVector.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/ValueMap.h" namespace llvm { -void convertConstantExprsToInstructions(Instruction *I, ConstantExpr *CE, - SmallPtrSetImpl<Instruction *> *Insts) { - // Collect all reachable paths to CE from constant exprssion operands of I. - std::map<Use *, std::vector<std::vector<ConstantExpr *>>> CEPaths; - collectConstantExprPaths(I, CE, CEPaths); - - // Convert all constant expressions to instructions which are collected at - // CEPaths. - convertConstantExprsToInstructions(I, CEPaths, Insts); +static bool isExpandableUser(User *U) { + return isa<ConstantExpr>(U) || isa<ConstantAggregate>(U); } -void convertConstantExprsToInstructions( - Instruction *I, - std::map<Use *, std::vector<std::vector<ConstantExpr *>>> &CEPaths, - SmallPtrSetImpl<Instruction *> *Insts) { - ValueMap<ConstantExpr *, Instruction *> Visited; - - for (Use &U : I->operands()) { - // The operand U is either not a constant expression operand or the - // constant expression paths do not belong to U, ignore U. - if (!CEPaths.count(&U)) - continue; - - // If the instruction I is a PHI instruction, then fix the instruction - // insertion point to the entry of the incoming basic block for operand U. - auto *BI = I; - if (auto *Phi = dyn_cast<PHINode>(I)) { - BasicBlock *BB = Phi->getIncomingBlock(U); - BI = &(*(BB->getFirstInsertionPt())); - } - - // Go through all the paths associated with operand U, and convert all the - // constant expressions along all the paths to corresponding instructions. - auto *II = I; - auto &Paths = CEPaths[&U]; - for (auto &Path : Paths) { - for (auto *CE : Path) { - // Instruction which is equivalent to CE. - Instruction *NI = nullptr; - - if (!Visited.count(CE)) { - // CE is encountered first time, convert it into a corresponding - // instruction NI, and appropriately insert NI before the parent - // instruction. - NI = CE->getAsInstruction(BI); - - // Mark CE as visited by mapping CE to NI. - Visited[CE] = NI; - - // If required collect NI. - if (Insts) - Insts->insert(NI); - } else { - // We had already encountered CE, the correponding instruction already - // exist, use it to replace CE. - NI = Visited[CE]; - } - - assert(NI && "Expected an instruction corresponding to constant " - "expression."); - - // Replace all uses of constant expression CE by the corresponding - // instruction NI within the current parent instruction. - II->replaceUsesOfWith(CE, NI); - BI = II = NI; - } - } +static Instruction *expandUser(Instruction *InsertPt, Constant *C) { + if (auto *CE = dyn_cast<ConstantExpr>(C)) { + return CE->getAsInstruction(InsertPt); + } else if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { + Value *V = PoisonValue::get(C->getType()); + for (auto [Idx, Op] : enumerate(C->operands())) + V = InsertValueInst::Create(V, Op, Idx, "", InsertPt); + return cast<Instruction>(V); + } else if (isa<ConstantVector>(C)) { + Type *IdxTy = Type::getInt32Ty(C->getContext()); + Value *V = PoisonValue::get(C->getType()); + for (auto [Idx, Op] : enumerate(C->operands())) + V = InsertElementInst::Create(V, Op, ConstantInt::get(IdxTy, Idx), "", + InsertPt); + return cast<Instruction>(V); + } else { + llvm_unreachable("Not an expandable user"); } - - // Remove all converted constant expressions which are dead by now. - for (auto Item : Visited) - Item.first->removeDeadConstantUsers(); } -void collectConstantExprPaths( - Instruction *I, ConstantExpr *CE, - std::map<Use *, std::vector<std::vector<ConstantExpr *>>> &CEPaths) { - for (Use &U : I->operands()) { - // If the operand U is not a constant expression operand, then ignore it. - auto *CE2 = dyn_cast<ConstantExpr>(U.get()); - if (!CE2) +bool convertUsersOfConstantsToInstructions(ArrayRef<Constant *> Consts) { + // Find all expandable direct users of Consts. + SmallVector<Constant *> Stack; + for (Constant *C : Consts) + for (User *U : C->users()) + if (isExpandableUser(U)) + Stack.push_back(cast<Constant>(U)); + + // Include transitive users. + SetVector<Constant *> ExpandableUsers; + while (!Stack.empty()) { + Constant *C = Stack.pop_back_val(); + if (!ExpandableUsers.insert(C)) continue; - // Holds all reachable paths from CE2 to CE. - std::vector<std::vector<ConstantExpr *>> Paths; - - // Collect all reachable paths from CE2 to CE. - std::vector<ConstantExpr *> Path{CE2}; - std::vector<std::vector<ConstantExpr *>> Stack{Path}; - while (!Stack.empty()) { - std::vector<ConstantExpr *> TPath = Stack.back(); - Stack.pop_back(); - auto *CE3 = TPath.back(); + for (auto *Nested : C->users()) + if (isExpandableUser(Nested)) + Stack.push_back(cast<Constant>(Nested)); + } - if (CE3 == CE) { - Paths.push_back(TPath); - continue; + // Find all instructions that use any of the expandable users + SetVector<Instruction *> InstructionWorklist; + for (Constant *C : ExpandableUsers) + for (User *U : C->users()) + if (auto *I = dyn_cast<Instruction>(U)) + InstructionWorklist.insert(I); + + // Replace those expandable operands with instructions + bool Changed = false; + while (!InstructionWorklist.empty()) { + Instruction *I = InstructionWorklist.pop_back_val(); + for (Use &U : I->operands()) { + auto *BI = I; + if (auto *Phi = dyn_cast<PHINode>(I)) { + BasicBlock *BB = Phi->getIncomingBlock(U); + BasicBlock::iterator It = BB->getFirstInsertionPt(); + assert(It != BB->end() && "Unexpected empty basic block"); + BI = &*It; } - for (auto &UU : CE3->operands()) { - if (auto *CE4 = dyn_cast<ConstantExpr>(UU.get())) { - std::vector<ConstantExpr *> NPath(TPath.begin(), TPath.end()); - NPath.push_back(CE4); - Stack.push_back(NPath); + if (auto *C = dyn_cast<Constant>(U.get())) { + if (ExpandableUsers.contains(C)) { + Changed = true; + Instruction *NI = expandUser(BI, C); + InstructionWorklist.insert(NI); + U.set(NI); } } } - - // Associate all the collected paths with U, and save it. - if (!Paths.empty()) - CEPaths[&U] = Paths; } + + for (Constant *C : Consts) + C->removeDeadConstantUsers(); + + return Changed; } } // namespace llvm diff --git a/llvm/lib/IR/SSAContext.cpp b/llvm/lib/IR/SSAContext.cpp index e758a3fbeac9..4790d19b74b5 100644 --- a/llvm/lib/IR/SSAContext.cpp +++ b/llvm/lib/IR/SSAContext.cpp @@ -22,8 +22,6 @@ using namespace llvm; -Value *SSAContext::ValueRefNull = nullptr; - void SSAContext::setFunction(Function &Fn) { F = &Fn; } BasicBlock *SSAContext::getEntryBlock(Function &F) { @@ -75,9 +73,9 @@ bool SSAContext::comesBefore(const Instruction *lhs, const Instruction *rhs) { return lhs->comesBefore(rhs); } -bool SSAContext::isConstantValuePhi(const Instruction &Instr) { +bool SSAContext::isConstantOrUndefValuePhi(const Instruction &Instr) { if (auto *Phi = dyn_cast<PHINode>(&Instr)) - return Phi->hasConstantValue(); + return Phi->hasConstantOrUndefValue(); return false; } diff --git a/llvm/lib/IR/SafepointIRVerifier.cpp b/llvm/lib/IR/SafepointIRVerifier.cpp index 5d3fa28f7d0a..ed99d05975c2 100644 --- a/llvm/lib/IR/SafepointIRVerifier.cpp +++ b/llvm/lib/IR/SafepointIRVerifier.cpp @@ -485,9 +485,7 @@ public: InstructionVerifier &Verifier); /// Returns true for reachable and live blocks. - bool isMapped(const BasicBlock *BB) const { - return BlockMap.find(BB) != BlockMap.end(); - } + bool isMapped(const BasicBlock *BB) const { return BlockMap.contains(BB); } private: /// Returns true if the instruction may be safely skipped during verification. diff --git a/llvm/lib/IR/StructuralHash.cpp b/llvm/lib/IR/StructuralHash.cpp index b6b9fe72cc35..6ea108d831a1 100644 --- a/llvm/lib/IR/StructuralHash.cpp +++ b/llvm/lib/IR/StructuralHash.cpp @@ -1,40 +1,41 @@ -//===-- StructuralHash.cpp - IR Hash for expensive checks -------*- C++ -*-===// +//===-- StructuralHash.cpp - IR Hashing -------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// -// - -#ifdef EXPENSIVE_CHECKS #include "llvm/IR/StructuralHash.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" using namespace llvm; -namespace details { +namespace { // Basic hashing mechanism to detect structural change to the IR, used to verify // pass return status consistency with actual change. Loosely copied from // llvm/lib/Transforms/Utils/FunctionComparator.cpp -class StructuralHash { - uint64_t Hash = 0x6acaa36bef8325c5ULL; +class StructuralHashImpl { + hash_code Hash; - void update(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); } + template <typename T> void hash(const T &V) { Hash = hash_combine(Hash, V); } public: - StructuralHash() = default; + StructuralHashImpl() : Hash(4) {} void update(const Function &F) { - if (F.empty()) + // Declarations don't affect analyses. + if (F.isDeclaration()) return; - update(F.isVarArg()); - update(F.arg_size()); + hash(12345); // Function header + + hash(F.isVarArg()); + hash(F.arg_size()); SmallVector<const BasicBlock *, 8> BBs; SmallPtrSet<const BasicBlock *, 16> VisitedBBs; @@ -43,9 +44,9 @@ public: VisitedBBs.insert(BBs[0]); while (!BBs.empty()) { const BasicBlock *BB = BBs.pop_back_val(); - update(45798); // Block header + hash(45798); // Block header for (auto &Inst : *BB) - update(Inst.getOpcode()); + hash(Inst.getOpcode()); const Instruction *Term = BB->getTerminator(); for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) { @@ -56,7 +57,19 @@ public: } } + void update(const GlobalVariable &GV) { + // Declarations and used/compiler.used don't affect analyses. + // Since there are several `llvm.*` metadata, like `llvm.embedded.object`, + // we ignore anything with the `.llvm` prefix + if (GV.isDeclaration() || GV.getName().starts_with("llvm.")) + return; + hash(23456); // Global header + hash(GV.getValueType()->getTypeID()); + } + void update(const Module &M) { + for (const GlobalVariable &GV : M.globals()) + update(GV); for (const Function &F : M) update(F); } @@ -64,18 +77,16 @@ public: uint64_t getHash() const { return Hash; } }; -} // namespace details +} // namespace uint64_t llvm::StructuralHash(const Function &F) { - ::details::StructuralHash H; + StructuralHashImpl H; H.update(F); return H.getHash(); } uint64_t llvm::StructuralHash(const Module &M) { - ::details::StructuralHash H; + StructuralHashImpl H; H.update(M); return H.getHash(); } - -#endif diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 8bb8c9d29a62..ba4d0f5dc18d 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -57,10 +57,12 @@ bool Type::isIntegerTy(unsigned Bitwidth) const { return isIntegerTy() && cast<IntegerType>(this)->getBitWidth() == Bitwidth; } -bool Type::isOpaquePointerTy() const { - if (auto *PTy = dyn_cast<PointerType>(this)) - return PTy->isOpaque(); - return false; +bool Type::isScalableTy() const { + if (const auto *STy = dyn_cast<StructType>(this)) { + SmallPtrSet<Type *, 4> Visited; + return STy->containsScalableVectorType(&Visited); + } + return getTypeID() == ScalableVectorTyID || isScalableTargetExtTy(); } const fltSemantics &Type::getFltSemantics() const { @@ -80,6 +82,12 @@ bool Type::isIEEE() const { return APFloat::getZero(getFltSemantics()).isIEEE(); } +bool Type::isScalableTargetExtTy() const { + if (auto *TT = dyn_cast<TargetExtType>(this)) + return isa<ScalableVectorType>(TT->getLayoutType()); + return false; +} + Type *Type::getFloatingPointTy(LLVMContext &C, const fltSemantics &S) { Type *Ty; if (&S == &APFloat::IEEEhalf()) @@ -306,6 +314,18 @@ PointerType *Type::getInt64PtrTy(LLVMContext &C, unsigned AS) { return getInt64Ty(C)->getPointerTo(AS); } +Type *Type::getWasm_ExternrefTy(LLVMContext &C) { + // opaque pointer in addrspace(10) + static PointerType *Ty = PointerType::get(C, 10); + return Ty; +} + +Type *Type::getWasm_FuncrefTy(LLVMContext &C) { + // opaque pointer in addrspace(20) + static PointerType *Ty = PointerType::get(C, 20); + return Ty; +} + //===----------------------------------------------------------------------===// // IntegerType Implementation //===----------------------------------------------------------------------===// @@ -432,18 +452,51 @@ StructType *StructType::get(LLVMContext &Context, ArrayRef<Type*> ETypes, return ST; } -bool StructType::containsScalableVectorType() const { +bool StructType::containsScalableVectorType( + SmallPtrSetImpl<Type *> *Visited) const { + if ((getSubclassData() & SCDB_ContainsScalableVector) != 0) + return true; + + if ((getSubclassData() & SCDB_NotContainsScalableVector) != 0) + return false; + + if (Visited && !Visited->insert(const_cast<StructType *>(this)).second) + return false; + for (Type *Ty : elements()) { - if (isa<ScalableVectorType>(Ty)) + if (isa<ScalableVectorType>(Ty)) { + const_cast<StructType *>(this)->setSubclassData( + getSubclassData() | SCDB_ContainsScalableVector); return true; - if (auto *STy = dyn_cast<StructType>(Ty)) - if (STy->containsScalableVectorType()) + } + if (auto *STy = dyn_cast<StructType>(Ty)) { + if (STy->containsScalableVectorType(Visited)) { + const_cast<StructType *>(this)->setSubclassData( + getSubclassData() | SCDB_ContainsScalableVector); return true; + } + } } + // For structures that are opaque, return false but do not set the + // SCDB_NotContainsScalableVector flag since it may gain scalable vector type + // when it becomes non-opaque. + if (!isOpaque()) + const_cast<StructType *>(this)->setSubclassData( + getSubclassData() | SCDB_NotContainsScalableVector); return false; } +bool StructType::containsHomogeneousScalableVectorTypes() const { + Type *FirstTy = getNumElements() > 0 ? elements()[0] : nullptr; + if (!FirstTy || !isa<ScalableVectorType>(FirstTy)) + return false; + for (Type *Ty : elements()) + if (Ty != FirstTy) + return false; + return true; +} + void StructType::setBody(ArrayRef<Type*> Elements, bool isPacked) { assert(isOpaque() && "Struct body already set!"); @@ -563,10 +616,19 @@ bool StructType::isSized(SmallPtrSetImpl<Type*> *Visited) const { // Okay, our struct is sized if all of the elements are, but if one of the // elements is opaque, the struct isn't sized *yet*, but may become sized in // the future, so just bail out without caching. + // The ONLY special case inside a struct that is considered sized is when the + // elements are homogeneous of a scalable vector type. + if (containsHomogeneousScalableVectorTypes()) { + const_cast<StructType *>(this)->setSubclassData(getSubclassData() | + SCDB_IsSized); + return true; + } for (Type *Ty : elements()) { // If the struct contains a scalable vector type, don't consider it sized. - // This prevents it from being used in loads/stores/allocas/GEPs. - if (isa<ScalableVectorType>(Ty)) + // This prevents it from being used in loads/stores/allocas/GEPs. The ONLY + // special case right now is a structure of homogenous scalable vector + // types and is handled by the if-statement before this for-loop. + if (Ty->isScalableTy()) return false; if (!Ty->isSized(Visited)) return false; @@ -730,46 +792,24 @@ PointerType *PointerType::get(Type *EltTy, unsigned AddressSpace) { assert(EltTy && "Can't get a pointer to <null> type!"); assert(isValidElementType(EltTy) && "Invalid type for pointer element!"); - LLVMContextImpl *CImpl = EltTy->getContext().pImpl; - // Automatically convert typed pointers to opaque pointers. - if (CImpl->getOpaquePointers()) - return get(EltTy->getContext(), AddressSpace); - - // Since AddressSpace #0 is the common case, we special case it. - PointerType *&Entry = AddressSpace == 0 ? CImpl->PointerTypes[EltTy] - : CImpl->ASPointerTypes[std::make_pair(EltTy, AddressSpace)]; - - if (!Entry) - Entry = new (CImpl->Alloc) PointerType(EltTy, AddressSpace); - return Entry; + return get(EltTy->getContext(), AddressSpace); } PointerType *PointerType::get(LLVMContext &C, unsigned AddressSpace) { LLVMContextImpl *CImpl = C.pImpl; - assert(CImpl->getOpaquePointers() && - "Can only create opaque pointers in opaque pointer mode"); // Since AddressSpace #0 is the common case, we special case it. - PointerType *&Entry = - AddressSpace == 0 - ? CImpl->PointerTypes[nullptr] - : CImpl->ASPointerTypes[std::make_pair(nullptr, AddressSpace)]; + PointerType *&Entry = AddressSpace == 0 ? CImpl->AS0PointerType + : CImpl->PointerTypes[AddressSpace]; if (!Entry) Entry = new (CImpl->Alloc) PointerType(C, AddressSpace); return Entry; } -PointerType::PointerType(Type *E, unsigned AddrSpace) - : Type(E->getContext(), PointerTyID), PointeeTy(E) { - ContainedTys = &PointeeTy; - NumContainedTys = 1; - setSubclassData(AddrSpace); -} - PointerType::PointerType(LLVMContext &C, unsigned AddrSpace) - : Type(C, PointerTyID), PointeeTy(nullptr) { + : Type(C, PointerTyID) { setSubclassData(AddrSpace); } @@ -850,10 +890,14 @@ struct TargetTypeInfo { static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) { LLVMContext &C = Ty->getContext(); StringRef Name = Ty->getName(); - if (Name.startswith("spirv.")) { + if (Name.startswith("spirv.")) return TargetTypeInfo(Type::getInt8PtrTy(C, 0), TargetExtType::HasZeroInit, TargetExtType::CanBeGlobal); - } + + // Opaque types in the AArch64 name space. + if (Name == "aarch64.svcount") + return TargetTypeInfo(ScalableVectorType::get(Type::getInt1Ty(C), 16)); + return TargetTypeInfo(Type::getVoidTy(C)); } diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index fa22065dcf36..41260a98e3ce 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -20,6 +20,7 @@ #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DerivedUser.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -315,8 +316,12 @@ StringRef Value::getName() const { } void Value::setNameImpl(const Twine &NewName) { + bool NeedNewName = + !getContext().shouldDiscardValueNames() || isa<GlobalValue>(this); + // Fast-path: LLVMContext can be set to strip out non-GlobalValue names - if (getContext().shouldDiscardValueNames() && !isa<GlobalValue>(this)) + // and there is no need to delete the old name. + if (!NeedNewName && !hasName()) return; // Fast path for common IRBuilder case of setName("") when there is no name. @@ -324,7 +329,7 @@ void Value::setNameImpl(const Twine &NewName) { return; SmallString<256> NameData; - StringRef NameRef = NewName.toStringRef(NameData); + StringRef NameRef = NeedNewName ? NewName.toStringRef(NameData) : ""; assert(NameRef.find_first_of(0) == StringRef::npos && "Null bytes are not allowed in names"); @@ -340,20 +345,17 @@ void Value::setNameImpl(const Twine &NewName) { return; // Cannot set a name on this value (e.g. constant). if (!ST) { // No symbol table to update? Just do the change. - if (NameRef.empty()) { - // Free the name for this value. - destroyValueName(); - return; - } - // NOTE: Could optimize for the case the name is shrinking to not deallocate // then reallocated. destroyValueName(); - // Create the new name. - MallocAllocator Allocator; - setValueName(ValueName::create(NameRef, Allocator)); - getValueName()->setValue(this); + if (!NameRef.empty()) { + // Create the new name. + assert(NeedNewName); + MallocAllocator Allocator; + setValueName(ValueName::create(NameRef, Allocator)); + getValueName()->setValue(this); + } return; } @@ -369,6 +371,7 @@ void Value::setNameImpl(const Twine &NewName) { } // Name is changing to something new. + assert(NeedNewName); setValueName(ST->createValueName(NameRef, this)); } @@ -737,7 +740,7 @@ const Value *Value::stripAndAccumulateConstantOffsets( // Stop traversal if the pointer offset wouldn't fit in the bit-width // provided by the Offset argument. This can happen due to AddrSpaceCast // stripping. - if (GEPOffset.getMinSignedBits() > BitWidth) + if (GEPOffset.getSignificantBits() > BitWidth) return V; // External Analysis can return a result higher/lower than the value @@ -972,7 +975,7 @@ Align Value::getPointerAlignment(const DataLayout &DL) const { if (auto *CstInt = dyn_cast_or_null<ConstantInt>(ConstantExpr::getPtrToInt( const_cast<Constant *>(CstPtr), DL.getIntPtrType(getType()), /*OnlyIfReduced=*/true))) { - size_t TrailingZeros = CstInt->getValue().countTrailingZeros(); + size_t TrailingZeros = CstInt->getValue().countr_zero(); // While the actual alignment may be large, elsewhere we have // an arbitrary upper alignmet limit, so let's clamp to it. return Align(TrailingZeros < Value::MaxAlignmentExponent @@ -983,6 +986,78 @@ Align Value::getPointerAlignment(const DataLayout &DL) const { return Align(1); } +static std::optional<int64_t> +getOffsetFromIndex(const GEPOperator *GEP, unsigned Idx, const DataLayout &DL) { + // Skip over the first indices. + gep_type_iterator GTI = gep_type_begin(GEP); + for (unsigned i = 1; i != Idx; ++i, ++GTI) + /*skip along*/; + + // Compute the offset implied by the rest of the indices. + int64_t Offset = 0; + for (unsigned i = Idx, e = GEP->getNumOperands(); i != e; ++i, ++GTI) { + ConstantInt *OpC = dyn_cast<ConstantInt>(GEP->getOperand(i)); + if (!OpC) + return std::nullopt; + if (OpC->isZero()) + continue; // No offset. + + // Handle struct indices, which add their field offset to the pointer. + if (StructType *STy = GTI.getStructTypeOrNull()) { + Offset += DL.getStructLayout(STy)->getElementOffset(OpC->getZExtValue()); + continue; + } + + // Otherwise, we have a sequential type like an array or fixed-length + // vector. Multiply the index by the ElementSize. + TypeSize Size = DL.getTypeAllocSize(GTI.getIndexedType()); + if (Size.isScalable()) + return std::nullopt; + Offset += Size.getFixedValue() * OpC->getSExtValue(); + } + + return Offset; +} + +std::optional<int64_t> Value::getPointerOffsetFrom(const Value *Other, + const DataLayout &DL) const { + const Value *Ptr1 = Other; + const Value *Ptr2 = this; + APInt Offset1(DL.getIndexTypeSizeInBits(Ptr1->getType()), 0); + APInt Offset2(DL.getIndexTypeSizeInBits(Ptr2->getType()), 0); + Ptr1 = Ptr1->stripAndAccumulateConstantOffsets(DL, Offset1, true); + Ptr2 = Ptr2->stripAndAccumulateConstantOffsets(DL, Offset2, true); + + // Handle the trivial case first. + if (Ptr1 == Ptr2) + return Offset2.getSExtValue() - Offset1.getSExtValue(); + + const GEPOperator *GEP1 = dyn_cast<GEPOperator>(Ptr1); + const GEPOperator *GEP2 = dyn_cast<GEPOperator>(Ptr2); + + // Right now we handle the case when Ptr1/Ptr2 are both GEPs with an identical + // base. After that base, they may have some number of common (and + // potentially variable) indices. After that they handle some constant + // offset, which determines their offset from each other. At this point, we + // handle no other case. + if (!GEP1 || !GEP2 || GEP1->getOperand(0) != GEP2->getOperand(0) || + GEP1->getSourceElementType() != GEP2->getSourceElementType()) + return std::nullopt; + + // Skip any common indices and track the GEP types. + unsigned Idx = 1; + for (; Idx != GEP1->getNumOperands() && Idx != GEP2->getNumOperands(); ++Idx) + if (GEP1->getOperand(Idx) != GEP2->getOperand(Idx)) + break; + + auto IOffset1 = getOffsetFromIndex(GEP1, Idx, DL); + auto IOffset2 = getOffsetFromIndex(GEP2, Idx, DL); + if (!IOffset1 || !IOffset2) + return std::nullopt; + return *IOffset2 - *IOffset1 + Offset2.getSExtValue() - + Offset1.getSExtValue(); +} + const Value *Value::DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const { auto *PN = dyn_cast<PHINode>(this); diff --git a/llvm/lib/IR/ValueSymbolTable.cpp b/llvm/lib/IR/ValueSymbolTable.cpp index cf85a571f9a0..52f7ddcdc65a 100644 --- a/llvm/lib/IR/ValueSymbolTable.cpp +++ b/llvm/lib/IR/ValueSymbolTable.cpp @@ -12,7 +12,6 @@ #include "llvm/IR/ValueSymbolTable.h" #include "llvm/ADT/SmallString.h" -#include "llvm/ADT/Triple.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Module.h" @@ -22,6 +21,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Triple.h" #include <cassert> #include <utility> diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp index e7be7a98a593..c07bc0561fba 100644 --- a/llvm/lib/IR/VectorBuilder.cpp +++ b/llvm/lib/IR/VectorBuilder.cpp @@ -32,9 +32,7 @@ Module &VectorBuilder::getModule() const { } Value *VectorBuilder::getAllTrueMask() { - auto *BoolTy = Builder.getInt1Ty(); - auto *MaskTy = VectorType::get(BoolTy, StaticVectorLength); - return ConstantInt::getAllOnesValue(MaskTy); + return Builder.getAllOnesMask(StaticVectorLength); } Value &VectorBuilder::requestMask() { diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 83e42bc184ff..1408ce293ca6 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -38,6 +38,11 @@ // * A landingpad instruction must be the first non-PHI instruction in the // block. // * Landingpad instructions must be in a function with a personality function. +// * Convergence control intrinsics are introduced in ConvergentOperations.rst. +// The applied restrictions are too numerous to list here. +// * The convergence entry intrinsic and the loop heart must be the first +// non-PHI instruction in their respective block. This does not conflict with +// the landing pads, since these two kinds cannot occur in the same block. // * All other things that are tested by asserts spread about the code... // //===----------------------------------------------------------------------===// @@ -48,6 +53,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" @@ -58,6 +64,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/Argument.h" +#include "llvm/IR/AttributeMask.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" @@ -66,12 +73,14 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/CycleInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/EHPersonalities.h" #include "llvm/IR/Function.h" #include "llvm/IR/GCStrategy.h" #include "llvm/IR/GlobalAlias.h" @@ -85,6 +94,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" +#include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsARM.h" #include "llvm/IR/IntrinsicsWebAssembly.h" #include "llvm/IR/LLVMContext.h" @@ -220,6 +230,8 @@ private: AL->print(*OS); } + void Write(Printable P) { *OS << P << '\n'; } + template <typename T> void Write(ArrayRef<T> Vs) { for (const T &V : Vs) Write(V); @@ -317,6 +329,13 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport { /// The current source language. dwarf::SourceLanguage CurrentSourceLang = dwarf::DW_LANG_lo_user; + /// Whether the current function has convergencectrl operand bundles. + enum { + ControlledConvergence, + UncontrolledConvergence, + NoConvergence + } ConvergenceKind = NoConvergence; + /// Whether source was present on the first DIFile encountered in each CU. DenseMap<const DICompileUnit *, bool> HasSourceDebugInfo; @@ -328,6 +347,10 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport { // terminators that indicate the unwind, used to detect cycles therein. MapVector<Instruction *, Instruction *> SiblingFuncletInfo; + /// Cache which blocks are in which funclet, if an EH funclet personality is + /// in use. Otherwise empty. + DenseMap<BasicBlock *, ColorVector> BlockEHFuncletColors; + /// Cache of constants visited in search of ConstantExprs. SmallPtrSet<const Constant *, 32> ConstantExprVisited; @@ -392,6 +415,8 @@ public: // FIXME: We strip const here because the inst visitor strips const. visit(const_cast<Function &>(F)); verifySiblingFuncletUnwinds(); + if (ConvergenceKind == ControlledConvergence) + verifyConvergenceControl(const_cast<Function &>(F)); InstsInThisBlock.clear(); DebugFnArgs.clear(); LandingPadResultTy = nullptr; @@ -399,6 +424,7 @@ public: SiblingFuncletInfo.clear(); verifyNoAliasScopeDecl(); NoAliasScopeDecls.clear(); + ConvergenceKind = NoConvergence; return !Broken; } @@ -467,6 +493,8 @@ private: void visitModuleFlagCGProfileEntry(const MDOperand &MDO); void visitFunction(const Function &F); void visitBasicBlock(BasicBlock &BB); + void verifyRangeMetadata(const Value &V, const MDNode *Range, Type *Ty, + bool IsAbsoluteSymbol); void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty); void visitDereferenceableMetadata(Instruction &I, MDNode *MD); void visitProfMetadata(Instruction &I, MDNode *MD); @@ -572,6 +600,7 @@ private: void verifyStatepoint(const CallBase &Call); void verifyFrameRecoverIndices(); void verifySiblingFuncletUnwinds(); + void verifyConvergenceControl(Function &F); void verifyFragmentExpression(const DbgVariableIntrinsic &I); template <typename ValueOrMetadata> @@ -653,7 +682,37 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { Check(A->value() <= Value::MaximumAlignment, "huge alignment values are unsupported", GO); } + + if (const MDNode *Associated = + GO->getMetadata(LLVMContext::MD_associated)) { + Check(Associated->getNumOperands() == 1, + "associated metadata must have one operand", &GV, Associated); + const Metadata *Op = Associated->getOperand(0).get(); + Check(Op, "associated metadata must have a global value", GO, Associated); + + const auto *VM = dyn_cast_or_null<ValueAsMetadata>(Op); + Check(VM, "associated metadata must be ValueAsMetadata", GO, Associated); + if (VM) { + Check(isa<PointerType>(VM->getValue()->getType()), + "associated value must be pointer typed", GV, Associated); + + const Value *Stripped = VM->getValue()->stripPointerCastsAndAliases(); + Check(isa<GlobalObject>(Stripped) || isa<Constant>(Stripped), + "associated metadata must point to a GlobalObject", GO, Stripped); + Check(Stripped != GO, + "global values should not associate to themselves", GO, + Associated); + } + } + + // FIXME: Why is getMetadata on GlobalValue protected? + if (const MDNode *AbsoluteSymbol = + GO->getMetadata(LLVMContext::MD_absolute_symbol)) { + verifyRangeMetadata(*GO, AbsoluteSymbol, DL.getIntPtrType(GO->getType()), + true); + } } + Check(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV), "Only global variables can have appending linkage!", &GV); @@ -748,10 +807,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { "the third field of the element type is mandatory, " "specify ptr null to migrate from the obsoleted 2-field form"); Type *ETy = STy->getTypeAtIndex(2); - Type *Int8Ty = Type::getInt8Ty(ETy->getContext()); - Check(ETy->isPointerTy() && - cast<PointerType>(ETy)->isOpaqueOrPointeeTypeMatches(Int8Ty), - "wrong type for intrinsic global variable", &GV); + Check(ETy->isPointerTy(), "wrong type for intrinsic global variable", + &GV); } } @@ -801,9 +858,11 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { Check(!isa<ScalableVectorType>(GV.getValueType()), "Globals cannot contain scalable vectors", &GV); - if (auto *STy = dyn_cast<StructType>(GV.getValueType())) - Check(!STy->containsScalableVectorType(), + if (auto *STy = dyn_cast<StructType>(GV.getValueType())) { + SmallPtrSet<Type *, 4> Visited; + Check(!STy->containsScalableVectorType(&Visited), "Globals cannot contain scalable vectors", &GV); + } // Check if it's a target extension type that disallows being used as a // global. @@ -1048,8 +1107,8 @@ void Verifier::visitDISubrange(const DISubrange &N) { isa<DIVariable>(CBound) || isa<DIExpression>(CBound), "Count must be signed constant or DIVariable or DIExpression", &N); auto Count = N.getCount(); - CheckDI(!Count || !Count.is<ConstantInt *>() || - Count.get<ConstantInt *>()->getSExtValue() >= -1, + CheckDI(!Count || !isa<ConstantInt *>(Count) || + cast<ConstantInt *>(Count)->getSExtValue() >= -1, "invalid subrange count", &N); auto *LBound = N.getRawLowerBound(); CheckDI(!LBound || isa<ConstantAsMetadata>(LBound) || @@ -1354,9 +1413,11 @@ void Verifier::visitDISubprogram(const DISubprogram &N) { auto *Node = dyn_cast<MDTuple>(RawNode); CheckDI(Node, "invalid retained nodes list", &N, RawNode); for (Metadata *Op : Node->operands()) { - CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op)), - "invalid retained nodes, expected DILocalVariable or DILabel", &N, - Node, Op); + CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) || + isa<DIImportedEntity>(Op)), + "invalid retained nodes, expected DILocalVariable, DILabel or " + "DIImportedEntity", + &N, Node, Op); } } CheckDI(!hasConflictingReferenceFlags(N.getFlags()), @@ -1373,6 +1434,8 @@ void Verifier::visitDISubprogram(const DISubprogram &N) { } else { // Subprogram declarations (part of the type hierarchy). CheckDI(!Unit, "subprogram declarations must not have a compile unit", &N); + CheckDI(!N.getRawDeclaration(), + "subprogram declaration must not have a declaration field"); } if (auto *RawThrownTypes = N.getRawThrownTypes()) { @@ -1875,7 +1938,7 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, } } - if (PointerType *PTy = dyn_cast<PointerType>(Ty)) { + if (isa<PointerType>(Ty)) { if (Attrs.hasAttribute(Attribute::ByVal)) { if (Attrs.hasAttribute(Attribute::Alignment)) { Align AttrAlign = Attrs.getAlignment().valueOrOne(); @@ -1902,38 +1965,14 @@ void Verifier::verifyParameterAttrs(AttributeSet Attrs, Type *Ty, Check(Attrs.getPreallocatedType()->isSized(&Visited), "Attribute 'preallocated' does not support unsized types!", V); } - if (!PTy->isOpaque()) { - if (!isa<PointerType>(PTy->getNonOpaquePointerElementType())) - Check(!Attrs.hasAttribute(Attribute::SwiftError), - "Attribute 'swifterror' only applies to parameters " - "with pointer to pointer type!", - V); - if (Attrs.hasAttribute(Attribute::ByRef)) { - Check(Attrs.getByRefType() == PTy->getNonOpaquePointerElementType(), - "Attribute 'byref' type does not match parameter!", V); - } - - if (Attrs.hasAttribute(Attribute::ByVal) && Attrs.getByValType()) { - Check(Attrs.getByValType() == PTy->getNonOpaquePointerElementType(), - "Attribute 'byval' type does not match parameter!", V); - } - - if (Attrs.hasAttribute(Attribute::Preallocated)) { - Check(Attrs.getPreallocatedType() == - PTy->getNonOpaquePointerElementType(), - "Attribute 'preallocated' type does not match parameter!", V); - } - - if (Attrs.hasAttribute(Attribute::InAlloca)) { - Check(Attrs.getInAllocaType() == PTy->getNonOpaquePointerElementType(), - "Attribute 'inalloca' type does not match parameter!", V); - } + } - if (Attrs.hasAttribute(Attribute::ElementType)) { - Check(Attrs.getElementType() == PTy->getNonOpaquePointerElementType(), - "Attribute 'elementtype' type does not match parameter!", V); - } - } + if (Attrs.hasAttribute(Attribute::NoFPClass)) { + uint64_t Val = Attrs.getAttribute(Attribute::NoFPClass).getValueAsInt(); + Check(Val != 0, "Attribute 'nofpclass' must have at least one test bit set", + V); + Check((Val & ~static_cast<unsigned>(fcAllFlags)) == 0, + "Invalid value for 'nofpclass' test mask", V); } } @@ -2142,10 +2181,13 @@ void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeList Attrs, unsigned VScaleMin = Attrs.getFnAttrs().getVScaleRangeMin(); if (VScaleMin == 0) CheckFailed("'vscale_range' minimum must be greater than 0", V); - + else if (!isPowerOf2_32(VScaleMin)) + CheckFailed("'vscale_range' minimum must be power-of-two value", V); std::optional<unsigned> VScaleMax = Attrs.getFnAttrs().getVScaleRangeMax(); if (VScaleMax && VScaleMin > VScaleMax) CheckFailed("'vscale_range' minimum cannot be greater than maximum", V); + else if (VScaleMax && !isPowerOf2_32(*VScaleMax)) + CheckFailed("'vscale_range' maximum must be power-of-two value", V); } if (Attrs.hasFnAttr("frame-pointer")) { @@ -2484,6 +2526,118 @@ void Verifier::verifySiblingFuncletUnwinds() { } } +void Verifier::verifyConvergenceControl(Function &F) { + DenseMap<BasicBlock *, SmallVector<CallBase *, 8>> LiveTokenMap; + DenseMap<const Cycle *, const CallBase *> CycleHearts; + + // Just like the DominatorTree, compute the CycleInfo locally so that we + // can run the verifier outside of a pass manager and we don't rely on + // potentially out-dated analysis results. + CycleInfo CI; + CI.compute(F); + + auto checkBundle = [&](OperandBundleUse &Bundle, CallBase *CB, + SmallVectorImpl<CallBase *> &LiveTokens) { + Check(Bundle.Inputs.size() == 1 && Bundle.Inputs[0]->getType()->isTokenTy(), + "The 'convergencectrl' bundle requires exactly one token use.", CB); + + Value *Token = Bundle.Inputs[0].get(); + auto *Def = dyn_cast<CallBase>(Token); + Check(Def != nullptr, + "Convergence control tokens can only be produced by call " + "instructions.", + Token); + + Check(llvm::is_contained(LiveTokens, Token), + "Convergence region is not well-nested.", Token, CB); + + while (LiveTokens.back() != Token) + LiveTokens.pop_back(); + + // Check static rules about cycles. + auto *BB = CB->getParent(); + auto *BBCycle = CI.getCycle(BB); + if (!BBCycle) + return; + + BasicBlock *DefBB = Def->getParent(); + if (DefBB == BB || BBCycle->contains(DefBB)) { + // degenerate occurrence of a loop intrinsic + return; + } + + auto *II = dyn_cast<IntrinsicInst>(CB); + Check(II && + II->getIntrinsicID() == Intrinsic::experimental_convergence_loop, + "Convergence token used by an instruction other than " + "llvm.experimental.convergence.loop in a cycle that does " + "not contain the token's definition.", + CB, CI.print(BBCycle)); + + while (true) { + auto *Parent = BBCycle->getParentCycle(); + if (!Parent || Parent->contains(DefBB)) + break; + BBCycle = Parent; + }; + + Check(BBCycle->isReducible() && BB == BBCycle->getHeader(), + "Cycle heart must dominate all blocks in the cycle.", CB, BB, + CI.print(BBCycle)); + Check(!CycleHearts.count(BBCycle), + "Two static convergence token uses in a cycle that does " + "not contain either token's definition.", + CB, CycleHearts[BBCycle], CI.print(BBCycle)); + CycleHearts[BBCycle] = CB; + }; + + ReversePostOrderTraversal<Function *> RPOT(&F); + SmallVector<CallBase *, 8> LiveTokens; + for (BasicBlock *BB : RPOT) { + LiveTokens.clear(); + auto LTIt = LiveTokenMap.find(BB); + if (LTIt != LiveTokenMap.end()) { + LiveTokens = std::move(LTIt->second); + LiveTokenMap.erase(LTIt); + } + + for (Instruction &I : *BB) { + CallBase *CB = dyn_cast<CallBase>(&I); + if (!CB) + continue; + + auto Bundle = CB->getOperandBundle(LLVMContext::OB_convergencectrl); + if (Bundle) + checkBundle(*Bundle, CB, LiveTokens); + + if (CB->getType()->isTokenTy()) + LiveTokens.push_back(CB); + } + + // Propagate token liveness + for (BasicBlock *Succ : successors(BB)) { + DomTreeNode *SuccNode = DT.getNode(Succ); + LTIt = LiveTokenMap.find(Succ); + if (LTIt == LiveTokenMap.end()) { + // We're the first predecessor: all tokens which dominate the + // successor are live for now. + LTIt = LiveTokenMap.try_emplace(Succ).first; + for (CallBase *LiveToken : LiveTokens) { + if (!DT.dominates(DT.getNode(LiveToken->getParent()), SuccNode)) + break; + LTIt->second.push_back(LiveToken); + } + } else { + // Compute the intersection of live tokens. + auto It = llvm::partition(LTIt->second, [&LiveTokens](CallBase *Token) { + return llvm::is_contained(LiveTokens, Token); + }); + LTIt->second.erase(It, LTIt->second.end()); + } + } + } +} + // visitFunction - Verify that a function is ok. // void Verifier::visitFunction(const Function &F) { @@ -2540,6 +2694,8 @@ void Verifier::visitFunction(const Function &F) { } case CallingConv::AMDGPU_KERNEL: case CallingConv::SPIR_KERNEL: + case CallingConv::AMDGPU_CS_Chain: + case CallingConv::AMDGPU_CS_ChainPreserve: Check(F.getReturnType()->isVoidTy(), "Calling convention requires void return type", &F); [[fallthrough]]; @@ -2630,6 +2786,9 @@ void Verifier::visitFunction(const Function &F) { F.getParent(), Per, Per->getParent()); } + // EH funclet coloring can be expensive, recompute on-demand + BlockEHFuncletColors.clear(); + if (F.isMaterializable()) { // Function has a body somewhere we can't see. Check(MDs.empty(), "unmaterialized function cannot have metadata", &F, @@ -3207,14 +3366,23 @@ void Verifier::visitPHINode(PHINode &PN) { visitInstruction(PN); } +static bool isControlledConvergent(const CallBase &Call) { + if (Call.getOperandBundle(LLVMContext::OB_convergencectrl)) + return true; + if (const auto *F = dyn_cast<Function>(Call.getCalledOperand())) { + switch (F->getIntrinsicID()) { + case Intrinsic::experimental_convergence_anchor: + case Intrinsic::experimental_convergence_entry: + case Intrinsic::experimental_convergence_loop: + return true; + } + } + return false; +} + void Verifier::visitCallBase(CallBase &Call) { Check(Call.getCalledOperand()->getType()->isPointerTy(), "Called function must be a pointer!", Call); - PointerType *FPTy = cast<PointerType>(Call.getCalledOperand()->getType()); - - Check(FPTy->isOpaqueOrPointeeTypeMatches(Call.getFunctionType()), - "Called function is not the same type as the call!", Call); - FunctionType *FTy = Call.getFunctionType(); // Verify that the correct number of arguments are being passed @@ -3243,6 +3411,15 @@ void Verifier::visitCallBase(CallBase &Call) { Check(Callee->getValueType() == FTy, "Intrinsic called with incompatible signature", Call); + // Disallow calls to functions with the amdgpu_cs_chain[_preserve] calling + // convention. + auto CC = Call.getCallingConv(); + Check(CC != CallingConv::AMDGPU_CS_Chain && + CC != CallingConv::AMDGPU_CS_ChainPreserve, + "Direct calls to amdgpu_cs_chain/amdgpu_cs_chain_preserve functions " + "not allowed. Please use the @llvm.amdgpu.cs.chain intrinsic instead.", + Call); + auto VerifyTypeAlign = [&](Type *Ty, const Twine &Message) { if (!Ty->isSized()) return; @@ -3496,6 +3673,23 @@ void Verifier::visitCallBase(CallBase &Call) { if (Call.isInlineAsm()) verifyInlineAsmCall(Call); + if (isControlledConvergent(Call)) { + Check(Call.isConvergent(), + "Expected convergent attribute on a controlled convergent call.", + Call); + Check(ConvergenceKind != UncontrolledConvergence, + "Cannot mix controlled and uncontrolled convergence in the same " + "function.", + Call); + ConvergenceKind = ControlledConvergence; + } else if (Call.isConvergent()) { + Check(ConvergenceKind != ControlledConvergence, + "Cannot mix controlled and uncontrolled convergence in the same " + "function.", + Call); + ConvergenceKind = UncontrolledConvergence; + } + visitInstruction(Call); } @@ -3796,6 +3990,14 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { "GEP base pointer is not a vector or a vector of pointers", &GEP); Check(GEP.getSourceElementType()->isSized(), "GEP into unsized type!", &GEP); + if (auto *STy = dyn_cast<StructType>(GEP.getSourceElementType())) { + SmallPtrSet<Type *, 4> Visited; + Check(!STy->containsScalableVectorType(&Visited), + "getelementptr cannot target structure that contains scalable vector" + "type", + &GEP); + } + SmallVector<Value *, 16> Idxs(GEP.indices()); Check( all_of(Idxs, [](Value *V) { return V->getType()->isIntOrIntVectorTy(); }), @@ -3839,10 +4041,10 @@ static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); } -void Verifier::visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty) { - assert(Range && Range == I.getMetadata(LLVMContext::MD_range) && - "precondition violation"); - +/// Verify !range and !absolute_symbol metadata. These have the same +/// restrictions, except !absolute_symbol allows the full set. +void Verifier::verifyRangeMetadata(const Value &I, const MDNode *Range, + Type *Ty, bool IsAbsoluteSymbol) { unsigned NumOperands = Range->getNumOperands(); Check(NumOperands % 2 == 0, "Unfinished range!", Range); unsigned NumRanges = NumOperands / 2; @@ -3856,13 +4058,20 @@ void Verifier::visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty) { ConstantInt *High = mdconst::dyn_extract<ConstantInt>(Range->getOperand(2 * i + 1)); Check(High, "The upper limit must be an integer!", High); - Check(High->getType() == Low->getType() && High->getType() == Ty, + Check(High->getType() == Low->getType() && + High->getType() == Ty->getScalarType(), "Range types must match instruction type!", &I); APInt HighV = High->getValue(); APInt LowV = Low->getValue(); + + // ConstantRange asserts if the ranges are the same except for the min/max + // value. Leave the cases it tolerates for the empty range error below. + Check(LowV != HighV || LowV.isMaxValue() || LowV.isMinValue(), + "The upper and lower limits cannot be the same value", &I); + ConstantRange CurRange(LowV, HighV); - Check(!CurRange.isEmptySet() && !CurRange.isFullSet(), + Check(!CurRange.isEmptySet() && (IsAbsoluteSymbol || !CurRange.isFullSet()), "Range must not be empty!", Range); if (i != 0) { Check(CurRange.intersectWith(LastRange).isEmptySet(), @@ -3887,6 +4096,12 @@ void Verifier::visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty) { } } +void Verifier::visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty) { + assert(Range && Range == I.getMetadata(LLVMContext::MD_range) && + "precondition violation"); + verifyRangeMetadata(I, Range, Ty, false); +} + void Verifier::checkAtomicMemAccessSize(Type *Ty, const Instruction *I) { unsigned Size = DL.getTypeSizeInBits(Ty); Check(Size >= 8, "atomic memory access' size must be byte-sized", Ty, I); @@ -3924,8 +4139,6 @@ void Verifier::visitStoreInst(StoreInst &SI) { PointerType *PTy = dyn_cast<PointerType>(SI.getOperand(1)->getType()); Check(PTy, "Store operand must be a pointer.", &SI); Type *ElTy = SI.getOperand(0)->getType(); - Check(PTy->isOpaqueOrPointeeTypeMatches(ElTy), - "Stored value type does not match pointer operand type!", &SI, ElTy); if (MaybeAlign A = SI.getAlign()) { Check(A->value() <= Value::MaximumAlignment, "huge alignment values are unsupported", &SI); @@ -4637,8 +4850,15 @@ void Verifier::visitAnnotationMetadata(MDNode *Annotation) { Check(isa<MDTuple>(Annotation), "annotation must be a tuple"); Check(Annotation->getNumOperands() >= 1, "annotation must have at least one operand"); - for (const MDOperand &Op : Annotation->operands()) - Check(isa<MDString>(Op.get()), "operands must be strings"); + for (const MDOperand &Op : Annotation->operands()) { + bool TupleOfStrings = + isa<MDTuple>(Op.get()) && + all_of(cast<MDTuple>(Op)->operands(), [](auto &Annotation) { + return isa<MDString>(Annotation.get()); + }); + Check(isa<MDString>(Op.get()) || TupleOfStrings, + "operands must be a string or a tuple of strings"); + } } void Verifier::visitAliasScopeMetadata(const MDNode *MD) { @@ -5038,7 +5258,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { } case Intrinsic::is_fpclass: { const ConstantInt *TestMask = cast<ConstantInt>(Call.getOperand(1)); - Check((TestMask->getZExtValue() & ~fcAllFlags) == 0, + Check((TestMask->getZExtValue() & ~static_cast<unsigned>(fcAllFlags)) == 0, "unsupported bits for llvm.is.fpclass test mask"); break; } @@ -5076,9 +5296,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "invalid llvm.dbg.declare intrinsic call 1", Call); visitDbgIntrinsic("declare", cast<DbgVariableIntrinsic>(Call)); break; - case Intrinsic::dbg_addr: // llvm.dbg.addr - visitDbgIntrinsic("addr", cast<DbgVariableIntrinsic>(Call)); - break; case Intrinsic::dbg_value: // llvm.dbg.value visitDbgIntrinsic("value", cast<DbgVariableIntrinsic>(Call)); break; @@ -5414,11 +5631,16 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Call); break; } + case Intrinsic::experimental_get_vector_length: { + ConstantInt *VF = cast<ConstantInt>(Call.getArgOperand(1)); + Check(!VF->isNegative() && !VF->isZero(), + "get_vector_length: VF must be positive", Call); + break; + } case Intrinsic::masked_load: { Check(Call.getType()->isVectorTy(), "masked_load: must return a vector", Call); - Value *Ptr = Call.getArgOperand(0); ConstantInt *Alignment = cast<ConstantInt>(Call.getArgOperand(1)); Value *Mask = Call.getArgOperand(2); Value *PassThru = Call.getArgOperand(3); @@ -5426,10 +5648,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Call); Check(Alignment->getValue().isPowerOf2(), "masked_load: alignment must be a power of 2", Call); - - PointerType *PtrTy = cast<PointerType>(Ptr->getType()); - Check(PtrTy->isOpaqueOrPointeeTypeMatches(Call.getType()), - "masked_load: return must match pointer type", Call); Check(PassThru->getType() == Call.getType(), "masked_load: pass through and return type must match", Call); Check(cast<VectorType>(Mask->getType())->getElementCount() == @@ -5439,17 +5657,12 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { } case Intrinsic::masked_store: { Value *Val = Call.getArgOperand(0); - Value *Ptr = Call.getArgOperand(1); ConstantInt *Alignment = cast<ConstantInt>(Call.getArgOperand(2)); Value *Mask = Call.getArgOperand(3); Check(Mask->getType()->isVectorTy(), "masked_store: mask must be vector", Call); Check(Alignment->getValue().isPowerOf2(), "masked_store: alignment must be a power of 2", Call); - - PointerType *PtrTy = cast<PointerType>(Ptr->getType()); - Check(PtrTy->isOpaqueOrPointeeTypeMatches(Val->getType()), - "masked_store: storee must match pointer type", Call); Check(cast<VectorType>(Mask->getType())->getElementCount() == cast<VectorType>(Val->getType())->getElementCount(), "masked_store: vector mask must be same length as value", Call); @@ -5600,15 +5813,28 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Type *Op0ElemTy = nullptr; Type *Op1ElemTy = nullptr; switch (ID) { - case Intrinsic::matrix_multiply: + case Intrinsic::matrix_multiply: { NumRows = cast<ConstantInt>(Call.getArgOperand(2)); + ConstantInt *N = cast<ConstantInt>(Call.getArgOperand(3)); NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); + Check(cast<FixedVectorType>(Call.getArgOperand(0)->getType()) + ->getNumElements() == + NumRows->getZExtValue() * N->getZExtValue(), + "First argument of a matrix operation does not match specified " + "shape!"); + Check(cast<FixedVectorType>(Call.getArgOperand(1)->getType()) + ->getNumElements() == + N->getZExtValue() * NumColumns->getZExtValue(), + "Second argument of a matrix operation does not match specified " + "shape!"); + ResultTy = cast<VectorType>(Call.getType()); Op0ElemTy = cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); Op1ElemTy = cast<VectorType>(Call.getArgOperand(1)->getType())->getElementType(); break; + } case Intrinsic::matrix_transpose: NumRows = cast<ConstantInt>(Call.getArgOperand(1)); NumColumns = cast<ConstantInt>(Call.getArgOperand(2)); @@ -5621,11 +5847,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { NumRows = cast<ConstantInt>(Call.getArgOperand(3)); NumColumns = cast<ConstantInt>(Call.getArgOperand(4)); ResultTy = cast<VectorType>(Call.getType()); - - PointerType *Op0PtrTy = - cast<PointerType>(Call.getArgOperand(0)->getType()); - if (!Op0PtrTy->isOpaque()) - Op0ElemTy = Op0PtrTy->getNonOpaquePointerElementType(); break; } case Intrinsic::matrix_column_major_store: { @@ -5635,11 +5856,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { ResultTy = cast<VectorType>(Call.getArgOperand(0)->getType()); Op0ElemTy = cast<VectorType>(Call.getArgOperand(0)->getType())->getElementType(); - - PointerType *Op1PtrTy = - cast<PointerType>(Call.getArgOperand(1)->getType()); - if (!Op1PtrTy->isOpaque()) - Op1ElemTy = Op1PtrTy->getNonOpaquePointerElementType(); break; } default: @@ -5794,7 +6010,102 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { "isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call); break; } + case Intrinsic::callbr_landingpad: { + const auto *CBR = dyn_cast<CallBrInst>(Call.getOperand(0)); + Check(CBR, "intrinstic requires callbr operand", &Call); + if (!CBR) + break; + + const BasicBlock *LandingPadBB = Call.getParent(); + const BasicBlock *PredBB = LandingPadBB->getUniquePredecessor(); + if (!PredBB) { + CheckFailed("Intrinsic in block must have 1 unique predecessor", &Call); + break; + } + if (!isa<CallBrInst>(PredBB->getTerminator())) { + CheckFailed("Intrinsic must have corresponding callbr in predecessor", + &Call); + break; + } + Check(llvm::any_of(CBR->getIndirectDests(), + [LandingPadBB](const BasicBlock *IndDest) { + return IndDest == LandingPadBB; + }), + "Intrinsic's corresponding callbr must have intrinsic's parent basic " + "block in indirect destination list", + &Call); + const Instruction &First = *LandingPadBB->begin(); + Check(&First == &Call, "No other instructions may proceed intrinsic", + &Call); + break; + } + case Intrinsic::amdgcn_cs_chain: { + auto CallerCC = Call.getCaller()->getCallingConv(); + switch (CallerCC) { + case CallingConv::AMDGPU_CS: + case CallingConv::AMDGPU_CS_Chain: + case CallingConv::AMDGPU_CS_ChainPreserve: + break; + default: + CheckFailed("Intrinsic can only be used from functions with the " + "amdgpu_cs, amdgpu_cs_chain or amdgpu_cs_chain_preserve " + "calling conventions", + &Call); + break; + } + break; + } + case Intrinsic::experimental_convergence_entry: + Check(Call.getFunction()->isConvergent(), + "Entry intrinsic can occur only in a convergent function.", &Call); + Check(Call.getParent()->isEntryBlock(), + "Entry intrinsic must occur in the entry block.", &Call); + Check(Call.getParent()->getFirstNonPHI() == &Call, + "Entry intrinsic must occur at the start of the basic block.", &Call); + LLVM_FALLTHROUGH; + case Intrinsic::experimental_convergence_anchor: + Check(!Call.getOperandBundle(LLVMContext::OB_convergencectrl), + "Entry or anchor intrinsic must not have a convergencectrl bundle.", + &Call); + break; + case Intrinsic::experimental_convergence_loop: + Check(Call.getOperandBundle(LLVMContext::OB_convergencectrl), + "Loop intrinsic must have a convergencectrl bundle.", &Call); + Check(Call.getParent()->getFirstNonPHI() == &Call, + "Loop intrinsic must occur at the start of the basic block.", &Call); + break; }; + + // Verify that there aren't any unmediated control transfers between funclets. + if (IntrinsicInst::mayLowerToFunctionCall(ID)) { + Function *F = Call.getParent()->getParent(); + if (F->hasPersonalityFn() && + isScopedEHPersonality(classifyEHPersonality(F->getPersonalityFn()))) { + // Run EH funclet coloring on-demand and cache results for other intrinsic + // calls in this function + if (BlockEHFuncletColors.empty()) + BlockEHFuncletColors = colorEHFunclets(*F); + + // Check for catch-/cleanup-pad in first funclet block + bool InEHFunclet = false; + BasicBlock *CallBB = Call.getParent(); + const ColorVector &CV = BlockEHFuncletColors.find(CallBB)->second; + assert(CV.size() > 0 && "Uncolored block"); + for (BasicBlock *ColorFirstBB : CV) + if (dyn_cast_or_null<FuncletPadInst>(ColorFirstBB->getFirstNonPHI())) + InEHFunclet = true; + + // Check for funclet operand bundle + bool HasToken = false; + for (unsigned I = 0, E = Call.getNumOperandBundles(); I != E; ++I) + if (Call.getOperandBundleAt(I).getTagID() == LLVMContext::OB_funclet) + HasToken = true; + + // This would cause silent code truncation in WinEHPrepare + if (InEHFunclet) + Check(HasToken, "Missing funclet token on intrinsic call", &Call); + } + } } /// Carefully grab the subprogram from a local scope. @@ -5961,20 +6272,20 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { case Intrinsic::experimental_constrained_fptosi: case Intrinsic::experimental_constrained_fptoui: { Value *Operand = FPI.getArgOperand(0); - uint64_t NumSrcElem = 0; + ElementCount SrcEC; Check(Operand->getType()->isFPOrFPVectorTy(), "Intrinsic first argument must be floating point", &FPI); if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) { - NumSrcElem = cast<FixedVectorType>(OperandT)->getNumElements(); + SrcEC = cast<VectorType>(OperandT)->getElementCount(); } Operand = &FPI; - Check((NumSrcElem > 0) == Operand->getType()->isVectorTy(), + Check(SrcEC.isNonZero() == Operand->getType()->isVectorTy(), "Intrinsic first argument and result disagree on vector use", &FPI); Check(Operand->getType()->isIntOrIntVectorTy(), "Intrinsic result must be an integer", &FPI); if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) { - Check(NumSrcElem == cast<FixedVectorType>(OperandT)->getNumElements(), + Check(SrcEC == cast<VectorType>(OperandT)->getElementCount(), "Intrinsic first argument and result vector lengths must be equal", &FPI); } @@ -5984,20 +6295,20 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { case Intrinsic::experimental_constrained_sitofp: case Intrinsic::experimental_constrained_uitofp: { Value *Operand = FPI.getArgOperand(0); - uint64_t NumSrcElem = 0; + ElementCount SrcEC; Check(Operand->getType()->isIntOrIntVectorTy(), "Intrinsic first argument must be integer", &FPI); if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) { - NumSrcElem = cast<FixedVectorType>(OperandT)->getNumElements(); + SrcEC = cast<VectorType>(OperandT)->getElementCount(); } Operand = &FPI; - Check((NumSrcElem > 0) == Operand->getType()->isVectorTy(), + Check(SrcEC.isNonZero() == Operand->getType()->isVectorTy(), "Intrinsic first argument and result disagree on vector use", &FPI); Check(Operand->getType()->isFPOrFPVectorTy(), "Intrinsic result must be a floating point", &FPI); if (auto *OperandT = dyn_cast<VectorType>(Operand->getType())) { - Check(NumSrcElem == cast<FixedVectorType>(OperandT)->getNumElements(), + Check(SrcEC == cast<VectorType>(OperandT)->getElementCount(), "Intrinsic first argument and result vector lengths must be equal", &FPI); } @@ -6016,8 +6327,8 @@ void Verifier::visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI) { Check(OperandTy->isVectorTy() == ResultTy->isVectorTy(), "Intrinsic first argument and result disagree on vector use", &FPI); if (OperandTy->isVectorTy()) { - Check(cast<FixedVectorType>(OperandTy)->getNumElements() == - cast<FixedVectorType>(ResultTy)->getNumElements(), + Check(cast<VectorType>(OperandTy)->getElementCount() == + cast<VectorType>(ResultTy)->getElementCount(), "Intrinsic first argument and result vector lengths must be equal", &FPI); } @@ -6221,7 +6532,17 @@ void Verifier::verifyNotEntryValue(const DbgVariableIntrinsic &I) { if (!E || !E->isValid()) return; - CheckDI(!E->isEntryValue(), "Entry values are only allowed in MIR", &I); + // We allow EntryValues for swift async arguments, as they have an + // ABI-guarantee to be turned into a specific register. + if (isa<ValueAsMetadata>(I.getRawLocation())) + if (auto *ArgLoc = dyn_cast_or_null<Argument>(I.getVariableLocationOp(0)); + ArgLoc && ArgLoc->hasAttribute(Attribute::SwiftAsync)) + return; + + CheckDI(!E->isEntryValue(), + "Entry values are only allowed in MIR unless they target a " + "swiftasync Argument", + &I); } void Verifier::verifyCompileUnits() { @@ -6680,6 +7001,9 @@ static bool isNewFormatTBAATypeNode(llvm::MDNode *Type) { } bool TBAAVerifier::visitTBAAMetadata(Instruction &I, const MDNode *MD) { + CheckTBAA(MD->getNumOperands() > 0, "TBAA metadata cannot have 0 operands", + &I, MD); + CheckTBAA(isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) || isa<VAArgInst>(I) || isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I), |