diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen')
16 files changed, 172 insertions, 155 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/CalcSpillWeights.cpp b/contrib/llvm-project/llvm/lib/CodeGen/CalcSpillWeights.cpp index 0377bc002067..5a005ba7b414 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/CalcSpillWeights.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/CalcSpillWeights.cpp @@ -97,7 +97,7 @@ bool VirtRegAuxInfo::isRematerializable(const LiveInterval &LI, // Trace copies introduced by live range splitting. The inline // spiller can rematerialize through these copies, so the spill // weight must reflect this. - while (TII.isFullCopyInstr(*MI)) { + while (MI->isFullCopy()) { // The copy destination must match the interval register. if (MI->getOperand(0).getReg() != Reg) return false; @@ -224,16 +224,7 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start, continue; NumInstr++; - bool identityCopy = false; - auto DestSrc = TII.isCopyInstr(*MI); - if (DestSrc) { - const MachineOperand *DestRegOp = DestSrc->Destination; - const MachineOperand *SrcRegOp = DestSrc->Source; - identityCopy = DestRegOp->getReg() == SrcRegOp->getReg() && - DestRegOp->getSubReg() == SrcRegOp->getSubReg(); - } - - if (identityCopy || MI->isImplicitDef()) + if (MI->isIdentityCopy() || MI->isImplicitDef()) continue; if (!Visited.insert(MI).second) continue; @@ -267,7 +258,7 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start, } // Get allocation hints from copies. - if (!TII.isCopyInstr(*MI)) + if (!MI->isCopy()) continue; Register HintReg = copyHint(MI, LI.reg(), TRI, MRI); if (!HintReg) diff --git a/contrib/llvm-project/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/contrib/llvm-project/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp index 02c67e500bdc..7979ac9a5fb7 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp @@ -226,6 +226,7 @@ private: const TargetLowering *TL = nullptr; const TargetLibraryInfo *TLI = nullptr; SmallVector<NodePtr> CompositeNodes; + DenseMap<std::pair<Value *, Value *>, NodePtr> CachedResult; SmallPtrSet<Instruction *, 16> FinalInstructions; @@ -292,17 +293,11 @@ private: NodePtr submitCompositeNode(NodePtr Node) { CompositeNodes.push_back(Node); + if (Node->Real && Node->Imag) + CachedResult[{Node->Real, Node->Imag}] = Node; return Node; } - NodePtr getContainingComposite(Value *R, Value *I) { - for (const auto &CN : CompositeNodes) { - if (CN->Real == R && CN->Imag == I) - return CN; - } - return nullptr; - } - /// Identifies a complex partial multiply pattern and its rotation, based on /// the following patterns /// @@ -900,9 +895,11 @@ ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) { LLVM_DEBUG(dbgs() << "identifyNode on " << *R << " / " << *I << "\n"); assert(R->getType() == I->getType() && "Real and imaginary parts should not have different types"); - if (NodePtr CN = getContainingComposite(R, I)) { + + auto It = CachedResult.find({R, I}); + if (It != CachedResult.end()) { LLVM_DEBUG(dbgs() << " - Folding to existing node\n"); - return CN; + return It->second; } if (NodePtr CN = identifySplat(R, I)) @@ -949,6 +946,7 @@ ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) { return CN; LLVM_DEBUG(dbgs() << " - Not recognised as a valid pattern.\n"); + CachedResult[{R, I}] = nullptr; return nullptr; } @@ -1426,7 +1424,17 @@ bool ComplexDeinterleavingGraph::identifyNodes(Instruction *RootI) { // CompositeNode we should choose only one either Real or Imag instruction to // use as an anchor for generating complex instruction. auto It = RootToNode.find(RootI); - if (It != RootToNode.end() && It->second->Real == RootI) { + if (It != RootToNode.end()) { + auto RootNode = It->second; + assert(RootNode->Operation == + ComplexDeinterleavingOperation::ReductionOperation); + // Find out which part, Real or Imag, comes later, and only if we come to + // the latest part, add it to OrderedRoots. + auto *R = cast<Instruction>(RootNode->Real); + auto *I = cast<Instruction>(RootNode->Imag); + auto *ReplacementAnchor = R->comesBefore(I) ? I : R; + if (ReplacementAnchor != RootI) + return false; OrderedRoots.push_back(RootI); return true; } diff --git a/contrib/llvm-project/llvm/lib/CodeGen/InlineSpiller.cpp b/contrib/llvm-project/llvm/lib/CodeGen/InlineSpiller.cpp index c62f3db9d321..277c6be418c5 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/InlineSpiller.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/InlineSpiller.cpp @@ -256,11 +256,11 @@ Spiller *llvm::createInlineSpiller(MachineFunctionPass &Pass, // This minimizes register pressure and maximizes the store-to-load distance for // spill slots which can be important in tight loops. -/// isFullCopyOf - If MI is a COPY to or from Reg, return the other register, -/// otherwise return 0. -static Register isCopyOf(const MachineInstr &MI, Register Reg, - const TargetInstrInfo &TII) { - if (!TII.isCopyInstr(MI)) +/// If MI is a COPY to or from Reg, return the other register, otherwise return +/// 0. +static Register isCopyOf(const MachineInstr &MI, Register Reg) { + assert(!MI.isBundled()); + if (!MI.isCopy()) return Register(); const MachineOperand &DstOp = MI.getOperand(0); @@ -277,10 +277,9 @@ static Register isCopyOf(const MachineInstr &MI, Register Reg, } /// Check for a copy bundle as formed by SplitKit. -static Register isCopyOfBundle(const MachineInstr &FirstMI, Register Reg, - const TargetInstrInfo &TII) { +static Register isCopyOfBundle(const MachineInstr &FirstMI, Register Reg) { if (!FirstMI.isBundled()) - return isCopyOf(FirstMI, Reg, TII); + return isCopyOf(FirstMI, Reg); assert(!FirstMI.isBundledWithPred() && FirstMI.isBundledWithSucc() && "expected to see first instruction in bundle"); @@ -289,12 +288,11 @@ static Register isCopyOfBundle(const MachineInstr &FirstMI, Register Reg, MachineBasicBlock::const_instr_iterator I = FirstMI.getIterator(); while (I->isBundledWithSucc()) { const MachineInstr &MI = *I; - auto CopyInst = TII.isCopyInstr(MI); - if (!CopyInst) + if (!MI.isCopy()) return Register(); - const MachineOperand &DstOp = *CopyInst->Destination; - const MachineOperand &SrcOp = *CopyInst->Source; + const MachineOperand &DstOp = MI.getOperand(0); + const MachineOperand &SrcOp = MI.getOperand(1); if (DstOp.getReg() == Reg) { if (!SnipReg) SnipReg = SrcOp.getReg(); @@ -360,7 +358,7 @@ bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) { MachineInstr &MI = *RI++; // Allow copies to/from Reg. - if (isCopyOfBundle(MI, Reg, TII)) + if (isCopyOfBundle(MI, Reg)) continue; // Allow stack slot loads. @@ -398,7 +396,7 @@ void InlineSpiller::collectRegsToSpill() { return; for (MachineInstr &MI : llvm::make_early_inc_range(MRI.reg_bundles(Reg))) { - Register SnipReg = isCopyOfBundle(MI, Reg, TII); + Register SnipReg = isCopyOfBundle(MI, Reg); if (!isSibling(SnipReg)) continue; LiveInterval &SnipLI = LIS.getInterval(SnipReg); @@ -521,14 +519,14 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { // Find all spills and copies of VNI. for (MachineInstr &MI : llvm::make_early_inc_range(MRI.use_nodbg_bundles(Reg))) { - if (!MI.mayStore() && !TII.isCopyInstr(MI)) + if (!MI.isCopy() && !MI.mayStore()) continue; SlotIndex Idx = LIS.getInstructionIndex(MI); if (LI->getVNInfoAt(Idx) != VNI) continue; // Follow sibling copies down the dominator tree. - if (Register DstReg = isCopyOfBundle(MI, Reg, TII)) { + if (Register DstReg = isCopyOfBundle(MI, Reg)) { if (isSibling(DstReg)) { LiveInterval &DstLI = LIS.getInterval(DstReg); VNInfo *DstVNI = DstLI.getVNInfoAt(Idx.getRegSlot()); @@ -872,7 +870,7 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops, if (Ops.back().first != MI || MI->isBundled()) return false; - bool WasCopy = TII.isCopyInstr(*MI).has_value(); + bool WasCopy = MI->isCopy(); Register ImpReg; // TII::foldMemoryOperand will do what we need here for statepoint @@ -1157,7 +1155,7 @@ void InlineSpiller::spillAroundUses(Register Reg) { Idx = VNI->def; // Check for a sibling copy. - Register SibReg = isCopyOfBundle(MI, Reg, TII); + Register SibReg = isCopyOfBundle(MI, Reg); if (SibReg && isSibling(SibReg)) { // This may actually be a copy between snippets. if (isRegToSpill(SibReg)) { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeEdit.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeEdit.cpp index ff49e080090c..c3477cd8ce34 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeEdit.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeEdit.cpp @@ -352,8 +352,7 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink) { // unlikely to change anything. We typically don't want to shrink the // PIC base register that has lots of uses everywhere. // Always shrink COPY uses that probably come from live range splitting. - if ((MI->readsVirtualRegister(Reg) && - (MO.isDef() || TII.isCopyInstr(*MI))) || + if ((MI->readsVirtualRegister(Reg) && (MI->isCopy() || MO.isDef())) || (MO.readsReg() && (MRI.hasOneNonDBGUse(Reg) || useIsKill(LI, MO)))) ToShrink.insert(&LI); else if (MO.readsReg()) diff --git a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp index af7d6c4403b8..93f5314539cd 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/LiveRangeShrink.cpp @@ -23,7 +23,6 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" @@ -110,7 +109,6 @@ bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) { return false; MachineRegisterInfo &MRI = MF.getRegInfo(); - const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); LLVM_DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n'); @@ -199,7 +197,7 @@ bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) { // is because it needs more accurate model to handle register // pressure correctly. MachineInstr &DefInstr = *MRI.def_instr_begin(Reg); - if (!TII.isCopyInstr(DefInstr)) + if (!DefInstr.isCopy()) NumEligibleUse++; Insert = FindDominatedInstruction(DefInstr, Insert, IOM); } else { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineLICM.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineLICM.cpp index 4e80e9b58c06..523e077fd9a2 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/MachineLICM.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineLICM.cpp @@ -538,6 +538,10 @@ void MachineLICMBase::HoistRegionPostRA() { PhysRegDefs.set(*AI); } + // Funclet entry blocks will clobber all registers + if (const uint32_t *Mask = BB->getBeginClobberMask(TRI)) + PhysRegClobbers.setBitsNotInMask(Mask); + SpeculationState = SpeculateUnknown; for (MachineInstr &MI : *BB) ProcessMI(&MI, PhysRegDefs, PhysRegClobbers, StoredFIs, Candidates); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/contrib/llvm-project/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 3448c56e4994..5b822b5d7b95 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -14,9 +14,10 @@ #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/Analysis/ObjCARCInstKind.h" #include "llvm/Analysis/ObjCARCUtil.h" -#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" @@ -26,6 +27,7 @@ #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" +#include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" using namespace llvm; @@ -41,19 +43,19 @@ static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt( namespace { struct PreISelIntrinsicLowering { + const TargetMachine &TM; const function_ref<TargetTransformInfo &(Function &)> LookupTTI; - const function_ref<TargetLibraryInfo &(Function &)> LookupLibInfo; /// If this is true, assume it's preferably to leave memory intrinsic calls /// for replacement with a library call later. Otherwise this depends on - /// TargetLibraryInfo availability of the corresponding function. + /// TargetLoweringInfo availability of the corresponding function. const bool UseMemIntrinsicLibFunc; explicit PreISelIntrinsicLowering( + const TargetMachine &TM_, function_ref<TargetTransformInfo &(Function &)> LookupTTI_, - function_ref<TargetLibraryInfo &(Function &)> LookupLibInfo_, bool UseMemIntrinsicLibFunc_ = true) - : LookupTTI(LookupTTI_), LookupLibInfo(LookupLibInfo_), + : TM(TM_), LookupTTI(LookupTTI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {} static bool shouldExpandMemIntrinsicWithSize(Value *Size, @@ -187,6 +189,13 @@ bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize( return SizeVal > Threshold || Threshold == 0; } +static bool canEmitLibcall(const TargetMachine &TM, Function *F, + RTLIB::Libcall LC) { + // TODO: Should this consider the address space of the memcpy? + const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); + return TLI->getLibcallName(LC) != nullptr; +} + // TODO: Handle atomic memcpy and memcpy.inline // TODO: Pass ScalarEvolution bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { @@ -203,9 +212,10 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) { if (UseMemIntrinsicLibFunc && - LookupLibInfo(*ParentFunc).has(LibFunc_memcpy)) + canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY)) break; + // TODO: For optsize, emit the loop into a separate function expandMemCpyAsLoop(Memcpy, TTI); Changed = true; Memcpy->eraseFromParent(); @@ -219,7 +229,7 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) { if (UseMemIntrinsicLibFunc && - LookupLibInfo(*ParentFunc).has(LibFunc_memmove)) + canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE)) break; if (expandMemMoveAsLoop(Memmove, TTI)) { @@ -236,7 +246,7 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) { if (UseMemIntrinsicLibFunc && - LookupLibInfo(*Memset->getFunction()).has(LibFunc_memset)) + canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET)) break; expandMemSetAsLoop(Memset); @@ -357,8 +367,8 @@ public: PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {} void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired<TargetLibraryInfoWrapperPass>(); AU.addRequired<TargetTransformInfoWrapperPass>(); + AU.addRequired<TargetPassConfig>(); } bool runOnModule(Module &M) override { @@ -366,11 +376,8 @@ public: return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); }; - auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & { - return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); - }; - - PreISelIntrinsicLowering Lowering(LookupTTI, LookupTLI); + const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); + PreISelIntrinsicLowering Lowering(TM, LookupTTI); return Lowering.lowerIntrinsics(M); } }; @@ -379,27 +386,28 @@ public: char PreISelIntrinsicLoweringLegacyPass::ID; -INITIALIZE_PASS(PreISelIntrinsicLoweringLegacyPass, - "pre-isel-intrinsic-lowering", "Pre-ISel Intrinsic Lowering", - false, false) +INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass, + "pre-isel-intrinsic-lowering", + "Pre-ISel Intrinsic Lowering", false, false) +INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) +INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass, + "pre-isel-intrinsic-lowering", + "Pre-ISel Intrinsic Lowering", false, false) ModulePass *llvm::createPreISelIntrinsicLoweringPass() { - return new PreISelIntrinsicLoweringLegacyPass; + return new PreISelIntrinsicLoweringLegacyPass(); } PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); - auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & { - return FAM.getResult<TargetLibraryAnalysis>(F); - }; - auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & { return FAM.getResult<TargetIRAnalysis>(F); }; - PreISelIntrinsicLowering Lowering(LookupTTI, LookupTLI); + PreISelIntrinsicLowering Lowering(TM, LookupTTI); if (!Lowering.lowerIntrinsics(M)) return PreservedAnalyses::all(); else diff --git a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp index 68f6ea3268a9..48187e575494 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -1282,12 +1282,10 @@ static LaneBitmask getInstReadLaneMask(const MachineRegisterInfo &MRI, /// VirtReg. static bool readsLaneSubset(const MachineRegisterInfo &MRI, const MachineInstr *MI, const LiveInterval &VirtReg, - const TargetRegisterInfo *TRI, SlotIndex Use, - const TargetInstrInfo *TII) { + const TargetRegisterInfo *TRI, SlotIndex Use) { // Early check the common case. - auto DestSrc = TII->isCopyInstr(*MI); - if (DestSrc && - DestSrc->Destination->getSubReg() == DestSrc->Source->getSubReg()) + if (MI->isCopy() && + MI->getOperand(0).getSubReg() == MI->getOperand(1).getSubReg()) return false; // FIXME: We're only considering uses, but should be consider defs too? @@ -1346,14 +1344,14 @@ unsigned RAGreedy::tryInstructionSplit(const LiveInterval &VirtReg, // the allocation. for (const SlotIndex Use : Uses) { if (const MachineInstr *MI = Indexes->getInstructionFromIndex(Use)) { - if (TII->isFullCopyInstr(*MI) || + if (MI->isFullCopy() || (SplitSubClass && SuperRCNumAllocatableRegs == getNumAllocatableRegsForConstraints(MI, VirtReg.reg(), SuperRC, TII, TRI, RegClassInfo)) || // TODO: Handle split for subranges with subclass constraints? (!SplitSubClass && VirtReg.hasSubRanges() && - !readsLaneSubset(*MRI, MI, VirtReg, TRI, Use, TII))) { + !readsLaneSubset(*MRI, MI, VirtReg, TRI, Use))) { LLVM_DEBUG(dbgs() << " skip:\t" << Use << '\t' << *MI); continue; } @@ -2140,7 +2138,7 @@ void RAGreedy::initializeCSRCost() { /// \p Out is not cleared before being populated. void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { for (const MachineInstr &Instr : MRI->reg_nodbg_instructions(Reg)) { - if (!TII->isFullCopyInstr(Instr)) + if (!Instr.isFullCopy()) continue; // Look for the other end of the copy. Register OtherReg = Instr.getOperand(0).getReg(); @@ -2455,10 +2453,9 @@ RAGreedy::RAGreedyStats RAGreedy::computeStats(MachineBasicBlock &MBB) { MI.getOpcode() == TargetOpcode::STATEPOINT; }; for (MachineInstr &MI : MBB) { - auto DestSrc = TII->isCopyInstr(MI); - if (DestSrc) { - const MachineOperand &Dest = *DestSrc->Destination; - const MachineOperand &Src = *DestSrc->Source; + if (MI.isCopy()) { + const MachineOperand &Dest = MI.getOperand(0); + const MachineOperand &Src = MI.getOperand(1); Register SrcReg = Src.getReg(); Register DestReg = Dest.getReg(); // Only count `COPY`s with a virtual register as source or destination. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index de909cc10795..235f0da86b90 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -20492,9 +20492,11 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) { SDValue Elt = Value.getOperand(1); SDValue Idx = Value.getOperand(2); - // If the element isn't byte sized then we can't compute an offset + // If the element isn't byte sized or is implicitly truncated then we can't + // compute an offset. EVT EltVT = Elt.getValueType(); - if (!EltVT.isByteSized()) + if (!EltVT.isByteSized() || + EltVT != Value.getOperand(0).getValueType().getVectorElementType()) return SDValue(); auto *Ld = dyn_cast<LoadSDNode>(Value.getOperand(0)); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 5c1b19eba1c1..30d202494320 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1945,6 +1945,9 @@ SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm, assert(MulImm.getBitWidth() == VT.getSizeInBits() && "APInt size does not match type size!"); + if (MulImm == 0) + return getConstant(0, DL, VT); + if (ConstantFold) { const MachineFunction &MF = getMachineFunction(); auto Attr = MF.getFunction().getFnAttribute(Attribute::VScaleRange); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 9595da9d0d8a..20c37eb4cb11 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4156,6 +4156,18 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects()); } +static const MDNode *getRangeMetadata(const Instruction &I) { + // If !noundef is not present, then !range violation results in a poison + // value rather than immediate undefined behavior. In theory, transferring + // these annotations to SDAG is fine, but in practice there are key SDAG + // transforms that are known not to be poison-safe, such as folding logical + // and/or to bitwise and/or. For now, only transfer !range if !noundef is + // also present. + if (!I.hasMetadata(LLVMContext::MD_noundef)) + return nullptr; + return I.getMetadata(LLVMContext::MD_range); +} + void SelectionDAGBuilder::visitLoad(const LoadInst &I) { if (I.isAtomic()) return visitAtomicLoad(I); @@ -4180,7 +4192,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { Type *Ty = I.getType(); SmallVector<EVT, 4> ValueVTs, MemVTs; - SmallVector<uint64_t, 4> Offsets; + SmallVector<TypeSize, 4> Offsets; ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets, 0); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) @@ -4188,7 +4200,7 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { Align Alignment = I.getAlign(); AAMDNodes AAInfo = I.getAAMetadata(); - const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(I); bool isVolatile = I.isVolatile(); MachineMemOperand::Flags MMOFlags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); @@ -4219,14 +4231,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { if (isVolatile) Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); - // An aggregate load cannot wrap around the address space, so offsets to its - // parts don't wrap either. - SDNodeFlags Flags; - Flags.setNoUnsignedWrap(true); - SmallVector<SDValue, 4> Values(NumValues); SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); - EVT PtrVT = Ptr.getValueType(); unsigned ChainI = 0; for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { @@ -4243,13 +4249,15 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { Root = Chain; ChainI = 0; } - SDValue A = DAG.getNode(ISD::ADD, dl, - PtrVT, Ptr, - DAG.getConstant(Offsets[i], dl, PtrVT), - Flags); - SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, - MachinePointerInfo(SV, Offsets[i]), Alignment, + // TODO: MachinePointerInfo only supports a fixed length offset. + MachinePointerInfo PtrInfo = + !Offsets[i].isScalable() || Offsets[i].isZero() + ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue()) + : MachinePointerInfo(); + + SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]); + SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment, MMOFlags, AAInfo, Ranges); Chains[ChainI] = L.getValue(1); @@ -4351,7 +4359,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { } SmallVector<EVT, 4> ValueVTs, MemVTs; - SmallVector<uint64_t, 4> Offsets; + SmallVector<TypeSize, 4> Offsets; ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), SrcV->getType(), ValueVTs, &MemVTs, &Offsets, 0); unsigned NumValues = ValueVTs.size(); @@ -4372,11 +4380,6 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); - // An aggregate load cannot wrap around the address space, so offsets to its - // parts don't wrap either. - SDNodeFlags Flags; - Flags.setNoUnsignedWrap(true); - unsigned ChainI = 0; for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { // See visitLoad comments. @@ -4386,14 +4389,19 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { Root = Chain; ChainI = 0; } - SDValue Add = - DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); + + // TODO: MachinePointerInfo only supports a fixed length offset. + MachinePointerInfo PtrInfo = + !Offsets[i].isScalable() || Offsets[i].isZero() + ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue()) + : MachinePointerInfo(); + + SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]); SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); if (MemVTs[i] != ValueVTs[i]) Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); SDValue St = - DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), - Alignment, MMOFlags, AAInfo); + DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo); Chains[ChainI] = St; } @@ -4607,7 +4615,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { Alignment = DAG.getEVTAlign(VT); AAMDNodes AAInfo = I.getAAMetadata(); - const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(I); // Do not serialize masked loads of constant memory with anything. MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); @@ -4641,7 +4649,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { ->getMaybeAlignValue() .value_or(DAG.getEVTAlign(VT.getScalarType())); - const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(I); SDValue Root = DAG.getRoot(); SDValue Base; @@ -7396,7 +7404,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, } SDValue TripCount = getValue(I.getOperand(1)); - auto VecTy = CCVT.changeVectorElementType(ElementVT); + EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT, + CCVT.getVectorElementCount()); SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index); SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount); @@ -7645,7 +7654,7 @@ void SelectionDAGBuilder::visitVPLoad( Value *PtrOperand = VPIntrin.getArgOperand(0); MaybeAlign Alignment = VPIntrin.getPointerAlignment(); AAMDNodes AAInfo = VPIntrin.getAAMetadata(); - const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(VPIntrin); SDValue LD; // Do not serialize variable-length loads of constant memory with // anything. @@ -7672,7 +7681,7 @@ void SelectionDAGBuilder::visitVPGather( Value *PtrOperand = VPIntrin.getArgOperand(0); MaybeAlign Alignment = VPIntrin.getPointerAlignment(); AAMDNodes AAInfo = VPIntrin.getAAMetadata(); - const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(VPIntrin); SDValue LD; if (!Alignment) Alignment = DAG.getEVTAlign(VT.getScalarType()); @@ -7779,7 +7788,7 @@ void SelectionDAGBuilder::visitVPStridedLoad( if (!Alignment) Alignment = DAG.getEVTAlign(VT.getScalarType()); AAMDNodes AAInfo = VPIntrin.getAAMetadata(); - const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); + const MDNode *Ranges = getRangeMetadata(VPIntrin); MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); @@ -9626,7 +9635,7 @@ void SelectionDAGBuilder::visitVACopy(const CallInst &I) { SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op) { - const MDNode *Range = I.getMetadata(LLVMContext::MD_range); + const MDNode *Range = getRangeMetadata(I); if (!Range) return Op; @@ -11310,8 +11319,32 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } } - if (FallthroughUnreachable) - JTH->FallthroughUnreachable = true; + // If the default clause is unreachable, propagate that knowledge into + // JTH->FallthroughUnreachable which will use it to suppress the range + // check. + // + // However, don't do this if we're doing branch target enforcement, + // because a table branch _without_ a range check can be a tempting JOP + // gadget - out-of-bounds inputs that are impossible in correct + // execution become possible again if an attacker can influence the + // control flow. So if an attacker doesn't already have a BTI bypass + // available, we don't want them to be able to get one out of this + // table branch. + if (FallthroughUnreachable) { + Function &CurFunc = CurMF->getFunction(); + bool HasBranchTargetEnforcement = false; + if (CurFunc.hasFnAttribute("branch-target-enforcement")) { + HasBranchTargetEnforcement = + CurFunc.getFnAttribute("branch-target-enforcement") + .getValueAsBool(); + } else { + HasBranchTargetEnforcement = + CurMF->getMMI().getModule()->getModuleFlag( + "branch-target-enforcement"); + } + if (!HasBranchTargetEnforcement) + JTH->FallthroughUnreachable = true; + } if (!JTH->FallthroughUnreachable) addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.cpp index 83964eced597..eee54f09fbad 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.cpp @@ -514,10 +514,10 @@ void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo &ParentVNI) { VFP = ValueForcePair(nullptr, true); } -SlotIndex SplitEditor::buildSingleSubRegCopy( - Register FromReg, Register ToReg, MachineBasicBlock &MBB, - MachineBasicBlock::iterator InsertBefore, unsigned SubIdx, - LiveInterval &DestLI, bool Late, SlotIndex Def, const MCInstrDesc &Desc) { +SlotIndex SplitEditor::buildSingleSubRegCopy(Register FromReg, Register ToReg, + MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, + unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def) { + const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY); bool FirstCopy = !Def.isValid(); MachineInstr *CopyMI = BuildMI(MBB, InsertBefore, DebugLoc(), Desc) .addReg(ToReg, RegState::Define | getUndefRegState(FirstCopy) @@ -536,8 +536,7 @@ SlotIndex SplitEditor::buildSingleSubRegCopy( SlotIndex SplitEditor::buildCopy(Register FromReg, Register ToReg, LaneBitmask LaneMask, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, bool Late, unsigned RegIdx) { - const MCInstrDesc &Desc = - TII.get(TII.getLiveRangeSplitOpcode(FromReg, *MBB.getParent())); + const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY); SlotIndexes &Indexes = *LIS.getSlotIndexes(); if (LaneMask.all() || LaneMask == MRI.getMaxLaneMaskForVReg(FromReg)) { // The full vreg is copied. @@ -565,7 +564,7 @@ SlotIndex SplitEditor::buildCopy(Register FromReg, Register ToReg, SlotIndex Def; for (unsigned BestIdx : SubIndexes) { Def = buildSingleSubRegCopy(FromReg, ToReg, MBB, InsertBefore, BestIdx, - DestLI, Late, Def, Desc); + DestLI, Late, Def); } BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator(); @@ -1585,9 +1584,7 @@ bool SplitAnalysis::shouldSplitSingleBlock(const BlockInfo &BI, if (BI.LiveIn && BI.LiveOut) return true; // No point in isolating a copy. It has no register class constraints. - MachineInstr *MI = LIS.getInstructionFromIndex(BI.FirstInstr); - bool copyLike = TII.isCopyInstr(*MI) || MI->isSubregToReg(); - if (copyLike) + if (LIS.getInstructionFromIndex(BI.FirstInstr)->isCopyLike()) return false; // Finally, don't isolate an end point that was created by earlier splits. return isOriginalEndpoint(BI.FirstInstr); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.h b/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.h index 1174e392e4e4..f764ffd4750c 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/SplitKit.h @@ -428,11 +428,8 @@ private: bool Late, unsigned RegIdx); SlotIndex buildSingleSubRegCopy(Register FromReg, Register ToReg, - MachineBasicBlock &MB, - MachineBasicBlock::iterator InsertBefore, - unsigned SubIdx, LiveInterval &DestLI, - bool Late, SlotIndex Def, - const MCInstrDesc &Desc); + MachineBasicBlock &MB, MachineBasicBlock::iterator InsertBefore, + unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def); public: /// Create a new SplitEditor for editing the LiveInterval analyzed by SA. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/TargetInstrInfo.cpp b/contrib/llvm-project/llvm/lib/CodeGen/TargetInstrInfo.cpp index 09dcddc17b06..b29404b42519 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -440,9 +440,8 @@ MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB, // If the COPY instruction in MI can be folded to a stack operation, return // the register class to use. static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI, - const TargetInstrInfo &TII, unsigned FoldIdx) { - assert(TII.isCopyInstr(MI) && "MI must be a COPY instruction"); + assert(MI.isCopy() && "MI must be a COPY instruction"); if (MI.getNumOperands() != 2) return nullptr; assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand"); @@ -631,10 +630,10 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, } // Straight COPY may fold as load/store. - if (!isCopyInstr(MI) || Ops.size() != 1) + if (!MI.isCopy() || Ops.size() != 1) return nullptr; - const TargetRegisterClass *RC = canFoldCopy(MI, *this, Ops[0]); + const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); if (!RC) return nullptr; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringBase.cpp b/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringBase.cpp index 1d5adf46e4a6..10c54560da5a 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -921,7 +921,7 @@ void TargetLoweringBase::initActions() { // Legal, in which case all fp constants are legal, or use isFPImmLegal() // to optimize expansions for certain constants. setOperationAction(ISD::ConstantFP, - {MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128}, + {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128}, Expand); // These library functions default to expand. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 3994552884c4..55fb522554fa 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -2426,23 +2426,6 @@ MCSection *TargetLoweringObjectFileXCOFF::SelectSectionForGlobal( Name, Kind, XCOFF::CsectProperties(SMC, XCOFF::XTY_CM)); } - if (Kind.isMergeableCString()) { - Align Alignment = GO->getParent()->getDataLayout().getPreferredAlign( - cast<GlobalVariable>(GO)); - - unsigned EntrySize = getEntrySizeForKind(Kind); - std::string SizeSpec = ".rodata.str" + utostr(EntrySize) + "."; - SmallString<128> Name; - Name = SizeSpec + utostr(Alignment.value()); - - if (TM.getDataSections()) - getNameWithPrefix(Name, GO, TM); - - return getContext().getXCOFFSection( - Name, Kind, XCOFF::CsectProperties(XCOFF::XMC_RO, XCOFF::XTY_SD), - /* MultiSymbolsAllowed*/ !TM.getDataSections()); - } - if (Kind.isText()) { if (TM.getFunctionSections()) { return cast<MCSymbolXCOFF>(getFunctionEntryPointSymbol(GO, TM)) @@ -2628,12 +2611,12 @@ MCSymbol *TargetLoweringObjectFileXCOFF::getFunctionEntryPointSymbol( // function entry point csect instead. And for function delcarations, the // undefined symbols gets treated as csect with XTY_ER property. if (((TM.getFunctionSections() && !Func->hasSection()) || - Func->isDeclaration()) && + Func->isDeclarationForLinker()) && isa<Function>(Func)) { return getContext() .getXCOFFSection( NameStr, SectionKind::getText(), - XCOFF::CsectProperties(XCOFF::XMC_PR, Func->isDeclaration() + XCOFF::CsectProperties(XCOFF::XMC_PR, Func->isDeclarationForLinker() ? XCOFF::XTY_ER : XCOFF::XTY_SD)) ->getQualNameSymbol(); |