aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/MachineSink.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/MachineSink.cpp')
-rw-r--r--llvm/lib/CodeGen/MachineSink.cpp208
1 files changed, 188 insertions, 20 deletions
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 27a2e7023f22..a4ba197b7a1d 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -15,6 +15,8 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
@@ -38,6 +40,7 @@
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/LLVMContext.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/BranchProbability.h"
@@ -105,6 +108,25 @@ namespace {
using AllSuccsCache =
std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
+ /// DBG_VALUE pointer and flag. The flag is true if this DBG_VALUE is
+ /// post-dominated by another DBG_VALUE of the same variable location.
+ /// This is necessary to detect sequences such as:
+ /// %0 = someinst
+ /// DBG_VALUE %0, !123, !DIExpression()
+ /// %1 = anotherinst
+ /// DBG_VALUE %1, !123, !DIExpression()
+ /// Where if %0 were to sink, the DBG_VAUE should not sink with it, as that
+ /// would re-order assignments.
+ using SeenDbgUser = PointerIntPair<MachineInstr *, 1>;
+
+ /// Record of DBG_VALUE uses of vregs in a block, so that we can identify
+ /// debug instructions to sink.
+ SmallDenseMap<unsigned, TinyPtrVector<SeenDbgUser>> SeenDbgUsers;
+
+ /// Record of debug variables that have had their locations set in the
+ /// current block.
+ DenseSet<DebugVariable> SeenDbgVars;
+
public:
static char ID; // Pass identification
@@ -132,6 +154,7 @@ namespace {
private:
bool ProcessBlock(MachineBasicBlock &MBB);
+ void ProcessDbgInst(MachineInstr &MI);
bool isWorthBreakingCriticalEdge(MachineInstr &MI,
MachineBasicBlock *From,
MachineBasicBlock *To);
@@ -153,8 +176,14 @@ namespace {
MachineBasicBlock *To,
bool BreakPHIEdge);
bool SinkInstruction(MachineInstr &MI, bool &SawStore,
-
AllSuccsCache &AllSuccessors);
+
+ /// If we sink a COPY inst, some debug users of it's destination may no
+ /// longer be dominated by the COPY, and will eventually be dropped.
+ /// This is easily rectified by forwarding the non-dominated debug uses
+ /// to the copy source.
+ void SalvageUnsunkDebugUsersOfCopy(MachineInstr &,
+ MachineBasicBlock *TargetBlock);
bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB,
MachineBasicBlock *DefMBB,
bool &BreakPHIEdge, bool &LocalUse) const;
@@ -367,8 +396,11 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
if (!ProcessedBegin)
--I;
- if (MI.isDebugInstr())
+ if (MI.isDebugInstr()) {
+ if (MI.isDebugValue())
+ ProcessDbgInst(MI);
continue;
+ }
bool Joined = PerformTrivialForwardCoalescing(MI, &MBB);
if (Joined) {
@@ -384,9 +416,29 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) {
// If we just processed the first instruction in the block, we're done.
} while (!ProcessedBegin);
+ SeenDbgUsers.clear();
+ SeenDbgVars.clear();
+
return MadeChange;
}
+void MachineSinking::ProcessDbgInst(MachineInstr &MI) {
+ // When we see DBG_VALUEs for registers, record any vreg it reads, so that
+ // we know what to sink if the vreg def sinks.
+ assert(MI.isDebugValue() && "Expected DBG_VALUE for processing");
+
+ DebugVariable Var(MI.getDebugVariable(), MI.getDebugExpression(),
+ MI.getDebugLoc()->getInlinedAt());
+ bool SeenBefore = SeenDbgVars.count(Var) != 0;
+
+ MachineOperand &MO = MI.getOperand(0);
+ if (MO.isReg() && MO.getReg().isVirtual())
+ SeenDbgUsers[MO.getReg()].push_back(SeenDbgUser(&MI, SeenBefore));
+
+ // Record the variable for any DBG_VALUE, to avoid re-ordering any of them.
+ SeenDbgVars.insert(Var);
+}
+
bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr &MI,
MachineBasicBlock *From,
MachineBasicBlock *To) {
@@ -731,18 +783,60 @@ static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI,
MBP.LHS.getReg() == BaseOp->getReg();
}
-/// Sink an instruction and its associated debug instructions. If the debug
-/// instructions to be sunk are already known, they can be provided in DbgVals.
+/// If the sunk instruction is a copy, try to forward the copy instead of
+/// leaving an 'undef' DBG_VALUE in the original location. Don't do this if
+/// there's any subregister weirdness involved. Returns true if copy
+/// propagation occurred.
+static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI) {
+ const MachineRegisterInfo &MRI = SinkInst.getMF()->getRegInfo();
+ const TargetInstrInfo &TII = *SinkInst.getMF()->getSubtarget().getInstrInfo();
+
+ // Copy DBG_VALUE operand and set the original to undef. We then check to
+ // see whether this is something that can be copy-forwarded. If it isn't,
+ // continue around the loop.
+ MachineOperand DbgMO = DbgMI.getOperand(0);
+
+ const MachineOperand *SrcMO = nullptr, *DstMO = nullptr;
+ auto CopyOperands = TII.isCopyInstr(SinkInst);
+ if (!CopyOperands)
+ return false;
+ SrcMO = CopyOperands->Source;
+ DstMO = CopyOperands->Destination;
+
+ // Check validity of forwarding this copy.
+ bool PostRA = MRI.getNumVirtRegs() == 0;
+
+ // Trying to forward between physical and virtual registers is too hard.
+ if (DbgMO.getReg().isVirtual() != SrcMO->getReg().isVirtual())
+ return false;
+
+ // Only try virtual register copy-forwarding before regalloc, and physical
+ // register copy-forwarding after regalloc.
+ bool arePhysRegs = !DbgMO.getReg().isVirtual();
+ if (arePhysRegs != PostRA)
+ return false;
+
+ // Pre-regalloc, only forward if all subregisters agree (or there are no
+ // subregs at all). More analysis might recover some forwardable copies.
+ if (!PostRA && (DbgMO.getSubReg() != SrcMO->getSubReg() ||
+ DbgMO.getSubReg() != DstMO->getSubReg()))
+ return false;
+
+ // Post-regalloc, we may be sinking a DBG_VALUE of a sub or super-register
+ // of this copy. Only forward the copy if the DBG_VALUE operand exactly
+ // matches the copy destination.
+ if (PostRA && DbgMO.getReg() != DstMO->getReg())
+ return false;
+
+ DbgMI.getOperand(0).setReg(SrcMO->getReg());
+ DbgMI.getOperand(0).setSubReg(SrcMO->getSubReg());
+ return true;
+}
+
+/// Sink an instruction and its associated debug instructions.
static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
MachineBasicBlock::iterator InsertPos,
- SmallVectorImpl<MachineInstr *> *DbgVals = nullptr) {
- // If debug values are provided use those, otherwise call collectDebugValues.
- SmallVector<MachineInstr *, 2> DbgValuesToSink;
- if (DbgVals)
- DbgValuesToSink.insert(DbgValuesToSink.begin(),
- DbgVals->begin(), DbgVals->end());
- else
- MI.collectDebugValues(DbgValuesToSink);
+ SmallVectorImpl<MachineInstr *> &DbgValuesToSink) {
// If we cannot find a location to use (merge with), then we erase the debug
// location to prevent debug-info driven tools from potentially reporting
@@ -758,13 +852,19 @@ static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo,
SuccToSinkTo.splice(InsertPos, ParentBlock, MI,
++MachineBasicBlock::iterator(MI));
- // Move previously adjacent debug value instructions to the insert position.
+ // Sink a copy of debug users to the insert position. Mark the original
+ // DBG_VALUE location as 'undef', indicating that any earlier variable
+ // location should be terminated as we've optimised away the value at this
+ // point.
for (SmallVectorImpl<MachineInstr *>::iterator DBI = DbgValuesToSink.begin(),
DBE = DbgValuesToSink.end();
DBI != DBE; ++DBI) {
MachineInstr *DbgMI = *DBI;
- SuccToSinkTo.splice(InsertPos, ParentBlock, DbgMI,
- ++MachineBasicBlock::iterator(DbgMI));
+ MachineInstr *NewDbgMI = DbgMI->getMF()->CloneMachineInstr(*DBI);
+ SuccToSinkTo.insert(InsertPos, NewDbgMI);
+
+ if (!attemptDebugCopyProp(MI, *DbgMI))
+ DbgMI->getOperand(0).setReg(0);
}
}
@@ -882,7 +982,36 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
++InsertPos;
- performSink(MI, *SuccToSinkTo, InsertPos);
+ // Collect debug users of any vreg that this inst defines.
+ SmallVector<MachineInstr *, 4> DbgUsersToSink;
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
+ continue;
+ if (!SeenDbgUsers.count(MO.getReg()))
+ continue;
+
+ // Sink any users that don't pass any other DBG_VALUEs for this variable.
+ auto &Users = SeenDbgUsers[MO.getReg()];
+ for (auto &User : Users) {
+ MachineInstr *DbgMI = User.getPointer();
+ if (User.getInt()) {
+ // This DBG_VALUE would re-order assignments. If we can't copy-propagate
+ // it, it can't be recovered. Set it undef.
+ if (!attemptDebugCopyProp(MI, *DbgMI))
+ DbgMI->getOperand(0).setReg(0);
+ } else {
+ DbgUsersToSink.push_back(DbgMI);
+ }
+ }
+ }
+
+ // After sinking, some debug users may not be dominated any more. If possible,
+ // copy-propagate their operands. As it's expensive, don't do this if there's
+ // no debuginfo in the program.
+ if (MI.getMF()->getFunction().getSubprogram() && MI.isCopy())
+ SalvageUnsunkDebugUsersOfCopy(MI, SuccToSinkTo);
+
+ performSink(MI, *SuccToSinkTo, InsertPos, DbgUsersToSink);
// Conservatively, clear any kill flags, since it's possible that they are no
// longer correct.
@@ -897,6 +1026,41 @@ bool MachineSinking::SinkInstruction(MachineInstr &MI, bool &SawStore,
return true;
}
+void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
+ MachineInstr &MI, MachineBasicBlock *TargetBlock) {
+ assert(MI.isCopy());
+ assert(MI.getOperand(1).isReg());
+
+ // Enumerate all users of vreg operands that are def'd. Skip those that will
+ // be sunk. For the rest, if they are not dominated by the block we will sink
+ // MI into, propagate the copy source to them.
+ SmallVector<MachineInstr *, 4> DbgDefUsers;
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
+ continue;
+ for (auto &User : MRI.use_instructions(MO.getReg())) {
+ if (!User.isDebugValue() || DT->dominates(TargetBlock, User.getParent()))
+ continue;
+
+ // If is in same block, will either sink or be use-before-def.
+ if (User.getParent() == MI.getParent())
+ continue;
+
+ assert(User.getOperand(0).isReg() &&
+ "DBG_VALUE user of vreg, but non reg operand?");
+ DbgDefUsers.push_back(&User);
+ }
+ }
+
+ // Point the users of this copy that are no longer dominated, at the source
+ // of the copy.
+ for (auto *User : DbgDefUsers) {
+ User->getOperand(0).setReg(MI.getOperand(1).getReg());
+ User->getOperand(0).setSubReg(MI.getOperand(1).getSubReg());
+ }
+}
+
//===----------------------------------------------------------------------===//
// This pass is not intended to be a replacement or a complete alternative
// for the pre-ra machine sink pass. It is only designed to sink COPY
@@ -1051,10 +1215,14 @@ static void updateLiveIn(MachineInstr *MI, MachineBasicBlock *SuccBB,
for (MCSubRegIterator S(DefReg, TRI, true); S.isValid(); ++S)
SuccBB->removeLiveIn(*S);
for (auto U : UsedOpsInCopy) {
- Register Reg = MI->getOperand(U).getReg();
- if (!SuccBB->isLiveIn(Reg))
- SuccBB->addLiveIn(Reg);
+ Register SrcReg = MI->getOperand(U).getReg();
+ LaneBitmask Mask;
+ for (MCRegUnitMaskIterator S(SrcReg, TRI); S.isValid(); ++S) {
+ Mask |= (*S).second;
+ }
+ SuccBB->addLiveIn(SrcReg, Mask.any() ? Mask : LaneBitmask::getAll());
}
+ SuccBB->sortUniqueLiveIns();
}
static bool hasRegisterDependency(MachineInstr *MI,
@@ -1206,7 +1374,7 @@ bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
// block.
clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI);
MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI();
- performSink(*MI, *SuccBB, InsertPos, &DbgValsToSink);
+ performSink(*MI, *SuccBB, InsertPos, DbgValsToSink);
updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy);
Changed = true;
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-28 06:25:12 +0000