diff options
Diffstat (limited to 'contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp')
-rw-r--r-- | contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp | 88 |
1 files changed, 73 insertions, 15 deletions
diff --git a/contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp index fa4d24a2f25a..5f182c5304c6 100644 --- a/contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp +++ b/contrib/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp @@ -14,6 +14,7 @@ #include "AMDGPU.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/Analysis/DivergenceAnalysis.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" @@ -34,17 +35,16 @@ typedef std::pair<BasicBlock *, Value *> StackEntry; typedef SmallVector<StackEntry, 16> StackVector; // Intrinsic names the control flow is annotated with -static const char *const IfIntrinsic = "llvm.SI.if"; -static const char *const ElseIntrinsic = "llvm.SI.else"; -static const char *const BreakIntrinsic = "llvm.SI.break"; -static const char *const IfBreakIntrinsic = "llvm.SI.if.break"; -static const char *const ElseBreakIntrinsic = "llvm.SI.else.break"; -static const char *const LoopIntrinsic = "llvm.SI.loop"; -static const char *const EndCfIntrinsic = "llvm.SI.end.cf"; +static const char *const IfIntrinsic = "llvm.amdgcn.if"; +static const char *const ElseIntrinsic = "llvm.amdgcn.else"; +static const char *const BreakIntrinsic = "llvm.amdgcn.break"; +static const char *const IfBreakIntrinsic = "llvm.amdgcn.if.break"; +static const char *const ElseBreakIntrinsic = "llvm.amdgcn.else.break"; +static const char *const LoopIntrinsic = "llvm.amdgcn.loop"; +static const char *const EndCfIntrinsic = "llvm.amdgcn.end.cf"; class SIAnnotateControlFlow : public FunctionPass { - - static char ID; + DivergenceAnalysis *DA; Type *Boolean; Type *Void; @@ -69,6 +69,8 @@ class SIAnnotateControlFlow : public FunctionPass { LoopInfo *LI; + bool isUniform(BranchInst *T); + bool isTopOfStack(BasicBlock *BB); Value *popSaved(); @@ -83,13 +85,16 @@ class SIAnnotateControlFlow : public FunctionPass { void insertElse(BranchInst *Term); - Value *handleLoopCondition(Value *Cond, PHINode *Broken, llvm::Loop *L); + Value *handleLoopCondition(Value *Cond, PHINode *Broken, + llvm::Loop *L, BranchInst *Term); void handleLoop(BranchInst *Term); void closeControlFlow(BasicBlock *BB); public: + static char ID; + SIAnnotateControlFlow(): FunctionPass(ID) { } @@ -104,6 +109,7 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<LoopInfoWrapperPass>(); AU.addRequired<DominatorTreeWrapperPass>(); + AU.addRequired<DivergenceAnalysis>(); AU.addPreserved<DominatorTreeWrapperPass>(); FunctionPass::getAnalysisUsage(AU); } @@ -112,6 +118,12 @@ public: } // end anonymous namespace +INITIALIZE_PASS_BEGIN(SIAnnotateControlFlow, DEBUG_TYPE, + "Annotate SI Control Flow", false, false) +INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis) +INITIALIZE_PASS_END(SIAnnotateControlFlow, DEBUG_TYPE, + "Annotate SI Control Flow", false, false) + char SIAnnotateControlFlow::ID = 0; /// \brief Initialize all the types and constants used in the pass @@ -152,6 +164,13 @@ bool SIAnnotateControlFlow::doInitialization(Module &M) { return false; } +/// \brief Is the branch condition uniform or did the StructurizeCFG pass +/// consider it as such? +bool SIAnnotateControlFlow::isUniform(BranchInst *T) { + return DA->isUniform(T->getCondition()) || + T->getMetadata("structurizecfg.uniform") != nullptr; +} + /// \brief Is BB the last block saved on the stack ? bool SIAnnotateControlFlow::isTopOfStack(BasicBlock *BB) { return !Stack.empty() && Stack.back().first == BB; @@ -194,6 +213,9 @@ void SIAnnotateControlFlow::eraseIfUnused(PHINode *Phi) { /// \brief Open a new "If" block void SIAnnotateControlFlow::openIf(BranchInst *Term) { + if (isUniform(Term)) { + return; + } Value *Ret = CallInst::Create(If, Term->getCondition(), "", Term); Term->setCondition(ExtractValueInst::Create(Ret, 0, "", Term)); push(Term->getSuccessor(1), ExtractValueInst::Create(Ret, 1, "", Term)); @@ -201,6 +223,9 @@ void SIAnnotateControlFlow::openIf(BranchInst *Term) { /// \brief Close the last "If" block and open a new "Else" block void SIAnnotateControlFlow::insertElse(BranchInst *Term) { + if (isUniform(Term)) { + return; + } Value *Ret = CallInst::Create(Else, popSaved(), "", Term); Term->setCondition(ExtractValueInst::Create(Ret, 0, "", Term)); push(Term->getSuccessor(1), ExtractValueInst::Create(Ret, 1, "", Term)); @@ -208,7 +233,7 @@ void SIAnnotateControlFlow::insertElse(BranchInst *Term) { /// \brief Recursively handle the condition leading to a loop Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken, - llvm::Loop *L) { + llvm::Loop *L, BranchInst *Term) { // Only search through PHI nodes which are inside the loop. If we try this // with PHI nodes that are outside of the loop, we end up inserting new PHI @@ -232,7 +257,7 @@ Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken, } Phi->setIncomingValue(i, BoolFalse); - Value *PhiArg = handleLoopCondition(Incoming, Broken, L); + Value *PhiArg = handleLoopCondition(Incoming, Broken, L, Term); NewPhi->addIncoming(PhiArg, From); } @@ -246,7 +271,23 @@ Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken, BasicBlock *From = Phi->getIncomingBlock(i); if (From == IDom) { + // We're in the following situation: + // IDom/From + // | \ + // | If-block + // | / + // Parent + // where we want to break out of the loop if the If-block is not taken. + // Due to the depth-first traversal, there should be an end.cf + // intrinsic in Parent, and we insert an else.break before it. + // + // Note that the end.cf need not be the first non-phi instruction + // of parent, particularly when we're dealing with a multi-level + // break, but it should occur within a group of intrinsic calls + // at the beginning of the block. CallInst *OldEnd = dyn_cast<CallInst>(Parent->getFirstInsertionPt()); + while (OldEnd && OldEnd->getCalledFunction() != EndCf) + OldEnd = dyn_cast<CallInst>(OldEnd->getNextNode()); if (OldEnd && OldEnd->getCalledFunction() == EndCf) { Value *Args[] = { OldEnd->getArgOperand(0), NewPhi }; Ret = CallInst::Create(ElseBreak, Args, "", OldEnd); @@ -271,14 +312,23 @@ Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken, Value *Args[] = { Cond, Broken }; return CallInst::Create(IfBreak, Args, "", Insert); + // Insert IfBreak before TERM for constant COND. + } else if (isa<ConstantInt>(Cond)) { + Value *Args[] = { Cond, Broken }; + return CallInst::Create(IfBreak, Args, "", Term); + } else { llvm_unreachable("Unhandled loop condition!"); } - return 0; + return nullptr; } /// \brief Handle a back edge (loop) void SIAnnotateControlFlow::handleLoop(BranchInst *Term) { + if (isUniform(Term)) { + return; + } + BasicBlock *BB = Term->getParent(); llvm::Loop *L = LI->getLoopFor(BB); BasicBlock *Target = Term->getSuccessor(1); @@ -286,7 +336,7 @@ void SIAnnotateControlFlow::handleLoop(BranchInst *Term) { Value *Cond = Term->getCondition(); Term->setCondition(BoolTrue); - Value *Arg = handleLoopCondition(Cond, Broken, L); + Value *Arg = handleLoopCondition(Cond, Broken, L, Term); for (pred_iterator PI = pred_begin(Target), PE = pred_end(Target); PI != PE; ++PI) { @@ -300,6 +350,8 @@ void SIAnnotateControlFlow::handleLoop(BranchInst *Term) { void SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) { llvm::Loop *L = LI->getLoopFor(BB); + assert(Stack.back().first == BB); + if (L && L->getHeader() == BB) { // We can't insert an EndCF call into a loop header, because it will // get executed on every iteration of the loop, when it should be @@ -315,14 +367,18 @@ void SIAnnotateControlFlow::closeControlFlow(BasicBlock *BB) { BB = llvm::SplitBlockPredecessors(BB, Preds, "endcf.split", DT, LI, false); } - CallInst::Create(EndCf, popSaved(), "", &*BB->getFirstInsertionPt()); + Value *Exec = popSaved(); + if (!isa<UndefValue>(Exec)) + CallInst::Create(EndCf, Exec, "", &*BB->getFirstInsertionPt()); } /// \brief Annotate the control flow with intrinsics so the backend can /// recognize if/then/else and loops. bool SIAnnotateControlFlow::runOnFunction(Function &F) { + DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); + DA = &getAnalysis<DivergenceAnalysis>(); for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()), E = df_end(&F.getEntryBlock()); I != E; ++I) { @@ -332,12 +388,14 @@ bool SIAnnotateControlFlow::runOnFunction(Function &F) { if (!Term || Term->isUnconditional()) { if (isTopOfStack(*I)) closeControlFlow(*I); + continue; } if (I.nodeVisited(Term->getSuccessor(1))) { if (isTopOfStack(*I)) closeControlFlow(*I); + handleLoop(Term); continue; } |