diff options
Diffstat (limited to 'llvm/lib/CodeGen')
-rw-r--r-- | llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp | 34 | ||||
-rw-r--r-- | llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h | 3 | ||||
-rw-r--r-- | llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 8 | ||||
-rw-r--r-- | llvm/lib/CodeGen/MachineFunction.cpp | 9 | ||||
-rw-r--r-- | llvm/lib/CodeGen/MachineInstr.cpp | 142 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 7 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp | 8 | ||||
-rw-r--r-- | llvm/lib/CodeGen/StackProtector.cpp | 74 |
9 files changed, 186 insertions, 101 deletions
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 932959c311fa..8fc9e980b5f1 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1127,15 +1127,9 @@ void CodeViewDebug::emitDebugInfoForFunction(const Function *GV, } for (auto HeapAllocSite : FI.HeapAllocSites) { - MCSymbol *BeginLabel = std::get<0>(HeapAllocSite); - MCSymbol *EndLabel = std::get<1>(HeapAllocSite); - - // The labels might not be defined if the instruction was replaced - // somewhere in the codegen pipeline. - if (!BeginLabel->isDefined() || !EndLabel->isDefined()) - continue; - - DIType *DITy = std::get<2>(HeapAllocSite); + const MCSymbol *BeginLabel = std::get<0>(HeapAllocSite); + const MCSymbol *EndLabel = std::get<1>(HeapAllocSite); + const DIType *DITy = std::get<2>(HeapAllocSite); MCSymbol *HeapAllocEnd = beginSymbolRecord(SymbolKind::S_HEAPALLOCSITE); OS.AddComment("Call site offset"); OS.EmitCOFFSecRel32(BeginLabel, /*Offset=*/0); @@ -1454,6 +1448,16 @@ void CodeViewDebug::beginFunctionImpl(const MachineFunction *MF) { DebugLoc FnStartDL = PrologEndLoc.getFnDebugLoc(); maybeRecordLocation(FnStartDL, MF); } + + // Find heap alloc sites and emit labels around them. + for (const auto &MBB : *MF) { + for (const auto &MI : MBB) { + if (MI.getHeapAllocMarker()) { + requestLabelBeforeInsn(&MI); + requestLabelAfterInsn(&MI); + } + } + } } static bool shouldEmitUdt(const DIType *T) { @@ -2888,8 +2892,18 @@ void CodeViewDebug::endFunctionImpl(const MachineFunction *MF) { return; } + // Find heap alloc sites and add to list. + for (const auto &MBB : *MF) { + for (const auto &MI : MBB) { + if (MDNode *MD = MI.getHeapAllocMarker()) { + CurFn->HeapAllocSites.push_back(std::make_tuple(getLabelBeforeInsn(&MI), + getLabelAfterInsn(&MI), + dyn_cast<DIType>(MD))); + } + } + } + CurFn->Annotations = MF->getCodeViewAnnotations(); - CurFn->HeapAllocSites = MF->getCodeViewHeapAllocSites(); CurFn->End = Asm->getFunctionEnd(); diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h index ce57b789d7fa..b56b9047e1a9 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h @@ -148,7 +148,8 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { SmallVector<LexicalBlock *, 1> ChildBlocks; std::vector<std::pair<MCSymbol *, MDNode *>> Annotations; - std::vector<std::tuple<MCSymbol *, MCSymbol *, DIType *>> HeapAllocSites; + std::vector<std::tuple<const MCSymbol *, const MCSymbol *, const DIType *>> + HeapAllocSites; const MCSymbol *Begin = nullptr; const MCSymbol *End = nullptr; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 6e99bdbd8264..16ac45398d06 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -588,8 +588,8 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB, Register CondRHS = getOrCreateVReg(*CB.CmpRHS); Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0); } else { - assert(CB.PredInfo.Pred == CmpInst::ICMP_ULE && - "Can only handle ULE ranges"); + assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE && + "Can only handle SLE ranges"); const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); @@ -598,7 +598,7 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB, if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { Register CondRHS = getOrCreateVReg(*CB.CmpRHS); Cond = - MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0); + MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0); } else { const LLT &CmpTy = MRI->getType(CmpOpReg); auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS); @@ -728,7 +728,7 @@ bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, MHS = nullptr; } else { // Check I->Low <= Cond <= I->High. - Pred = CmpInst::ICMP_ULE; + Pred = CmpInst::ICMP_SLE; LHS = I->Low; MHS = Cond; RHS = I->High; diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index 4df5ce2dcedc..faae944f5913 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -451,7 +451,14 @@ MachineFunction::createMIExtraInfo(ArrayRef<MachineMemOperand *> MMOs, MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol) { return MachineInstr::ExtraInfo::create(Allocator, MMOs, PreInstrSymbol, - PostInstrSymbol); + PostInstrSymbol, nullptr); +} + +MachineInstr::ExtraInfo *MachineFunction::createMIExtraInfoWithMarker( + ArrayRef<MachineMemOperand *> MMOs, MCSymbol *PreInstrSymbol, + MCSymbol *PostInstrSymbol, MDNode *HeapAllocMarker) { + return MachineInstr::ExtraInfo::create(Allocator, MMOs, PreInstrSymbol, + PostInstrSymbol, HeapAllocMarker); } const char *MachineFunction::createExternalSymbolName(StringRef Name) { diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index e5c398a2d10c..d81f5bbf78c5 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -316,27 +316,48 @@ void MachineInstr::RemoveOperand(unsigned OpNo) { --NumOperands; } -void MachineInstr::dropMemRefs(MachineFunction &MF) { - if (memoperands_empty()) - return; - - // See if we can just drop all of our extra info. - if (!getPreInstrSymbol() && !getPostInstrSymbol()) { +void MachineInstr::setExtraInfo(MachineFunction &MF, + ArrayRef<MachineMemOperand *> MMOs, + MCSymbol *PreInstrSymbol, + MCSymbol *PostInstrSymbol, + MDNode *HeapAllocMarker) { + bool HasPreInstrSymbol = PreInstrSymbol != nullptr; + bool HasPostInstrSymbol = PostInstrSymbol != nullptr; + bool HasHeapAllocMarker = HeapAllocMarker != nullptr; + int NumPointers = + MMOs.size() + HasPreInstrSymbol + HasPostInstrSymbol + HasHeapAllocMarker; + + // Drop all extra info if there is none. + if (NumPointers <= 0) { Info.clear(); return; } - if (!getPostInstrSymbol()) { - Info.set<EIIK_PreInstrSymbol>(getPreInstrSymbol()); + + // If more than one pointer, then store out of line. Store heap alloc markers + // out of line because PointerSumType cannot hold more than 4 tag types with + // 32-bit pointers. + // FIXME: Maybe we should make the symbols in the extra info mutable? + else if (NumPointers > 1 || HasHeapAllocMarker) { + Info.set<EIIK_OutOfLine>(MF.createMIExtraInfoWithMarker( + MMOs, PreInstrSymbol, PostInstrSymbol, HeapAllocMarker)); return; } - if (!getPreInstrSymbol()) { - Info.set<EIIK_PostInstrSymbol>(getPostInstrSymbol()); + + // Otherwise store the single pointer inline. + if (HasPreInstrSymbol) + Info.set<EIIK_PreInstrSymbol>(PreInstrSymbol); + else if (HasPostInstrSymbol) + Info.set<EIIK_PostInstrSymbol>(PostInstrSymbol); + else + Info.set<EIIK_MMO>(MMOs[0]); +} + +void MachineInstr::dropMemRefs(MachineFunction &MF) { + if (memoperands_empty()) return; - } - // Otherwise allocate a fresh extra info with just these symbols. - Info.set<EIIK_OutOfLine>( - MF.createMIExtraInfo({}, getPreInstrSymbol(), getPostInstrSymbol())); + setExtraInfo(MF, {}, getPreInstrSymbol(), getPostInstrSymbol(), + getHeapAllocMarker()); } void MachineInstr::setMemRefs(MachineFunction &MF, @@ -346,15 +367,8 @@ void MachineInstr::setMemRefs(MachineFunction &MF, return; } - // Try to store a single MMO inline. - if (MMOs.size() == 1 && !getPreInstrSymbol() && !getPostInstrSymbol()) { - Info.set<EIIK_MMO>(MMOs[0]); - return; - } - - // Otherwise create an extra info struct with all of our info. - Info.set<EIIK_OutOfLine>( - MF.createMIExtraInfo(MMOs, getPreInstrSymbol(), getPostInstrSymbol())); + setExtraInfo(MF, MMOs, getPreInstrSymbol(), getPostInstrSymbol(), + getHeapAllocMarker()); } void MachineInstr::addMemOperand(MachineFunction &MF, @@ -376,7 +390,8 @@ void MachineInstr::cloneMemRefs(MachineFunction &MF, const MachineInstr &MI) { // instruction. We can do this whenever the pre- and post-instruction symbols // are the same (including null). if (getPreInstrSymbol() == MI.getPreInstrSymbol() && - getPostInstrSymbol() == MI.getPostInstrSymbol()) { + getPostInstrSymbol() == MI.getPostInstrSymbol() && + getHeapAllocMarker() == MI.getHeapAllocMarker()) { Info = MI.Info; return; } @@ -450,67 +465,42 @@ void MachineInstr::cloneMergedMemRefs(MachineFunction &MF, } void MachineInstr::setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { - MCSymbol *OldSymbol = getPreInstrSymbol(); - if (OldSymbol == Symbol) + // Do nothing if old and new symbols are the same. + if (Symbol == getPreInstrSymbol()) return; - if (OldSymbol && !Symbol) { - // We're removing a symbol rather than adding one. Try to clean up any - // extra info carried around. - if (Info.is<EIIK_PreInstrSymbol>()) { - Info.clear(); - return; - } - if (memoperands_empty()) { - assert(getPostInstrSymbol() && - "Should never have only a single symbol allocated out-of-line!"); - Info.set<EIIK_PostInstrSymbol>(getPostInstrSymbol()); - return; - } - - // Otherwise fallback on the generic update. - } else if (!Info || Info.is<EIIK_PreInstrSymbol>()) { - // If we don't have any other extra info, we can store this inline. - Info.set<EIIK_PreInstrSymbol>(Symbol); + // If there was only one symbol and we're removing it, just clear info. + if (!Symbol && Info.is<EIIK_PreInstrSymbol>()) { + Info.clear(); return; } - // Otherwise, allocate a full new set of extra info. - // FIXME: Maybe we should make the symbols in the extra info mutable? - Info.set<EIIK_OutOfLine>( - MF.createMIExtraInfo(memoperands(), Symbol, getPostInstrSymbol())); + setExtraInfo(MF, memoperands(), Symbol, getPostInstrSymbol(), + getHeapAllocMarker()); } void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { - MCSymbol *OldSymbol = getPostInstrSymbol(); - if (OldSymbol == Symbol) + // Do nothing if old and new symbols are the same. + if (Symbol == getPostInstrSymbol()) return; - if (OldSymbol && !Symbol) { - // We're removing a symbol rather than adding one. Try to clean up any - // extra info carried around. - if (Info.is<EIIK_PostInstrSymbol>()) { - Info.clear(); - return; - } - if (memoperands_empty()) { - assert(getPreInstrSymbol() && - "Should never have only a single symbol allocated out-of-line!"); - Info.set<EIIK_PreInstrSymbol>(getPreInstrSymbol()); - return; - } - - // Otherwise fallback on the generic update. - } else if (!Info || Info.is<EIIK_PostInstrSymbol>()) { - // If we don't have any other extra info, we can store this inline. - Info.set<EIIK_PostInstrSymbol>(Symbol); + // If there was only one symbol and we're removing it, just clear info. + if (!Symbol && Info.is<EIIK_PostInstrSymbol>()) { + Info.clear(); return; } - // Otherwise, allocate a full new set of extra info. - // FIXME: Maybe we should make the symbols in the extra info mutable? - Info.set<EIIK_OutOfLine>( - MF.createMIExtraInfo(memoperands(), getPreInstrSymbol(), Symbol)); + setExtraInfo(MF, memoperands(), getPreInstrSymbol(), Symbol, + getHeapAllocMarker()); +} + +void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) { + // Do nothing if old and new symbols are the same. + if (Marker == getHeapAllocMarker()) + return; + + setExtraInfo(MF, memoperands(), getPreInstrSymbol(), getPostInstrSymbol(), + Marker); } void MachineInstr::cloneInstrSymbols(MachineFunction &MF, @@ -524,6 +514,7 @@ void MachineInstr::cloneInstrSymbols(MachineFunction &MF, setPreInstrSymbol(MF, MI.getPreInstrSymbol()); setPostInstrSymbol(MF, MI.getPostInstrSymbol()); + setHeapAllocMarker(MF, MI.getHeapAllocMarker()); } uint16_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const { @@ -1707,6 +1698,13 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << " post-instr-symbol "; MachineOperand::printSymbol(OS, *PostInstrSymbol); } + if (MDNode *HeapAllocMarker = getHeapAllocMarker()) { + if (!FirstOp) { + FirstOp = false; + OS << ','; + } + OS << " heap-alloc-marker"; + } if (!SkipDebugLoc) { if (const DebugLoc &DL = getDebugLoc()) { diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 49c922f560fa..2530beabac81 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -18878,7 +18878,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { // build_vector. if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) { int SplatIndex = SVN->getSplatIndex(); - if (TLI.isExtractVecEltCheap(VT, SplatIndex) && + if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) && TLI.isBinOp(N0.getOpcode()) && N0.getNode()->getNumValues() == 1) { // splat (vector_bo L, R), Index --> // splat (scalar_bo (extelt L, Index), (extelt R, Index)) diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 22c23ba877e8..5ac3606dc662 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1237,10 +1237,9 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) { updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs); // Set labels for heapallocsite call. - if (CLI.CS && CLI.CS->getInstruction()->getMetadata("heapallocsite")) { - MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite"); - MF->addCodeViewHeapAllocSite(CLI.Call, MD); - } + if (CLI.CS) + if (MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite")) + CLI.Call->setHeapAllocMarker(*MF, MD); return true; } diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index e09f2e760f55..25e451d88992 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -910,10 +910,9 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { if (HasDbg) ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen, NewInsn); - if (MDNode *MD = DAG->getHeapAllocSite(N)) { + if (MDNode *MD = DAG->getHeapAllocSite(N)) if (NewInsn && NewInsn->isCall()) - MF.addCodeViewHeapAllocSite(NewInsn, MD); - } + NewInsn->setHeapAllocMarker(MF, MD); GluedNodes.pop_back(); } @@ -923,9 +922,10 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { if (HasDbg) ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders, Seen, NewInsn); + if (MDNode *MD = DAG->getHeapAllocSite(SU->getNode())) { if (NewInsn && NewInsn->isCall()) - MF.addCodeViewHeapAllocSite(NewInsn, MD); + NewInsn->setHeapAllocMarker(MF, MD); } } diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp index 809960c7fdf9..baa57e2c6311 100644 --- a/llvm/lib/CodeGen/StackProtector.cpp +++ b/llvm/lib/CodeGen/StackProtector.cpp @@ -17,7 +17,6 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/Passes.h" @@ -157,6 +156,69 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge, return NeedsProtector; } +bool StackProtector::HasAddressTaken(const Instruction *AI, + SmallPtrSetImpl<const PHINode *> &VisitedPHIs) { + for (const User *U : AI->users()) { + const auto *I = cast<Instruction>(U); + switch (I->getOpcode()) { + case Instruction::Store: + if (AI == cast<StoreInst>(I)->getValueOperand()) + return true; + break; + case Instruction::AtomicCmpXchg: + // cmpxchg conceptually includes both a load and store from the same + // location. So, like store, the value being stored is what matters. + if (AI == cast<AtomicCmpXchgInst>(I)->getNewValOperand()) + return true; + break; + case Instruction::PtrToInt: + if (AI == cast<PtrToIntInst>(I)->getOperand(0)) + return true; + break; + case Instruction::Call: { + // Ignore intrinsics that do not become real instructions. + // TODO: Narrow this to intrinsics that have store-like effects. + const auto *CI = cast<CallInst>(I); + if (!isa<DbgInfoIntrinsic>(CI) && !CI->isLifetimeStartOrEnd()) + return true; + break; + } + case Instruction::Invoke: + return true; + case Instruction::BitCast: + case Instruction::GetElementPtr: + case Instruction::Select: + case Instruction::AddrSpaceCast: + if (HasAddressTaken(I, VisitedPHIs)) + return true; + break; + case Instruction::PHI: { + // Keep track of what PHI nodes we have already visited to ensure + // they are only visited once. + const auto *PN = cast<PHINode>(I); + if (VisitedPHIs.insert(PN).second) + if (HasAddressTaken(PN, VisitedPHIs)) + return true; + break; + } + case Instruction::Load: + case Instruction::AtomicRMW: + case Instruction::Ret: + // These instructions take an address operand, but have load-like or + // other innocuous behavior that should not trigger a stack protector. + // atomicrmw conceptually has both load and store semantics, but the + // value being stored must be integer; so if a pointer is being stored, + // we'll catch it in the PtrToInt case above. + break; + default: + // Conservatively return true for any instruction that takes an address + // operand, but is not handled above. + return true; + } + } + return false; +} + /// Search for the first call to the llvm.stackprotector intrinsic and return it /// if present. static const CallInst *findStackProtectorIntrinsic(Function &F) { @@ -211,6 +273,12 @@ bool StackProtector::RequiresStackProtector() { else if (!F->hasFnAttribute(Attribute::StackProtect)) return false; + /// VisitedPHIs - The set of PHI nodes visited when determining + /// if a variable's reference has been taken. This set + /// is maintained to ensure we don't visit the same PHI node multiple + /// times. + SmallPtrSet<const PHINode *, 16> VisitedPHIs; + for (const BasicBlock &BB : *F) { for (const Instruction &I : BB) { if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) { @@ -264,9 +332,7 @@ bool StackProtector::RequiresStackProtector() { continue; } - if (Strong && PointerMayBeCaptured(AI, - /* ReturnCaptures */ false, - /* StoreCaptures */ true)) { + if (Strong && HasAddressTaken(AI, VisitedPHIs)) { ++NumAddrTaken; Layout.insert(std::make_pair(AI, MachineFrameInfo::SSPLK_AddrOf)); ORE.emit([&]() { |