diff options
277 files changed, 9435 insertions, 2819 deletions
diff --git a/include/llvm/ADT/SmallVector.h b/include/llvm/ADT/SmallVector.h index bd24eab93b50..35c255002001 100644 --- a/include/llvm/ADT/SmallVector.h +++ b/include/llvm/ADT/SmallVector.h @@ -415,6 +415,9 @@ public: append(IL.begin(), IL.end()); } + // FIXME: Consider assigning over existing elements, rather than clearing & + // re-initializing them - for all assign(...) variants. + void assign(size_type NumElts, const T &Elt) { clear(); if (this->capacity() < NumElts) @@ -423,6 +426,11 @@ public: std::uninitialized_fill(this->begin(), this->end(), Elt); } + template <typename in_iter> void assign(in_iter in_start, in_iter in_end) { + clear(); + append(in_start, in_end); + } + void assign(std::initializer_list<T> IL) { clear(); append(IL); diff --git a/include/llvm/Analysis/OrderedBasicBlock.h b/include/llvm/Analysis/OrderedBasicBlock.h index 5aa813eb4832..2e716af1f60d 100644 --- a/include/llvm/Analysis/OrderedBasicBlock.h +++ b/include/llvm/Analysis/OrderedBasicBlock.h @@ -58,6 +58,7 @@ public: /// comes before \p B in \p BB. This is a simplification that considers /// cached instruction positions and ignores other basic blocks, being /// only relevant to compare relative instructions positions inside \p BB. + /// Returns false for A == B. bool dominates(const Instruction *A, const Instruction *B); }; diff --git a/include/llvm/Analysis/RegionPass.h b/include/llvm/Analysis/RegionPass.h index b5f38139abf2..515b362e5407 100644 --- a/include/llvm/Analysis/RegionPass.h +++ b/include/llvm/Analysis/RegionPass.h @@ -78,6 +78,11 @@ public: return PMT_RegionPassManager; } //@} + +protected: + /// Optional passes call this function to check whether the pass should be + /// skipped. This is the case when optimization bisect is over the limit. + bool skipRegion(Region &R) const; }; /// @brief The pass manager to schedule RegionPasses. diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index 1d715b590ab7..8ee9712b93d8 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -636,7 +636,7 @@ private: /// @} public: - BackedgeTakenInfo() : MaxAndComplete(nullptr, 0) {} + BackedgeTakenInfo() : MaxAndComplete(nullptr, 0), MaxOrZero(false) {} BackedgeTakenInfo(BackedgeTakenInfo &&) = default; BackedgeTakenInfo &operator=(BackedgeTakenInfo &&) = default; diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h index 21f847c7e5ba..8394b58d0a16 100644 --- a/include/llvm/CodeGen/MachineRegionInfo.h +++ b/include/llvm/CodeGen/MachineRegionInfo.h @@ -10,83 +10,77 @@ #ifndef LLVM_CODEGEN_MACHINEREGIONINFO_H #define LLVM_CODEGEN_MACHINEREGIONINFO_H +#include "llvm/ADT/DepthFirstIterator.h" #include "llvm/Analysis/RegionInfo.h" #include "llvm/Analysis/RegionIterator.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominanceFrontier.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" - +#include <cassert> namespace llvm { -class MachineDominatorTree; struct MachinePostDominatorTree; class MachineRegion; class MachineRegionNode; class MachineRegionInfo; -template<> -struct RegionTraits<MachineFunction> { - typedef MachineFunction FuncT; - typedef MachineBasicBlock BlockT; - typedef MachineRegion RegionT; - typedef MachineRegionNode RegionNodeT; - typedef MachineRegionInfo RegionInfoT; - typedef MachineDominatorTree DomTreeT; - typedef MachineDomTreeNode DomTreeNodeT; - typedef MachinePostDominatorTree PostDomTreeT; - typedef MachineDominanceFrontier DomFrontierT; - typedef MachineInstr InstT; - typedef MachineLoop LoopT; - typedef MachineLoopInfo LoopInfoT; +template <> struct RegionTraits<MachineFunction> { + using FuncT = MachineFunction; + using BlockT = MachineBasicBlock; + using RegionT = MachineRegion; + using RegionNodeT = MachineRegionNode; + using RegionInfoT = MachineRegionInfo; + using DomTreeT = MachineDominatorTree; + using DomTreeNodeT = MachineDomTreeNode; + using PostDomTreeT = MachinePostDominatorTree; + using DomFrontierT = MachineDominanceFrontier; + using InstT = MachineInstr; + using LoopT = MachineLoop; + using LoopInfoT = MachineLoopInfo; static unsigned getNumSuccessors(MachineBasicBlock *BB) { return BB->succ_size(); } }; - class MachineRegionNode : public RegionNodeBase<RegionTraits<MachineFunction>> { public: - inline MachineRegionNode(MachineRegion *Parent, - MachineBasicBlock *Entry, + inline MachineRegionNode(MachineRegion *Parent, MachineBasicBlock *Entry, bool isSubRegion = false) - : RegionNodeBase<RegionTraits<MachineFunction>>(Parent, Entry, isSubRegion) { - - } + : RegionNodeBase<RegionTraits<MachineFunction>>(Parent, Entry, + isSubRegion) {} bool operator==(const MachineRegion &RN) const { - return this == reinterpret_cast<const MachineRegionNode*>(&RN); + return this == reinterpret_cast<const MachineRegionNode *>(&RN); } }; class MachineRegion : public RegionBase<RegionTraits<MachineFunction>> { public: MachineRegion(MachineBasicBlock *Entry, MachineBasicBlock *Exit, - MachineRegionInfo* RI, - MachineDominatorTree *DT, MachineRegion *Parent = nullptr); + MachineRegionInfo *RI, MachineDominatorTree *DT, + MachineRegion *Parent = nullptr); ~MachineRegion(); bool operator==(const MachineRegionNode &RN) const { - return &RN == reinterpret_cast<const MachineRegionNode*>(this); + return &RN == reinterpret_cast<const MachineRegionNode *>(this); } }; class MachineRegionInfo : public RegionInfoBase<RegionTraits<MachineFunction>> { public: explicit MachineRegionInfo(); - ~MachineRegionInfo() override; // updateStatistics - Update statistic about created regions. void updateStatistics(MachineRegion *R) final; - void recalculate(MachineFunction &F, - MachineDominatorTree *DT, - MachinePostDominatorTree *PDT, - MachineDominanceFrontier *DF); + void recalculate(MachineFunction &F, MachineDominatorTree *DT, + MachinePostDominatorTree *PDT, MachineDominanceFrontier *DF); }; class MachineRegionInfoPass : public MachineFunctionPass { @@ -94,17 +88,13 @@ class MachineRegionInfoPass : public MachineFunctionPass { public: static char ID; - explicit MachineRegionInfoPass(); + explicit MachineRegionInfoPass(); ~MachineRegionInfoPass() override; - MachineRegionInfo &getRegionInfo() { - return RI; - } + MachineRegionInfo &getRegionInfo() { return RI; } - const MachineRegionInfo &getRegionInfo() const { - return RI; - } + const MachineRegionInfo &getRegionInfo() const { return RI; } /// @name MachineFunctionPass interface //@{ @@ -117,66 +107,76 @@ public: //@} }; - template <> template <> -inline MachineBasicBlock* RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineBasicBlock>() const { +inline MachineBasicBlock * +RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineBasicBlock>() + const { assert(!isSubRegion() && "This is not a MachineBasicBlock RegionNode!"); return getEntry(); } -template<> -template<> -inline MachineRegion* RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineRegion>() const { +template <> +template <> +inline MachineRegion * +RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineRegion>() + const { assert(isSubRegion() && "This is not a subregion RegionNode!"); - auto Unconst = const_cast<RegionNodeBase<RegionTraits<MachineFunction>>*>(this); - return reinterpret_cast<MachineRegion*>(Unconst); + auto Unconst = + const_cast<RegionNodeBase<RegionTraits<MachineFunction>> *>(this); + return reinterpret_cast<MachineRegion *>(Unconst); } - RegionNodeGraphTraits(MachineRegionNode, MachineBasicBlock, MachineRegion); -RegionNodeGraphTraits(const MachineRegionNode, MachineBasicBlock, MachineRegion); +RegionNodeGraphTraits(const MachineRegionNode, MachineBasicBlock, + MachineRegion); RegionGraphTraits(MachineRegion, MachineRegionNode); RegionGraphTraits(const MachineRegion, const MachineRegionNode); -template <> struct GraphTraits<MachineRegionInfo*> - : public GraphTraits<FlatIt<MachineRegionNode*> > { - typedef df_iterator<NodeRef, df_iterator_default_set<NodeRef>, false, - GraphTraits<FlatIt<NodeRef>>> - nodes_iterator; +template <> +struct GraphTraits<MachineRegionInfo *> + : public GraphTraits<FlatIt<MachineRegionNode *>> { + using nodes_iterator = df_iterator<NodeRef, df_iterator_default_set<NodeRef>, + false, GraphTraits<FlatIt<NodeRef>>>; static NodeRef getEntryNode(MachineRegionInfo *RI) { - return GraphTraits<FlatIt<MachineRegion*> >::getEntryNode(RI->getTopLevelRegion()); + return GraphTraits<FlatIt<MachineRegion *>>::getEntryNode( + RI->getTopLevelRegion()); } - static nodes_iterator nodes_begin(MachineRegionInfo* RI) { + + static nodes_iterator nodes_begin(MachineRegionInfo *RI) { return nodes_iterator::begin(getEntryNode(RI)); } + static nodes_iterator nodes_end(MachineRegionInfo *RI) { return nodes_iterator::end(getEntryNode(RI)); } }; -template <> struct GraphTraits<MachineRegionInfoPass*> - : public GraphTraits<MachineRegionInfo *> { - typedef df_iterator<NodeRef, df_iterator_default_set<NodeRef>, false, - GraphTraits<FlatIt<NodeRef>>> - nodes_iterator; +template <> +struct GraphTraits<MachineRegionInfoPass *> + : public GraphTraits<MachineRegionInfo *> { + using nodes_iterator = df_iterator<NodeRef, df_iterator_default_set<NodeRef>, + false, GraphTraits<FlatIt<NodeRef>>>; static NodeRef getEntryNode(MachineRegionInfoPass *RI) { - return GraphTraits<MachineRegionInfo*>::getEntryNode(&RI->getRegionInfo()); + return GraphTraits<MachineRegionInfo *>::getEntryNode(&RI->getRegionInfo()); } - static nodes_iterator nodes_begin(MachineRegionInfoPass* RI) { - return GraphTraits<MachineRegionInfo*>::nodes_begin(&RI->getRegionInfo()); + + static nodes_iterator nodes_begin(MachineRegionInfoPass *RI) { + return GraphTraits<MachineRegionInfo *>::nodes_begin(&RI->getRegionInfo()); } + static nodes_iterator nodes_end(MachineRegionInfoPass *RI) { - return GraphTraits<MachineRegionInfo*>::nodes_end(&RI->getRegionInfo()); + return GraphTraits<MachineRegionInfo *>::nodes_end(&RI->getRegionInfo()); } }; extern template class RegionBase<RegionTraits<MachineFunction>>; extern template class RegionNodeBase<RegionTraits<MachineFunction>>; extern template class RegionInfoBase<RegionTraits<MachineFunction>>; -} -#endif +} // end namespace llvm + +#endif // LLVM_CODEGEN_MACHINEREGIONINFO_H diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index 1026654da3d7..c027783aae55 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -14,11 +14,13 @@ #ifndef LLVM_CODEGEN_MACHINEREGISTERINFO_H #define LLVM_CODEGEN_MACHINEREGISTERINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/iterator_range.h" #include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -41,8 +43,8 @@ namespace llvm { class PSetIterator; /// Convenient type to represent either a register class or a register bank. -typedef PointerUnion<const TargetRegisterClass *, const RegisterBank *> - RegClassOrRegBank; +using RegClassOrRegBank = + PointerUnion<const TargetRegisterClass *, const RegisterBank *>; /// MachineRegisterInfo - Keep track of information for virtual and physical /// registers, including vreg register classes, use/def chains for registers, @@ -125,7 +127,7 @@ private: /// started. BitVector ReservedRegs; - typedef DenseMap<unsigned, LLT> VRegToTypeMap; + using VRegToTypeMap = DenseMap<unsigned, LLT>; /// Map generic virtual registers to their actual size. mutable std::unique_ptr<VRegToTypeMap> VRegToType; @@ -266,8 +268,8 @@ public: /// reg_iterator/reg_begin/reg_end - Walk all defs and uses of the specified /// register. - typedef defusechain_iterator<true,true,false,true,false,false> - reg_iterator; + using reg_iterator = + defusechain_iterator<true, true, false, true, false, false>; reg_iterator reg_begin(unsigned RegNo) const { return reg_iterator(getRegUseDefListHead(RegNo)); } @@ -279,8 +281,8 @@ public: /// reg_instr_iterator/reg_instr_begin/reg_instr_end - Walk all defs and uses /// of the specified register, stepping by MachineInstr. - typedef defusechain_instr_iterator<true,true,false,false,true,false> - reg_instr_iterator; + using reg_instr_iterator = + defusechain_instr_iterator<true, true, false, false, true, false>; reg_instr_iterator reg_instr_begin(unsigned RegNo) const { return reg_instr_iterator(getRegUseDefListHead(RegNo)); } @@ -295,8 +297,8 @@ public: /// reg_bundle_iterator/reg_bundle_begin/reg_bundle_end - Walk all defs and uses /// of the specified register, stepping by bundle. - typedef defusechain_instr_iterator<true,true,false,false,false,true> - reg_bundle_iterator; + using reg_bundle_iterator = + defusechain_instr_iterator<true, true, false, false, false, true>; reg_bundle_iterator reg_bundle_begin(unsigned RegNo) const { return reg_bundle_iterator(getRegUseDefListHead(RegNo)); } @@ -314,8 +316,8 @@ public: /// reg_nodbg_iterator/reg_nodbg_begin/reg_nodbg_end - Walk all defs and uses /// of the specified register, skipping those marked as Debug. - typedef defusechain_iterator<true,true,true,true,false,false> - reg_nodbg_iterator; + using reg_nodbg_iterator = + defusechain_iterator<true, true, true, true, false, false>; reg_nodbg_iterator reg_nodbg_begin(unsigned RegNo) const { return reg_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -331,8 +333,8 @@ public: /// reg_instr_nodbg_iterator/reg_instr_nodbg_begin/reg_instr_nodbg_end - Walk /// all defs and uses of the specified register, stepping by MachineInstr, /// skipping those marked as Debug. - typedef defusechain_instr_iterator<true,true,true,false,true,false> - reg_instr_nodbg_iterator; + using reg_instr_nodbg_iterator = + defusechain_instr_iterator<true, true, true, false, true, false>; reg_instr_nodbg_iterator reg_instr_nodbg_begin(unsigned RegNo) const { return reg_instr_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -348,8 +350,8 @@ public: /// reg_bundle_nodbg_iterator/reg_bundle_nodbg_begin/reg_bundle_nodbg_end - Walk /// all defs and uses of the specified register, stepping by bundle, /// skipping those marked as Debug. - typedef defusechain_instr_iterator<true,true,true,false,false,true> - reg_bundle_nodbg_iterator; + using reg_bundle_nodbg_iterator = + defusechain_instr_iterator<true, true, true, false, false, true>; reg_bundle_nodbg_iterator reg_bundle_nodbg_begin(unsigned RegNo) const { return reg_bundle_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -369,8 +371,8 @@ public: } /// def_iterator/def_begin/def_end - Walk all defs of the specified register. - typedef defusechain_iterator<false,true,false,true,false,false> - def_iterator; + using def_iterator = + defusechain_iterator<false, true, false, true, false, false>; def_iterator def_begin(unsigned RegNo) const { return def_iterator(getRegUseDefListHead(RegNo)); } @@ -382,8 +384,8 @@ public: /// def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the /// specified register, stepping by MachineInst. - typedef defusechain_instr_iterator<false,true,false,false,true,false> - def_instr_iterator; + using def_instr_iterator = + defusechain_instr_iterator<false, true, false, false, true, false>; def_instr_iterator def_instr_begin(unsigned RegNo) const { return def_instr_iterator(getRegUseDefListHead(RegNo)); } @@ -398,8 +400,8 @@ public: /// def_bundle_iterator/def_bundle_begin/def_bundle_end - Walk all defs of the /// specified register, stepping by bundle. - typedef defusechain_instr_iterator<false,true,false,false,false,true> - def_bundle_iterator; + using def_bundle_iterator = + defusechain_instr_iterator<false, true, false, false, false, true>; def_bundle_iterator def_bundle_begin(unsigned RegNo) const { return def_bundle_iterator(getRegUseDefListHead(RegNo)); } @@ -425,8 +427,8 @@ public: } /// use_iterator/use_begin/use_end - Walk all uses of the specified register. - typedef defusechain_iterator<true,false,false,true,false,false> - use_iterator; + using use_iterator = + defusechain_iterator<true, false, false, true, false, false>; use_iterator use_begin(unsigned RegNo) const { return use_iterator(getRegUseDefListHead(RegNo)); } @@ -438,8 +440,8 @@ public: /// use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the /// specified register, stepping by MachineInstr. - typedef defusechain_instr_iterator<true,false,false,false,true,false> - use_instr_iterator; + using use_instr_iterator = + defusechain_instr_iterator<true, false, false, false, true, false>; use_instr_iterator use_instr_begin(unsigned RegNo) const { return use_instr_iterator(getRegUseDefListHead(RegNo)); } @@ -454,8 +456,8 @@ public: /// use_bundle_iterator/use_bundle_begin/use_bundle_end - Walk all uses of the /// specified register, stepping by bundle. - typedef defusechain_instr_iterator<true,false,false,false,false,true> - use_bundle_iterator; + using use_bundle_iterator = + defusechain_instr_iterator<true, false, false, false, false, true>; use_bundle_iterator use_bundle_begin(unsigned RegNo) const { return use_bundle_iterator(getRegUseDefListHead(RegNo)); } @@ -482,8 +484,8 @@ public: /// use_nodbg_iterator/use_nodbg_begin/use_nodbg_end - Walk all uses of the /// specified register, skipping those marked as Debug. - typedef defusechain_iterator<true,false,true,true,false,false> - use_nodbg_iterator; + using use_nodbg_iterator = + defusechain_iterator<true, false, true, true, false, false>; use_nodbg_iterator use_nodbg_begin(unsigned RegNo) const { return use_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -499,8 +501,8 @@ public: /// use_instr_nodbg_iterator/use_instr_nodbg_begin/use_instr_nodbg_end - Walk /// all uses of the specified register, stepping by MachineInstr, skipping /// those marked as Debug. - typedef defusechain_instr_iterator<true,false,true,false,true,false> - use_instr_nodbg_iterator; + using use_instr_nodbg_iterator = + defusechain_instr_iterator<true, false, true, false, true, false>; use_instr_nodbg_iterator use_instr_nodbg_begin(unsigned RegNo) const { return use_instr_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -516,8 +518,8 @@ public: /// use_bundle_nodbg_iterator/use_bundle_nodbg_begin/use_bundle_nodbg_end - Walk /// all uses of the specified register, stepping by bundle, skipping /// those marked as Debug. - typedef defusechain_instr_iterator<true,false,true,false,false,true> - use_bundle_nodbg_iterator; + using use_bundle_nodbg_iterator = + defusechain_instr_iterator<true, false, true, false, false, true>; use_bundle_nodbg_iterator use_bundle_nodbg_begin(unsigned RegNo) const { return use_bundle_nodbg_iterator(getRegUseDefListHead(RegNo)); } @@ -593,7 +595,6 @@ public: /// Return the register class of the specified virtual register. /// This shouldn't be used directly unless \p Reg has a register class. /// \see getRegClassOrNull when this might happen. - /// const TargetRegisterClass *getRegClass(unsigned Reg) const { assert(VRegInfo[Reg].first.is<const TargetRegisterClass *>() && "Register class not set, wrong accessor"); @@ -620,7 +621,6 @@ public: /// a register bank or has been assigned a register class. /// \note It is possible to get the register bank from the register class via /// RegisterBankInfo::getRegBankFromRegClass. - /// const RegisterBank *getRegBankOrNull(unsigned Reg) const { const RegClassOrRegBank &Val = VRegInfo[Reg].first; return Val.dyn_cast<const RegisterBank *>(); @@ -629,17 +629,14 @@ public: /// Return the register bank or register class of \p Reg. /// \note Before the register bank gets assigned (i.e., before the /// RegBankSelect pass) \p Reg may not have either. - /// const RegClassOrRegBank &getRegClassOrRegBank(unsigned Reg) const { return VRegInfo[Reg].first; } /// setRegClass - Set the register class of the specified virtual register. - /// void setRegClass(unsigned Reg, const TargetRegisterClass *RC); /// Set the register bank to \p RegBank for \p Reg. - /// void setRegBank(unsigned Reg, const RegisterBank &RegBank); void setRegClassOrRegBank(unsigned Reg, @@ -653,7 +650,6 @@ public: /// new register class, or NULL if no such class exists. /// This should only be used when the constraint is known to be trivial, like /// GR32 -> GR32_NOSP. Beware of increasing register pressure. - /// const TargetRegisterClass *constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs = 0); @@ -665,12 +661,10 @@ public: /// This method can be used after constraints have been removed from a /// virtual register, for example after removing instructions or splitting /// the live range. - /// bool recomputeRegClass(unsigned Reg); /// createVirtualRegister - Create and return a new virtual register in the /// function with the specified register class. - /// unsigned createVirtualRegister(const TargetRegisterClass *RegClass); /// Accessor for VRegToType. This accessor should only be used @@ -704,7 +698,6 @@ public: unsigned createIncompleteVirtualRegister(); /// getNumVirtRegs - Return the number of virtual registers created. - /// unsigned getNumVirtRegs() const { return VRegInfo.size(); } /// clearVirtRegs - Remove all virtual registers (after physreg assignment). @@ -810,7 +803,6 @@ public: /// /// Reserved registers may belong to an allocatable register class, but the /// target has explicitly requested that they are not used. - /// bool isReserved(unsigned PhysReg) const { return getReservedRegs().test(PhysReg); } @@ -838,8 +830,8 @@ public: // Iteration support for the live-ins set. It's kept in sorted order // by register number. - typedef std::vector<std::pair<unsigned,unsigned>>::const_iterator - livein_iterator; + using livein_iterator = + std::vector<std::pair<unsigned,unsigned>>::const_iterator; livein_iterator livein_begin() const { return LiveIns.begin(); } livein_iterator livein_end() const { return LiveIns.end(); } bool livein_empty() const { return LiveIns.empty(); } @@ -910,10 +902,10 @@ public: } public: - typedef std::iterator<std::forward_iterator_tag, - MachineInstr, ptrdiff_t>::reference reference; - typedef std::iterator<std::forward_iterator_tag, - MachineInstr, ptrdiff_t>::pointer pointer; + using reference = std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::reference; + using pointer = std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::pointer; defusechain_iterator() = default; @@ -1016,10 +1008,10 @@ public: } public: - typedef std::iterator<std::forward_iterator_tag, - MachineInstr, ptrdiff_t>::reference reference; - typedef std::iterator<std::forward_iterator_tag, - MachineInstr, ptrdiff_t>::pointer pointer; + using reference = std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::reference; + using pointer = std::iterator<std::forward_iterator_tag, + MachineInstr, ptrdiff_t>::pointer; defusechain_instr_iterator() = default; diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 6b2a16e1d36e..3b02ec400aba 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -104,10 +104,15 @@ extern cl::opt<bool> ForceBottomUp; class LiveIntervals; class MachineDominatorTree; +class MachineFunction; +class MachineInstr; class MachineLoopInfo; class RegisterClassInfo; class SchedDFSResult; class ScheduleHazardRecognizer; +class TargetInstrInfo; +class TargetPassConfig; +class TargetRegisterInfo; /// MachineSchedContext provides enough context from the MachineScheduler pass /// for the target to instantiate a scheduler. @@ -129,10 +134,10 @@ struct MachineSchedContext { /// schedulers. class MachineSchedRegistry : public MachinePassRegistryNode { public: - typedef ScheduleDAGInstrs *(*ScheduleDAGCtor)(MachineSchedContext *); + using ScheduleDAGCtor = ScheduleDAGInstrs *(*)(MachineSchedContext *); // RegisterPassParser requires a (misnamed) FunctionPassCtor type. - typedef ScheduleDAGCtor FunctionPassCtor; + using FunctionPassCtor = ScheduleDAGCtor; static MachinePassRegistry Registry; @@ -527,7 +532,7 @@ public: unsigned size() const { return Queue.size(); } - typedef std::vector<SUnit*>::iterator iterator; + using iterator = std::vector<SUnit*>::iterator; iterator begin() { return Queue.begin(); } diff --git a/include/llvm/CodeGen/PBQP/CostAllocator.h b/include/llvm/CodeGen/PBQP/CostAllocator.h index 02d39fe383f1..bde451ae1fcc 100644 --- a/include/llvm/CodeGen/PBQP/CostAllocator.h +++ b/include/llvm/CodeGen/PBQP/CostAllocator.h @@ -1,4 +1,4 @@ -//===---------- CostAllocator.h - PBQP Cost Allocator -----------*- C++ -*-===// +//===- CostAllocator.h - PBQP Cost Allocator --------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -19,26 +19,28 @@ #define LLVM_CODEGEN_PBQP_COSTALLOCATOR_H #include "llvm/ADT/DenseSet.h" +#include <algorithm> +#include <cstdint> #include <memory> -#include <type_traits> namespace llvm { namespace PBQP { -template <typename ValueT> -class ValuePool { +template <typename ValueT> class ValuePool { public: - typedef std::shared_ptr<const ValueT> PoolRef; + using PoolRef = std::shared_ptr<const ValueT>; private: - class PoolEntry : public std::enable_shared_from_this<PoolEntry> { public: template <typename ValueKeyT> PoolEntry(ValuePool &Pool, ValueKeyT Value) : Pool(Pool), Value(std::move(Value)) {} + ~PoolEntry() { Pool.removeEntry(this); } - const ValueT& getValue() const { return Value; } + + const ValueT &getValue() const { return Value; } + private: ValuePool &Pool; ValueT Value; @@ -46,10 +48,10 @@ private: class PoolEntryDSInfo { public: - static inline PoolEntry* getEmptyKey() { return nullptr; } + static inline PoolEntry *getEmptyKey() { return nullptr; } - static inline PoolEntry* getTombstoneKey() { - return reinterpret_cast<PoolEntry*>(static_cast<uintptr_t>(1)); + static inline PoolEntry *getTombstoneKey() { + return reinterpret_cast<PoolEntry *>(static_cast<uintptr_t>(1)); } template <typename ValueKeyT> @@ -66,8 +68,7 @@ private: } template <typename ValueKeyT1, typename ValueKeyT2> - static - bool isEqual(const ValueKeyT1 &C1, const ValueKeyT2 &C2) { + static bool isEqual(const ValueKeyT1 &C1, const ValueKeyT2 &C2) { return C1 == C2; } @@ -83,10 +84,9 @@ private: return P1 == P2; return isEqual(P1->getValue(), P2); } - }; - typedef DenseSet<PoolEntry*, PoolEntryDSInfo> EntrySetT; + using EntrySetT = DenseSet<PoolEntry *, PoolEntryDSInfo>; EntrySetT EntrySet; @@ -105,28 +105,31 @@ public: } }; -template <typename VectorT, typename MatrixT> -class PoolCostAllocator { +template <typename VectorT, typename MatrixT> class PoolCostAllocator { private: - typedef ValuePool<VectorT> VectorCostPool; - typedef ValuePool<MatrixT> MatrixCostPool; + using VectorCostPool = ValuePool<VectorT>; + using MatrixCostPool = ValuePool<MatrixT>; + public: - typedef VectorT Vector; - typedef MatrixT Matrix; - typedef typename VectorCostPool::PoolRef VectorPtr; - typedef typename MatrixCostPool::PoolRef MatrixPtr; + using Vector = VectorT; + using Matrix = MatrixT; + using VectorPtr = typename VectorCostPool::PoolRef; + using MatrixPtr = typename MatrixCostPool::PoolRef; + + template <typename VectorKeyT> VectorPtr getVector(VectorKeyT v) { + return VectorPool.getValue(std::move(v)); + } - template <typename VectorKeyT> - VectorPtr getVector(VectorKeyT v) { return VectorPool.getValue(std::move(v)); } + template <typename MatrixKeyT> MatrixPtr getMatrix(MatrixKeyT m) { + return MatrixPool.getValue(std::move(m)); + } - template <typename MatrixKeyT> - MatrixPtr getMatrix(MatrixKeyT m) { return MatrixPool.getValue(std::move(m)); } private: VectorCostPool VectorPool; MatrixCostPool MatrixPool; }; -} // namespace PBQP -} // namespace llvm +} // end namespace PBQP +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_PBQP_COSTALLOCATOR_H diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h index 83487e6a808a..e94878ced10d 100644 --- a/include/llvm/CodeGen/PBQP/Graph.h +++ b/include/llvm/CodeGen/PBQP/Graph.h @@ -1,4 +1,4 @@ -//===-------------------- Graph.h - PBQP Graph ------------------*- C++ -*-===// +//===- Graph.h - PBQP Graph -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,16 +11,14 @@ // //===----------------------------------------------------------------------===// - #ifndef LLVM_CODEGEN_PBQP_GRAPH_H #define LLVM_CODEGEN_PBQP_GRAPH_H #include "llvm/ADT/STLExtras.h" -#include "llvm/Support/Debug.h" #include <algorithm> #include <cassert> +#include <iterator> #include <limits> -#include <utility> #include <vector> namespace llvm { @@ -28,8 +26,8 @@ namespace PBQP { class GraphBase { public: - typedef unsigned NodeId; - typedef unsigned EdgeId; + using NodeId = unsigned; + using EdgeId = unsigned; /// @brief Returns a value representing an invalid (non-existent) node. static NodeId invalidNodeId() { @@ -48,32 +46,32 @@ namespace PBQP { template <typename SolverT> class Graph : public GraphBase { private: - typedef typename SolverT::CostAllocator CostAllocator; + using CostAllocator = typename SolverT::CostAllocator; + public: - typedef typename SolverT::RawVector RawVector; - typedef typename SolverT::RawMatrix RawMatrix; - typedef typename SolverT::Vector Vector; - typedef typename SolverT::Matrix Matrix; - typedef typename CostAllocator::VectorPtr VectorPtr; - typedef typename CostAllocator::MatrixPtr MatrixPtr; - typedef typename SolverT::NodeMetadata NodeMetadata; - typedef typename SolverT::EdgeMetadata EdgeMetadata; - typedef typename SolverT::GraphMetadata GraphMetadata; + using RawVector = typename SolverT::RawVector; + using RawMatrix = typename SolverT::RawMatrix; + using Vector = typename SolverT::Vector; + using Matrix = typename SolverT::Matrix; + using VectorPtr = typename CostAllocator::VectorPtr; + using MatrixPtr = typename CostAllocator::MatrixPtr; + using NodeMetadata = typename SolverT::NodeMetadata; + using EdgeMetadata = typename SolverT::EdgeMetadata; + using GraphMetadata = typename SolverT::GraphMetadata; private: - class NodeEntry { public: - typedef std::vector<EdgeId> AdjEdgeList; - typedef AdjEdgeList::size_type AdjEdgeIdx; - typedef AdjEdgeList::const_iterator AdjEdgeItr; + using AdjEdgeList = std::vector<EdgeId>; + using AdjEdgeIdx = AdjEdgeList::size_type; + using AdjEdgeItr = AdjEdgeList::const_iterator; + + NodeEntry(VectorPtr Costs) : Costs(std::move(Costs)) {} static AdjEdgeIdx getInvalidAdjEdgeIdx() { return std::numeric_limits<AdjEdgeIdx>::max(); } - NodeEntry(VectorPtr Costs) : Costs(std::move(Costs)) {} - AdjEdgeIdx addAdjEdgeId(EdgeId EId) { AdjEdgeIdx Idx = AdjEdgeIds.size(); AdjEdgeIds.push_back(EId); @@ -96,6 +94,7 @@ namespace PBQP { VectorPtr Costs; NodeMetadata Metadata; + private: AdjEdgeList AdjEdgeIds; }; @@ -150,8 +149,10 @@ namespace PBQP { NodeId getN1Id() const { return NIds[0]; } NodeId getN2Id() const { return NIds[1]; } + MatrixPtr Costs; EdgeMetadata Metadata; + private: NodeId NIds[2]; typename NodeEntry::AdjEdgeIdx ThisEdgeAdjIdxs[2]; @@ -161,18 +162,20 @@ namespace PBQP { GraphMetadata Metadata; CostAllocator CostAlloc; - SolverT *Solver; + SolverT *Solver = nullptr; - typedef std::vector<NodeEntry> NodeVector; - typedef std::vector<NodeId> FreeNodeVector; + using NodeVector = std::vector<NodeEntry>; + using FreeNodeVector = std::vector<NodeId>; NodeVector Nodes; FreeNodeVector FreeNodeIds; - typedef std::vector<EdgeEntry> EdgeVector; - typedef std::vector<EdgeId> FreeEdgeVector; + using EdgeVector = std::vector<EdgeEntry>; + using FreeEdgeVector = std::vector<EdgeId>; EdgeVector Edges; FreeEdgeVector FreeEdgeIds; + Graph(const Graph &Other) {} + // ----- INTERNAL METHODS ----- NodeEntry &getNode(NodeId NId) { @@ -220,20 +223,18 @@ namespace PBQP { return EId; } - Graph(const Graph &Other) {} void operator=(const Graph &Other) {} public: - - typedef typename NodeEntry::AdjEdgeItr AdjEdgeItr; + using AdjEdgeItr = typename NodeEntry::AdjEdgeItr; class NodeItr { public: - typedef std::forward_iterator_tag iterator_category; - typedef NodeId value_type; - typedef int difference_type; - typedef NodeId* pointer; - typedef NodeId& reference; + using iterator_category = std::forward_iterator_tag; + using value_type = NodeId; + using difference_type = int; + using pointer = NodeId *; + using reference = NodeId &; NodeItr(NodeId CurNId, const Graph &G) : CurNId(CurNId), EndNId(G.Nodes.size()), FreeNodeIds(G.FreeNodeIds) { @@ -283,53 +284,65 @@ namespace PBQP { class NodeIdSet { public: - NodeIdSet(const Graph &G) : G(G) { } + NodeIdSet(const Graph &G) : G(G) {} + NodeItr begin() const { return NodeItr(0, G); } NodeItr end() const { return NodeItr(G.Nodes.size(), G); } + bool empty() const { return G.Nodes.empty(); } + typename NodeVector::size_type size() const { return G.Nodes.size() - G.FreeNodeIds.size(); } + private: const Graph& G; }; class EdgeIdSet { public: - EdgeIdSet(const Graph &G) : G(G) { } + EdgeIdSet(const Graph &G) : G(G) {} + EdgeItr begin() const { return EdgeItr(0, G); } EdgeItr end() const { return EdgeItr(G.Edges.size(), G); } + bool empty() const { return G.Edges.empty(); } + typename NodeVector::size_type size() const { return G.Edges.size() - G.FreeEdgeIds.size(); } + private: const Graph& G; }; class AdjEdgeIdSet { public: - AdjEdgeIdSet(const NodeEntry &NE) : NE(NE) { } + AdjEdgeIdSet(const NodeEntry &NE) : NE(NE) {} + typename NodeEntry::AdjEdgeItr begin() const { return NE.getAdjEdgeIds().begin(); } + typename NodeEntry::AdjEdgeItr end() const { return NE.getAdjEdgeIds().end(); } + bool empty() const { return NE.getAdjEdgeIds().empty(); } + typename NodeEntry::AdjEdgeList::size_type size() const { return NE.getAdjEdgeIds().size(); } + private: const NodeEntry &NE; }; /// @brief Construct an empty PBQP graph. - Graph() : Solver(nullptr) {} + Graph() = default; /// @brief Construct an empty PBQP graph with the given graph metadata. - Graph(GraphMetadata Metadata) - : Metadata(std::move(Metadata)), Solver(nullptr) {} + Graph(GraphMetadata Metadata) : Metadata(std::move(Metadata)) {} /// @brief Get a reference to the graph metadata. GraphMetadata& getMetadata() { return Metadata; } @@ -656,7 +669,7 @@ namespace PBQP { } }; -} // namespace PBQP -} // namespace llvm +} // end namespace PBQP +} // end namespace llvm #endif // LLVM_CODEGEN_PBQP_GRAPH_HPP diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h index 278787550a43..ba405e816d10 100644 --- a/include/llvm/CodeGen/PBQP/Math.h +++ b/include/llvm/CodeGen/PBQP/Math.h @@ -1,4 +1,4 @@ -//===------ Math.h - PBQP Vector and Matrix classes -------------*- C++ -*-===// +//===- Math.h - PBQP Vector and Matrix classes ------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -11,20 +11,22 @@ #define LLVM_CODEGEN_PBQP_MATH_H #include "llvm/ADT/Hashing.h" +#include "llvm/ADT/STLExtras.h" #include <algorithm> #include <cassert> #include <functional> +#include <memory> namespace llvm { namespace PBQP { -typedef float PBQPNum; +using PBQPNum = float; /// \brief PBQP Vector class. class Vector { friend hash_code hash_value(const Vector &); -public: +public: /// \brief Construct a PBQP vector of the given size. explicit Vector(unsigned Length) : Length(Length), Data(llvm::make_unique<PBQPNum []>(Length)) {} @@ -120,8 +122,8 @@ OStream& operator<<(OStream &OS, const Vector &V) { class Matrix { private: friend hash_code hash_value(const Matrix &); -public: +public: /// \brief Construct a PBQP Matrix with the given dimensions. Matrix(unsigned Rows, unsigned Cols) : Rows(Rows), Cols(Cols), Data(llvm::make_unique<PBQPNum []>(Rows * Cols)) { @@ -253,9 +255,11 @@ OStream& operator<<(OStream &OS, const Matrix &M) { template <typename Metadata> class MDVector : public Vector { public: - MDVector(const Vector &v) : Vector(v), md(*this) { } + MDVector(const Vector &v) : Vector(v), md(*this) {} MDVector(Vector &&v) : Vector(std::move(v)), md(*this) { } + const Metadata& getMetadata() const { return md; } + private: Metadata md; }; @@ -268,9 +272,11 @@ inline hash_code hash_value(const MDVector<Metadata> &V) { template <typename Metadata> class MDMatrix : public Matrix { public: - MDMatrix(const Matrix &m) : Matrix(m), md(*this) { } + MDMatrix(const Matrix &m) : Matrix(m), md(*this) {} MDMatrix(Matrix &&m) : Matrix(std::move(m)), md(*this) { } + const Metadata& getMetadata() const { return md; } + private: Metadata md; }; @@ -280,7 +286,7 @@ inline hash_code hash_value(const MDMatrix<Metadata> &M) { return hash_value(static_cast<const Matrix&>(M)); } -} // namespace PBQP -} // namespace llvm +} // end namespace PBQP +} // end namespace llvm #endif // LLVM_CODEGEN_PBQP_MATH_H diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h index d4a544bfe721..8aeb51936760 100644 --- a/include/llvm/CodeGen/PBQP/ReductionRules.h +++ b/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -1,4 +1,4 @@ -//===----------- ReductionRules.h - Reduction Rules -------------*- C++ -*-===// +//===- ReductionRules.h - Reduction Rules -----------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -17,6 +17,8 @@ #include "Graph.h" #include "Math.h" #include "Solution.h" +#include <cassert> +#include <limits> namespace llvm { namespace PBQP { @@ -27,11 +29,11 @@ namespace PBQP { /// neighbor. Notify the problem domain. template <typename GraphT> void applyR1(GraphT &G, typename GraphT::NodeId NId) { - typedef typename GraphT::NodeId NodeId; - typedef typename GraphT::EdgeId EdgeId; - typedef typename GraphT::Vector Vector; - typedef typename GraphT::Matrix Matrix; - typedef typename GraphT::RawVector RawVector; + using NodeId = typename GraphT::NodeId; + using EdgeId = typename GraphT::EdgeId; + using Vector = typename GraphT::Vector; + using Matrix = typename GraphT::Matrix; + using RawVector = typename GraphT::RawVector; assert(G.getNodeDegree(NId) == 1 && "R1 applied to node with degree != 1."); @@ -71,11 +73,11 @@ namespace PBQP { template <typename GraphT> void applyR2(GraphT &G, typename GraphT::NodeId NId) { - typedef typename GraphT::NodeId NodeId; - typedef typename GraphT::EdgeId EdgeId; - typedef typename GraphT::Vector Vector; - typedef typename GraphT::Matrix Matrix; - typedef typename GraphT::RawMatrix RawMatrix; + using NodeId = typename GraphT::NodeId; + using EdgeId = typename GraphT::EdgeId; + using Vector = typename GraphT::Vector; + using Matrix = typename GraphT::Matrix; + using RawMatrix = typename GraphT::RawMatrix; assert(G.getNodeDegree(NId) == 2 && "R2 applied to node with degree != 2."); @@ -177,9 +179,9 @@ namespace PBQP { // state. template <typename GraphT, typename StackT> Solution backpropagate(GraphT& G, StackT stack) { - typedef GraphBase::NodeId NodeId; - typedef typename GraphT::Matrix Matrix; - typedef typename GraphT::RawVector RawVector; + using NodeId = GraphBase::NodeId; + using Matrix = typename GraphT::Matrix; + using RawVector = typename GraphT::RawVector; Solution s; @@ -215,7 +217,7 @@ namespace PBQP { return s; } -} // namespace PBQP -} // namespace llvm +} // end namespace PBQP +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_PBQP_REDUCTIONRULES_H diff --git a/include/llvm/CodeGen/PBQP/Solution.h b/include/llvm/CodeGen/PBQP/Solution.h index d96b5eac4520..8d5d2374679d 100644 --- a/include/llvm/CodeGen/PBQP/Solution.h +++ b/include/llvm/CodeGen/PBQP/Solution.h @@ -26,7 +26,7 @@ namespace PBQP { /// To get the selection for each node in the problem use the getSelection method. class Solution { private: - typedef std::map<GraphBase::NodeId, unsigned> SelectionsMap; + using SelectionsMap = std::map<GraphBase::NodeId, unsigned>; SelectionsMap selections; unsigned r0Reductions = 0; diff --git a/include/llvm/CodeGen/PBQPRAConstraint.h b/include/llvm/CodeGen/PBQPRAConstraint.h index 833b9bad613f..269b7a7b3a35 100644 --- a/include/llvm/CodeGen/PBQPRAConstraint.h +++ b/include/llvm/CodeGen/PBQPRAConstraint.h @@ -1,4 +1,4 @@ -//===-- RegAllocPBQP.h ------------------------------------------*- C++ -*-===// +//===- RegAllocPBQP.h -------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -16,23 +16,22 @@ #ifndef LLVM_CODEGEN_PBQPRACONSTRAINT_H #define LLVM_CODEGEN_PBQPRACONSTRAINT_H +#include <algorithm> #include <memory> #include <vector> namespace llvm { + namespace PBQP { namespace RegAlloc { + // Forward declare PBQP graph class. class PBQPRAGraph; -} -} -class LiveIntervals; -class MachineBlockFrequencyInfo; -class MachineFunction; -class TargetRegisterInfo; +} // end namespace RegAlloc +} // end namespace PBQP -typedef PBQP::RegAlloc::PBQPRAGraph PBQPRAGraph; +using PBQPRAGraph = PBQP::RegAlloc::PBQPRAGraph; /// @brief Abstract base for classes implementing PBQP register allocation /// constraints (e.g. Spill-costs, interference, coalescing). @@ -40,6 +39,7 @@ class PBQPRAConstraint { public: virtual ~PBQPRAConstraint() = 0; virtual void apply(PBQPRAGraph &G) = 0; + private: virtual void anchor(); }; @@ -59,11 +59,13 @@ public: if (C) Constraints.push_back(std::move(C)); } + private: std::vector<std::unique_ptr<PBQPRAConstraint>> Constraints; + void anchor() override; }; -} +} // end namespace llvm -#endif /* LLVM_CODEGEN_PBQPRACONSTRAINT_H */ +#endif // LLVM_CODEGEN_PBQPRACONSTRAINT_H diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index 3bcfc1c4254b..96cfce5b84df 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -140,6 +140,9 @@ namespace llvm { /// Greedy register allocator. extern char &RAGreedyID; + /// Basic register allocator. + extern char &RABasicID; + /// VirtRegRewriter pass. Rewrite virtual registers to physical registers as /// assigned in VirtRegMap. extern char &VirtRegRewriterID; diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h index 8872a5dc54a1..5b342863eb50 100644 --- a/include/llvm/CodeGen/RegAllocPBQP.h +++ b/include/llvm/CodeGen/RegAllocPBQP.h @@ -130,10 +130,10 @@ inline hash_code hash_value(const AllowedRegVector &OptRegs) { /// \brief Holds graph-level metadata relevant to PBQP RA problems. class GraphMetadata { private: - typedef ValuePool<AllowedRegVector> AllowedRegVecPool; + using AllowedRegVecPool = ValuePool<AllowedRegVector>; public: - typedef AllowedRegVecPool::PoolRef AllowedRegVecRef; + using AllowedRegVecRef = AllowedRegVecPool::PoolRef; GraphMetadata(MachineFunction &MF, LiveIntervals &LIS, @@ -167,17 +167,17 @@ private: /// \brief Holds solver state and other metadata relevant to each PBQP RA node. class NodeMetadata { public: - typedef RegAlloc::AllowedRegVector AllowedRegVector; + using AllowedRegVector = RegAlloc::AllowedRegVector; // The node's reduction state. The order in this enum is important, // as it is assumed nodes can only progress up (i.e. towards being // optimally reducible) when reducing the graph. - typedef enum { + using ReductionState = enum { Unprocessed, NotProvablyAllocatable, ConservativelyAllocatable, OptimallyReducible - } ReductionState; + }; NodeMetadata() = default; @@ -267,23 +267,23 @@ private: class RegAllocSolverImpl { private: - typedef MDMatrix<MatrixMetadata> RAMatrix; + using RAMatrix = MDMatrix<MatrixMetadata>; public: - typedef PBQP::Vector RawVector; - typedef PBQP::Matrix RawMatrix; - typedef PBQP::Vector Vector; - typedef RAMatrix Matrix; - typedef PBQP::PoolCostAllocator<Vector, Matrix> CostAllocator; + using RawVector = PBQP::Vector; + using RawMatrix = PBQP::Matrix; + using Vector = PBQP::Vector; + using Matrix = RAMatrix; + using CostAllocator = PBQP::PoolCostAllocator<Vector, Matrix>; - typedef GraphBase::NodeId NodeId; - typedef GraphBase::EdgeId EdgeId; + using NodeId = GraphBase::NodeId; + using EdgeId = GraphBase::EdgeId; - typedef RegAlloc::NodeMetadata NodeMetadata; - struct EdgeMetadata { }; - typedef RegAlloc::GraphMetadata GraphMetadata; + using NodeMetadata = RegAlloc::NodeMetadata; + struct EdgeMetadata {}; + using GraphMetadata = RegAlloc::GraphMetadata; - typedef PBQP::Graph<RegAllocSolverImpl> Graph; + using Graph = PBQP::Graph<RegAllocSolverImpl>; RegAllocSolverImpl(Graph &G) : G(G) {} @@ -426,7 +426,7 @@ private: std::vector<GraphBase::NodeId> reduce() { assert(!G.empty() && "Cannot reduce empty graph."); - typedef GraphBase::NodeId NodeId; + using NodeId = GraphBase::NodeId; std::vector<NodeId> NodeStack; // Consume worklists. @@ -459,7 +459,6 @@ private: ConservativelyAllocatableNodes.erase(NItr); NodeStack.push_back(NId); G.disconnectAllNeighborsFromNode(NId); - } else if (!NotProvablyAllocatableNodes.empty()) { NodeSet::iterator NItr = std::min_element(NotProvablyAllocatableNodes.begin(), @@ -493,7 +492,7 @@ private: }; Graph& G; - typedef std::set<NodeId> NodeSet; + using NodeSet = std::set<NodeId>; NodeSet OptimallyReducibleNodes; NodeSet ConservativelyAllocatableNodes; NodeSet NotProvablyAllocatableNodes; @@ -501,7 +500,7 @@ private: class PBQPRAGraph : public PBQP::Graph<RegAllocSolverImpl> { private: - typedef PBQP::Graph<RegAllocSolverImpl> BaseT; + using BaseT = PBQP::Graph<RegAllocSolverImpl>; public: PBQPRAGraph(GraphMetadata Metadata) : BaseT(std::move(Metadata)) {} diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h index 1f939e72e139..ad1efe18c72d 100644 --- a/include/llvm/CodeGen/RegisterScavenging.h +++ b/include/llvm/CodeGen/RegisterScavenging.h @@ -204,6 +204,10 @@ private: void setLiveInsUsed(const MachineBasicBlock &MBB); }; +/// Replaces all frame index virtual registers with physical registers. Uses the +/// register scavenger to find an appropriate register to use. +void scavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger &RS); + } // end namespace llvm #endif // LLVM_CODEGEN_REGISTERSCAVENGING_H diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index f5f5bfd45e79..d62bb9bf0b75 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -1,4 +1,4 @@ -//==- ScheduleDAGInstrs.h - MachineInstr Scheduling --------------*- C++ -*-==// +//===- ScheduleDAGInstrs.h - MachineInstr Scheduling ------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,22 +15,38 @@ #ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H #define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H -#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/PointerIntPair.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseMultiSet.h" #include "llvm/ADT/SparseSet.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/TargetSchedule.h" -#include "llvm/Support/Compiler.h" +#include "llvm/MC/LaneBitmask.h" #include "llvm/Target/TargetRegisterInfo.h" +#include <cassert> +#include <cstdint> #include <list> +#include <utility> +#include <vector> namespace llvm { + + class LiveIntervals; class MachineFrameInfo; + class MachineFunction; + class MachineInstr; class MachineLoopInfo; - class MachineDominatorTree; - class RegPressureTracker; + class MachineOperand; + struct MCSchedClassDesc; class PressureDiffs; + class PseudoSourceValue; + class RegPressureTracker; + class UndefValue; + class Value; /// An individual mapping from virtual register number to SUnit. struct VReg2SUnit { @@ -70,31 +86,34 @@ namespace llvm { /// Use a SparseMultiSet to track physical registers. Storage is only /// allocated once for the pass. It can be cleared in constant time and reused /// without any frees. - typedef SparseMultiSet<PhysRegSUOper, llvm::identity<unsigned>, uint16_t> - Reg2SUnitsMap; + using Reg2SUnitsMap = + SparseMultiSet<PhysRegSUOper, identity<unsigned>, uint16_t>; /// Use SparseSet as a SparseMap by relying on the fact that it never /// compares ValueT's, only unsigned keys. This allows the set to be cleared /// between scheduling regions in constant time as long as ValueT does not /// require a destructor. - typedef SparseSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMap; + using VReg2SUnitMap = SparseSet<VReg2SUnit, VirtReg2IndexFunctor>; /// Track local uses of virtual registers. These uses are gathered by the DAG /// builder and may be consulted by the scheduler to avoid iterating an entire /// vreg use list. - typedef SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor> VReg2SUnitMultiMap; + using VReg2SUnitMultiMap = SparseMultiSet<VReg2SUnit, VirtReg2IndexFunctor>; + + using VReg2SUnitOperIdxMultiMap = + SparseMultiSet<VReg2SUnitOperIdx, VirtReg2IndexFunctor>; - typedef SparseMultiSet<VReg2SUnitOperIdx, VirtReg2IndexFunctor> - VReg2SUnitOperIdxMultiMap; + using ValueType = PointerUnion<const Value *, const PseudoSourceValue *>; - typedef PointerUnion<const Value *, const PseudoSourceValue *> ValueType; struct UnderlyingObject : PointerIntPair<ValueType, 1, bool> { UnderlyingObject(ValueType V, bool MayAlias) : PointerIntPair<ValueType, 1, bool>(V, MayAlias) {} + ValueType getValue() const { return getPointer(); } bool mayAlias() const { return getInt(); } }; - typedef SmallVector<UnderlyingObject, 4> UnderlyingObjectsVector; + + using UnderlyingObjectsVector = SmallVector<UnderlyingObject, 4>; /// A ScheduleDAG for scheduling lists of MachineInstr. class ScheduleDAGInstrs : public ScheduleDAG { @@ -114,10 +133,10 @@ namespace llvm { /// reordering. A specialized scheduler can override /// TargetInstrInfo::isSchedulingBoundary then enable this flag to indicate /// it has taken responsibility for scheduling the terminator correctly. - bool CanHandleTerminators; + bool CanHandleTerminators = false; /// Whether lane masks should get tracked. - bool TrackLaneMasks; + bool TrackLaneMasks = false; // State specific to the current scheduling region. // ------------------------------------------------ @@ -155,12 +174,12 @@ namespace llvm { /// Tracks the last instructions in this region using each virtual register. VReg2SUnitOperIdxMultiMap CurrentVRegUses; - AliasAnalysis *AAForDep; + AliasAnalysis *AAForDep = nullptr; /// Remember a generic side-effecting instruction as we proceed. /// No other SU ever gets scheduled around it (except in the special /// case of a huge region that gets reduced). - SUnit *BarrierChain; + SUnit *BarrierChain = nullptr; public: /// A list of SUnits, used in Value2SUsMap, during DAG construction. @@ -168,7 +187,7 @@ namespace llvm { /// implementation of this data structure, such as a singly linked list /// with a memory pool (SmallVector was tried but slow and SparseSet is not /// applicable). - typedef std::list<SUnit *> SUList; + using SUList = std::list<SUnit *>; protected: /// \brief A map from ValueType to SUList, used during DAG construction, as @@ -216,13 +235,13 @@ namespace llvm { /// For an unanalyzable memory access, this Value is used in maps. UndefValue *UnknownValue; - typedef std::vector<std::pair<MachineInstr *, MachineInstr *>> - DbgValueVector; + using DbgValueVector = + std::vector<std::pair<MachineInstr *, MachineInstr *>>; /// Remember instruction that precedes DBG_VALUE. /// These are generated by buildSchedGraph but persist so they can be /// referenced when emitting the final schedule. DbgValueVector DbgValues; - MachineInstr *FirstDbgValue; + MachineInstr *FirstDbgValue = nullptr; /// Set of live physical registers for updating kill flags. LivePhysRegs LiveRegs; @@ -232,7 +251,7 @@ namespace llvm { const MachineLoopInfo *mli, bool RemoveKillFlags = false); - ~ScheduleDAGInstrs() override {} + ~ScheduleDAGInstrs() override = default; /// Gets the machine model for instruction scheduling. const TargetSchedModel *getSchedModel() const { return &SchedModel; } @@ -354,6 +373,7 @@ namespace llvm { return nullptr; return I->second; } + } // end namespace llvm -#endif +#endif // LLVM_CODEGEN_SCHEDULEDAGINSTRS_H diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 493122b15704..4b1a375abd57 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ---------*- C++ -*-===// +//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,35 +15,72 @@ #ifndef LLVM_CODEGEN_SELECTIONDAG_H #define LLVM_CODEGEN_SELECTIONDAG_H +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/ilist.h" +#include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/ilist.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/DAGCombine.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" +#include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" +#include "llvm/Support/AtomicOrdering.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/RecyclingAllocator.h" -#include "llvm/Target/TargetMachine.h" +#include <algorithm> #include <cassert> +#include <cstdint> +#include <functional> #include <map> #include <string> +#include <tuple> +#include <utility> #include <vector> namespace llvm { +class BlockAddress; +class Constant; +class ConstantFP; +class ConstantInt; +class DataLayout; +struct fltSemantics; +class GlobalValue; struct KnownBits; +class LLVMContext; +class MachineBasicBlock; class MachineConstantPoolValue; -class MachineFunction; -class MDNode; +class MCSymbol; class OptimizationRemarkEmitter; class SDDbgValue; -class TargetLowering; +class SelectionDAG; class SelectionDAGTargetInfo; +class TargetLowering; +class TargetMachine; +class TargetSubtargetInfo; +class Value; class SDVTListNode : public FoldingSetNode { friend struct FoldingSetTrait<SDVTListNode>; + /// A reference to an Interned FoldingSetNodeID for this node. /// The Allocator in SelectionDAG holds the data. /// SDVTList contains all types which are frequently accessed in SelectionDAG. @@ -55,11 +92,13 @@ class SDVTListNode : public FoldingSetNode { /// The hash value for SDVTList is fixed, so cache it to avoid /// hash calculation. unsigned HashValue; + public: SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) : FastID(ID), VTs(VT), NumVTs(Num) { HashValue = ID.ComputeHash(); } + SDVTList getSDVTList() { SDVTList result = {VTs, NumVTs}; return result; @@ -72,12 +111,14 @@ template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTLis static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) { ID = X.FastID; } + static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID, unsigned IDHash, FoldingSetNodeID &TempID) { if (X.HashValue != IDHash) return false; return ID == X.FastID; } + static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) { return X.HashValue; } @@ -104,13 +145,13 @@ class SDDbgInfo { BumpPtrAllocator Alloc; SmallVector<SDDbgValue*, 32> DbgValues; SmallVector<SDDbgValue*, 32> ByvalParmDbgValues; - typedef DenseMap<const SDNode*, SmallVector<SDDbgValue*, 2> > DbgValMapType; + using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>; DbgValMapType DbgValMap; - void operator=(const SDDbgInfo&) = delete; - SDDbgInfo(const SDDbgInfo&) = delete; public: - SDDbgInfo() {} + SDDbgInfo() = default; + SDDbgInfo(const SDDbgInfo &) = delete; + SDDbgInfo &operator=(const SDDbgInfo &) = delete; void add(SDDbgValue *V, const SDNode *Node, bool isParameter) { if (isParameter) { @@ -144,14 +185,14 @@ public: return ArrayRef<SDDbgValue*>(); } - typedef SmallVectorImpl<SDDbgValue*>::iterator DbgIterator; + using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator; + DbgIterator DbgBegin() { return DbgValues.begin(); } DbgIterator DbgEnd() { return DbgValues.end(); } DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); } DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); } }; -class SelectionDAG; void checkForCycles(const SelectionDAG *DAG, bool force = false); /// This is used to represent a portion of an LLVM function in a low-level @@ -167,8 +208,8 @@ void checkForCycles(const SelectionDAG *DAG, bool force = false); /// class SelectionDAG { const TargetMachine &TM; - const SelectionDAGTargetInfo *TSI; - const TargetLowering *TLI; + const SelectionDAGTargetInfo *TSI = nullptr; + const TargetLowering *TLI = nullptr; MachineFunction *MF; LLVMContext *Context; CodeGenOpt::Level OptLevel; @@ -188,9 +229,9 @@ class SelectionDAG { /// The AllocatorType for allocating SDNodes. We use /// pool allocation with recycling. - typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode), - alignof(MostAlignedSDNode)> - NodeAllocatorType; + using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode, + sizeof(LargestSDNode), + alignof(MostAlignedSDNode)>; /// Pool allocation for nodes. NodeAllocatorType NodeAllocator; @@ -243,9 +284,11 @@ public: struct DAGNodeDeletedListener : public DAGUpdateListener { std::function<void(SDNode *, SDNode *)> Callback; + DAGNodeDeletedListener(SelectionDAG &DAG, std::function<void(SDNode *, SDNode *)> Callback) : DAGUpdateListener(DAG), Callback(std::move(Callback)) {} + void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); } }; @@ -254,7 +297,7 @@ public: /// have legal types. This is important after type legalization since /// any illegally typed nodes generated after this point will not experience /// type legalization. - bool NewNodesMustHaveLegalTypes; + bool NewNodesMustHaveLegalTypes = false; private: /// DAGUpdateListener is a friend so it can manipulate the listener stack. @@ -262,7 +305,7 @@ private: /// Linked list of registered DAGUpdateListener instances. /// This stack is maintained by DAGUpdateListener RAII. - DAGUpdateListener *UpdateListeners; + DAGUpdateListener *UpdateListeners = nullptr; /// Implementation of setSubgraphColor. /// Return whether we had to truncate the search. @@ -316,11 +359,10 @@ private: Node->OperandList = nullptr; } - void operator=(const SelectionDAG&) = delete; - SelectionDAG(const SelectionDAG&) = delete; - public: - explicit SelectionDAG(const TargetMachine &TM, llvm::CodeGenOpt::Level); + explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level); + SelectionDAG(const SelectionDAG &) = delete; + SelectionDAG &operator=(const SelectionDAG &) = delete; ~SelectionDAG(); /// Prepare this SelectionDAG to process code in the given MachineFunction. @@ -364,12 +406,16 @@ public: /// Convenience for setting subgraph color attribute. void setSubgraphColor(SDNode *N, const char *Color); - typedef ilist<SDNode>::const_iterator allnodes_const_iterator; + using allnodes_const_iterator = ilist<SDNode>::const_iterator; + allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); } allnodes_const_iterator allnodes_end() const { return AllNodes.end(); } - typedef ilist<SDNode>::iterator allnodes_iterator; + + using allnodes_iterator = ilist<SDNode>::iterator; + allnodes_iterator allnodes_begin() { return AllNodes.begin(); } allnodes_iterator allnodes_end() { return AllNodes.end(); } + ilist<SDNode>::size_type allnodes_size() const { return AllNodes.size(); } @@ -475,7 +521,6 @@ public: //===--------------------------------------------------------------------===// // Node creation methods. - // /// \brief Create a ConstantSDNode wrapping a constant value. /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR. @@ -1251,9 +1296,11 @@ public: SDDbgInfo::DbgIterator DbgBegin() { return DbgInfo->DbgBegin(); } SDDbgInfo::DbgIterator DbgEnd() { return DbgInfo->DbgEnd(); } + SDDbgInfo::DbgIterator ByvalParmDbgBegin() { return DbgInfo->ByvalParmDbgBegin(); } + SDDbgInfo::DbgIterator ByvalParmDbgEnd() { return DbgInfo->ByvalParmDbgEnd(); } @@ -1479,10 +1526,12 @@ private: }; template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> { - typedef pointer_iterator<SelectionDAG::allnodes_iterator> nodes_iterator; + using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>; + static nodes_iterator nodes_begin(SelectionDAG *G) { return nodes_iterator(G->allnodes_begin()); } + static nodes_iterator nodes_end(SelectionDAG *G) { return nodes_iterator(G->allnodes_end()); } @@ -1493,7 +1542,6 @@ SDValue SelectionDAG::getTargetMemSDNode(SDVTList VTs, ArrayRef<SDValue> Ops, const SDLoc &dl, EVT MemVT, MachineMemOperand *MMO) { - /// Compose node ID and try to find an existing node. FoldingSetNodeID ID; unsigned Opcode = @@ -1524,6 +1572,6 @@ SDValue SelectionDAG::getTargetMemSDNode(SDVTList VTs, return SDValue(N, 0); } -} // end namespace llvm +} // end namespace llvm -#endif +#endif // LLVM_CODEGEN_SELECTIONDAG_H diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 973c5aac5281..3a4feb322092 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -37,6 +37,7 @@ #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Metadata.h" #include "llvm/Support/AlignOf.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Casting.h" @@ -53,14 +54,18 @@ namespace llvm { -class SelectionDAG; +class APInt; +class Constant; +template <typename T> struct DenseMapInfo; class GlobalValue; class MachineBasicBlock; class MachineConstantPoolValue; +class MCSymbol; +class raw_ostream; class SDNode; +class SelectionDAG; +class Type; class Value; -class MCSymbol; -template <typename T> struct DenseMapInfo; void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, bool force = false); @@ -229,13 +234,15 @@ template <> struct isPodLike<SDValue> { static const bool value = true; }; /// Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. template<> struct simplify_type<SDValue> { - typedef SDNode* SimpleType; + using SimpleType = SDNode *; + static SimpleType getSimplifiedValue(SDValue &Val) { return Val.getNode(); } }; template<> struct simplify_type<const SDValue> { - typedef /*const*/ SDNode* SimpleType; + using SimpleType = /*const*/ SDNode *; + static SimpleType getSimplifiedValue(const SDValue &Val) { return Val.getNode(); } @@ -330,7 +337,8 @@ private: /// simplify_type specializations - Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. template<> struct simplify_type<SDUse> { - typedef SDNode* SimpleType; + using SimpleType = SDNode *; + static SimpleType getSimplifiedValue(SDUse &Val) { return Val.getNode(); } @@ -695,10 +703,10 @@ public: explicit use_iterator(SDUse *op) : Op(op) {} public: - typedef std::iterator<std::forward_iterator_tag, - SDUse, ptrdiff_t>::reference reference; - typedef std::iterator<std::forward_iterator_tag, - SDUse, ptrdiff_t>::pointer pointer; + using reference = std::iterator<std::forward_iterator_tag, + SDUse, ptrdiff_t>::reference; + using pointer = std::iterator<std::forward_iterator_tag, + SDUse, ptrdiff_t>::pointer; use_iterator() = default; use_iterator(const use_iterator &I) : Op(I.Op) {} @@ -824,7 +832,7 @@ public: return OperandList[Num]; } - typedef SDUse* op_iterator; + using op_iterator = SDUse *; op_iterator op_begin() const { return OperandList; } op_iterator op_end() const { return OperandList+NumOperands; } @@ -896,7 +904,8 @@ public: return getValueType(ResNo).getSizeInBits(); } - typedef const EVT* value_iterator; + using value_iterator = const EVT *; + value_iterator value_begin() const { return ValueList; } value_iterator value_end() const { return ValueList+NumValues; } @@ -1822,8 +1831,7 @@ class BlockAddressSDNode : public SDNode { BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, int64_t o, unsigned char Flags) : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), - BA(ba), Offset(o), TargetFlags(Flags) { - } + BA(ba), Offset(o), TargetFlags(Flags) {} public: const BlockAddress *getBlockAddress() const { return BA; } @@ -2154,7 +2162,7 @@ public: /// instruction selection proper phase. class MachineSDNode : public SDNode { public: - typedef MachineMemOperand **mmo_iterator; + using mmo_iterator = MachineMemOperand **; private: friend class SelectionDAG; @@ -2226,8 +2234,8 @@ public: }; template <> struct GraphTraits<SDNode*> { - typedef SDNode *NodeRef; - typedef SDNodeIterator ChildIteratorType; + using NodeRef = SDNode *; + using ChildIteratorType = SDNodeIterator; static NodeRef getEntryNode(SDNode *N) { return N; } @@ -2244,12 +2252,12 @@ template <> struct GraphTraits<SDNode*> { /// /// This needs to be a union because the largest node differs on 32 bit systems /// with 4 and 8 byte pointer alignment, respectively. -typedef AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, - BlockAddressSDNode, GlobalAddressSDNode> - LargestSDNode; +using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, + BlockAddressSDNode, + GlobalAddressSDNode>; /// The SDNode class with the greatest alignment requirement. -typedef GlobalAddressSDNode MostAlignedSDNode; +using MostAlignedSDNode = GlobalAddressSDNode; namespace ISD { diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 14fc3a499a08..a275b2721b44 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -20,17 +20,26 @@ #define LLVM_CODEGEN_SLOTINDEXES_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/ilist.h" #include "llvm/ADT/IntervalMap.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/ilist.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBundle.h" +#include "llvm/Pass.h" #include "llvm/Support/Allocator.h" +#include <algorithm> +#include <cassert> +#include <iterator> +#include <utility> namespace llvm { +class raw_ostream; + /// This class represents an entry in the slot index list held in the /// SlotIndexes pass. It should not be used directly. See the /// SlotIndex & SlotIndexes classes for the public interface to this @@ -40,7 +49,6 @@ namespace llvm { unsigned index; public: - IndexListEntry(MachineInstr *mi, unsigned index) : mi(mi), index(index) {} MachineInstr* getInstr() const { return mi; } @@ -301,7 +309,7 @@ namespace llvm { return os; } - typedef std::pair<SlotIndex, MachineBasicBlock*> IdxMBBPair; + using IdxMBBPair = std::pair<SlotIndex, MachineBasicBlock *>; inline bool operator<(SlotIndex V, const IdxMBBPair &IM) { return V < IM.first; @@ -325,7 +333,7 @@ namespace llvm { // IndexListEntry allocator. BumpPtrAllocator ileAllocator; - typedef ilist<IndexListEntry> IndexList; + using IndexList = ilist<IndexListEntry>; IndexList indexList; #ifdef EXPENSIVE_CHECKS @@ -334,7 +342,7 @@ namespace llvm { MachineFunction *mf; - typedef DenseMap<const MachineInstr*, SlotIndex> Mi2IndexMap; + using Mi2IndexMap = DenseMap<const MachineInstr *, SlotIndex>; Mi2IndexMap mi2iMap; /// MBBRanges - Map MBB number to (start, stop) indexes. @@ -436,7 +444,7 @@ namespace llvm { const MachineBasicBlock *MBB = MI.getParent(); assert(MBB && "MI must be inserted inna basic block"); MachineBasicBlock::const_iterator I = MI, B = MBB->begin(); - for (;;) { + while (true) { if (I == B) return getMBBStartIdx(MBB); --I; @@ -453,7 +461,7 @@ namespace llvm { const MachineBasicBlock *MBB = MI.getParent(); assert(MBB && "MI must be inserted inna basic block"); MachineBasicBlock::const_iterator I = MI, E = MBB->end(); - for (;;) { + while (true) { ++I; if (I == E) return getMBBEndIdx(MBB); @@ -497,21 +505,25 @@ namespace llvm { /// Iterator over the idx2MBBMap (sorted pairs of slot index of basic block /// begin and basic block) - typedef SmallVectorImpl<IdxMBBPair>::const_iterator MBBIndexIterator; + using MBBIndexIterator = SmallVectorImpl<IdxMBBPair>::const_iterator; + /// Move iterator to the next IdxMBBPair where the SlotIndex is greater or /// equal to \p To. MBBIndexIterator advanceMBBIndex(MBBIndexIterator I, SlotIndex To) const { return std::lower_bound(I, idx2MBBMap.end(), To); } + /// Get an iterator pointing to the IdxMBBPair with the biggest SlotIndex /// that is greater or equal to \p Idx. MBBIndexIterator findMBBIndex(SlotIndex Idx) const { return advanceMBBIndex(idx2MBBMap.begin(), Idx); } + /// Returns an iterator for the begin of the idx2MBBMap. MBBIndexIterator MBBIndexBegin() const { return idx2MBBMap.begin(); } + /// Return an iterator for the end of the idx2MBBMap. MBBIndexIterator MBBIndexEnd() const { return idx2MBBMap.end(); diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index a18936feea7b..8263946ed928 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -145,21 +145,27 @@ public: /// /// Statepoint operands take the form: /// <id>, <num patch bytes >, <num call arguments>, <call target>, -/// [call arguments], <StackMaps::ConstantOp>, <calling convention>, +/// [call arguments...], +/// <StackMaps::ConstantOp>, <calling convention>, /// <StackMaps::ConstantOp>, <statepoint flags>, -/// <StackMaps::ConstantOp>, <num other args>, [other args], -/// [gc values] +/// <StackMaps::ConstantOp>, <num deopt args>, [deopt args...], +/// <gc base/derived pairs...> <gc allocas...> +/// Note that the last two sets of arguments are not currently length +/// prefixed. class StatepointOpers { -private: + // TODO:: we should change the STATEPOINT representation so that CC and + // Flags should be part of meta operands, with args and deopt operands, and + // gc operands all prefixed by their length and a type code. This would be + // much more consistent. +public: // These values are aboolute offsets into the operands of the statepoint // instruction. enum { IDPos, NBytesPos, NCallArgsPos, CallTargetPos, MetaEnd }; // These values are relative offests from the start of the statepoint meta // arguments (i.e. the end of the call arguments). - enum { CCOffset = 1, FlagsOffset = 3, NumVMSArgsOffset = 5 }; + enum { CCOffset = 1, FlagsOffset = 3, NumDeoptOperandsOffset = 5 }; -public: explicit StatepointOpers(const MachineInstr *MI) : MI(MI) {} /// Get starting index of non call related arguments @@ -220,7 +226,7 @@ public: // OpTypes are used to encode information about the following logical // operand (which may consist of several MachineOperands) for the // OpParser. - typedef enum { DirectMemRefOp, IndirectMemRefOp, ConstantOp } OpType; + using OpType = enum { DirectMemRefOp, IndirectMemRefOp, ConstantOp }; StackMaps(AsmPrinter &AP); @@ -248,9 +254,10 @@ public: private: static const char *WSMP; - typedef SmallVector<Location, 8> LocationVec; - typedef SmallVector<LiveOutReg, 8> LiveOutVec; - typedef MapVector<uint64_t, uint64_t> ConstantPool; + + using LocationVec = SmallVector<Location, 8>; + using LiveOutVec = SmallVector<LiveOutReg, 8>; + using ConstantPool = MapVector<uint64_t, uint64_t>; struct FunctionInfo { uint64_t StackSize = 0; @@ -273,8 +280,8 @@ private: LiveOuts(std::move(LiveOuts)) {} }; - typedef MapVector<const MCSymbol *, FunctionInfo> FnInfoMap; - typedef std::vector<CallsiteInfo> CallsiteInfoList; + using FnInfoMap = MapVector<const MCSymbol *, FunctionInfo>; + using CallsiteInfoList = std::vector<CallsiteInfo>; AsmPrinter &AP; CallsiteInfoList CSInfos; diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index 1992412120aa..4365fca74bf1 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -55,6 +55,9 @@ public: /// Return the MCSchedClassDesc for this instruction. const MCSchedClassDesc *resolveSchedClass(const MachineInstr *MI) const; + /// \brief TargetSubtargetInfo getter. + const TargetSubtargetInfo *getSubtargetInfo() const { return STI; } + /// \brief TargetInstrInfo getter. const TargetInstrInfo *getInstrInfo() const { return TII; } diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h index dd730495a5f6..8043024626a0 100644 --- a/include/llvm/CodeGen/WinEHFuncInfo.h +++ b/include/llvm/CodeGen/WinEHFuncInfo.h @@ -1,4 +1,4 @@ -//===-- llvm/CodeGen/WinEHFuncInfo.h ----------------------------*- C++ -*-===// +//===- llvm/CodeGen/WinEHFuncInfo.h -----------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -17,28 +17,26 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/TinyPtrVector.h" -#include "llvm/IR/Instructions.h" +#include <cstdint> +#include <limits> +#include <utility> namespace llvm { + class AllocaInst; class BasicBlock; -class CatchReturnInst; -class Constant; +class FuncletPadInst; class Function; class GlobalVariable; +class Instruction; class InvokeInst; -class IntrinsicInst; -class LandingPadInst; -class MCExpr; -class MCSymbol; class MachineBasicBlock; -class Value; +class MCSymbol; // The following structs respresent the .xdata tables for various // Windows-related EH personalities. -typedef PointerUnion<const BasicBlock *, MachineBasicBlock *> MBBOrBasicBlock; +using MBBOrBasicBlock = PointerUnion<const BasicBlock *, MachineBasicBlock *>; struct CxxUnwindMapEntry { int ToState; @@ -99,18 +97,18 @@ struct WinEHFuncInfo { SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap; SmallVector<SEHUnwindMapEntry, 4> SEHUnwindMap; SmallVector<ClrEHUnwindMapEntry, 4> ClrEHUnwindMap; - int UnwindHelpFrameIdx = INT_MAX; - int PSPSymFrameIdx = INT_MAX; + int UnwindHelpFrameIdx = std::numeric_limits<int>::max(); + int PSPSymFrameIdx = std::numeric_limits<int>::max(); int getLastStateNumber() const { return CxxUnwindMap.size() - 1; } void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd); - int EHRegNodeFrameIndex = INT_MAX; - int EHRegNodeEndOffset = INT_MAX; - int EHGuardFrameIndex = INT_MAX; - int SEHSetFrameOffset = INT_MAX; + int EHRegNodeFrameIndex = std::numeric_limits<int>::max(); + int EHRegNodeEndOffset = std::numeric_limits<int>::max(); + int EHGuardFrameIndex = std::numeric_limits<int>::max(); + int SEHSetFrameOffset = std::numeric_limits<int>::max(); WinEHFuncInfo(); }; @@ -125,5 +123,7 @@ void calculateSEHStateNumbers(const Function *ParentFn, WinEHFuncInfo &FuncInfo); void calculateClrEHStateNumbers(const Function *Fn, WinEHFuncInfo &FuncInfo); -} + +} // end namespace llvm + #endif // LLVM_CODEGEN_WINEHFUNCINFO_H diff --git a/include/llvm/DebugInfo/CodeView/CodeView.h b/include/llvm/DebugInfo/CodeView/CodeView.h index 4e8c8feb7a12..9890263ae2d2 100644 --- a/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/include/llvm/DebugInfo/CodeView/CodeView.h @@ -574,6 +574,14 @@ struct FrameData { IsFunctionStart = 1 << 2, }; }; + +enum class CodeViewContainer { ObjectFile, Pdb }; + +inline uint32_t alignOf(CodeViewContainer Container) { + if (Container == CodeViewContainer::ObjectFile) + return 1; + return 4; +} } } diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h index b3976826a316..db944c7057f7 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h +++ b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h @@ -136,6 +136,7 @@ public: Error mapByteVectorTail(ArrayRef<uint8_t> &Bytes); Error mapByteVectorTail(std::vector<uint8_t> &Bytes); + Error padToAlignment(uint32_t Align); Error skipPadding(); private: diff --git a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h index e7036033d2d9..c958a95ee6de 100644 --- a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h @@ -60,8 +60,8 @@ public: Error initialize(BinaryStreamReader Reader); Error initialize(BinaryStreamRef Stream); - Iterator begin() { return Checksums.begin(); } - Iterator end() { return Checksums.end(); } + Iterator begin() const { return Checksums.begin(); } + Iterator end() const { return Checksums.end(); } const FileChecksumArray &getArray() const { return Checksums; } diff --git a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h index e2cfc3c99233..60440700c265 100644 --- a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h @@ -74,8 +74,13 @@ private: class DebugInlineeLinesSubsection final : public DebugSubsection { public: + struct Entry { + std::vector<support::ulittle32_t> ExtraFiles; + InlineeSourceLineHeader Header; + }; + DebugInlineeLinesSubsection(DebugChecksumsSubsection &Checksums, - bool HasExtraFiles); + bool HasExtraFiles = false); static bool classof(const DebugSubsection *S) { return S->kind() == DebugSubsectionKind::InlineeLines; @@ -87,16 +92,18 @@ public: void addInlineSite(TypeIndex FuncId, StringRef FileName, uint32_t SourceLine); void addExtraFile(StringRef FileName); + bool hasExtraFiles() const { return HasExtraFiles; } + void setHasExtraFiles(bool Has) { HasExtraFiles = Has; } + + std::vector<Entry>::const_iterator begin() const { return Entries.begin(); } + std::vector<Entry>::const_iterator end() const { return Entries.end(); } + private: DebugChecksumsSubsection &Checksums; bool HasExtraFiles = false; uint32_t ExtraFileCount = 0; - struct Entry { - std::vector<support::ulittle32_t> ExtraFiles; - InlineeSourceLineHeader Header; - }; std::vector<Entry> Entries; }; } diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h index b2e1131e5968..847259c5ceac 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h +++ b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h @@ -31,28 +31,32 @@ struct DebugSubsectionHeader { class DebugSubsectionRecord { public: DebugSubsectionRecord(); - DebugSubsectionRecord(DebugSubsectionKind Kind, BinaryStreamRef Data); + DebugSubsectionRecord(DebugSubsectionKind Kind, BinaryStreamRef Data, + CodeViewContainer Container); - static Error initialize(BinaryStreamRef Stream, DebugSubsectionRecord &Info); + static Error initialize(BinaryStreamRef Stream, DebugSubsectionRecord &Info, + CodeViewContainer Container); uint32_t getRecordLength() const; DebugSubsectionKind kind() const; BinaryStreamRef getRecordData() const; private: + CodeViewContainer Container; DebugSubsectionKind Kind; BinaryStreamRef Data; }; class DebugSubsectionRecordBuilder { public: - DebugSubsectionRecordBuilder(DebugSubsectionKind Kind, DebugSubsection &Frag); + DebugSubsectionRecordBuilder(std::unique_ptr<DebugSubsection> Subsection, + CodeViewContainer Container); uint32_t calculateSerializedLength(); Error commit(BinaryStreamWriter &Writer); private: - DebugSubsectionKind Kind; - DebugSubsection &Frag; + std::unique_ptr<DebugSubsection> Subsection; + CodeViewContainer Container; }; } // namespace codeview @@ -62,7 +66,12 @@ template <> struct VarStreamArrayExtractor<codeview::DebugSubsectionRecord> { static Error extract(BinaryStreamRef Stream, uint32_t &Length, codeview::DebugSubsectionRecord &Info) { - if (auto EC = codeview::DebugSubsectionRecord::initialize(Stream, Info)) + // FIXME: We need to pass the container type through to this function, but + // VarStreamArray doesn't easily support stateful contexts. In practice + // this isn't super important since the subsection header describes its + // length and we can just skip it. It's more important when writing. + if (auto EC = codeview::DebugSubsectionRecord::initialize( + Stream, Info, codeview::CodeViewContainer::Pdb)) return EC; Length = Info.getRecordLength(); return Error::success(); diff --git a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h index 428ff153d5d1..7080b0480757 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h @@ -24,9 +24,9 @@ namespace codeview { class SymbolVisitorDelegate; class SymbolDeserializer : public SymbolVisitorCallbacks { struct MappingInfo { - explicit MappingInfo(ArrayRef<uint8_t> RecordData) + MappingInfo(ArrayRef<uint8_t> RecordData, CodeViewContainer Container) : Stream(RecordData, llvm::support::little), Reader(Stream), - Mapping(Reader) {} + Mapping(Reader, Container) {} BinaryByteStream Stream; BinaryStreamReader Reader; @@ -35,7 +35,9 @@ class SymbolDeserializer : public SymbolVisitorCallbacks { public: template <typename T> static Error deserializeAs(CVSymbol Symbol, T &Record) { - SymbolDeserializer S(nullptr); + // If we're just deserializing one record, then don't worry about alignment + // as there's nothing that comes after. + SymbolDeserializer S(nullptr, CodeViewContainer::ObjectFile); if (auto EC = S.visitSymbolBegin(Symbol)) return EC; if (auto EC = S.visitKnownRecord(Symbol, Record)) @@ -45,12 +47,13 @@ public: return Error::success(); } - explicit SymbolDeserializer(SymbolVisitorDelegate *Delegate) - : Delegate(Delegate) {} + explicit SymbolDeserializer(SymbolVisitorDelegate *Delegate, + CodeViewContainer Container) + : Delegate(Delegate), Container(Container) {} Error visitSymbolBegin(CVSymbol &Record) override { assert(!Mapping && "Already in a symbol mapping!"); - Mapping = llvm::make_unique<MappingInfo>(Record.content()); + Mapping = llvm::make_unique<MappingInfo>(Record.content(), Container); return Mapping->Mapping.visitSymbolBegin(Record); } Error visitSymbolEnd(CVSymbol &Record) override { @@ -78,6 +81,7 @@ private: } SymbolVisitorDelegate *Delegate; + CodeViewContainer Container; std::unique_ptr<MappingInfo> Mapping; }; } diff --git a/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/include/llvm/DebugInfo/CodeView/SymbolDumper.h index e91065dcf87e..293daa851bdd 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDumper.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDumper.h @@ -26,9 +26,11 @@ class TypeCollection; class CVSymbolDumper { public: CVSymbolDumper(ScopedPrinter &W, TypeCollection &Types, + CodeViewContainer Container, std::unique_ptr<SymbolDumpDelegate> ObjDelegate, bool PrintRecordBytes) - : W(W), Types(Types), ObjDelegate(std::move(ObjDelegate)), + : W(W), Types(Types), Container(Container), + ObjDelegate(std::move(ObjDelegate)), PrintRecordBytes(PrintRecordBytes) {} /// Dumps one type record. Returns false if there was a type parsing error, @@ -44,6 +46,7 @@ public: private: ScopedPrinter &W; TypeCollection &Types; + CodeViewContainer Container; std::unique_ptr<SymbolDumpDelegate> ObjDelegate; bool PrintRecordBytes; diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h index 5d072a3b2723..391e8f127665 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h +++ b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h @@ -20,8 +20,12 @@ class BinaryStreamWriter; namespace codeview { class SymbolRecordMapping : public SymbolVisitorCallbacks { public: - explicit SymbolRecordMapping(BinaryStreamReader &Reader) : IO(Reader) {} - explicit SymbolRecordMapping(BinaryStreamWriter &Writer) : IO(Writer) {} + explicit SymbolRecordMapping(BinaryStreamReader &Reader, + CodeViewContainer Container) + : IO(Reader), Container(Container) {} + explicit SymbolRecordMapping(BinaryStreamWriter &Writer, + CodeViewContainer Container) + : IO(Writer), Container(Container) {} Error visitSymbolBegin(CVSymbol &Record) override; Error visitSymbolEnd(CVSymbol &Record) override; @@ -35,6 +39,7 @@ private: Optional<SymbolKind> Kind; CodeViewRecordIO IO; + CodeViewContainer Container; }; } } diff --git a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h index a8fe1a3ae1d0..42adbdb4e20f 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h +++ b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h @@ -46,17 +46,18 @@ class SymbolSerializer : public SymbolVisitorCallbacks { public: template <typename SymType> - static CVSymbol writeOneSymbol(SymType &Sym, BumpPtrAllocator &Storage) { + static CVSymbol writeOneSymbol(SymType &Sym, BumpPtrAllocator &Storage, + CodeViewContainer Container) { CVSymbol Result; Result.Type = static_cast<SymbolKind>(Sym.Kind); - SymbolSerializer Serializer(Storage); + SymbolSerializer Serializer(Storage, Container); consumeError(Serializer.visitSymbolBegin(Result)); consumeError(Serializer.visitKnownRecord(Result, Sym)); consumeError(Serializer.visitSymbolEnd(Result)); return Result; } - explicit SymbolSerializer(BumpPtrAllocator &Storage); + SymbolSerializer(BumpPtrAllocator &Storage, CodeViewContainer Container); virtual Error visitSymbolBegin(CVSymbol &Record) override; virtual Error visitSymbolEnd(CVSymbol &Record) override; diff --git a/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/include/llvm/DebugInfo/MSF/MappedBlockStream.h index d68f5f70c83e..36dce393fc66 100644 --- a/include/llvm/DebugInfo/MSF/MappedBlockStream.h +++ b/include/llvm/DebugInfo/MSF/MappedBlockStream.h @@ -44,17 +44,19 @@ class MappedBlockStream : public BinaryStream { public: static std::unique_ptr<MappedBlockStream> createStream(uint32_t BlockSize, const MSFStreamLayout &Layout, - BinaryStreamRef MsfData); + BinaryStreamRef MsfData, BumpPtrAllocator &Allocator); static std::unique_ptr<MappedBlockStream> createIndexedStream(const MSFLayout &Layout, BinaryStreamRef MsfData, - uint32_t StreamIndex); + uint32_t StreamIndex, BumpPtrAllocator &Allocator); static std::unique_ptr<MappedBlockStream> - createFpmStream(const MSFLayout &Layout, BinaryStreamRef MsfData); + createFpmStream(const MSFLayout &Layout, BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator); static std::unique_ptr<MappedBlockStream> - createDirectoryStream(const MSFLayout &Layout, BinaryStreamRef MsfData); + createDirectoryStream(const MSFLayout &Layout, BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator); llvm::support::endianness getEndian() const override { return llvm::support::little; @@ -67,9 +69,7 @@ public: uint32_t getLength() override; - uint32_t getNumBytesCopied() const; - - llvm::BumpPtrAllocator &getAllocator() { return Pool; } + llvm::BumpPtrAllocator &getAllocator() { return Allocator; } void invalidateCache(); @@ -79,7 +79,7 @@ public: protected: MappedBlockStream(uint32_t BlockSize, const MSFStreamLayout &StreamLayout, - BinaryStreamRef MsfData); + BinaryStreamRef MsfData, BumpPtrAllocator &Allocator); private: const MSFStreamLayout &getStreamLayout() const { return StreamLayout; } @@ -94,7 +94,15 @@ private: BinaryStreamRef MsfData; typedef MutableArrayRef<uint8_t> CacheEntry; - llvm::BumpPtrAllocator Pool; + + // We just store the allocator by reference. We use this to allocate + // contiguous memory for things like arrays or strings that cross a block + // boundary, and this memory is expected to outlive the stream. For example, + // someone could create a stream, read some stuff, then close the stream, and + // we would like outstanding references to fields to remain valid since the + // entire file is mapped anyway. Because of that, the user must supply the + // allocator to allocate broken records from. + BumpPtrAllocator &Allocator; DenseMap<uint32_t, std::vector<CacheEntry>> CacheMap; }; @@ -102,18 +110,20 @@ class WritableMappedBlockStream : public WritableBinaryStream { public: static std::unique_ptr<WritableMappedBlockStream> createStream(uint32_t BlockSize, const MSFStreamLayout &Layout, - WritableBinaryStreamRef MsfData); + WritableBinaryStreamRef MsfData, BumpPtrAllocator &Allocator); static std::unique_ptr<WritableMappedBlockStream> createIndexedStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData, - uint32_t StreamIndex); + uint32_t StreamIndex, BumpPtrAllocator &Allocator); static std::unique_ptr<WritableMappedBlockStream> createDirectoryStream(const MSFLayout &Layout, - WritableBinaryStreamRef MsfData); + WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator); static std::unique_ptr<WritableMappedBlockStream> - createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData); + createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator); llvm::support::endianness getEndian() const override { return llvm::support::little; @@ -139,7 +149,8 @@ public: protected: WritableMappedBlockStream(uint32_t BlockSize, const MSFStreamLayout &StreamLayout, - WritableBinaryStreamRef MsfData); + WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator); private: MappedBlockStream ReadInterface; diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h index e5858d0f45e3..2ff166b24e68 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h @@ -49,11 +49,8 @@ public: void setObjFileName(StringRef Name); void addSymbol(codeview::CVSymbol Symbol); - void addC13Fragment(std::unique_ptr<codeview::DebugLinesSubsection> Lines); - void addC13Fragment( - std::unique_ptr<codeview::DebugInlineeLinesSubsection> Inlinees); - void setC13FileChecksums( - std::unique_ptr<codeview::DebugChecksumsSubsection> Checksums); + void + addDebugSubsection(std::unique_ptr<codeview::DebugSubsection> Subsection); uint16_t getStreamIndex() const; StringRef getModuleName() const { return ModuleName; } @@ -83,10 +80,6 @@ private: std::vector<std::string> SourceFiles; std::vector<codeview::CVSymbol> Symbols; - std::unique_ptr<codeview::DebugChecksumsSubsection> ChecksumInfo; - std::vector<std::unique_ptr<codeview::DebugLinesSubsection>> LineInfo; - std::vector<std::unique_ptr<codeview::DebugInlineeLinesSubsection>> Inlinees; - std::vector<std::unique_ptr<codeview::DebugSubsectionRecordBuilder>> C13Builders; diff --git a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h index 822ce3ce13d3..a8121978d882 100644 --- a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h +++ b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h @@ -12,6 +12,7 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" +#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" @@ -25,7 +26,7 @@ class PDBFile; class DbiModuleDescriptor; class ModuleDebugStreamRef { - typedef codeview::DebugSubsectionArray::Iterator LinesAndChecksumsIterator; + typedef codeview::DebugSubsectionArray::Iterator DebugSubsectionIterator; public: ModuleDebugStreamRef(const DbiModuleDescriptor &Module, @@ -39,12 +40,15 @@ public: iterator_range<codeview::CVSymbolArray::Iterator> symbols(bool *HadError) const; - llvm::iterator_range<LinesAndChecksumsIterator> linesAndChecksums() const; + llvm::iterator_range<DebugSubsectionIterator> subsections() const; - bool hasLineInfo() const; + bool hasDebugSubsections() const; Error commit(); + Expected<codeview::DebugChecksumsSubsectionRef> + findChecksumsSubsection() const; + private: const DbiModuleDescriptor &Mod; @@ -57,7 +61,7 @@ private: BinaryStreamRef C13LinesSubstream; BinaryStreamRef GlobalRefsSubstream; - codeview::DebugSubsectionArray LinesAndChecksums; + codeview::DebugSubsectionArray Subsections; }; } } diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h index 6aeb0a5479cb..28a14d7356d2 100644 --- a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h +++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h @@ -45,6 +45,8 @@ public: FixedStreamArray<support::ulittle32_t> name_ids() const; + codeview::DebugStringTableSubsectionRef getStringTable() const; + private: Error readHeader(BinaryStreamReader &Reader); Error readStrings(BinaryStreamReader &Reader); diff --git a/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/include/llvm/DebugInfo/PDB/Native/TpiStream.h index 17fba9991c2e..0ee697696ca5 100644 --- a/include/llvm/DebugInfo/PDB/Native/TpiStream.h +++ b/include/llvm/DebugInfo/PDB/Native/TpiStream.h @@ -34,8 +34,7 @@ class TpiStream { friend class TpiStreamBuilder; public: - TpiStream(const PDBFile &File, - std::unique_ptr<msf::MappedBlockStream> Stream); + TpiStream(PDBFile &File, std::unique_ptr<msf::MappedBlockStream> Stream); ~TpiStream(); Error reload(); @@ -61,7 +60,7 @@ public: Error commit(); private: - const PDBFile &Pdb; + PDBFile &Pdb; std::unique_ptr<msf::MappedBlockStream> Stream; std::unique_ptr<codeview::LazyRandomTypeCollection> Types; diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h index 4afb5d9d63b2..8e6bb4baccaf 100644 --- a/include/llvm/IR/DIBuilder.h +++ b/include/llvm/IR/DIBuilder.h @@ -86,6 +86,10 @@ namespace llvm { /// Construct any deferred debug info descriptors. void finalize(); + /// Finalize a specific subprogram - no new variables may be added to this + /// subprogram afterwards. + void finalizeSubprogram(DISubprogram *SP); + /// A CompileUnit provides an anchor for all debugging /// information generated during this instance of compilation. /// \param Lang Source programming language, eg. dwarf::DW_LANG_C99 diff --git a/include/llvm/IR/DebugLoc.h b/include/llvm/IR/DebugLoc.h index aa74f361cda2..eef1212abc4b 100644 --- a/include/llvm/IR/DebugLoc.h +++ b/include/llvm/IR/DebugLoc.h @@ -90,12 +90,6 @@ namespace llvm { DenseMap<const MDNode *, MDNode *> &Cache, bool ReplaceLast = false); - /// Reparent all debug locations referenced by \c I that belong to \c OrigSP - /// to become (possibly indirect) children of \c NewSP. - static void reparentDebugInfo(Instruction &I, DISubprogram *OrigSP, - DISubprogram *NewSP, - DenseMap<const MDNode *, MDNode *> &Cache); - unsigned getLine() const; unsigned getCol() const; MDNode *getScope() const; diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h index c46c609609e2..757ddf6cf46b 100644 --- a/include/llvm/IR/ModuleSummaryIndex.h +++ b/include/llvm/IR/ModuleSummaryIndex.h @@ -134,16 +134,18 @@ public: /// be renamed or references something that can't be renamed). unsigned NotEligibleToImport : 1; - /// Indicate that the global value must be considered a live root for - /// index-based liveness analysis. Used for special LLVM values such as - /// llvm.global_ctors that the linker does not know about. - unsigned LiveRoot : 1; + /// In per-module summary, indicate that the global value must be considered + /// a live root for index-based liveness analysis. Used for special LLVM + /// values such as llvm.global_ctors that the linker does not know about. + /// + /// In combined summary, indicate that the global value is live. + unsigned Live : 1; /// Convenience Constructors explicit GVFlags(GlobalValue::LinkageTypes Linkage, - bool NotEligibleToImport, bool LiveRoot) + bool NotEligibleToImport, bool Live) : Linkage(Linkage), NotEligibleToImport(NotEligibleToImport), - LiveRoot(LiveRoot) {} + Live(Live) {} }; private: @@ -172,6 +174,8 @@ private: /// are listed in the derived FunctionSummary object. std::vector<ValueInfo> RefEdgeList; + bool isLive() const { return Flags.Live; } + protected: GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector<ValueInfo> Refs) : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {} @@ -213,19 +217,17 @@ public: /// Return true if this global value can't be imported. bool notEligibleToImport() const { return Flags.NotEligibleToImport; } - /// Return true if this global value must be considered a root for live - /// value analysis on the index. - bool liveRoot() const { return Flags.LiveRoot; } - - /// Flag that this global value must be considered a root for live - /// value analysis on the index. - void setLiveRoot() { Flags.LiveRoot = true; } + void setLive(bool Live) { Flags.Live = Live; } /// Flag that this global value cannot be imported. void setNotEligibleToImport() { Flags.NotEligibleToImport = true; } /// Return the list of values referenced by this global value definition. ArrayRef<ValueInfo> refs() const { return RefEdgeList; } + + friend class ModuleSummaryIndex; + friend void computeDeadSymbols(class ModuleSummaryIndex &, + const DenseSet<GlobalValue::GUID> &); }; /// \brief Alias summary information. @@ -535,6 +537,11 @@ private: /// GUIDs, it will be mapped to 0. std::map<GlobalValue::GUID, GlobalValue::GUID> OidGuidMap; + /// Indicates that summary-based GlobalValue GC has run, and values with + /// GVFlags::Live==false are really dead. Otherwise, all values must be + /// considered live. + bool WithGlobalValueDeadStripping = false; + // YAML I/O support. friend yaml::MappingTraits<ModuleSummaryIndex>; @@ -550,6 +557,17 @@ public: const_gvsummary_iterator end() const { return GlobalValueMap.end(); } size_t size() const { return GlobalValueMap.size(); } + bool withGlobalValueDeadStripping() const { + return WithGlobalValueDeadStripping; + } + void setWithGlobalValueDeadStripping() { + WithGlobalValueDeadStripping = true; + } + + bool isGlobalValueLive(const GlobalValueSummary *GVS) const { + return !WithGlobalValueDeadStripping || GVS->isLive(); + } + /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo(). ValueInfo getValueInfo(GlobalValue::GUID GUID) const { auto I = GlobalValueMap.find(GUID); diff --git a/include/llvm/IR/ModuleSummaryIndexYAML.h b/include/llvm/IR/ModuleSummaryIndexYAML.h index 78fdb602027d..891d84c2dbca 100644 --- a/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -128,6 +128,8 @@ template <> struct MappingTraits<TypeIdSummary> { }; struct FunctionSummaryYaml { + unsigned Linkage; + bool NotEligibleToImport, Live; std::vector<uint64_t> TypeTests; std::vector<FunctionSummary::VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls; @@ -168,6 +170,9 @@ namespace yaml { template <> struct MappingTraits<FunctionSummaryYaml> { static void mapping(IO &io, FunctionSummaryYaml& summary) { + io.mapOptional("Linkage", summary.Linkage); + io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport); + io.mapOptional("Live", summary.Live); io.mapOptional("TypeTests", summary.TypeTests); io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls); io.mapOptional("TypeCheckedLoadVCalls", summary.TypeCheckedLoadVCalls); @@ -199,12 +204,12 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { } auto &Elem = V[KeyInt]; for (auto &FSum : FSums) { - GlobalValueSummary::GVFlags GVFlags(GlobalValue::ExternalLinkage, false, - false); Elem.SummaryList.push_back(llvm::make_unique<FunctionSummary>( - GVFlags, 0, ArrayRef<ValueInfo>{}, - ArrayRef<FunctionSummary::EdgeTy>{}, std::move(FSum.TypeTests), - std::move(FSum.TypeTestAssumeVCalls), + GlobalValueSummary::GVFlags( + static_cast<GlobalValue::LinkageTypes>(FSum.Linkage), + FSum.NotEligibleToImport, FSum.Live), + 0, ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{}, + std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), std::move(FSum.TypeCheckedLoadVCalls), std::move(FSum.TypeTestAssumeConstVCalls), std::move(FSum.TypeCheckedLoadConstVCalls))); @@ -216,8 +221,10 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> { for (auto &Sum : P.second.SummaryList) { if (auto *FSum = dyn_cast<FunctionSummary>(Sum.get())) FSums.push_back(FunctionSummaryYaml{ - FSum->type_tests(), FSum->type_test_assume_vcalls(), - FSum->type_checked_load_vcalls(), + FSum->flags().Linkage, + static_cast<bool>(FSum->flags().NotEligibleToImport), + static_cast<bool>(FSum->flags().Live), FSum->type_tests(), + FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(), FSum->type_test_assume_const_vcalls(), FSum->type_checked_load_const_vcalls()}); } @@ -231,6 +238,8 @@ template <> struct MappingTraits<ModuleSummaryIndex> { static void mapping(IO &io, ModuleSummaryIndex& index) { io.mapOptional("GlobalValueMap", index.GlobalValueMap); io.mapOptional("TypeIdMap", index.TypeIdMap); + io.mapOptional("WithGlobalValueDeadStripping", + index.WithGlobalValueDeadStripping); } }; diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h index f01607614a0c..a5f0130f79f4 100644 --- a/include/llvm/IR/Statepoint.h +++ b/include/llvm/IR/Statepoint.h @@ -228,24 +228,24 @@ public: return cast<ConstantInt>(NumVMSArgs)->getZExtValue(); } - typename CallSiteTy::arg_iterator vm_state_begin() const { + typename CallSiteTy::arg_iterator deopt_begin() const { auto I = gc_transition_args_end() + 1; assert((getCallSite().arg_end() - I) >= 0); return I; } - typename CallSiteTy::arg_iterator vm_state_end() const { - auto I = vm_state_begin() + getNumTotalVMSArgs(); + typename CallSiteTy::arg_iterator deopt_end() const { + auto I = deopt_begin() + getNumTotalVMSArgs(); assert((getCallSite().arg_end() - I) >= 0); return I; } /// range adapter for vm state arguments - iterator_range<arg_iterator> vm_state_args() const { - return make_range(vm_state_begin(), vm_state_end()); + iterator_range<arg_iterator> deopt_operands() const { + return make_range(deopt_begin(), deopt_end()); } typename CallSiteTy::arg_iterator gc_args_begin() const { - return vm_state_end(); + return deopt_end(); } typename CallSiteTy::arg_iterator gc_args_end() const { return getCallSite().arg_end(); @@ -289,8 +289,8 @@ public: (void)arg_end(); (void)gc_transition_args_begin(); (void)gc_transition_args_end(); - (void)vm_state_begin(); - (void)vm_state_end(); + (void)deopt_begin(); + (void)deopt_end(); (void)gc_args_begin(); (void)gc_args_end(); } diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index 5b9796d4fba6..abb0aa3e3caf 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -86,7 +86,6 @@ void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&); void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&); void initializeCFGPrinterLegacyPassPass(PassRegistry&); void initializeCFGSimplifyPassPass(PassRegistry&); -void initializeLateCFGSimplifyPassPass(PassRegistry&); void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); @@ -144,8 +143,8 @@ void initializeGCMachineCodeAnalysisPass(PassRegistry&); void initializeGCModuleInfoPass(PassRegistry&); void initializeGCOVProfilerLegacyPassPass(PassRegistry&); void initializeGVNHoistLegacyPassPass(PassRegistry&); -void initializeGVNSinkLegacyPassPass(PassRegistry&); void initializeGVNLegacyPassPass(PassRegistry&); +void initializeGVNSinkLegacyPassPass(PassRegistry&); void initializeGlobalDCELegacyPassPass(PassRegistry&); void initializeGlobalMergePass(PassRegistry&); void initializeGlobalOptLegacyPassPass(PassRegistry&); @@ -175,13 +174,14 @@ void initializeIntervalPartitionPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); void initializeLCSSAVerificationPassPass(PassRegistry&); void initializeLCSSAWrapperPassPass(PassRegistry&); +void initializeLateCFGSimplifyPassPass(PassRegistry&); void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&); void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&); void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&); +void initializeLazyValueInfoPrinterPass(PassRegistry&); void initializeLazyValueInfoWrapperPassPass(PassRegistry&); void initializeLegacyLICMPassPass(PassRegistry&); void initializeLegacyLoopSinkPassPass(PassRegistry&); -void initializeLazyValueInfoPrinterPass(PassRegistry&); void initializeLegalizerPass(PassRegistry&); void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&); void initializeLintPass(PassRegistry&); @@ -195,8 +195,8 @@ void initializeLiveVariablesPass(PassRegistry&); void initializeLoadCombinePass(PassRegistry&); void initializeLoadStoreVectorizerPass(PassRegistry&); void initializeLoaderPassPass(PassRegistry&); -void initializeLocalizerPass(PassRegistry&); void initializeLocalStackSlotPassPass(PassRegistry&); +void initializeLocalizerPass(PassRegistry&); void initializeLoopAccessLegacyAnalysisPass(PassRegistry&); void initializeLoopDataPrefetchLegacyPassPass(PassRegistry&); void initializeLoopDeletionLegacyPassPass(PassRegistry&); @@ -304,6 +304,7 @@ void initializeProcessImplicitDefsPass(PassRegistry&); void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&); void initializePromoteLegacyPassPass(PassRegistry&); void initializePruneEHPass(PassRegistry&); +void initializeRABasicPass(PassRegistry&); void initializeRAGreedyPass(PassRegistry&); void initializeReassociateLegacyPassPass(PassRegistry&); void initializeRegBankSelectPass(PassRegistry&); @@ -327,8 +328,9 @@ void initializeSafeStackLegacyPassPass(PassRegistry&); void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&); void initializeSanitizerCoverageModulePass(PassRegistry&); void initializeScalarEvolutionWrapperPassPass(PassRegistry&); -void initializeScalarizerPass(PassRegistry&); void initializeScalarizeMaskedMemIntrinPass(PassRegistry&); +void initializeScalarizerPass(PassRegistry&); +void initializeScavengerTestPass(PassRegistry&); void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&); void initializeSeparateConstOffsetFromGEPPass(PassRegistry&); void initializeShadowStackGCLoweringPass(PassRegistry&); diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h index 5ba8492db8f5..73106f77ca55 100644 --- a/include/llvm/LTO/Config.h +++ b/include/llvm/LTO/Config.h @@ -46,6 +46,9 @@ struct Config { unsigned OptLevel = 2; bool DisableVerify = false; + /// Use the new pass manager + bool UseNewPM = false; + /// Disable entirely the optimizer, including importing for ThinLTO bool CodeGenOnly = false; diff --git a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h index 6ddae2e2b41c..a6d4d404415f 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h @@ -17,12 +17,20 @@ #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/DebugInfo/CodeView/DebugSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include "llvm/ObjectYAML/YAML.h" namespace llvm { + +namespace codeview { +class DebugStringTableSubsection; +class DebugStringTableSubsectionRef; +class DebugChecksumsSubsectionRef; +} namespace CodeViewYAML { + namespace detail { -struct C13FragmentBase; +struct YAMLSubsectionBase; } struct SourceLineEntry { @@ -74,18 +82,24 @@ struct InlineeInfo { std::vector<InlineeSite> Sites; }; -struct SourceFileInfo { - std::vector<SourceFileChecksumEntry> FileChecksums; - std::vector<SourceLineInfo> LineFragments; - std::vector<InlineeInfo> Inlinees; -}; +struct YAMLDebugSubsection { + static Expected<YAMLDebugSubsection> + fromCodeViewSubection(const codeview::DebugStringTableSubsectionRef &Strings, + const codeview::DebugChecksumsSubsectionRef &Checksums, + const codeview::DebugSubsectionRecord &SS); -struct C13DebugSection { - std::vector<detail::C13FragmentBase> Fragments; + std::shared_ptr<detail::YAMLSubsectionBase> Subsection; }; + +Expected<std::vector<std::unique_ptr<codeview::DebugSubsection>>> +convertSubsectionList(ArrayRef<YAMLDebugSubsection> Subsections, + codeview::DebugStringTableSubsection &Strings); + } // namespace CodeViewYAML } // namespace llvm -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceFileInfo) +LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::YAMLDebugSubsection) + +LLVM_YAML_IS_SEQUENCE_VECTOR(CodeViewYAML::YAMLDebugSubsection) #endif diff --git a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h index ee4e2ac9d404..9b411e8b074f 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h @@ -28,7 +28,9 @@ struct SymbolRecordBase; struct SymbolRecord { std::shared_ptr<detail::SymbolRecordBase> Symbol; - codeview::CVSymbol toCodeViewSymbol(BumpPtrAllocator &Allocator) const; + codeview::CVSymbol + toCodeViewSymbol(BumpPtrAllocator &Allocator, + codeview::CodeViewContainer Container) const; static Expected<SymbolRecord> fromCodeViewSymbol(codeview::CVSymbol Symbol); }; diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h index 3c181f0e511b..5c3bf88fbbfa 100644 --- a/include/llvm/TableGen/Record.h +++ b/include/llvm/TableGen/Record.h @@ -1361,10 +1361,6 @@ public: return false; } - bool isTemplateArg(StringRef Name) const { - return isTemplateArg(StringInit::get(Name)); - } - const RecordVal *getValue(const Init *Name) const { for (const RecordVal &Val : Values) if (Val.Name == Name) return &Val; @@ -1388,10 +1384,6 @@ public: TemplateArgs.push_back(Name); } - void addTemplateArg(StringRef Name) { - addTemplateArg(StringInit::get(Name)); - } - void addValue(const RecordVal &RV) { assert(getValue(RV.getNameInit()) == nullptr && "Value already added!"); Values.push_back(RV); diff --git a/include/llvm/Transforms/IPO/FunctionImport.h b/include/llvm/Transforms/IPO/FunctionImport.h index d66b6edc7a4f..de35cdf052e1 100644 --- a/include/llvm/Transforms/IPO/FunctionImport.h +++ b/include/llvm/Transforms/IPO/FunctionImport.h @@ -81,15 +81,11 @@ public: /// \p ExportLists contains for each Module the set of globals (GUID) that will /// be imported by another module, or referenced by such a function. I.e. this /// is the set of globals that need to be promoted/renamed appropriately. -/// -/// \p DeadSymbols (optional) contains a list of GUID that are deemed "dead" and -/// will be ignored for the purpose of importing. void ComputeCrossModuleImport( const ModuleSummaryIndex &Index, const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, StringMap<FunctionImporter::ImportMapTy> &ImportLists, - StringMap<FunctionImporter::ExportSetTy> &ExportLists, - const DenseSet<GlobalValue::GUID> *DeadSymbols = nullptr); + StringMap<FunctionImporter::ExportSetTy> &ExportLists); /// Compute all the imports for the given module using the Index. /// @@ -102,9 +98,9 @@ void ComputeCrossModuleImportForModule( /// Compute all the symbols that are "dead": i.e these that can't be reached /// in the graph from any of the given symbols listed in /// \p GUIDPreservedSymbols. -DenseSet<GlobalValue::GUID> -computeDeadSymbols(const ModuleSummaryIndex &Index, - const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols); +void computeDeadSymbols( + ModuleSummaryIndex &Index, + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols); /// Compute the set of summaries needed for a ThinLTO backend compilation of /// \p ModulePath. diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h index 023d7af7f729..b6c6c091631d 100644 --- a/include/llvm/Transforms/Instrumentation.h +++ b/include/llvm/Transforms/Instrumentation.h @@ -177,6 +177,7 @@ struct SanitizerCoverageOptions { bool Use8bitCounters = false; bool TracePC = false; bool TracePCGuard = false; + bool Inline8bitCounters = false; bool NoPrune = false; SanitizerCoverageOptions() = default; diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h index 91c9d255302f..2a8b89d86282 100644 --- a/include/llvm/Transforms/Utils/Cloning.h +++ b/include/llvm/Transforms/Utils/Cloning.h @@ -36,6 +36,7 @@ class BasicBlock; class BlockFrequencyInfo; class CallInst; class CallGraph; +class DebugInfoFinder; class DominatorTree; class Function; class Instruction; @@ -110,7 +111,8 @@ struct ClonedCodeInfo { /// BasicBlock *CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix = "", Function *F = nullptr, - ClonedCodeInfo *CodeInfo = nullptr); + ClonedCodeInfo *CodeInfo = nullptr, + DebugInfoFinder *DIFinder = nullptr); /// CloneFunction - Return a copy of the specified function and add it to that /// function's module. Also, any references specified in the VMap are changed diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp index 6a1af87450c9..a906770dbb34 100644 --- a/lib/Analysis/ConstantFolding.cpp +++ b/lib/Analysis/ConstantFolding.cpp @@ -1170,7 +1170,9 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, const DataLayout &DL, const TargetLibraryInfo *TLI) { // fold: icmp (inttoptr x), null -> icmp x, 0 + // fold: icmp null, (inttoptr x) -> icmp 0, x // fold: icmp (ptrtoint x), 0 -> icmp x, null + // fold: icmp 0, (ptrtoint x) -> icmp null, x // fold: icmp (inttoptr x), (inttoptr y) -> icmp trunc/zext x, trunc/zext y // fold: icmp (ptrtoint x), (ptrtoint y) -> icmp x, y // @@ -1240,6 +1242,11 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, Predicate == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or; return ConstantFoldBinaryOpOperands(OpC, LHS, RHS, DL); } + } else if (isa<ConstantExpr>(Ops1)) { + // If RHS is a constant expression, but the left side isn't, swap the + // operands and try again. + Predicate = ICmpInst::getSwappedPredicate((ICmpInst::Predicate)Predicate); + return ConstantFoldCompareInstOperands(Predicate, Ops1, Ops0, DL, TLI); } return ConstantExpr::getCompare(Predicate, Ops0, Ops1); diff --git a/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/lib/Analysis/IndirectCallPromotionAnalysis.cpp index 3da33ac71421..ed233d201537 100644 --- a/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -43,7 +43,7 @@ static cl::opt<unsigned> // The percent threshold for the direct-call target (this call site vs the // total call count) for it to be considered as the promotion target. static cl::opt<unsigned> - ICPPercentThreshold("icp-percent-threshold", cl::init(33), cl::Hidden, + ICPPercentThreshold("icp-percent-threshold", cl::init(30), cl::Hidden, cl::ZeroOrMore, cl::desc("The percentage threshold for the promotion")); diff --git a/lib/Analysis/InlineCost.cpp b/lib/Analysis/InlineCost.cpp index 4702569126c6..77c87928728a 100644 --- a/lib/Analysis/InlineCost.cpp +++ b/lib/Analysis/InlineCost.cpp @@ -54,11 +54,6 @@ static cl::opt<int> cl::init(45), cl::desc("Threshold for inlining cold callsites")); -static cl::opt<bool> - EnableGenericSwitchCost("inline-generic-switch-cost", cl::Hidden, - cl::init(false), - cl::desc("Enable generic switch cost model")); - // We introduce this threshold to help performance of instrumentation based // PGO before we actually hook up inliner with analysis passes such as BPI and // BFI. @@ -1015,83 +1010,68 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { if (isa<ConstantInt>(V)) return true; - if (EnableGenericSwitchCost) { - // Assume the most general case where the swith is lowered into - // either a jump table, bit test, or a balanced binary tree consisting of - // case clusters without merging adjacent clusters with the same - // destination. We do not consider the switches that are lowered with a mix - // of jump table/bit test/binary search tree. The cost of the switch is - // proportional to the size of the tree or the size of jump table range. - - // Exit early for a large switch, assuming one case needs at least one - // instruction. - // FIXME: This is not true for a bit test, but ignore such case for now to - // save compile-time. - int64_t CostLowerBound = - std::min((int64_t)INT_MAX, - (int64_t)SI.getNumCases() * InlineConstants::InstrCost + Cost); - - if (CostLowerBound > Threshold) { - Cost = CostLowerBound; - return false; - } + // Assume the most general case where the swith is lowered into + // either a jump table, bit test, or a balanced binary tree consisting of + // case clusters without merging adjacent clusters with the same + // destination. We do not consider the switches that are lowered with a mix + // of jump table/bit test/binary search tree. The cost of the switch is + // proportional to the size of the tree or the size of jump table range. + // + // NB: We convert large switches which are just used to initialize large phi + // nodes to lookup tables instead in simplify-cfg, so this shouldn't prevent + // inlining those. It will prevent inlining in cases where the optimization + // does not (yet) fire. - unsigned JumpTableSize = 0; - unsigned NumCaseCluster = - TTI.getEstimatedNumberOfCaseClusters(SI, JumpTableSize); + // Exit early for a large switch, assuming one case needs at least one + // instruction. + // FIXME: This is not true for a bit test, but ignore such case for now to + // save compile-time. + int64_t CostLowerBound = + std::min((int64_t)INT_MAX, + (int64_t)SI.getNumCases() * InlineConstants::InstrCost + Cost); - // If suitable for a jump table, consider the cost for the table size and - // branch to destination. - if (JumpTableSize) { - int64_t JTCost = (int64_t)JumpTableSize * InlineConstants::InstrCost + - 4 * InlineConstants::InstrCost; - Cost = std::min((int64_t)INT_MAX, JTCost + Cost); - return false; - } + if (CostLowerBound > Threshold) { + Cost = CostLowerBound; + return false; + } - // Considering forming a binary search, we should find the number of nodes - // which is same as the number of comparisons when lowered. For a given - // number of clusters, n, we can define a recursive function, f(n), to find - // the number of nodes in the tree. The recursion is : - // f(n) = 1 + f(n/2) + f (n - n/2), when n > 3, - // and f(n) = n, when n <= 3. - // This will lead a binary tree where the leaf should be either f(2) or f(3) - // when n > 3. So, the number of comparisons from leaves should be n, while - // the number of non-leaf should be : - // 2^(log2(n) - 1) - 1 - // = 2^log2(n) * 2^-1 - 1 - // = n / 2 - 1. - // Considering comparisons from leaf and non-leaf nodes, we can estimate the - // number of comparisons in a simple closed form : - // n + n / 2 - 1 = n * 3 / 2 - 1 - if (NumCaseCluster <= 3) { - // Suppose a comparison includes one compare and one conditional branch. - Cost += NumCaseCluster * 2 * InlineConstants::InstrCost; - return false; - } - int64_t ExpectedNumberOfCompare = 3 * (uint64_t)NumCaseCluster / 2 - 1; - uint64_t SwitchCost = - ExpectedNumberOfCompare * 2 * InlineConstants::InstrCost; - Cost = std::min((uint64_t)INT_MAX, SwitchCost + Cost); + unsigned JumpTableSize = 0; + unsigned NumCaseCluster = + TTI.getEstimatedNumberOfCaseClusters(SI, JumpTableSize); + + // If suitable for a jump table, consider the cost for the table size and + // branch to destination. + if (JumpTableSize) { + int64_t JTCost = (int64_t)JumpTableSize * InlineConstants::InstrCost + + 4 * InlineConstants::InstrCost; + Cost = std::min((int64_t)INT_MAX, JTCost + Cost); return false; } - // Use a simple switch cost model where we accumulate a cost proportional to - // the number of distinct successor blocks. This fan-out in the CFG cannot - // be represented for free even if we can represent the core switch as a - // jumptable that takes a single instruction. - /// - // NB: We convert large switches which are just used to initialize large phi - // nodes to lookup tables instead in simplify-cfg, so this shouldn't prevent - // inlining those. It will prevent inlining in cases where the optimization - // does not (yet) fire. - SmallPtrSet<BasicBlock *, 8> SuccessorBlocks; - SuccessorBlocks.insert(SI.getDefaultDest()); - for (auto Case : SI.cases()) - SuccessorBlocks.insert(Case.getCaseSuccessor()); - // Add cost corresponding to the number of distinct destinations. The first - // we model as free because of fallthrough. - Cost += (SuccessorBlocks.size() - 1) * InlineConstants::InstrCost; + // Considering forming a binary search, we should find the number of nodes + // which is same as the number of comparisons when lowered. For a given + // number of clusters, n, we can define a recursive function, f(n), to find + // the number of nodes in the tree. The recursion is : + // f(n) = 1 + f(n/2) + f (n - n/2), when n > 3, + // and f(n) = n, when n <= 3. + // This will lead a binary tree where the leaf should be either f(2) or f(3) + // when n > 3. So, the number of comparisons from leaves should be n, while + // the number of non-leaf should be : + // 2^(log2(n) - 1) - 1 + // = 2^log2(n) * 2^-1 - 1 + // = n / 2 - 1. + // Considering comparisons from leaf and non-leaf nodes, we can estimate the + // number of comparisons in a simple closed form : + // n + n / 2 - 1 = n * 3 / 2 - 1 + if (NumCaseCluster <= 3) { + // Suppose a comparison includes one compare and one conditional branch. + Cost += NumCaseCluster * 2 * InlineConstants::InstrCost; + return false; + } + int64_t ExpectedNumberOfCompare = 3 * (uint64_t)NumCaseCluster / 2 - 1; + uint64_t SwitchCost = + ExpectedNumberOfCompare * 2 * InlineConstants::InstrCost; + Cost = std::min((uint64_t)INT_MAX, SwitchCost + Cost); return false; } diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp index a2b9015a8a1d..6a9ae6440ace 100644 --- a/lib/Analysis/LazyValueInfo.cpp +++ b/lib/Analysis/LazyValueInfo.cpp @@ -662,13 +662,13 @@ namespace { bool solveBlockValuePHINode(LVILatticeVal &BBLV, PHINode *PN, BasicBlock *BB); bool solveBlockValueSelect(LVILatticeVal &BBLV, SelectInst *S, BasicBlock *BB); - bool solveBlockValueBinaryOp(LVILatticeVal &BBLV, Instruction *BBI, + bool solveBlockValueBinaryOp(LVILatticeVal &BBLV, BinaryOperator *BBI, BasicBlock *BB); - bool solveBlockValueCast(LVILatticeVal &BBLV, Instruction *BBI, + bool solveBlockValueCast(LVILatticeVal &BBLV, CastInst *CI, BasicBlock *BB); void intersectAssumeOrGuardBlockValueConstantRange(Value *Val, LVILatticeVal &BBLV, - Instruction *BBI); + Instruction *BBI); void solve(); @@ -849,12 +849,12 @@ bool LazyValueInfoImpl::solveBlockValueImpl(LVILatticeVal &Res, return true; } if (BBI->getType()->isIntegerTy()) { - if (isa<CastInst>(BBI)) - return solveBlockValueCast(Res, BBI, BB); - + if (auto *CI = dyn_cast<CastInst>(BBI)) + return solveBlockValueCast(Res, CI, BB); + BinaryOperator *BO = dyn_cast<BinaryOperator>(BBI); if (BO && isa<ConstantInt>(BO->getOperand(1))) - return solveBlockValueBinaryOp(Res, BBI, BB); + return solveBlockValueBinaryOp(Res, BO, BB); } DEBUG(dbgs() << " compute BB '" << BB->getName() @@ -1168,9 +1168,9 @@ bool LazyValueInfoImpl::solveBlockValueSelect(LVILatticeVal &BBLV, } bool LazyValueInfoImpl::solveBlockValueCast(LVILatticeVal &BBLV, - Instruction *BBI, - BasicBlock *BB) { - if (!BBI->getOperand(0)->getType()->isSized()) { + CastInst *CI, + BasicBlock *BB) { + if (!CI->getOperand(0)->getType()->isSized()) { // Without knowing how wide the input is, we can't analyze it in any useful // way. BBLV = LVILatticeVal::getOverdefined(); @@ -1180,7 +1180,7 @@ bool LazyValueInfoImpl::solveBlockValueCast(LVILatticeVal &BBLV, // Filter out casts we don't know how to reason about before attempting to // recurse on our operand. This can cut a long search short if we know we're // not going to be able to get any useful information anways. - switch (BBI->getOpcode()) { + switch (CI->getOpcode()) { case Instruction::Trunc: case Instruction::SExt: case Instruction::ZExt: @@ -1197,44 +1197,43 @@ bool LazyValueInfoImpl::solveBlockValueCast(LVILatticeVal &BBLV, // Figure out the range of the LHS. If that fails, we still apply the // transfer rule on the full set since we may be able to locally infer // interesting facts. - if (!hasBlockValue(BBI->getOperand(0), BB)) - if (pushBlockValue(std::make_pair(BB, BBI->getOperand(0)))) + if (!hasBlockValue(CI->getOperand(0), BB)) + if (pushBlockValue(std::make_pair(BB, CI->getOperand(0)))) // More work to do before applying this transfer rule. return false; const unsigned OperandBitWidth = - DL.getTypeSizeInBits(BBI->getOperand(0)->getType()); + DL.getTypeSizeInBits(CI->getOperand(0)->getType()); ConstantRange LHSRange = ConstantRange(OperandBitWidth); - if (hasBlockValue(BBI->getOperand(0), BB)) { - LVILatticeVal LHSVal = getBlockValue(BBI->getOperand(0), BB); - intersectAssumeOrGuardBlockValueConstantRange(BBI->getOperand(0), LHSVal, - BBI); + if (hasBlockValue(CI->getOperand(0), BB)) { + LVILatticeVal LHSVal = getBlockValue(CI->getOperand(0), BB); + intersectAssumeOrGuardBlockValueConstantRange(CI->getOperand(0), LHSVal, + CI); if (LHSVal.isConstantRange()) LHSRange = LHSVal.getConstantRange(); } - const unsigned ResultBitWidth = - cast<IntegerType>(BBI->getType())->getBitWidth(); + const unsigned ResultBitWidth = CI->getType()->getIntegerBitWidth(); // NOTE: We're currently limited by the set of operations that ConstantRange // can evaluate symbolically. Enhancing that set will allows us to analyze // more definitions. - auto CastOp = (Instruction::CastOps) BBI->getOpcode(); - BBLV = LVILatticeVal::getRange(LHSRange.castOp(CastOp, ResultBitWidth)); + BBLV = LVILatticeVal::getRange(LHSRange.castOp(CI->getOpcode(), + ResultBitWidth)); return true; } bool LazyValueInfoImpl::solveBlockValueBinaryOp(LVILatticeVal &BBLV, - Instruction *BBI, + BinaryOperator *BO, BasicBlock *BB) { - assert(BBI->getOperand(0)->getType()->isSized() && + assert(BO->getOperand(0)->getType()->isSized() && "all operands to binary operators are sized"); // Filter out operators we don't know how to reason about before attempting to // recurse on our operand(s). This can cut a long search short if we know - // we're not going to be able to get any useful information anways. - switch (BBI->getOpcode()) { + // we're not going to be able to get any useful information anyways. + switch (BO->getOpcode()) { case Instruction::Add: case Instruction::Sub: case Instruction::Mul: @@ -1256,29 +1255,29 @@ bool LazyValueInfoImpl::solveBlockValueBinaryOp(LVILatticeVal &BBLV, // Figure out the range of the LHS. If that fails, use a conservative range, // but apply the transfer rule anyways. This lets us pick up facts from // expressions like "and i32 (call i32 @foo()), 32" - if (!hasBlockValue(BBI->getOperand(0), BB)) - if (pushBlockValue(std::make_pair(BB, BBI->getOperand(0)))) + if (!hasBlockValue(BO->getOperand(0), BB)) + if (pushBlockValue(std::make_pair(BB, BO->getOperand(0)))) // More work to do before applying this transfer rule. return false; const unsigned OperandBitWidth = - DL.getTypeSizeInBits(BBI->getOperand(0)->getType()); + DL.getTypeSizeInBits(BO->getOperand(0)->getType()); ConstantRange LHSRange = ConstantRange(OperandBitWidth); - if (hasBlockValue(BBI->getOperand(0), BB)) { - LVILatticeVal LHSVal = getBlockValue(BBI->getOperand(0), BB); - intersectAssumeOrGuardBlockValueConstantRange(BBI->getOperand(0), LHSVal, - BBI); + if (hasBlockValue(BO->getOperand(0), BB)) { + LVILatticeVal LHSVal = getBlockValue(BO->getOperand(0), BB); + intersectAssumeOrGuardBlockValueConstantRange(BO->getOperand(0), LHSVal, + BO); if (LHSVal.isConstantRange()) LHSRange = LHSVal.getConstantRange(); } - ConstantInt *RHS = cast<ConstantInt>(BBI->getOperand(1)); + ConstantInt *RHS = cast<ConstantInt>(BO->getOperand(1)); ConstantRange RHSRange = ConstantRange(RHS->getValue()); // NOTE: We're currently limited by the set of operations that ConstantRange // can evaluate symbolically. Enhancing that set will allows us to analyze // more definitions. - auto BinOp = (Instruction::BinaryOps) BBI->getOpcode(); + Instruction::BinaryOps BinOp = BO->getOpcode(); BBLV = LVILatticeVal::getRange(LHSRange.binaryOp(BinOp, RHSRange)); return true; } diff --git a/lib/Analysis/ModuleSummaryAnalysis.cpp b/lib/Analysis/ModuleSummaryAnalysis.cpp index 26706f5509ba..3253f27c010d 100644 --- a/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -275,7 +275,7 @@ computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, // FIXME: refactor this to use the same code that inliner is using. F.isVarArg(); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, - /* LiveRoot = */ false); + /* Live = */ false); auto FuncSummary = llvm::make_unique<FunctionSummary>( Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), @@ -295,7 +295,7 @@ computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, findRefEdges(Index, &V, RefEdges, Visited); bool NonRenamableLocal = isNonRenamableLocal(V); GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal, - /* LiveRoot = */ false); + /* Live = */ false); auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector()); if (NonRenamableLocal) @@ -308,7 +308,7 @@ computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet<GlobalValue::GUID> &CantBePromoted) { bool NonRenamableLocal = isNonRenamableLocal(A); GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal, - /* LiveRoot = */ false); + /* Live = */ false); auto AS = llvm::make_unique<AliasSummary>(Flags, ArrayRef<ValueInfo>{}); auto *Aliasee = A.getBaseObject(); auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee); @@ -323,7 +323,7 @@ computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name))) for (auto &Summary : VI.getSummaryList()) - Summary->setLiveRoot(); + Summary->setLive(true); } ModuleSummaryIndex llvm::buildModuleSummaryIndex( @@ -423,8 +423,8 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( return; assert(GV->isDeclaration() && "Def in module asm already has definition"); GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage, - /* NotEligibleToImport */ true, - /* LiveRoot */ true); + /* NotEligibleToImport = */ true, + /* Live = */ true); CantBePromoted.insert(GlobalValue::getGUID(Name)); // Create the appropriate summary type. if (isa<Function>(GV)) { diff --git a/lib/Analysis/OrderedBasicBlock.cpp b/lib/Analysis/OrderedBasicBlock.cpp index 0f0016f22cc0..a04c0aef04be 100644 --- a/lib/Analysis/OrderedBasicBlock.cpp +++ b/lib/Analysis/OrderedBasicBlock.cpp @@ -55,7 +55,7 @@ bool OrderedBasicBlock::comesBefore(const Instruction *A, assert(II != IE && "Instruction not found?"); assert((Inst == A || Inst == B) && "Should find A or B"); LastInstFound = II; - return Inst == A; + return Inst != B; } /// \brief Find out whether \p A dominates \p B, meaning whether \p A diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp index 82107cb18025..b38e6225c840 100644 --- a/lib/Analysis/RegionPass.cpp +++ b/lib/Analysis/RegionPass.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/RegionPass.h" #include "llvm/Analysis/RegionIterator.h" +#include "llvm/IR/OptBisect.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" @@ -280,3 +281,18 @@ Pass *RegionPass::createPrinterPass(raw_ostream &O, const std::string &Banner) const { return new PrintRegionPass(Banner, O); } + +bool RegionPass::skipRegion(Region &R) const { + Function &F = *R.getEntry()->getParent(); + if (!F.getContext().getOptBisect().shouldRunPass(this, R)) + return true; + + if (F.hasFnAttribute(Attribute::OptimizeNone)) { + // Report this only once per function. + if (R.getEntry() == &F.getEntryBlock()) + DEBUG(dbgs() << "Skipping pass '" << getPassName() + << "' on function " << F.getName() << "\n"); + return true; + } + return false; +} diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index 686c94687669..fffa9045b2fd 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -865,11 +865,11 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, auto Linkage = GlobalValue::LinkageTypes(RawFlags & 0xF); // 4 bits RawFlags = RawFlags >> 4; bool NotEligibleToImport = (RawFlags & 0x1) || Version < 3; - // The LiveRoot flag wasn't introduced until version 3. For dead stripping + // The Live flag wasn't introduced until version 3. For dead stripping // to work correctly on earlier versions, we must conservatively treat all // values as live. - bool LiveRoot = (RawFlags & 0x2) || Version < 3; - return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, LiveRoot); + bool Live = (RawFlags & 0x2) || Version < 3; + return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index a402b4ddd462..9043b8c12d25 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -351,7 +351,8 @@ public: /// Calls the callback for each value GUID and summary to be written to /// bitcode. This hides the details of whether they are being pulled from the /// entire index or just those in a provided ModuleToSummariesForIndex map. - void forEachSummary(std::function<void(GVInfo)> Callback) { + template<typename Functor> + void forEachSummary(Functor Callback) { if (ModuleToSummariesForIndex) { for (auto &M : *ModuleToSummariesForIndex) for (auto &Summary : M.second) @@ -363,6 +364,29 @@ public: } } + /// Calls the callback for each entry in the modulePaths StringMap that + /// should be written to the module path string table. This hides the details + /// of whether they are being pulled from the entire index or just those in a + /// provided ModuleToSummariesForIndex map. + template <typename Functor> void forEachModule(Functor Callback) { + if (ModuleToSummariesForIndex) { + for (const auto &M : *ModuleToSummariesForIndex) { + const auto &MPI = Index.modulePaths().find(M.first); + if (MPI == Index.modulePaths().end()) { + // This should only happen if the bitcode file was empty, in which + // case we shouldn't be importing (the ModuleToSummariesForIndex + // would only include the module we are writing and index for). + assert(ModuleToSummariesForIndex->size() == 1); + continue; + } + Callback(*MPI); + } + } else { + for (const auto &MPSE : Index.modulePaths()) + Callback(MPSE); + } + } + /// Main entry point for writing a combined index to bitcode. void write(); @@ -370,14 +394,6 @@ private: void writeModStrings(); void writeCombinedGlobalValueSummary(); - /// Indicates whether the provided \p ModulePath should be written into - /// the module string table, e.g. if full index written or if it is in - /// the provided subset. - bool doIncludeModule(StringRef ModulePath) { - return !ModuleToSummariesForIndex || - ModuleToSummariesForIndex->count(ModulePath); - } - Optional<unsigned> getValueId(GlobalValue::GUID ValGUID) { auto VMI = GUIDToValueIdMap.find(ValGUID); if (VMI == GUIDToValueIdMap.end()) @@ -864,7 +880,7 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { uint64_t RawFlags = 0; RawFlags |= Flags.NotEligibleToImport; // bool - RawFlags |= (Flags.LiveRoot << 1); + RawFlags |= (Flags.Live << 1); // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into // account here as well. @@ -968,19 +984,18 @@ void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() { enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 }; /// Determine the encoding to use for the given string name and length. -static StringEncoding getStringEncoding(const char *Str, unsigned StrLen) { +static StringEncoding getStringEncoding(StringRef Str) { bool isChar6 = true; - for (const char *C = Str, *E = C + StrLen; C != E; ++C) { + for (char C : Str) { if (isChar6) - isChar6 = BitCodeAbbrevOp::isChar6(*C); - if ((unsigned char)*C & 128) + isChar6 = BitCodeAbbrevOp::isChar6(C); + if ((unsigned char)C & 128) // don't bother scanning the rest. return SE_Fixed8; } if (isChar6) return SE_Char6; - else - return SE_Fixed7; + return SE_Fixed7; } /// Emit top-level description of module, including target triple, inline asm, @@ -1073,8 +1088,7 @@ void ModuleBitcodeWriter::writeModuleInfo() { SmallVector<unsigned, 64> Vals; // Emit the module's source file name. { - StringEncoding Bits = getStringEncoding(M.getSourceFileName().data(), - M.getSourceFileName().size()); + StringEncoding Bits = getStringEncoding(M.getSourceFileName()); BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8); if (Bits == SE_Char6) AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6); @@ -2790,8 +2804,7 @@ void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable( for (const ValueName &Name : VST) { // Figure out the encoding to use for the name. - StringEncoding Bits = - getStringEncoding(Name.getKeyData(), Name.getKeyLength()); + StringEncoding Bits = getStringEncoding(Name.getKey()); unsigned AbbrevToUse = VST_ENTRY_8_ABBREV; NameVals.push_back(VE.getValueID(Name.getValue())); @@ -3149,41 +3162,33 @@ void IndexBitcodeWriter::writeModStrings() { unsigned AbbrevHash = Stream.EmitAbbrev(std::move(Abbv)); SmallVector<unsigned, 64> Vals; - for (const auto &MPSE : Index.modulePaths()) { - if (!doIncludeModule(MPSE.getKey())) - continue; - StringEncoding Bits = - getStringEncoding(MPSE.getKey().data(), MPSE.getKey().size()); - unsigned AbbrevToUse = Abbrev8Bit; - if (Bits == SE_Char6) - AbbrevToUse = Abbrev6Bit; - else if (Bits == SE_Fixed7) - AbbrevToUse = Abbrev7Bit; - - Vals.push_back(MPSE.getValue().first); - - for (const auto P : MPSE.getKey()) - Vals.push_back((unsigned char)P); - - // Emit the finished record. - Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); - - Vals.clear(); - // Emit an optional hash for the module now - auto &Hash = MPSE.getValue().second; - bool AllZero = true; // Detect if the hash is empty, and do not generate it - for (auto Val : Hash) { - if (Val) - AllZero = false; - Vals.push_back(Val); - } - if (!AllZero) { - // Emit the hash record. - Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); - } + forEachModule( + [&](const StringMapEntry<std::pair<uint64_t, ModuleHash>> &MPSE) { + StringRef Key = MPSE.getKey(); + const auto &Value = MPSE.getValue(); + StringEncoding Bits = getStringEncoding(Key); + unsigned AbbrevToUse = Abbrev8Bit; + if (Bits == SE_Char6) + AbbrevToUse = Abbrev6Bit; + else if (Bits == SE_Fixed7) + AbbrevToUse = Abbrev7Bit; + + Vals.push_back(Value.first); + Vals.append(Key.begin(), Key.end()); + + // Emit the finished record. + Stream.EmitRecord(bitc::MST_CODE_ENTRY, Vals, AbbrevToUse); + + // Emit an optional hash for the module now + const auto &Hash = Value.second; + if (llvm::any_of(Hash, [](uint32_t H) { return H; })) { + Vals.assign(Hash.begin(), Hash.end()); + // Emit the hash record. + Stream.EmitRecord(bitc::MST_CODE_HASH, Vals, AbbrevHash); + } - Vals.clear(); - } + Vals.clear(); + }); Stream.ExitBlock(); } diff --git a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp index 20e1467b30c3..c2ad9db81cfd 100644 --- a/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp +++ b/lib/CodeGen/AsmPrinter/DbgValueHistoryCalculator.cpp @@ -194,6 +194,10 @@ void llvm::calculateDbgValueHistory(const MachineFunction *MF, // some variables. for (const MachineOperand &MO : MI.operands()) { if (MO.isReg() && MO.isDef() && MO.getReg()) { + // Ignore call instructions that claim to clobber SP. The AArch64 + // backend does this for aggregate function arguments. + if (MI.isCall() && MO.getReg() == SP) + continue; // If this is a virtual register, only clobber it since it doesn't // have aliases. if (TRI->isVirtualRegister(MO.getReg())) diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 4d30c6574b12..256a0c95d365 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -77,6 +77,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializePostRASchedulerPass(Registry); initializePreISelIntrinsicLoweringLegacyPassPass(Registry); initializeProcessImplicitDefsPass(Registry); + initializeRABasicPass(Registry); initializeRAGreedyPass(Registry); initializeRegisterCoalescerPass(Registry); initializeRenameIndependentSubregsPass(Registry); diff --git a/lib/CodeGen/GlobalMerge.cpp b/lib/CodeGen/GlobalMerge.cpp index 23812a2a2344..3603f9b7ed93 100644 --- a/lib/CodeGen/GlobalMerge.cpp +++ b/lib/CodeGen/GlobalMerge.cpp @@ -556,6 +556,10 @@ bool GlobalMerge::doInitialization(Module &M) { if (GV.isDeclaration() || GV.isThreadLocal() || GV.hasSection()) continue; + // It's not safe to merge globals that may be preempted + if (TM && !TM->shouldAssumeDSOLocal(M, &GV)) + continue; + if (!(MergeExternalGlobals && GV.hasExternalLinkage()) && !GV.hasInternalLinkage()) continue; diff --git a/lib/CodeGen/LivePhysRegs.cpp b/lib/CodeGen/LivePhysRegs.cpp index 0dc1079b2ad4..cde6ccd29dfd 100644 --- a/lib/CodeGen/LivePhysRegs.cpp +++ b/lib/CodeGen/LivePhysRegs.cpp @@ -198,13 +198,12 @@ void LivePhysRegs::addLiveOutsNoPristines(const MachineBasicBlock &MBB) { } void LivePhysRegs::addLiveOuts(const MachineBasicBlock &MBB) { + const MachineFunction &MF = *MBB.getParent(); if (!MBB.succ_empty()) { - const MachineFunction &MF = *MBB.getParent(); addPristines(*this, MF); addLiveOutsNoPristines(MBB); } else if (MBB.isReturnBlock()) { // For the return block: Add all callee saved registers. - const MachineFunction &MF = *MBB.getParent(); const MachineFrameInfo &MFI = MF.getFrameInfo(); if (MFI.isCalleeSavedInfoValid()) addCalleeSavedRegs(*this, MF); diff --git a/lib/CodeGen/LiveRegUnits.cpp b/lib/CodeGen/LiveRegUnits.cpp index dff555f49565..3746b74e0528 100644 --- a/lib/CodeGen/LiveRegUnits.cpp +++ b/lib/CodeGen/LiveRegUnits.cpp @@ -12,11 +12,13 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/LiveRegUnits.h" + #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -81,46 +83,50 @@ void LiveRegUnits::accumulateBackward(const MachineInstr &MI) { } /// Add live-in registers of basic block \p MBB to \p LiveUnits. -static void addLiveIns(LiveRegUnits &LiveUnits, const MachineBasicBlock &MBB) { +static void addBlockLiveIns(LiveRegUnits &LiveUnits, + const MachineBasicBlock &MBB) { for (const auto &LI : MBB.liveins()) LiveUnits.addRegMasked(LI.PhysReg, LI.LaneMask); } -static void addLiveOuts(LiveRegUnits &LiveUnits, const MachineBasicBlock &MBB) { - // To get the live-outs we simply merge the live-ins of all successors. - for (const MachineBasicBlock *Succ : MBB.successors()) - addLiveIns(LiveUnits, *Succ); +/// Adds all callee saved registers to \p LiveUnits. +static void addCalleeSavedRegs(LiveRegUnits &LiveUnits, + const MachineFunction &MF) { + const MachineRegisterInfo &MRI = MF.getRegInfo(); + for (const MCPhysReg *CSR = MRI.getCalleeSavedRegs(); CSR && *CSR; ++CSR) + LiveUnits.addReg(*CSR); } -/// Add pristine registers to the given \p LiveUnits. This function removes -/// actually saved callee save registers when \p InPrologueEpilogue is false. -static void removeSavedRegs(LiveRegUnits &LiveUnits, const MachineFunction &MF, - const MachineFrameInfo &MFI, - const TargetRegisterInfo &TRI) { +/// Adds pristine registers to the given \p LiveUnits. Pristine registers are +/// callee saved registers that are unused in the function. +static void addPristines(LiveRegUnits &LiveUnits, const MachineFunction &MF) { + const MachineFrameInfo &MFI = MF.getFrameInfo(); + if (!MFI.isCalleeSavedInfoValid()) + return; + /// Add all callee saved regs, then remove the ones that are saved+restored. + addCalleeSavedRegs(LiveUnits, MF); + /// Remove the ones that are not saved/restored; they are pristine. for (const CalleeSavedInfo &Info : MFI.getCalleeSavedInfo()) LiveUnits.removeReg(Info.getReg()); } void LiveRegUnits::addLiveOuts(const MachineBasicBlock &MBB) { const MachineFunction &MF = *MBB.getParent(); - const MachineFrameInfo &MFI = MF.getFrameInfo(); - if (MFI.isCalleeSavedInfoValid()) { - for (const MCPhysReg *I = TRI->getCalleeSavedRegs(&MF); *I; ++I) - addReg(*I); - if (!MBB.isReturnBlock()) - removeSavedRegs(*this, MF, MFI, *TRI); + if (!MBB.succ_empty()) { + addPristines(*this, MF); + // To get the live-outs we simply merge the live-ins of all successors. + for (const MachineBasicBlock *Succ : MBB.successors()) + addBlockLiveIns(*this, *Succ); + } else if (MBB.isReturnBlock()) { + // For the return block: Add all callee saved registers. + const MachineFrameInfo &MFI = MF.getFrameInfo(); + if (MFI.isCalleeSavedInfoValid()) + addCalleeSavedRegs(*this, MF); } - ::addLiveOuts(*this, MBB); } void LiveRegUnits::addLiveIns(const MachineBasicBlock &MBB) { const MachineFunction &MF = *MBB.getParent(); - const MachineFrameInfo &MFI = MF.getFrameInfo(); - if (MFI.isCalleeSavedInfoValid()) { - for (const MCPhysReg *I = TRI->getCalleeSavedRegs(&MF); *I; ++I) - addReg(*I); - if (&MBB != &MF.front()) - removeSavedRegs(*this, MF, MFI, *TRI); - } - ::addLiveIns(*this, MBB); + addPristines(*this, MF); + addBlockLiveIns(*this, MBB); } diff --git a/lib/CodeGen/MachineRegionInfo.cpp b/lib/CodeGen/MachineRegionInfo.cpp index 71ad4e6aa7f5..2402ffdbbcb1 100644 --- a/lib/CodeGen/MachineRegionInfo.cpp +++ b/lib/CodeGen/MachineRegionInfo.cpp @@ -1,7 +1,19 @@ -#include "llvm/CodeGen/MachineRegionInfo.h" +//===- lib/Codegen/MachineRegionInfo.cpp ----------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/RegionInfoImpl.h" #include "llvm/CodeGen/MachinePostDominators.h" +#include "llvm/CodeGen/MachineRegionInfo.h" +#include "llvm/Pass.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" #define DEBUG_TYPE "machine-region-info" @@ -11,36 +23,29 @@ STATISTIC(numMachineRegions, "The # of machine regions"); STATISTIC(numMachineSimpleRegions, "The # of simple machine regions"); namespace llvm { + template class RegionBase<RegionTraits<MachineFunction>>; template class RegionNodeBase<RegionTraits<MachineFunction>>; template class RegionInfoBase<RegionTraits<MachineFunction>>; -} + +} // end namespace llvm //===----------------------------------------------------------------------===// // MachineRegion implementation -// MachineRegion::MachineRegion(MachineBasicBlock *Entry, MachineBasicBlock *Exit, MachineRegionInfo* RI, MachineDominatorTree *DT, MachineRegion *Parent) : - RegionBase<RegionTraits<MachineFunction>>(Entry, Exit, RI, DT, Parent) { + RegionBase<RegionTraits<MachineFunction>>(Entry, Exit, RI, DT, Parent) {} -} - -MachineRegion::~MachineRegion() { } +MachineRegion::~MachineRegion() = default; //===----------------------------------------------------------------------===// // MachineRegionInfo implementation -// - -MachineRegionInfo::MachineRegionInfo() : - RegionInfoBase<RegionTraits<MachineFunction>>() { - -} -MachineRegionInfo::~MachineRegionInfo() { +MachineRegionInfo::MachineRegionInfo() = default; -} +MachineRegionInfo::~MachineRegionInfo() = default; void MachineRegionInfo::updateStatistics(MachineRegion *R) { ++numMachineRegions; @@ -73,9 +78,7 @@ MachineRegionInfoPass::MachineRegionInfoPass() : MachineFunctionPass(ID) { initializeMachineRegionInfoPassPass(*PassRegistry::getPassRegistry()); } -MachineRegionInfoPass::~MachineRegionInfoPass() { - -} +MachineRegionInfoPass::~MachineRegionInfoPass() = default; bool MachineRegionInfoPass::runOnMachineFunction(MachineFunction &F) { releaseMemory(); @@ -137,8 +140,9 @@ INITIALIZE_PASS_END(MachineRegionInfoPass, DEBUG_TYPE, // the link time optimization. namespace llvm { - FunctionPass *createMachineRegionInfoPass() { - return new MachineRegionInfoPass(); - } + +FunctionPass *createMachineRegionInfoPass() { + return new MachineRegionInfoPass(); } +} // end namespace llvm diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 265f93c363ca..f6dbf667cf02 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -36,6 +36,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" @@ -909,17 +910,43 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { } } - // Generic loads and stores must have a single MachineMemOperand - // describing that access. - if ((MI->getOpcode() == TargetOpcode::G_LOAD || - MI->getOpcode() == TargetOpcode::G_STORE) && - !MI->hasOneMemOperand()) - report("Generic instruction accessing memory must have one mem operand", - MI); - StringRef ErrorInfo; if (!TII->verifyInstruction(*MI, ErrorInfo)) report(ErrorInfo.data(), MI); + + // Verify properties of various specific instruction types + switch(MI->getOpcode()) { + default: + break; + case TargetOpcode::G_LOAD: + case TargetOpcode::G_STORE: + // Generic loads and stores must have a single MachineMemOperand + // describing that access. + if (!MI->hasOneMemOperand()) + report("Generic instruction accessing memory must have one mem operand", + MI); + break; + case TargetOpcode::STATEPOINT: + if (!MI->getOperand(StatepointOpers::IDPos).isImm() || + !MI->getOperand(StatepointOpers::NBytesPos).isImm() || + !MI->getOperand(StatepointOpers::NCallArgsPos).isImm()) + report("meta operands to STATEPOINT not constant!", MI); + break; + + auto VerifyStackMapConstant = [&](unsigned Offset) { + if (!MI->getOperand(Offset).isImm() || + MI->getOperand(Offset).getImm() != StackMaps::ConstantOp || + !MI->getOperand(Offset + 1).isImm()) + report("stack map constant to STATEPOINT not well formed!", MI); + }; + const unsigned VarStart = StatepointOpers(MI).getVarIdx(); + VerifyStackMapConstant(VarStart + StatepointOpers::CCOffset); + VerifyStackMapConstant(VarStart + StatepointOpers::FlagsOffset); + VerifyStackMapConstant(VarStart + StatepointOpers::NumDeoptOperandsOffset); + + // TODO: verify we have properly encoded deopt arguments + + }; } void diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index a9813e534c5f..e9f8d43fe643 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -54,8 +54,6 @@ static void doSpillCalleeSavedRegs(MachineFunction &MF, RegScavenger *RS, const MBBVector &SaveBlocks, const MBBVector &RestoreBlocks); -static void doScavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger *RS); - namespace { class PEI : public MachineFunctionPass { public: @@ -84,7 +82,7 @@ private: const MBBVector &SaveBlocks, const MBBVector &RestoreBlocks)> SpillCalleeSavedRegisters; - std::function<void(MachineFunction &MF, RegScavenger *RS)> + std::function<void(MachineFunction &MF, RegScavenger &RS)> ScavengeFrameVirtualRegs; bool UsesCalleeSaves = false; @@ -142,7 +140,6 @@ MachineFunctionPass *llvm::createPrologEpilogInserterPass() { return new PEI(); } -STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); STATISTIC(NumBytesStackSpace, "Number of bytes used for stack in all functions"); @@ -168,10 +165,10 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { SpillCalleeSavedRegisters = [](MachineFunction &, RegScavenger *, unsigned &, unsigned &, const MBBVector &, const MBBVector &) {}; - ScavengeFrameVirtualRegs = [](MachineFunction &, RegScavenger *) {}; + ScavengeFrameVirtualRegs = [](MachineFunction &, RegScavenger &) {}; } else { SpillCalleeSavedRegisters = doSpillCalleeSavedRegs; - ScavengeFrameVirtualRegs = doScavengeFrameVirtualRegs; + ScavengeFrameVirtualRegs = scavengeFrameVirtualRegs; UsesCalleeSaves = true; } } @@ -222,7 +219,7 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { // post-pass, scavenge the virtual registers that frame index elimination // inserted. if (TRI->requiresRegisterScavenging(Fn) && FrameIndexVirtualScavenging) { - ScavengeFrameVirtualRegs(Fn, RS); + ScavengeFrameVirtualRegs(Fn, *RS); // Clear any vregs created by virtual scavenging. Fn.getRegInfo().clearVirtRegs(); @@ -1153,92 +1150,3 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn, RS->forward(MI); } } - -/// doScavengeFrameVirtualRegs - Replace all frame index virtual registers -/// with physical registers. Use the register scavenger to find an -/// appropriate register to use. -/// -/// FIXME: Iterating over the instruction stream is unnecessary. We can simply -/// iterate over the vreg use list, which at this point only contains machine -/// operands for which eliminateFrameIndex need a new scratch reg. -static void -doScavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger *RS) { - // Run through the instructions and find any virtual registers. - MachineRegisterInfo &MRI = MF.getRegInfo(); - for (MachineBasicBlock &MBB : MF) { - RS->enterBasicBlock(MBB); - - int SPAdj = 0; - - // The instruction stream may change in the loop, so check MBB.end() - // directly. - for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ) { - // We might end up here again with a NULL iterator if we scavenged a - // register for which we inserted spill code for definition by what was - // originally the first instruction in MBB. - if (I == MachineBasicBlock::iterator(nullptr)) - I = MBB.begin(); - - const MachineInstr &MI = *I; - MachineBasicBlock::iterator J = std::next(I); - MachineBasicBlock::iterator P = - I == MBB.begin() ? MachineBasicBlock::iterator(nullptr) - : std::prev(I); - - // RS should process this instruction before we might scavenge at this - // location. This is because we might be replacing a virtual register - // defined by this instruction, and if so, registers killed by this - // instruction are available, and defined registers are not. - RS->forward(I); - - for (const MachineOperand &MO : MI.operands()) { - if (!MO.isReg()) - continue; - unsigned Reg = MO.getReg(); - if (!TargetRegisterInfo::isVirtualRegister(Reg)) - continue; - - // When we first encounter a new virtual register, it - // must be a definition. - assert(MO.isDef() && "frame index virtual missing def!"); - // Scavenge a new scratch register - const TargetRegisterClass *RC = MRI.getRegClass(Reg); - unsigned ScratchReg = RS->scavengeRegister(RC, J, SPAdj); - - ++NumScavengedRegs; - - // Replace this reference to the virtual register with the - // scratch register. - assert(ScratchReg && "Missing scratch register!"); - MRI.replaceRegWith(Reg, ScratchReg); - - // Because this instruction was processed by the RS before this - // register was allocated, make sure that the RS now records the - // register as being used. - RS->setRegUsed(ScratchReg); - } - - // If the scavenger needed to use one of its spill slots, the - // spill code will have been inserted in between I and J. This is a - // problem because we need the spill code before I: Move I to just - // prior to J. - if (I != std::prev(J)) { - MBB.splice(J, &MBB, I); - - // Before we move I, we need to prepare the RS to visit I again. - // Specifically, RS will assert if it sees uses of registers that - // it believes are undefined. Because we have already processed - // register kills in I, when it visits I again, it will believe that - // those registers are undefined. To avoid this situation, unprocess - // the instruction I. - assert(RS->getCurrentPosition() == I && - "The register scavenger has an unexpected position"); - I = P; - RS->unprocess(P); - } else - ++I; - } - } - - MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs); -} diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index a87fed3a687e..24be7ea98d82 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -58,8 +58,9 @@ namespace { /// whenever a register is unavailable. This is not practical in production but /// provides a useful baseline both for measuring other allocators and comparing /// the speed of the basic algorithm against other styles of allocators. -class RABasic : public MachineFunctionPass, public RegAllocBase -{ +class RABasic : public MachineFunctionPass, + public RegAllocBase, + private LiveRangeEdit::Delegate { // context MachineFunction *MF; @@ -72,6 +73,9 @@ class RABasic : public MachineFunctionPass, public RegAllocBase // selectOrSplit(). BitVector UsableRegs; + bool LRE_CanEraseVirtReg(unsigned) override; + void LRE_WillShrinkVirtReg(unsigned) override; + public: RABasic(); @@ -121,17 +125,46 @@ char RABasic::ID = 0; } // end anonymous namespace +char &llvm::RABasicID = RABasic::ID; + +INITIALIZE_PASS_BEGIN(RABasic, "regallocbasic", "Basic Register Allocator", + false, false) +INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables) +INITIALIZE_PASS_DEPENDENCY(SlotIndexes) +INITIALIZE_PASS_DEPENDENCY(LiveIntervals) +INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer) +INITIALIZE_PASS_DEPENDENCY(MachineScheduler) +INITIALIZE_PASS_DEPENDENCY(LiveStacks) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(VirtRegMap) +INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix) +INITIALIZE_PASS_END(RABasic, "regallocbasic", "Basic Register Allocator", false, + false) + +bool RABasic::LRE_CanEraseVirtReg(unsigned VirtReg) { + if (VRM->hasPhys(VirtReg)) { + LiveInterval &LI = LIS->getInterval(VirtReg); + Matrix->unassign(LI); + aboutToRemoveInterval(LI); + return true; + } + // Unassigned virtreg is probably in the priority queue. + // RegAllocBase will erase it after dequeueing. + return false; +} + +void RABasic::LRE_WillShrinkVirtReg(unsigned VirtReg) { + if (!VRM->hasPhys(VirtReg)) + return; + + // Register is assigned, put it back on the queue for reassignment. + LiveInterval &LI = LIS->getInterval(VirtReg); + Matrix->unassign(LI); + enqueue(&LI); +} + RABasic::RABasic(): MachineFunctionPass(ID) { - initializeLiveDebugVariablesPass(*PassRegistry::getPassRegistry()); - initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); - initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); - initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); - initializeMachineSchedulerPass(*PassRegistry::getPassRegistry()); - initializeLiveStacksPass(*PassRegistry::getPassRegistry()); - initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry()); - initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); - initializeVirtRegMapPass(*PassRegistry::getPassRegistry()); - initializeLiveRegMatrixPass(*PassRegistry::getPassRegistry()); } void RABasic::getAnalysisUsage(AnalysisUsage &AU) const { @@ -200,7 +233,7 @@ bool RABasic::spillInterferences(LiveInterval &VirtReg, unsigned PhysReg, Matrix->unassign(Spill); // Spill the extracted interval. - LiveRangeEdit LRE(&Spill, SplitVRegs, *MF, *LIS, VRM, nullptr, &DeadRemats); + LiveRangeEdit LRE(&Spill, SplitVRegs, *MF, *LIS, VRM, this, &DeadRemats); spiller().spill(LRE); } return true; @@ -259,7 +292,7 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg, DEBUG(dbgs() << "spilling: " << VirtReg << '\n'); if (!VirtReg.isSpillable()) return ~0u; - LiveRangeEdit LRE(&VirtReg, SplitVRegs, *MF, *LIS, VRM, nullptr, &DeadRemats); + LiveRangeEdit LRE(&VirtReg, SplitVRegs, *MF, *LIS, VRM, this, &DeadRemats); spiller().spill(LRE); // The live virtual register requesting allocation was spilled, so tell diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 3b5964eef55e..b2dfef91add5 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -49,9 +49,11 @@ #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PBQP/Graph.h" +#include "llvm/CodeGen/PBQP/Math.h" #include "llvm/CodeGen/PBQP/Solution.h" #include "llvm/CodeGen/PBQPRAConstraint.h" #include "llvm/CodeGen/RegAllocPBQP.h" @@ -139,13 +141,13 @@ public: } private: - typedef std::map<const LiveInterval*, unsigned> LI2NodeMap; - typedef std::vector<const LiveInterval*> Node2LIMap; - typedef std::vector<unsigned> AllowedSet; - typedef std::vector<AllowedSet> AllowedSetMap; - typedef std::pair<unsigned, unsigned> RegPair; - typedef std::map<RegPair, PBQP::PBQPNum> CoalesceMap; - typedef std::set<unsigned> RegSet; + using LI2NodeMap = std::map<const LiveInterval *, unsigned>; + using Node2LIMap = std::vector<const LiveInterval *>; + using AllowedSet = std::vector<unsigned>; + using AllowedSetMap = std::vector<AllowedSet>; + using RegPair = std::pair<unsigned, unsigned>; + using CoalesceMap = std::map<RegPair, PBQP::PBQPNum>; + using RegSet = std::set<unsigned>; char *customPassID; @@ -212,12 +214,12 @@ public: /// @brief Add interference edges between overlapping vregs. class Interference : public PBQPRAConstraint { private: - typedef const PBQP::RegAlloc::AllowedRegVector* AllowedRegVecPtr; - typedef std::pair<AllowedRegVecPtr, AllowedRegVecPtr> IKey; - typedef DenseMap<IKey, PBQPRAGraph::MatrixPtr> IMatrixCache; - typedef DenseSet<IKey> DisjointAllowedRegsCache; - typedef std::pair<PBQP::GraphBase::NodeId, PBQP::GraphBase::NodeId> IEdgeKey; - typedef DenseSet<IEdgeKey> IEdgeCache; + using AllowedRegVecPtr = const PBQP::RegAlloc::AllowedRegVector *; + using IKey = std::pair<AllowedRegVecPtr, AllowedRegVecPtr>; + using IMatrixCache = DenseMap<IKey, PBQPRAGraph::MatrixPtr>; + using DisjointAllowedRegsCache = DenseSet<IKey>; + using IEdgeKey = std::pair<PBQP::GraphBase::NodeId, PBQP::GraphBase::NodeId>; + using IEdgeCache = DenseSet<IEdgeKey>; bool haveDisjointAllowedRegs(const PBQPRAGraph &G, PBQPRAGraph::NodeId NId, PBQPRAGraph::NodeId MId, @@ -252,8 +254,8 @@ private: // for the fast interference graph construction algorithm. The last is there // to save us from looking up node ids via the VRegToNode map in the graph // metadata. - typedef std::tuple<LiveInterval*, size_t, PBQP::GraphBase::NodeId> - IntervalInfo; + using IntervalInfo = + std::tuple<LiveInterval*, size_t, PBQP::GraphBase::NodeId>; static SlotIndex getStartPoint(const IntervalInfo &I) { return std::get<0>(I)->segments[std::get<1>(I)].start; @@ -320,9 +322,10 @@ public: // Cache known disjoint allowed registers pairs DisjointAllowedRegsCache D; - typedef std::set<IntervalInfo, decltype(&lowestEndPoint)> IntervalSet; - typedef std::priority_queue<IntervalInfo, std::vector<IntervalInfo>, - decltype(&lowestStartPoint)> IntervalQueue; + using IntervalSet = std::set<IntervalInfo, decltype(&lowestEndPoint)>; + using IntervalQueue = + std::priority_queue<IntervalInfo, std::vector<IntervalInfo>, + decltype(&lowestStartPoint)>; IntervalSet Active(lowestEndPoint); IntervalQueue Inactive(lowestStartPoint); @@ -658,7 +661,6 @@ void RegAllocPBQP::spillVReg(unsigned VReg, SmallVectorImpl<unsigned> &NewIntervals, MachineFunction &MF, LiveIntervals &LIS, VirtRegMap &VRM, Spiller &VRegSpiller) { - VRegsToAlloc.erase(VReg); LiveRangeEdit LRE(&LIS.getInterval(VReg), NewIntervals, MF, LIS, &VRM, nullptr, &DeadRemats); diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp index 0635e5c0a63c..1aed58c36e17 100644 --- a/lib/CodeGen/RegisterScavenging.cpp +++ b/lib/CodeGen/RegisterScavenging.cpp @@ -15,18 +15,23 @@ // //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/RegisterScavenging.h" + #include "llvm/ADT/BitVector.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/MC/MCRegisterInfo.h" +#include "llvm/PassSupport.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" @@ -39,6 +44,8 @@ using namespace llvm; #define DEBUG_TYPE "reg-scavenging" +STATISTIC(NumScavengedRegs, "Number of frame index regs scavenged"); + void RegScavenger::setRegUsed(unsigned Reg, LaneBitmask LaneMask) { LiveUnits.addRegMasked(Reg, LaneMask); } @@ -469,3 +476,120 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, return SReg; } + +void llvm::scavengeFrameVirtualRegs(MachineFunction &MF, RegScavenger &RS) { + // FIXME: Iterating over the instruction stream is unnecessary. We can simply + // iterate over the vreg use list, which at this point only contains machine + // operands for which eliminateFrameIndex need a new scratch reg. + + // Run through the instructions and find any virtual registers. + MachineRegisterInfo &MRI = MF.getRegInfo(); + for (MachineBasicBlock &MBB : MF) { + RS.enterBasicBlock(MBB); + + int SPAdj = 0; + + // The instruction stream may change in the loop, so check MBB.end() + // directly. + for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ) { + // We might end up here again with a NULL iterator if we scavenged a + // register for which we inserted spill code for definition by what was + // originally the first instruction in MBB. + if (I == MachineBasicBlock::iterator(nullptr)) + I = MBB.begin(); + + const MachineInstr &MI = *I; + MachineBasicBlock::iterator J = std::next(I); + MachineBasicBlock::iterator P = + I == MBB.begin() ? MachineBasicBlock::iterator(nullptr) + : std::prev(I); + + // RS should process this instruction before we might scavenge at this + // location. This is because we might be replacing a virtual register + // defined by this instruction, and if so, registers killed by this + // instruction are available, and defined registers are not. + RS.forward(I); + + for (const MachineOperand &MO : MI.operands()) { + if (!MO.isReg()) + continue; + unsigned Reg = MO.getReg(); + if (!TargetRegisterInfo::isVirtualRegister(Reg)) + continue; + + // When we first encounter a new virtual register, it + // must be a definition. + assert(MO.isDef() && "frame index virtual missing def!"); + // Scavenge a new scratch register + const TargetRegisterClass *RC = MRI.getRegClass(Reg); + unsigned ScratchReg = RS.scavengeRegister(RC, J, SPAdj); + + ++NumScavengedRegs; + + // Replace this reference to the virtual register with the + // scratch register. + assert(ScratchReg && "Missing scratch register!"); + MRI.replaceRegWith(Reg, ScratchReg); + + // Because this instruction was processed by the RS before this + // register was allocated, make sure that the RS now records the + // register as being used. + RS.setRegUsed(ScratchReg); + } + + // If the scavenger needed to use one of its spill slots, the + // spill code will have been inserted in between I and J. This is a + // problem because we need the spill code before I: Move I to just + // prior to J. + if (I != std::prev(J)) { + MBB.splice(J, &MBB, I); + + // Before we move I, we need to prepare the RS to visit I again. + // Specifically, RS will assert if it sees uses of registers that + // it believes are undefined. Because we have already processed + // register kills in I, when it visits I again, it will believe that + // those registers are undefined. To avoid this situation, unprocess + // the instruction I. + assert(RS.getCurrentPosition() == I && + "The register scavenger has an unexpected position"); + I = P; + RS.unprocess(P); + } else + ++I; + } + } + + MRI.clearVirtRegs(); + MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs); +} + +namespace { +/// This class runs register scavenging independ of the PrologEpilogInserter. +/// This is used in for testing. +class ScavengerTest : public MachineFunctionPass { +public: + static char ID; + ScavengerTest() : MachineFunctionPass(ID) {} + bool runOnMachineFunction(MachineFunction &MF) { + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const TargetFrameLowering &TFL = *STI.getFrameLowering(); + + RegScavenger RS; + // Let's hope that calling those outside of PrologEpilogueInserter works + // well enough to initialize the scavenger with some emergency spillslots + // for the target. + BitVector SavedRegs; + TFL.determineCalleeSaves(MF, SavedRegs, &RS); + TFL.processFunctionBeforeFrameFinalized(MF, &RS); + + // Let's scavenge the current function + scavengeFrameVirtualRegs(MF, RS); + return true; + } +}; +char ScavengerTest::ID; + +} // end anonymous namespace + +INITIALIZE_PASS(ScavengerTest, "scavenger-test", + "Scavenge virtual registers inside basic blocks", false, false) diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 8035ea80364b..3fdbd2459361 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -12,32 +12,54 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/ADT/IntEqClasses.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SparseSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/LivePhysRegs.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/RegisterPressure.h" +#include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/CodeGen/ScheduleDAGInstrs.h" #include "llvm/CodeGen/ScheduleDFS.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" -#include "llvm/IR/Type.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/MC/LaneBitmask.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" +#include <algorithm> +#include <cassert> +#include <iterator> +#include <string> +#include <utility> +#include <vector> using namespace llvm; @@ -90,11 +112,9 @@ ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf, const MachineLoopInfo *mli, bool RemoveKillFlags) : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), - RemoveKillFlags(RemoveKillFlags), CanHandleTerminators(false), - TrackLaneMasks(false), AAForDep(nullptr), BarrierChain(nullptr), + RemoveKillFlags(RemoveKillFlags), UnknownValue(UndefValue::get( - Type::getVoidTy(mf.getFunction()->getContext()))), - FirstDbgValue(nullptr) { + Type::getVoidTy(mf.getFunction()->getContext()))) { DbgValues.clear(); const TargetSubtargetInfo &ST = mf.getSubtarget(); @@ -126,7 +146,7 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) { return V; } assert(V->getType()->isIntegerTy() && "Unexpected operand type!"); - } while (1); + } while (true); } /// This is a wrapper around GetUnderlyingObjects and adds support for basic @@ -563,7 +583,7 @@ void ScheduleDAGInstrs::initSUnits() { // which is contained within a basic block. SUnits.reserve(NumRegionInstrs); - for (MachineInstr &MI : llvm::make_range(RegionBegin, RegionEnd)) { + for (MachineInstr &MI : make_range(RegionBegin, RegionEnd)) { if (MI.isDebugValue()) continue; @@ -606,13 +626,13 @@ void ScheduleDAGInstrs::initSUnits() { class ScheduleDAGInstrs::Value2SUsMap : public MapVector<ValueType, SUList> { /// Current total number of SUs in map. - unsigned NumNodes; + unsigned NumNodes = 0; /// 1 for loads, 0 for stores. (see comment in SUList) unsigned TrueMemOrderLatency; public: - Value2SUsMap(unsigned lat = 0) : NumNodes(0), TrueMemOrderLatency(lat) {} + Value2SUsMap(unsigned lat = 0) : TrueMemOrderLatency(lat) {} /// To keep NumNodes up to date, insert() is used instead of /// this operator w/ push_back(). @@ -630,7 +650,7 @@ public: void inline clearList(ValueType V) { iterator Itr = find(V); if (Itr != end()) { - assert (NumNodes >= Itr->second.size()); + assert(NumNodes >= Itr->second.size()); NumNodes -= Itr->second.size(); Itr->second.clear(); @@ -646,7 +666,7 @@ public: unsigned inline size() const { return NumNodes; } /// Counts the number of SUs in this map after a reduction. - void reComputeSize(void) { + void reComputeSize() { NumNodes = 0; for (auto &I : *this) NumNodes += I.second.size(); @@ -676,7 +696,7 @@ void ScheduleDAGInstrs::addChainDependencies(SUnit *SU, } void ScheduleDAGInstrs::addBarrierChain(Value2SUsMap &map) { - assert (BarrierChain != nullptr); + assert(BarrierChain != nullptr); for (auto &I : map) { SUList &sus = I.second; @@ -687,7 +707,7 @@ void ScheduleDAGInstrs::addBarrierChain(Value2SUsMap &map) { } void ScheduleDAGInstrs::insertBarrierChain(Value2SUsMap &map) { - assert (BarrierChain != nullptr); + assert(BarrierChain != nullptr); // Go through all lists of SUs. for (Value2SUsMap::iterator I = map.begin(), EE = map.end(); I != EE;) { @@ -1028,7 +1048,7 @@ void ScheduleDAGInstrs::reduceHugeMemNodeMaps(Value2SUsMap &stores, // The N last elements in NodeNums will be removed, and the SU with // the lowest NodeNum of them will become the new BarrierChain to // let the not yet seen SUs have a dependency to the removed SUs. - assert (N <= NodeNums.size()); + assert(N <= NodeNums.size()); SUnit *newBarrierChain = &SUnits[*(NodeNums.end() - N)]; if (BarrierChain) { // The aliasing and non-aliasing maps reduce independently of each @@ -1156,6 +1176,7 @@ std::string ScheduleDAGInstrs::getDAGName() const { //===----------------------------------------------------------------------===// namespace llvm { + /// Internal state used to compute SchedDFSResult. class SchedDFSImpl { SchedDFSResult &R; @@ -1163,16 +1184,16 @@ class SchedDFSImpl { /// Join DAG nodes into equivalence classes by their subtree. IntEqClasses SubtreeClasses; /// List PredSU, SuccSU pairs that represent data edges between subtrees. - std::vector<std::pair<const SUnit*, const SUnit*> > ConnectionPairs; + std::vector<std::pair<const SUnit *, const SUnit*>> ConnectionPairs; struct RootData { unsigned NodeID; unsigned ParentNodeID; ///< Parent node (member of the parent subtree). - unsigned SubInstrCount; ///< Instr count in this tree only, not children. + unsigned SubInstrCount = 0; ///< Instr count in this tree only, not + /// children. RootData(unsigned id): NodeID(id), - ParentNodeID(SchedDFSResult::InvalidSubtreeID), - SubInstrCount(0) {} + ParentNodeID(SchedDFSResult::InvalidSubtreeID) {} unsigned getSparseSetIndex() const { return NodeID; } }; @@ -1340,12 +1361,15 @@ protected: } while (FromTree != SchedDFSResult::InvalidSubtreeID); } }; + } // end namespace llvm namespace { + /// Manage the stack used by a reverse depth-first search over the DAG. class SchedDAGReverseDFS { - std::vector<std::pair<const SUnit*, SUnit::const_pred_iterator> > DFSStack; + std::vector<std::pair<const SUnit *, SUnit::const_pred_iterator>> DFSStack; + public: bool isComplete() const { return DFSStack.empty(); } @@ -1367,7 +1391,8 @@ public: return getCurr()->Preds.end(); } }; -} // anonymous + +} // end anonymous namespace static bool hasDataSucc(const SUnit *SU) { for (const SDep &SuccDep : SU->Succs) { @@ -1392,7 +1417,7 @@ void SchedDFSResult::compute(ArrayRef<SUnit> SUnits) { SchedDAGReverseDFS DFS; Impl.visitPreorder(&SU); DFS.follow(&SU); - for (;;) { + while (true) { // Traverse the leftmost path as far as possible. while (DFS.getPred() != DFS.getPredEnd()) { const SDep &PredDep = *DFS.getPred(); @@ -1457,4 +1482,5 @@ raw_ostream &operator<<(raw_ostream &OS, const ILPValue &Val) { } } // end namespace llvm + #endif diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 5f167f8de1cf..9355dbe77f94 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -225,6 +225,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { } return TranslateLegalizeResults(Op, Lowered); } + LLVM_FALLTHROUGH; case TargetLowering::Expand: Changed = true; return LegalizeOp(ExpandLoad(Op)); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 177898e1e950..80a03ea4eea0 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1,4 +1,4 @@ -//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// +//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// // // The LLVM Compiler Infrastructure // @@ -11,29 +11,46 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/SelectionDAG.h" #include "SDNodeDbgValue.h" +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/APSInt.h" -#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/None.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ADT/Twine.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineValueType.h" +#include "llvm/CodeGen/RuntimeLibcalls.h" +#include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" -#include "llvm/IR/CallingConv.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" -#include "llvm/IR/GlobalAlias.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" @@ -41,16 +58,20 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" #include <algorithm> -#include <cmath> +#include <cassert> +#include <cstdint> +#include <cstdlib> +#include <limits> +#include <set> +#include <string> #include <utility> +#include <vector> using namespace llvm; @@ -269,7 +290,6 @@ ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { return ISD::CondCode(Operation); } - /// For an integer comparison, return 1 if the comparison is a signed operation /// and 2 if the result is an unsigned comparison. Return zero if the operation /// does not depend on the sign of the input (setne and seteq). @@ -338,7 +358,6 @@ ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, //===----------------------------------------------------------------------===// /// AddNodeIDOpcode - Add the node opcode to the NodeID data. -/// static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { ID.AddInteger(OpC); } @@ -350,7 +369,6 @@ static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { } /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. -/// static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef<SDValue> Ops) { for (auto& Op : Ops) { @@ -360,7 +378,6 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID, } /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. -/// static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef<SDUse> Ops) { for (auto& Op : Ops) { @@ -392,10 +409,9 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { break; } case ISD::TargetConstantFP: - case ISD::ConstantFP: { + case ISD::ConstantFP: ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); break; - } case ISD::TargetGlobalAddress: case ISD::GlobalAddress: case ISD::TargetGlobalTLSAddress: @@ -770,7 +786,6 @@ bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { /// maps and modified in place. Add it back to the CSE maps, unless an identical /// node already exists, in which case transfer all its users to the existing /// node. This transfer can potentially trigger recursive merging. -/// void SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { // For node types that aren't CSE'd, just act as if no identical node @@ -835,7 +850,6 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, return Node; } - /// FindModifiedNodeSlot - Find a slot for the specified node if its operands /// were replaced with those specified. If this node is never memoized, /// return null, otherwise return a pointer to the slot it would take. If a @@ -864,10 +878,9 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { // EntryNode could meaningfully have debug info if we can find it... SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) - : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL), + : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), - Root(getEntryNode()), NewNodesMustHaveLegalTypes(false), - UpdateListeners(nullptr) { + Root(getEntryNode()) { InsertNode(&EntryNode); DbgInfo = new SDDbgInfo(); } @@ -1038,7 +1051,6 @@ SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, } /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). -/// SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { EVT EltVT = VT.getScalarType(); SDValue NegOne = @@ -1317,7 +1329,6 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, return SDValue(N, 0); } - SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, unsigned Alignment, int Offset, bool isTarget, @@ -1451,7 +1462,7 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, // Validate that all indices in Mask are within the range of the elements // input to the shuffle. int NElts = Mask.size(); - assert(all_of(Mask, [&](int M) { return M < (NElts * 2); }) && + assert(llvm::all_of(Mask, [&](int M) { return M < (NElts * 2); }) && "Index out of range"); // Copy the mask so we can do any needed cleanup. @@ -2918,7 +2929,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, else DemandedRHS.setBit((unsigned)M % NumElts); } - Tmp = UINT_MAX; + Tmp = std::numeric_limits<unsigned>::max(); if (!!DemandedLHS) Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); if (!!DemandedRHS) { @@ -3122,7 +3133,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, unsigned EltIdx = CEltNo->getZExtValue(); // If we demand the inserted element then get its sign bits. - Tmp = UINT_MAX; + Tmp = std::numeric_limits<unsigned>::max(); if (DemandedElts[EltIdx]) { // TODO - handle implicit truncation of inserted elements. if (InVal.getScalarValueSizeInBits() != VTBits) @@ -3188,7 +3199,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, case ISD::CONCAT_VECTORS: // Determine the minimum number of sign bits across all demanded // elts of the input vectors. Early out if the result is already 1. - Tmp = UINT_MAX; + Tmp = std::numeric_limits<unsigned>::max(); EVT SubVectorVT = Op.getOperand(0).getValueType(); unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); unsigned NumSubVectors = Op.getNumOperands(); @@ -3327,7 +3338,7 @@ bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, - llvm::SelectionDAG &DAG) { + SelectionDAG &DAG) { assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); assert(llvm::all_of(Ops, [Ops](SDValue Op) { @@ -3836,8 +3847,9 @@ bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { return true; return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && - any_of(Divisor->op_values(), - [](SDValue V) { return V.isUndef() || isNullConstant(V); }); + llvm::any_of(Divisor->op_values(), + [](SDValue V) { return V.isUndef() || + isNullConstant(V); }); // TODO: Handle signed overflow. } // TODO: Handle oversized shifts. @@ -3948,8 +3960,8 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, // All operands must be vector types with the same number of elements as // the result type and must be either UNDEF or a build vector of constant // or UNDEF scalars. - if (!all_of(Ops, IsConstantBuildVectorOrUndef) || - !all_of(Ops, IsScalarOrSameVectorSize)) + if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) || + !llvm::all_of(Ops, IsScalarOrSameVectorSize)) return SDValue(); // If we are comparing vectors, then the result needs to be a i1 boolean @@ -5550,7 +5562,7 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, Opcode == ISD::PREFETCH || Opcode == ISD::LIFETIME_START || Opcode == ISD::LIFETIME_END || - (Opcode <= INT_MAX && + ((int)Opcode <= std::numeric_limits<int>::max() && (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && "Opcode is not a memory-accessing opcode!"); @@ -5884,7 +5896,6 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType ExtTy, bool isExpanding) { - SDVTList VTs = getVTList(VT, MVT::Other); SDValue Ops[] = { Chain, Ptr, Mask, Src0 }; FoldingSetNodeID ID; @@ -6038,13 +6049,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, switch (Opcode) { default: break; - case ISD::CONCAT_VECTORS: { + case ISD::CONCAT_VECTORS: // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) return V; break; - } - case ISD::SELECT_CC: { + case ISD::SELECT_CC: assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); assert(Ops[0].getValueType() == Ops[1].getValueType() && "LHS and RHS of condition must have same type!"); @@ -6053,14 +6063,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, assert(Ops[2].getValueType() == VT && "select_cc node must be of same type as true and false value!"); break; - } - case ISD::BR_CC: { + case ISD::BR_CC: assert(NumOps == 5 && "BR_CC takes 5 operands!"); assert(Ops[2].getValueType() == Ops[3].getValueType() && "LHS/RHS of comparison should match types!"); break; } - } // Memoize nodes. SDNode *N; @@ -6599,7 +6607,6 @@ SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { return Res; } - /// getMachineNode - These are used for target selectors to create a new node /// with specified return type(s), MachineInstr opcode, and operands. /// @@ -6812,7 +6819,7 @@ public: : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} }; -} +} // end anonymous namespace /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. /// This can cause recursive merging of nodes in the DAG. @@ -6858,7 +6865,6 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { AddModifiedNodeToCSEMaps(User); } - // If we just RAUW'd the root, take note. if (FromN == getRoot()) setRoot(To); @@ -7028,6 +7034,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ } namespace { + /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith /// to record information about a use. struct UseMemo { @@ -7040,7 +7047,8 @@ namespace { bool operator<(const UseMemo &L, const UseMemo &R) { return (intptr_t)L.User < (intptr_t)R.User; } -} + +} // end anonymous namespace /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving /// uses of other values produced by From.getNode() alone. The same value @@ -7106,7 +7114,6 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, /// based on their topological order. It returns the maximum id and a vector /// of the SDNodes* in assigned order by reference. unsigned SelectionDAG::AssignTopologicalOrder() { - unsigned DAGSize = 0; // SortedPos tracks the progress of the algorithm. Nodes before it are @@ -7333,6 +7340,7 @@ void SDNode::Profile(FoldingSetNodeID &ID) const { } namespace { + struct EVTArray { std::vector<EVT> VTs; @@ -7342,11 +7350,12 @@ namespace { VTs.push_back(MVT((MVT::SimpleValueType)i)); } }; -} -static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; +} // end anonymous namespace + +static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs; static ManagedStatic<EVTArray> SimpleVTArray; -static ManagedStatic<sys::SmartMutex<true> > VTMutex; +static ManagedStatic<sys::SmartMutex<true>> VTMutex; /// getValueTypeList - Return a pointer to the specified value type. /// @@ -7380,7 +7389,6 @@ bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { return NUses == 0; } - /// hasAnyUseOfValue - Return true if there are any use of the indicated /// value. This method ignores uses of other values defined by this operation. bool SDNode::hasAnyUseOfValue(unsigned Value) const { @@ -7393,9 +7401,7 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const { return false; } - /// isOnlyUserOf - Return true if this node is the only use of N. -/// bool SDNode::isOnlyUserOf(const SDNode *N) const { bool Seen = false; for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { @@ -7425,7 +7431,6 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { } /// isOperand - Return true if this node is an operand of N. -/// bool SDValue::isOperandOf(const SDNode *N) const { for (const SDValue &Op : N->op_values()) if (*this == Op) @@ -7475,7 +7480,7 @@ bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, } // Next, try a deep search: check whether every operand of the TokenFactor // reaches Dest. - return all_of((*this)->ops(), [=](SDValue Op) { + return llvm::all_of((*this)->ops(), [=](SDValue Op) { return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); }); } @@ -7627,7 +7632,6 @@ bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, return false; } - /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if /// it cannot be inferred. unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { @@ -7718,7 +7722,6 @@ unsigned GlobalAddressSDNode::getAddressSpace() const { return getGlobal()->getType()->getAddressSpace(); } - Type *ConstantPoolSDNode::getType() const { if (isMachineConstantPoolEntry()) return Val.MachineCPVal->getType(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 687b882c5e4d..b5ccd64ee76c 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -2022,7 +2022,7 @@ static SDNode *findGlueUse(SDNode *N) { } /// findNonImmUse - Return true if "Use" is a non-immediate use of "Def". -/// This function recursively traverses up the operand chain, ignoring +/// This function iteratively traverses up the operand chain, ignoring /// certain nodes. static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, SDNode *Root, SmallPtrSetImpl<SDNode*> &Visited, @@ -2035,30 +2035,36 @@ static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, // The Use may be -1 (unassigned) if it is a newly allocated node. This can // happen because we scan down to newly selected nodes in the case of glue // uses. - if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1)) - return false; + std::vector<SDNode *> WorkList; + WorkList.push_back(Use); - // Don't revisit nodes if we already scanned it and didn't fail, we know we - // won't fail if we scan it again. - if (!Visited.insert(Use).second) - return false; + while (!WorkList.empty()) { + Use = WorkList.back(); + WorkList.pop_back(); + if (Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1) + continue; - for (const SDValue &Op : Use->op_values()) { - // Ignore chain uses, they are validated by HandleMergeInputChains. - if (Op.getValueType() == MVT::Other && IgnoreChains) + // Don't revisit nodes if we already scanned it and didn't fail, we know we + // won't fail if we scan it again. + if (!Visited.insert(Use).second) continue; - SDNode *N = Op.getNode(); - if (N == Def) { - if (Use == ImmedUse || Use == Root) - continue; // We are not looking for immediate use. - assert(N != Root); - return true; - } + for (const SDValue &Op : Use->op_values()) { + // Ignore chain uses, they are validated by HandleMergeInputChains. + if (Op.getValueType() == MVT::Other && IgnoreChains) + continue; - // Traverse up the operand chain. - if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains)) - return true; + SDNode *N = Op.getNode(); + if (N == Def) { + if (Use == ImmedUse || Use == Root) + continue; // We are not looking for immediate use. + assert(N != Root); + return true; + } + + // Traverse up the operand chain. + WorkList.push_back(N); + } } return false; } diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 1c66649cae01..eed667dbe7e0 100644 --- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -818,7 +818,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, SI.GCTransitionArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end()); SI.ID = ISP.getID(); - SI.DeoptState = ArrayRef<const Use>(ISP.vm_state_begin(), ISP.vm_state_end()); + SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end()); SI.StatepointFlags = ISP.getFlags(); SI.NumPatchBytes = ISP.getNumPatchBytes(); SI.EHPadBB = EHPadBB; diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 0dffffee9976..adb2b188265b 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1493,8 +1493,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, } } - // Ensure that the constant occurs on the RHS, and fold constant - // comparisons. + // Ensure that the constant occurs on the RHS and fold constant comparisons. ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond); if (isa<ConstantSDNode>(N0.getNode()) && (DCI.isBeforeLegalizeOps() || @@ -1638,14 +1637,13 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), TopSetCC.getOperand(1), InvCond); - } } } - // If the LHS is '(and load, const)', the RHS is 0, - // the test is for equality or unsigned, and all 1 bits of the const are - // in the same partial word, see if we can shorten the load. + // If the LHS is '(and load, const)', the RHS is 0, the test is for + // equality or unsigned, and all 1 bits of the const are in the same + // partial word, see if we can shorten the load. if (DCI.isBeforeLegalize() && !ISD::isSignedIntSetCC(Cond) && N0.getOpcode() == ISD::AND && C1 == 0 && @@ -1669,10 +1667,10 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, APInt newMask = APInt::getLowBitsSet(maskWidth, width); for (unsigned offset=0; offset<origWidth/width; offset++) { if ((newMask & Mask) == Mask) { - if (!DAG.getDataLayout().isLittleEndian()) - bestOffset = (origWidth/width - offset - 1) * (width/8); - else + if (DAG.getDataLayout().isLittleEndian()) bestOffset = (uint64_t)offset * (width/8); + else + bestOffset = (origWidth/width - offset - 1) * (width/8); bestMask = Mask.lshr(offset * (width/8) * 8); bestWidth = width; break; @@ -1713,10 +1711,12 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, switch (Cond) { case ISD::SETUGT: case ISD::SETUGE: - case ISD::SETEQ: return DAG.getConstant(0, dl, VT); + case ISD::SETEQ: + return DAG.getConstant(0, dl, VT); case ISD::SETULT: case ISD::SETULE: - case ISD::SETNE: return DAG.getConstant(1, dl, VT); + case ISD::SETNE: + return DAG.getConstant(1, dl, VT); case ISD::SETGT: case ISD::SETGE: // True if the sign bit of C1 is set. @@ -1816,9 +1816,9 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, BitWidth-1))) { // Okay, get the un-inverted input value. SDValue Val; - if (N0.getOpcode() == ISD::XOR) + if (N0.getOpcode() == ISD::XOR) { Val = N0.getOperand(0); - else { + } else { assert(N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR); // ((X^1)&1)^1 -> X & 1 @@ -1883,7 +1883,10 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // Canonicalize GE/LE comparisons to use GT/LT comparisons. if (Cond == ISD::SETGE || Cond == ISD::SETUGE) { - if (C1 == MinVal) return DAG.getConstant(1, dl, VT); // X >= MIN --> true + // X >= MIN --> true + if (C1 == MinVal) + return DAG.getConstant(1, dl, VT); + // X >= C0 --> X > (C0 - 1) APInt C = C1 - 1; ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT; @@ -1898,7 +1901,10 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, } if (Cond == ISD::SETLE || Cond == ISD::SETULE) { - if (C1 == MaxVal) return DAG.getConstant(1, dl, VT); // X <= MAX --> true + // X <= MAX --> true + if (C1 == MaxVal) + return DAG.getConstant(1, dl, VT); + // X <= C0 --> X < (C0 + 1) APInt C = C1 + 1; ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT; diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index 900c0318b179..c43a5e18ad23 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -1456,6 +1456,7 @@ void TargetLoweringBase::computeRegisterProperties( } if (IsLegalWiderType) break; + LLVM_FALLTHROUGH; } case TypeWidenVector: { // Try to widen the vector. @@ -1473,6 +1474,7 @@ void TargetLoweringBase::computeRegisterProperties( } if (IsLegalWiderType) break; + LLVM_FALLTHROUGH; } case TypeSplitVector: case TypeScalarizeVector: { diff --git a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp index 282e3103adc9..711144fc2faa 100644 --- a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp +++ b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp @@ -27,6 +27,14 @@ Error CodeViewRecordIO::beginRecord(Optional<uint32_t> MaxLength) { Error CodeViewRecordIO::endRecord() { assert(!Limits.empty() && "Not in a record!"); Limits.pop_back(); + // We would like to assert that we actually read / wrote all the bytes that we + // expected to for this record, but unfortunately we can't do this. Some + // producers such as MASM over-allocate for certain types of records and + // commit the extraneous data, so when reading we can't be sure every byte + // will have been read. And when writing we over-allocate temporarily since + // we don't know how big the record is until we're finished writing it, so + // even though we don't commit the extraneous data, we still can't guarantee + // we're at the end of the allocated data. return Error::success(); } @@ -49,6 +57,12 @@ uint32_t CodeViewRecordIO::maxFieldLength() const { return *Min; } +Error CodeViewRecordIO::padToAlignment(uint32_t Align) { + if (isReading()) + return Reader->padToAlignment(Align); + return Writer->padToAlignment(Align); +} + Error CodeViewRecordIO::skipPadding() { assert(!isWriting() && "Cannot skip padding while writing!"); diff --git a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp index b8741eb0b675..2e72242181b0 100644 --- a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp @@ -72,7 +72,7 @@ Error DebugStringTableSubsection::commit(BinaryStreamWriter &Writer) const { uint32_t DebugStringTableSubsection::size() const { return Strings.size(); } uint32_t DebugStringTableSubsection::getStringId(StringRef S) const { - auto P = Strings.find(S); - assert(P != Strings.end()); - return P->second; + auto Iter = Strings.find(S); + assert(Iter != Strings.end()); + return Iter->second; } diff --git a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp index 511f36d0020a..cfd1c5d3ab0c 100644 --- a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp +++ b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp @@ -16,14 +16,17 @@ using namespace llvm; using namespace llvm::codeview; DebugSubsectionRecord::DebugSubsectionRecord() - : Kind(DebugSubsectionKind::None) {} + : Container(CodeViewContainer::ObjectFile), + Kind(DebugSubsectionKind::None) {} DebugSubsectionRecord::DebugSubsectionRecord(DebugSubsectionKind Kind, - BinaryStreamRef Data) - : Kind(Kind), Data(Data) {} + BinaryStreamRef Data, + CodeViewContainer Container) + : Container(Container), Kind(Kind), Data(Data) {} Error DebugSubsectionRecord::initialize(BinaryStreamRef Stream, - DebugSubsectionRecord &Info) { + DebugSubsectionRecord &Info, + CodeViewContainer Container) { const DebugSubsectionHeader *Header; BinaryStreamReader Reader(Stream); if (auto EC = Reader.readObject(Header)) @@ -41,13 +44,14 @@ Error DebugSubsectionRecord::initialize(BinaryStreamRef Stream, } if (auto EC = Reader.readStreamRef(Info.Data, Header->Length)) return EC; + Info.Container = Container; Info.Kind = Kind; return Error::success(); } uint32_t DebugSubsectionRecord::getRecordLength() const { uint32_t Result = sizeof(DebugSubsectionHeader) + Data.getLength(); - assert(Result % 4 == 0); + assert(Result % alignOf(Container) == 0); return Result; } @@ -56,25 +60,29 @@ DebugSubsectionKind DebugSubsectionRecord::kind() const { return Kind; } BinaryStreamRef DebugSubsectionRecord::getRecordData() const { return Data; } DebugSubsectionRecordBuilder::DebugSubsectionRecordBuilder( - DebugSubsectionKind Kind, DebugSubsection &Frag) - : Kind(Kind), Frag(Frag) {} + std::unique_ptr<DebugSubsection> Subsection, CodeViewContainer Container) + : Subsection(std::move(Subsection)), Container(Container) {} uint32_t DebugSubsectionRecordBuilder::calculateSerializedLength() { - uint32_t Size = sizeof(DebugSubsectionHeader) + - alignTo(Frag.calculateSerializedSize(), 4); + uint32_t Size = + sizeof(DebugSubsectionHeader) + + alignTo(Subsection->calculateSerializedSize(), alignOf(Container)); return Size; } Error DebugSubsectionRecordBuilder::commit(BinaryStreamWriter &Writer) { + assert(Writer.getOffset() % alignOf(Container) == 0 && + "Debug Subsection not properly aligned"); + DebugSubsectionHeader Header; - Header.Kind = uint32_t(Kind); + Header.Kind = uint32_t(Subsection->kind()); Header.Length = calculateSerializedLength() - sizeof(DebugSubsectionHeader); if (auto EC = Writer.writeObject(Header)) return EC; - if (auto EC = Frag.commit(Writer)) + if (auto EC = Subsection->commit(Writer)) return EC; - if (auto EC = Writer.padToAlignment(4)) + if (auto EC = Writer.padToAlignment(alignOf(Container))) return EC; return Error::success(); diff --git a/lib/DebugInfo/CodeView/SymbolDumper.cpp b/lib/DebugInfo/CodeView/SymbolDumper.cpp index 3d49a7198d1a..66045933ce9b 100644 --- a/lib/DebugInfo/CodeView/SymbolDumper.cpp +++ b/lib/DebugInfo/CodeView/SymbolDumper.cpp @@ -668,7 +668,7 @@ Error CVSymbolDumperImpl::visitUnknownSymbol(CVSymbol &CVR) { Error CVSymbolDumper::dump(CVRecord<SymbolKind> &Record) { SymbolVisitorCallbackPipeline Pipeline; - SymbolDeserializer Deserializer(ObjDelegate.get()); + SymbolDeserializer Deserializer(ObjDelegate.get(), Container); CVSymbolDumperImpl Dumper(Types, ObjDelegate.get(), W, PrintRecordBytes); Pipeline.addCallbackToPipeline(Deserializer); @@ -679,7 +679,7 @@ Error CVSymbolDumper::dump(CVRecord<SymbolKind> &Record) { Error CVSymbolDumper::dump(const CVSymbolArray &Symbols) { SymbolVisitorCallbackPipeline Pipeline; - SymbolDeserializer Deserializer(ObjDelegate.get()); + SymbolDeserializer Deserializer(ObjDelegate.get(), Container); CVSymbolDumperImpl Dumper(Types, ObjDelegate.get(), W, PrintRecordBytes); Pipeline.addCallbackToPipeline(Deserializer); diff --git a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp index bb1731465495..ea46841a70f6 100644 --- a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp +++ b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp @@ -40,6 +40,7 @@ Error SymbolRecordMapping::visitSymbolBegin(CVSymbol &Record) { } Error SymbolRecordMapping::visitSymbolEnd(CVSymbol &Record) { + error(IO.padToAlignment(alignOf(Container))); error(IO.endRecord()); return Error::success(); } diff --git a/lib/DebugInfo/CodeView/SymbolSerializer.cpp b/lib/DebugInfo/CodeView/SymbolSerializer.cpp index 251cc431f52b..9f2d619d1a1c 100644 --- a/lib/DebugInfo/CodeView/SymbolSerializer.cpp +++ b/lib/DebugInfo/CodeView/SymbolSerializer.cpp @@ -12,9 +12,11 @@ using namespace llvm; using namespace llvm::codeview; -SymbolSerializer::SymbolSerializer(BumpPtrAllocator &Allocator) - : Storage(Allocator), RecordBuffer(MaxRecordLength), Stream(RecordBuffer, llvm::support::little), - Writer(Stream), Mapping(Writer) { } +SymbolSerializer::SymbolSerializer(BumpPtrAllocator &Allocator, + CodeViewContainer Container) + : Storage(Allocator), RecordBuffer(MaxRecordLength), + Stream(RecordBuffer, llvm::support::little), Writer(Stream), + Mapping(Writer, Container) {} Error SymbolSerializer::visitSymbolBegin(CVSymbol &Record) { assert(!CurrentSymbol.hasValue() && "Already in a symbol mapping!"); diff --git a/lib/DebugInfo/MSF/MappedBlockStream.cpp b/lib/DebugInfo/MSF/MappedBlockStream.cpp index dfdeb8414212..faf2442bc94b 100644 --- a/lib/DebugInfo/MSF/MappedBlockStream.cpp +++ b/lib/DebugInfo/MSF/MappedBlockStream.cpp @@ -47,42 +47,46 @@ static Interval intersect(const Interval &I1, const Interval &I2) { MappedBlockStream::MappedBlockStream(uint32_t BlockSize, const MSFStreamLayout &Layout, - BinaryStreamRef MsfData) - : BlockSize(BlockSize), StreamLayout(Layout), MsfData(MsfData) {} - -std::unique_ptr<MappedBlockStream> -MappedBlockStream::createStream(uint32_t BlockSize, - const MSFStreamLayout &Layout, - BinaryStreamRef MsfData) { + BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) + : BlockSize(BlockSize), StreamLayout(Layout), MsfData(MsfData), + Allocator(Allocator) {} + +std::unique_ptr<MappedBlockStream> MappedBlockStream::createStream( + uint32_t BlockSize, const MSFStreamLayout &Layout, BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { return llvm::make_unique<MappedBlockStreamImpl<MappedBlockStream>>( - BlockSize, Layout, MsfData); + BlockSize, Layout, MsfData, Allocator); } std::unique_ptr<MappedBlockStream> MappedBlockStream::createIndexedStream( - const MSFLayout &Layout, BinaryStreamRef MsfData, uint32_t StreamIndex) { + const MSFLayout &Layout, BinaryStreamRef MsfData, uint32_t StreamIndex, + BumpPtrAllocator &Allocator) { assert(StreamIndex < Layout.StreamMap.size() && "Invalid stream index"); MSFStreamLayout SL; SL.Blocks = Layout.StreamMap[StreamIndex]; SL.Length = Layout.StreamSizes[StreamIndex]; return llvm::make_unique<MappedBlockStreamImpl<MappedBlockStream>>( - Layout.SB->BlockSize, SL, MsfData); + Layout.SB->BlockSize, SL, MsfData, Allocator); } std::unique_ptr<MappedBlockStream> MappedBlockStream::createDirectoryStream(const MSFLayout &Layout, - BinaryStreamRef MsfData) { + BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { MSFStreamLayout SL; SL.Blocks = Layout.DirectoryBlocks; SL.Length = Layout.SB->NumDirectoryBytes; - return createStream(Layout.SB->BlockSize, SL, MsfData); + return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator); } std::unique_ptr<MappedBlockStream> MappedBlockStream::createFpmStream(const MSFLayout &Layout, - BinaryStreamRef MsfData) { + BinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { MSFStreamLayout SL; initializeFpmStreamLayout(Layout, SL); - return createStream(Layout.SB->BlockSize, SL, MsfData); + return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator); } Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size, @@ -148,7 +152,7 @@ Error MappedBlockStream::readBytes(uint32_t Offset, uint32_t Size, // into it, and return an ArrayRef to that. Do not touch existing pool // allocations, as existing clients may be holding a pointer which must // not be invalidated. - uint8_t *WriteBuffer = static_cast<uint8_t *>(Pool.Allocate(Size, 8)); + uint8_t *WriteBuffer = static_cast<uint8_t *>(Allocator.Allocate(Size, 8)); if (auto EC = readBytes(Offset, MutableArrayRef<uint8_t>(WriteBuffer, Size))) return EC; @@ -269,10 +273,6 @@ Error MappedBlockStream::readBytes(uint32_t Offset, return Error::success(); } -uint32_t MappedBlockStream::getNumBytesCopied() const { - return static_cast<uint32_t>(Pool.getBytesAllocated()); -} - void MappedBlockStream::invalidateCache() { CacheMap.shrink_and_clear(); } void MappedBlockStream::fixCacheAfterWrite(uint32_t Offset, @@ -313,43 +313,48 @@ void MappedBlockStream::fixCacheAfterWrite(uint32_t Offset, WritableMappedBlockStream::WritableMappedBlockStream( uint32_t BlockSize, const MSFStreamLayout &Layout, - WritableBinaryStreamRef MsfData) - : ReadInterface(BlockSize, Layout, MsfData), WriteInterface(MsfData) {} + WritableBinaryStreamRef MsfData, BumpPtrAllocator &Allocator) + : ReadInterface(BlockSize, Layout, MsfData, Allocator), + WriteInterface(MsfData) {} std::unique_ptr<WritableMappedBlockStream> WritableMappedBlockStream::createStream(uint32_t BlockSize, const MSFStreamLayout &Layout, - WritableBinaryStreamRef MsfData) { + WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { return llvm::make_unique<MappedBlockStreamImpl<WritableMappedBlockStream>>( - BlockSize, Layout, MsfData); + BlockSize, Layout, MsfData, Allocator); } std::unique_ptr<WritableMappedBlockStream> WritableMappedBlockStream::createIndexedStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData, - uint32_t StreamIndex) { + uint32_t StreamIndex, + BumpPtrAllocator &Allocator) { assert(StreamIndex < Layout.StreamMap.size() && "Invalid stream index"); MSFStreamLayout SL; SL.Blocks = Layout.StreamMap[StreamIndex]; SL.Length = Layout.StreamSizes[StreamIndex]; - return createStream(Layout.SB->BlockSize, SL, MsfData); + return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator); } std::unique_ptr<WritableMappedBlockStream> WritableMappedBlockStream::createDirectoryStream( - const MSFLayout &Layout, WritableBinaryStreamRef MsfData) { + const MSFLayout &Layout, WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { MSFStreamLayout SL; SL.Blocks = Layout.DirectoryBlocks; SL.Length = Layout.SB->NumDirectoryBytes; - return createStream(Layout.SB->BlockSize, SL, MsfData); + return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator); } std::unique_ptr<WritableMappedBlockStream> WritableMappedBlockStream::createFpmStream(const MSFLayout &Layout, - WritableBinaryStreamRef MsfData) { + WritableBinaryStreamRef MsfData, + BumpPtrAllocator &Allocator) { MSFStreamLayout SL; initializeFpmStreamLayout(Layout, SL); - return createStream(Layout.SB->BlockSize, SL, MsfData); + return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator); } Error WritableMappedBlockStream::readBytes(uint32_t Offset, uint32_t Size, diff --git a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp index b28ec2ff33ac..22c2ef31bd71 100644 --- a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp @@ -66,7 +66,11 @@ void DbiModuleDescriptorBuilder::setObjFileName(StringRef Name) { void DbiModuleDescriptorBuilder::addSymbol(CVSymbol Symbol) { Symbols.push_back(Symbol); - SymbolByteSize += Symbol.data().size(); + // Symbols written to a PDB file are required to be 4 byte aligned. The same + // is not true of object files. + assert(Symbol.length() % alignOf(CodeViewContainer::Pdb) == 0 && + "Invalid Symbol alignment!"); + SymbolByteSize += Symbol.length(); } void DbiModuleDescriptorBuilder::addSourceFile(StringRef Path) { @@ -140,7 +144,7 @@ Error DbiModuleDescriptorBuilder::commit(BinaryStreamWriter &ModiWriter, if (Layout.ModDiStream != kInvalidStreamIndex) { auto NS = WritableMappedBlockStream::createIndexedStream( - MsfLayout, MsfBuffer, Layout.ModDiStream); + MsfLayout, MsfBuffer, Layout.ModDiStream, MSF.getAllocator()); WritableBinaryStreamRef Ref(*NS); BinaryStreamWriter SymbolWriter(Ref); // Write the symbols. @@ -153,7 +157,8 @@ Error DbiModuleDescriptorBuilder::commit(BinaryStreamWriter &ModiWriter, if (auto EC = SymbolWriter.writeStreamRef(RecordsRef)) return EC; // TODO: Write C11 Line data - + assert(SymbolWriter.getOffset() % alignOf(CodeViewContainer::Pdb) == 0 && + "Invalid debug section alignment!"); for (const auto &Builder : C13Builders) { assert(Builder && "Empty C13 Fragment Builder!"); if (auto EC = Builder->commit(SymbolWriter)) @@ -169,42 +174,9 @@ Error DbiModuleDescriptorBuilder::commit(BinaryStreamWriter &ModiWriter, return Error::success(); } -void DbiModuleDescriptorBuilder::addC13Fragment( - std::unique_ptr<DebugLinesSubsection> Lines) { - DebugLinesSubsection &Frag = *Lines; - - // File Checksums have to come first, so push an empty entry on if this - // is the first. - if (C13Builders.empty()) - C13Builders.push_back(nullptr); - - this->LineInfo.push_back(std::move(Lines)); - C13Builders.push_back( - llvm::make_unique<DebugSubsectionRecordBuilder>(Frag.kind(), Frag)); -} - -void DbiModuleDescriptorBuilder::addC13Fragment( - std::unique_ptr<codeview::DebugInlineeLinesSubsection> Inlinees) { - DebugInlineeLinesSubsection &Frag = *Inlinees; - - // File Checksums have to come first, so push an empty entry on if this - // is the first. - if (C13Builders.empty()) - C13Builders.push_back(nullptr); - - this->Inlinees.push_back(std::move(Inlinees)); - C13Builders.push_back( - llvm::make_unique<DebugSubsectionRecordBuilder>(Frag.kind(), Frag)); -} - -void DbiModuleDescriptorBuilder::setC13FileChecksums( - std::unique_ptr<DebugChecksumsSubsection> Checksums) { - assert(!ChecksumInfo && "Can't have more than one checksum info!"); - - if (C13Builders.empty()) - C13Builders.push_back(nullptr); - - ChecksumInfo = std::move(Checksums); - C13Builders[0] = llvm::make_unique<DebugSubsectionRecordBuilder>( - ChecksumInfo->kind(), *ChecksumInfo); +void DbiModuleDescriptorBuilder::addDebugSubsection( + std::unique_ptr<DebugSubsection> Subsection) { + assert(Subsection); + C13Builders.push_back(llvm::make_unique<DebugSubsectionRecordBuilder>( + std::move(Subsection), CodeViewContainer::Pdb)); } diff --git a/lib/DebugInfo/PDB/Native/DbiStream.cpp b/lib/DebugInfo/PDB/Native/DbiStream.cpp index 2f4fb6cc295d..320b11dc5cab 100644 --- a/lib/DebugInfo/PDB/Native/DbiStream.cpp +++ b/lib/DebugInfo/PDB/Native/DbiStream.cpp @@ -252,7 +252,7 @@ Error DbiStream::initializeSectionHeadersData() { return make_error<RawError>(raw_error_code::no_stream); auto SHS = MappedBlockStream::createIndexedStream( - Pdb.getMsfLayout(), Pdb.getMsfBuffer(), StreamNum); + Pdb.getMsfLayout(), Pdb.getMsfBuffer(), StreamNum, Pdb.getAllocator()); size_t StreamLen = SHS->getLength(); if (StreamLen % sizeof(object::coff_section)) @@ -284,7 +284,7 @@ Error DbiStream::initializeFpoRecords() { return make_error<RawError>(raw_error_code::no_stream); auto FS = MappedBlockStream::createIndexedStream( - Pdb.getMsfLayout(), Pdb.getMsfBuffer(), StreamNum); + Pdb.getMsfLayout(), Pdb.getMsfBuffer(), StreamNum, Pdb.getAllocator()); size_t StreamLen = FS->getLength(); if (StreamLen % sizeof(object::FpoData)) diff --git a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp index 23c7456d7772..55c20fdb9af6 100644 --- a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp @@ -357,8 +357,8 @@ Error DbiStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = finalize()) return EC; - auto DbiS = WritableMappedBlockStream::createIndexedStream(Layout, MsfBuffer, - StreamDBI); + auto DbiS = WritableMappedBlockStream::createIndexedStream( + Layout, MsfBuffer, StreamDBI, Allocator); BinaryStreamWriter Writer(*DbiS); if (auto EC = Writer.writeObject(*Header)) @@ -396,7 +396,7 @@ Error DbiStreamBuilder::commit(const msf::MSFLayout &Layout, if (Stream.StreamNumber == kInvalidStreamIndex) continue; auto WritableStream = WritableMappedBlockStream::createIndexedStream( - Layout, MsfBuffer, Stream.StreamNumber); + Layout, MsfBuffer, Stream.StreamNumber, Allocator); BinaryStreamWriter DbgStreamWriter(*WritableStream); if (auto EC = DbgStreamWriter.writeArray(Stream.Data)) return EC; diff --git a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp index f019d410328a..707128f7efd4 100644 --- a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp @@ -50,8 +50,8 @@ Error InfoStreamBuilder::finalizeMsfLayout() { Error InfoStreamBuilder::commit(const msf::MSFLayout &Layout, WritableBinaryStreamRef Buffer) const { - auto InfoS = - WritableMappedBlockStream::createIndexedStream(Layout, Buffer, StreamPDB); + auto InfoS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, StreamPDB, Msf.getAllocator()); BinaryStreamWriter Writer(*InfoS); InfoStreamHeader H; diff --git a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp index d7a203746a0d..c4ff30011a17 100644 --- a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp +++ b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp @@ -55,9 +55,9 @@ Error ModuleDebugStreamRef::reload() { if (auto EC = Reader.readStreamRef(C13LinesSubstream, C13Size)) return EC; - BinaryStreamReader LineReader(C13LinesSubstream); - if (auto EC = - LineReader.readArray(LinesAndChecksums, LineReader.bytesRemaining())) + BinaryStreamReader SubsectionsReader(C13LinesSubstream); + if (auto EC = SubsectionsReader.readArray(Subsections, + SubsectionsReader.bytesRemaining())) return EC; uint32_t GlobalRefsSize; @@ -77,13 +77,27 @@ ModuleDebugStreamRef::symbols(bool *HadError) const { return make_range(SymbolsSubstream.begin(HadError), SymbolsSubstream.end()); } -llvm::iterator_range<ModuleDebugStreamRef::LinesAndChecksumsIterator> -ModuleDebugStreamRef::linesAndChecksums() const { - return make_range(LinesAndChecksums.begin(), LinesAndChecksums.end()); +llvm::iterator_range<ModuleDebugStreamRef::DebugSubsectionIterator> +ModuleDebugStreamRef::subsections() const { + return make_range(Subsections.begin(), Subsections.end()); } -bool ModuleDebugStreamRef::hasLineInfo() const { +bool ModuleDebugStreamRef::hasDebugSubsections() const { return C13LinesSubstream.getLength() > 0; } Error ModuleDebugStreamRef::commit() { return Error::success(); } + +Expected<codeview::DebugChecksumsSubsectionRef> +ModuleDebugStreamRef::findChecksumsSubsection() const { + for (const auto &SS : subsections()) { + if (SS.kind() != DebugSubsectionKind::FileChecksums) + continue; + + codeview::DebugChecksumsSubsectionRef Result; + if (auto EC = Result.initialize(SS.getRecordData())) + return std::move(EC); + return Result; + } + return make_error<RawError>(raw_error_code::no_entry); +} diff --git a/lib/DebugInfo/PDB/Native/PDBFile.cpp b/lib/DebugInfo/PDB/Native/PDBFile.cpp index 859295d2c7d3..1254e23c73eb 100644 --- a/lib/DebugInfo/PDB/Native/PDBFile.cpp +++ b/lib/DebugInfo/PDB/Native/PDBFile.cpp @@ -146,7 +146,8 @@ Error PDBFile::parseFileHeaders() { // at getBlockSize() intervals, so we have to be compatible. // See the function fpmPn() for more information: // https://github.com/Microsoft/microsoft-pdb/blob/master/PDB/msf/msf.cpp#L489 - auto FpmStream = MappedBlockStream::createFpmStream(ContainerLayout, *Buffer); + auto FpmStream = + MappedBlockStream::createFpmStream(ContainerLayout, *Buffer, Allocator); BinaryStreamReader FpmReader(*FpmStream); ArrayRef<uint8_t> FpmBytes; if (auto EC = FpmReader.readBytes(FpmBytes, @@ -184,7 +185,8 @@ Error PDBFile::parseStreamData() { // is exactly what we are attempting to parse. By specifying a custom // subclass of IPDBStreamData which only accesses the fields that have already // been parsed, we can avoid this and reuse MappedBlockStream. - auto DS = MappedBlockStream::createDirectoryStream(ContainerLayout, *Buffer); + auto DS = MappedBlockStream::createDirectoryStream(ContainerLayout, *Buffer, + Allocator); BinaryStreamReader Reader(*DS); if (auto EC = Reader.readInteger(NumStreams)) return EC; @@ -407,5 +409,6 @@ PDBFile::safelyCreateIndexedStream(const MSFLayout &Layout, uint32_t StreamIndex) const { if (StreamIndex >= getNumStreams()) return make_error<RawError>(raw_error_code::no_stream); - return MappedBlockStream::createIndexedStream(Layout, MsfData, StreamIndex); + return MappedBlockStream::createIndexedStream(Layout, MsfData, StreamIndex, + Allocator); } diff --git a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp index c6568029ec55..2c6465e6fb2a 100644 --- a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp @@ -140,8 +140,8 @@ Error PDBFileBuilder::commit(StringRef Filename) { if (auto EC = Writer.writeArray(Layout.DirectoryBlocks)) return EC; - auto DirStream = - WritableMappedBlockStream::createDirectoryStream(Layout, Buffer); + auto DirStream = WritableMappedBlockStream::createDirectoryStream( + Layout, Buffer, Allocator); BinaryStreamWriter DW(*DirStream); if (auto EC = DW.writeInteger<uint32_t>(Layout.StreamSizes.size())) return EC; @@ -158,8 +158,8 @@ Error PDBFileBuilder::commit(StringRef Filename) { if (!ExpectedSN) return ExpectedSN.takeError(); - auto NS = WritableMappedBlockStream::createIndexedStream(Layout, Buffer, - *ExpectedSN); + auto NS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, *ExpectedSN, Allocator); BinaryStreamWriter NSWriter(*NS); if (auto EC = Strings.commit(NSWriter)) return EC; diff --git a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp index e84573fe07b8..6013c342cf02 100644 --- a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp +++ b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp @@ -56,6 +56,10 @@ Error PDBStringTable::readStrings(BinaryStreamReader &Reader) { return Error::success(); } +codeview::DebugStringTableSubsectionRef PDBStringTable::getStringTable() const { + return Strings; +} + Error PDBStringTable::readHashTable(BinaryStreamReader &Reader) { const support::ulittle32_t *HashCount; if (auto EC = Reader.readObject(HashCount)) diff --git a/lib/DebugInfo/PDB/Native/TpiStream.cpp b/lib/DebugInfo/PDB/Native/TpiStream.cpp index 623afb371b50..67c803d3124e 100644 --- a/lib/DebugInfo/PDB/Native/TpiStream.cpp +++ b/lib/DebugInfo/PDB/Native/TpiStream.cpp @@ -32,8 +32,7 @@ using namespace llvm::support; using namespace llvm::msf; using namespace llvm::pdb; -TpiStream::TpiStream(const PDBFile &File, - std::unique_ptr<MappedBlockStream> Stream) +TpiStream::TpiStream(PDBFile &File, std::unique_ptr<MappedBlockStream> Stream) : Pdb(File), Stream(std::move(Stream)) {} TpiStream::~TpiStream() = default; @@ -77,7 +76,8 @@ Error TpiStream::reload() { "Invalid TPI hash stream index."); auto HS = MappedBlockStream::createIndexedStream( - Pdb.getMsfLayout(), Pdb.getMsfBuffer(), Header->HashStreamIndex); + Pdb.getMsfLayout(), Pdb.getMsfBuffer(), Header->HashStreamIndex, + Pdb.getAllocator()); BinaryStreamReader HSR(*HS); // There should be a hash value for every type record, or no hashes at all. diff --git a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp index 20456cc97823..9e943c7f114d 100644 --- a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp @@ -147,8 +147,8 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = finalize()) return EC; - auto InfoS = - WritableMappedBlockStream::createIndexedStream(Layout, Buffer, Idx); + auto InfoS = WritableMappedBlockStream::createIndexedStream(Layout, Buffer, + Idx, Allocator); BinaryStreamWriter Writer(*InfoS); if (auto EC = Writer.writeObject(*Header)) @@ -159,8 +159,8 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout, return EC; if (HashStreamIndex != kInvalidStreamIndex) { - auto HVS = WritableMappedBlockStream::createIndexedStream(Layout, Buffer, - HashStreamIndex); + auto HVS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, HashStreamIndex, Allocator); BinaryStreamWriter HW(*HVS); if (HashValueStream) { if (auto EC = HW.writeStreamRef(*HashValueStream)) diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp index 7e6f9a7804b9..7754ac03b43d 100644 --- a/lib/IR/DIBuilder.cpp +++ b/lib/IR/DIBuilder.cpp @@ -39,6 +39,21 @@ void DIBuilder::trackIfUnresolved(MDNode *N) { UnresolvedNodes.emplace_back(N); } +void DIBuilder::finalizeSubprogram(DISubprogram *SP) { + MDTuple *Temp = SP->getVariables().get(); + if (!Temp || !Temp->isTemporary()) + return; + + SmallVector<Metadata *, 4> Variables; + + auto PV = PreservedVariables.find(SP); + if (PV != PreservedVariables.end()) + Variables.append(PV->second.begin(), PV->second.end()); + + DINodeArray AV = getOrCreateArray(Variables); + TempMDTuple(Temp)->replaceAllUsesWith(AV.get()); +} + void DIBuilder::finalize() { if (!CUNode) { assert(!AllowUnresolvedNodes && @@ -62,25 +77,11 @@ void DIBuilder::finalize() { CUNode->replaceRetainedTypes(MDTuple::get(VMContext, RetainValues)); DISubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms); - auto resolveVariables = [&](DISubprogram *SP) { - MDTuple *Temp = SP->getVariables().get(); - if (!Temp) - return; - - SmallVector<Metadata *, 4> Variables; - - auto PV = PreservedVariables.find(SP); - if (PV != PreservedVariables.end()) - Variables.append(PV->second.begin(), PV->second.end()); - - DINodeArray AV = getOrCreateArray(Variables); - TempMDTuple(Temp)->replaceAllUsesWith(AV.get()); - }; for (auto *SP : SPs) - resolveVariables(SP); + finalizeSubprogram(SP); for (auto *N : RetainValues) if (auto *SP = dyn_cast<DISubprogram>(N)) - resolveVariables(SP); + finalizeSubprogram(SP); if (!AllGVs.empty()) CUNode->replaceGlobalVariables(MDTuple::get(VMContext, AllGVs)); diff --git a/lib/IR/DebugLoc.cpp b/lib/IR/DebugLoc.cpp index b7e3f0c6779e..0485fece7c42 100644 --- a/lib/IR/DebugLoc.cpp +++ b/lib/IR/DebugLoc.cpp @@ -99,87 +99,6 @@ DebugLoc DebugLoc::appendInlinedAt(DebugLoc DL, DILocation *InlinedAt, return Last; } -/// Reparent \c Scope from \c OrigSP to \c NewSP. -static DIScope *reparentScope(LLVMContext &Ctx, DIScope *Scope, - DISubprogram *OrigSP, DISubprogram *NewSP, - DenseMap<const MDNode *, MDNode *> &Cache) { - SmallVector<DIScope *, 3> ScopeChain; - DIScope *Last = NewSP; - DIScope *CurScope = Scope; - do { - if (auto *SP = dyn_cast<DISubprogram>(CurScope)) { - // Don't rewrite this scope chain if it doesn't lead to the replaced SP. - if (SP != OrigSP) - return Scope; - Cache.insert({OrigSP, NewSP}); - break; - } - if (auto *Found = Cache[CurScope]) { - Last = cast<DIScope>(Found); - break; - } - ScopeChain.push_back(CurScope); - } while ((CurScope = CurScope->getScope().resolve())); - - // Starting from the top, rebuild the nodes to point to the new inlined-at - // location (then rebuilding the rest of the chain behind it) and update the - // map of already-constructed inlined-at nodes. - for (const DIScope *MD : reverse(ScopeChain)) { - if (auto *LB = dyn_cast<DILexicalBlock>(MD)) - Cache[MD] = Last = DILexicalBlock::getDistinct( - Ctx, Last, LB->getFile(), LB->getLine(), LB->getColumn()); - else if (auto *LB = dyn_cast<DILexicalBlockFile>(MD)) - Cache[MD] = Last = DILexicalBlockFile::getDistinct( - Ctx, Last, LB->getFile(), LB->getDiscriminator()); - else - llvm_unreachable("illegal parent scope"); - } - return Last; -} - -void DebugLoc::reparentDebugInfo(Instruction &I, DISubprogram *OrigSP, - DISubprogram *NewSP, - DenseMap<const MDNode *, MDNode *> &Cache) { - auto DL = I.getDebugLoc(); - if (!OrigSP || !NewSP || OrigSP == NewSP || !DL) - return; - - // Reparent the debug location. - auto &Ctx = I.getContext(); - DILocation *InlinedAt = DL->getInlinedAt(); - if (InlinedAt) { - while (auto *IA = InlinedAt->getInlinedAt()) - InlinedAt = IA; - auto NewScope = - reparentScope(Ctx, InlinedAt->getScope(), OrigSP, NewSP, Cache); - InlinedAt = - DebugLoc::get(InlinedAt->getLine(), InlinedAt->getColumn(), NewScope); - } - I.setDebugLoc( - DebugLoc::get(DL.getLine(), DL.getCol(), - reparentScope(Ctx, DL->getScope(), OrigSP, NewSP, Cache), - DebugLoc::appendInlinedAt(DL, InlinedAt, Ctx, Cache, - ReplaceLastInlinedAt))); - - // Fix up debug variables to point to NewSP. - auto reparentVar = [&](DILocalVariable *Var) { - return DILocalVariable::get( - Ctx, - cast<DILocalScope>( - reparentScope(Ctx, Var->getScope(), OrigSP, NewSP, Cache)), - Var->getName(), Var->getFile(), Var->getLine(), Var->getType(), - Var->getArg(), Var->getFlags(), Var->getAlignInBits()); - }; - if (auto *DbgValue = dyn_cast<DbgValueInst>(&I)) { - auto *Var = DbgValue->getVariable(); - I.setOperand(2, MetadataAsValue::get(Ctx, reparentVar(Var))); - } else if (auto *DbgDeclare = dyn_cast<DbgDeclareInst>(&I)) { - auto *Var = DbgDeclare->getVariable(); - I.setOperand(1, MetadataAsValue::get(Ctx, reparentVar(Var))); - } -} - - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void DebugLoc::dump() const { if (!Loc) diff --git a/lib/IR/OptBisect.cpp b/lib/IR/OptBisect.cpp index b670c817569a..a03a6fb62237 100644 --- a/lib/IR/OptBisect.cpp +++ b/lib/IR/OptBisect.cpp @@ -16,6 +16,7 @@ #include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/RegionInfo.h" #include "llvm/IR/Module.h" #include "llvm/IR/OptBisect.h" #include "llvm/Pass.h" @@ -53,13 +54,20 @@ static std::string getDescription(const BasicBlock &BB) { } static std::string getDescription(const Loop &L) { - // FIXME: I'd like to be able to provide a better description here, but - // calling L->getHeader() would introduce a new dependency on the - // LLVMCore library. + // FIXME: Move into LoopInfo so we can get a better description + // (and avoid a circular dependency between IR and Analysis). return "loop"; } +static std::string getDescription(const Region &R) { + // FIXME: Move into RegionInfo so we can get a better description + // (and avoid a circular dependency between IR and Analysis). + return "region"; +} + static std::string getDescription(const CallGraphSCC &SCC) { + // FIXME: Move into CallGraphSCCPass to avoid circular dependency between + // IR and Analysis. std::string Desc = "SCC ("; bool First = true; for (CallGraphNode *CGN : SCC) { @@ -83,6 +91,7 @@ template bool OptBisect::shouldRunPass(const Pass *, const Function &); template bool OptBisect::shouldRunPass(const Pass *, const BasicBlock &); template bool OptBisect::shouldRunPass(const Pass *, const Loop &); template bool OptBisect::shouldRunPass(const Pass *, const CallGraphSCC &); +template bool OptBisect::shouldRunPass(const Pass *, const Region &); template <class UnitT> bool OptBisect::shouldRunPass(const Pass *P, const UnitT &U) { diff --git a/lib/LTO/LTO.cpp b/lib/LTO/LTO.cpp index 9efc095f9fcf..92145aaf667a 100644 --- a/lib/LTO/LTO.cpp +++ b/lib/LTO/LTO.cpp @@ -122,6 +122,7 @@ static void computeCacheKey( AddUnsigned(Conf.CGOptLevel); AddUnsigned(Conf.CGFileType); AddUnsigned(Conf.OptLevel); + AddUnsigned(Conf.UseNewPM); AddString(Conf.OptPipeline); AddString(Conf.AAPipeline); AddString(Conf.OverrideTriple); @@ -621,6 +622,19 @@ unsigned LTO::getMaxTasks() const { } Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) { + // Compute "dead" symbols, we don't want to import/export these! + DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; + for (auto &Res : GlobalResolutions) { + if (Res.second.VisibleOutsideThinLTO && + // IRName will be defined if we have seen the prevailing copy of + // this value. If not, no need to preserve any ThinLTO copies. + !Res.second.IRName.empty()) + GUIDPreservedSymbols.insert(GlobalValue::getGUID( + GlobalValue::dropLLVMManglingEscape(Res.second.IRName))); + } + + computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols); + // Save the status of having a regularLTO combined module, as // this is needed for generating the ThinLTO Task ID, and // the CombinedModule will be moved at the end of runRegularLTO. @@ -930,6 +944,17 @@ ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix, }; } +static bool IsLiveByGUID(const ModuleSummaryIndex &Index, + GlobalValue::GUID GUID) { + auto VI = Index.getValueInfo(GUID); + if (!VI) + return false; + for (auto &I : VI.getSummaryList()) + if (Index.isGlobalValueLive(I.get())) + return true; + return false; +} + Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, bool HasRegularLTO) { if (ThinLTO.ModuleMap.empty()) @@ -962,22 +987,8 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; if (Conf.OptLevel > 0) { - // Compute "dead" symbols, we don't want to import/export these! - DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; - for (auto &Res : GlobalResolutions) { - if (Res.second.VisibleOutsideThinLTO && - // IRName will be defined if we have seen the prevailing copy of - // this value. If not, no need to preserve any ThinLTO copies. - !Res.second.IRName.empty()) - GUIDPreservedSymbols.insert(GlobalValue::getGUID( - GlobalValue::dropLLVMManglingEscape(Res.second.IRName))); - } - - auto DeadSymbols = - computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols); - ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries, - ImportLists, ExportLists, &DeadSymbols); + ImportLists, ExportLists); std::set<GlobalValue::GUID> ExportedGUIDs; for (auto &Res : GlobalResolutions) { @@ -992,7 +1003,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, auto GUID = GlobalValue::getGUID( GlobalValue::dropLLVMManglingEscape(Res.second.IRName)); // Mark exported unless index-based analysis determined it to be dead. - if (!DeadSymbols.count(GUID)) + if (IsLiveByGUID(ThinLTO.CombinedIndex, GUID)) ExportedGUIDs.insert(GUID); } diff --git a/lib/LTO/LTOBackend.cpp b/lib/LTO/LTOBackend.cpp index f9c41f5c9744..3f72e446cdf2 100644 --- a/lib/LTO/LTOBackend.cpp +++ b/lib/LTO/LTOBackend.cpp @@ -42,11 +42,6 @@ using namespace llvm; using namespace lto; -static cl::opt<bool> - LTOUseNewPM("lto-use-new-pm", - cl::desc("Run LTO passes using the new pass manager"), - cl::init(false), cl::Hidden); - LLVM_ATTRIBUTE_NORETURN static void reportOpenError(StringRef Path, Twine Msg) { errs() << "failed to open " << Path << ": " << Msg << '\n'; errs().flush(); @@ -266,7 +261,7 @@ bool opt(Config &Conf, TargetMachine *TM, unsigned Task, Module &Mod, if (!Conf.OptPipeline.empty()) runNewPMCustomPasses(Mod, TM, Conf.OptPipeline, Conf.AAPipeline, Conf.DisableVerify); - else if (LTOUseNewPM) + else if (Conf.UseNewPM) runNewPMPasses(Mod, TM, Conf.OptLevel, IsThinLTO); else runOldPMPasses(Conf, Mod, TM, IsThinLTO, ExportSummary, ImportSummary); diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp index ca3fc60f9501..6b221a347c17 100644 --- a/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/lib/LTO/ThinLTOCodeGenerator.cpp @@ -628,13 +628,13 @@ void ThinLTOCodeGenerator::promote(Module &TheModule, PreservedSymbols, Triple(TheModule.getTargetTriple())); // Compute "dead" symbols, we don't want to import/export these! - auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols); + computeDeadSymbols(Index, GUIDPreservedSymbols); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, - ExportLists, &DeadSymbols); + ExportLists); // Resolve LinkOnce/Weak symbols. StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; @@ -673,13 +673,13 @@ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule, PreservedSymbols, Triple(TheModule.getTargetTriple())); // Compute "dead" symbols, we don't want to import/export these! - auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols); + computeDeadSymbols(Index, GUIDPreservedSymbols); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, - ExportLists, &DeadSymbols); + ExportLists); auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); @@ -750,13 +750,13 @@ void ThinLTOCodeGenerator::internalize(Module &TheModule, Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Compute "dead" symbols, we don't want to import/export these! - auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols); + computeDeadSymbols(Index, GUIDPreservedSymbols); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, - ExportLists, &DeadSymbols); + ExportLists); auto &ExportList = ExportLists[ModuleIdentifier]; // Be friendly and don't nuke totally the module when the client didn't @@ -902,14 +902,14 @@ void ThinLTOCodeGenerator::run() { computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple); // Compute "dead" symbols, we don't want to import/export these! - auto DeadSymbols = computeDeadSymbols(*Index, GUIDPreservedSymbols); + computeDeadSymbols(*Index, GUIDPreservedSymbols); // Collect the import/export lists for all modules from the call-graph in the // combined index. StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists, - ExportLists, &DeadSymbols); + ExportLists); // We use a std::map here to be able to have a defined ordering when // producing a hash for the cache entry. diff --git a/lib/MC/WasmObjectWriter.cpp b/lib/MC/WasmObjectWriter.cpp index 8c3df36cfb48..9b2031f05043 100644 --- a/lib/MC/WasmObjectWriter.cpp +++ b/lib/MC/WasmObjectWriter.cpp @@ -40,6 +40,7 @@ using namespace llvm; #define DEBUG_TYPE "reloc-info" namespace { + // For patching purposes, we need to remember where each section starts, both // for patching up the section size field, and for patching up references to // locations within the section. @@ -50,6 +51,82 @@ struct SectionBookkeeping { uint64_t ContentsOffset; }; +// The signature of a wasm function, in a struct capable of being used as a +// DenseMap key. +struct WasmFunctionType { + // Support empty and tombstone instances, needed by DenseMap. + enum { Plain, Empty, Tombstone } State; + + // The return types of the function. + SmallVector<wasm::ValType, 1> Returns; + + // The parameter types of the function. + SmallVector<wasm::ValType, 4> Params; + + WasmFunctionType() : State(Plain) {} + + bool operator==(const WasmFunctionType &Other) const { + return State == Other.State && Returns == Other.Returns && + Params == Other.Params; + } +}; + +// Traits for using WasmFunctionType in a DenseMap. +struct WasmFunctionTypeDenseMapInfo { + static WasmFunctionType getEmptyKey() { + WasmFunctionType FuncTy; + FuncTy.State = WasmFunctionType::Empty; + return FuncTy; + } + static WasmFunctionType getTombstoneKey() { + WasmFunctionType FuncTy; + FuncTy.State = WasmFunctionType::Tombstone; + return FuncTy; + } + static unsigned getHashValue(const WasmFunctionType &FuncTy) { + uintptr_t Value = FuncTy.State; + for (wasm::ValType Ret : FuncTy.Returns) + Value += DenseMapInfo<int32_t>::getHashValue(int32_t(Ret)); + for (wasm::ValType Param : FuncTy.Params) + Value += DenseMapInfo<int32_t>::getHashValue(int32_t(Param)); + return Value; + } + static bool isEqual(const WasmFunctionType &LHS, + const WasmFunctionType &RHS) { + return LHS == RHS; + } +}; + +// A wasm import to be written into the import section. +struct WasmImport { + StringRef ModuleName; + StringRef FieldName; + unsigned Kind; + int32_t Type; +}; + +// A wasm function to be written into the function section. +struct WasmFunction { + int32_t Type; + const MCSymbolWasm *Sym; +}; + +// A wasm export to be written into the export section. +struct WasmExport { + StringRef FieldName; + unsigned Kind; + uint32_t Index; +}; + +// A wasm global to be written into the global section. +struct WasmGlobal { + wasm::ValType Type; + bool IsMutable; + bool HasImport; + uint64_t InitialValue; + uint32_t ImportIndex; +}; + class WasmObjectWriter : public MCObjectWriter { /// Helper struct for containing some precomputed information on symbols. struct WasmSymbolData { @@ -91,18 +168,10 @@ public: : MCObjectWriter(OS, /*IsLittleEndian=*/true), TargetObjectWriter(MOTW) {} private: - void reset() override { - MCObjectWriter::reset(); - } - ~WasmObjectWriter() override; void writeHeader(const MCAssembler &Asm); - void writeValueType(wasm::ValType Ty) { - encodeSLEB128(int32_t(Ty), getStream()); - } - void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout, const MCFragment *Fragment, const MCFixup &Fixup, MCValue Target, bool &IsPCRel, @@ -112,7 +181,37 @@ private: const MCAsmLayout &Layout) override; void writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override; + + void writeValueType(wasm::ValType Ty) { + encodeSLEB128(int32_t(Ty), getStream()); + } + + void writeTypeSection(const SmallVector<WasmFunctionType, 4> &FunctionTypes); + void writeImportSection(const SmallVector<WasmImport, 4> &Imports); + void writeFunctionSection(const SmallVector<WasmFunction, 4> &Functions); + void writeTableSection(const SmallVector<uint32_t, 4> &TableElems); + void writeMemorySection(const SmallVector<char, 0> &DataBytes); + void writeGlobalSection(const SmallVector<WasmGlobal, 4> &Globals); + void writeExportSection(const SmallVector<WasmExport, 4> &Exports); + void writeElemSection(const SmallVector<uint32_t, 4> &TableElems); + void writeCodeSection(const MCAssembler &Asm, const MCAsmLayout &Layout, + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices, + const SmallVector<WasmFunction, 4> &Functions); + uint64_t + writeDataSection(const SmallVector<char, 0> &DataBytes, + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices); + void writeNameSection(const SmallVector<WasmFunction, 4> &Functions, + const SmallVector<WasmImport, 4> &Imports, + uint32_t NumFuncImports); + void writeCodeRelocSection( + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices); + void writeDataRelocSection( + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices, + uint64_t DataSectionHeaderSize); + void writeLinkingMetaDataSection(bool HasStackPointer, + uint32_t StackPointerGlobal); }; + } // end anonymous namespace WasmObjectWriter::~WasmObjectWriter() {} @@ -278,86 +377,6 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm, DataRelocations.push_back(Rec); } -namespace { - -// The signature of a wasm function, in a struct capable of being used as a -// DenseMap key. -struct WasmFunctionType { - // Support empty and tombstone instances, needed by DenseMap. - enum { Plain, Empty, Tombstone } State; - - // The return types of the function. - SmallVector<wasm::ValType, 1> Returns; - - // The parameter types of the function. - SmallVector<wasm::ValType, 4> Params; - - WasmFunctionType() : State(Plain) {} - - bool operator==(const WasmFunctionType &Other) const { - return State == Other.State && Returns == Other.Returns && - Params == Other.Params; - } -}; - -// Traits for using WasmFunctionType in a DenseMap. -struct WasmFunctionTypeDenseMapInfo { - static WasmFunctionType getEmptyKey() { - WasmFunctionType FuncTy; - FuncTy.State = WasmFunctionType::Empty; - return FuncTy; - } - static WasmFunctionType getTombstoneKey() { - WasmFunctionType FuncTy; - FuncTy.State = WasmFunctionType::Tombstone; - return FuncTy; - } - static unsigned getHashValue(const WasmFunctionType &FuncTy) { - uintptr_t Value = FuncTy.State; - for (wasm::ValType Ret : FuncTy.Returns) - Value += DenseMapInfo<int32_t>::getHashValue(int32_t(Ret)); - for (wasm::ValType Param : FuncTy.Params) - Value += DenseMapInfo<int32_t>::getHashValue(int32_t(Param)); - return Value; - } - static bool isEqual(const WasmFunctionType &LHS, - const WasmFunctionType &RHS) { - return LHS == RHS; - } -}; - -// A wasm import to be written into the import section. -struct WasmImport { - StringRef ModuleName; - StringRef FieldName; - unsigned Kind; - int32_t Type; -}; - -// A wasm function to be written into the function section. -struct WasmFunction { - int32_t Type; - const MCSymbolWasm *Sym; -}; - -// A wasm export to be written into the export section. -struct WasmExport { - StringRef FieldName; - unsigned Kind; - uint32_t Index; -}; - -// A wasm global to be written into the global section. -struct WasmGlobal { - wasm::ValType Type; - bool IsMutable; - bool HasImport; - uint64_t InitialValue; - uint32_t ImportIndex; -}; - -} // end anonymous namespace - // Write X as an (unsigned) LEB value at offset Offset in Stream, padded // to allow patching. static void @@ -529,6 +548,367 @@ static void WriteTypeRelocations( } } +void WasmObjectWriter::writeTypeSection( + const SmallVector<WasmFunctionType, 4> &FunctionTypes) { + if (FunctionTypes.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_TYPE); + + encodeULEB128(FunctionTypes.size(), getStream()); + + for (const WasmFunctionType &FuncTy : FunctionTypes) { + encodeSLEB128(wasm::WASM_TYPE_FUNC, getStream()); + encodeULEB128(FuncTy.Params.size(), getStream()); + for (wasm::ValType Ty : FuncTy.Params) + writeValueType(Ty); + encodeULEB128(FuncTy.Returns.size(), getStream()); + for (wasm::ValType Ty : FuncTy.Returns) + writeValueType(Ty); + } + + endSection(Section); +} + +void WasmObjectWriter::writeImportSection( + const SmallVector<WasmImport, 4> &Imports) { + if (Imports.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_IMPORT); + + encodeULEB128(Imports.size(), getStream()); + for (const WasmImport &Import : Imports) { + StringRef ModuleName = Import.ModuleName; + encodeULEB128(ModuleName.size(), getStream()); + writeBytes(ModuleName); + + StringRef FieldName = Import.FieldName; + encodeULEB128(FieldName.size(), getStream()); + writeBytes(FieldName); + + encodeULEB128(Import.Kind, getStream()); + + switch (Import.Kind) { + case wasm::WASM_EXTERNAL_FUNCTION: + encodeULEB128(Import.Type, getStream()); + break; + case wasm::WASM_EXTERNAL_GLOBAL: + encodeSLEB128(int32_t(Import.Type), getStream()); + encodeULEB128(0, getStream()); // mutability + break; + default: + llvm_unreachable("unsupported import kind"); + } + } + + endSection(Section); +} + +void WasmObjectWriter::writeFunctionSection( + const SmallVector<WasmFunction, 4> &Functions) { + if (Functions.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_FUNCTION); + + encodeULEB128(Functions.size(), getStream()); + for (const WasmFunction &Func : Functions) + encodeULEB128(Func.Type, getStream()); + + endSection(Section); +} + +void WasmObjectWriter::writeTableSection( + const SmallVector<uint32_t, 4> &TableElems) { + // For now, always emit the table section, since indirect calls are not + // valid without it. In the future, we could perhaps be more clever and omit + // it if there are no indirect calls. + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_TABLE); + + // The number of tables, fixed to 1 for now. + encodeULEB128(1, getStream()); + + encodeSLEB128(wasm::WASM_TYPE_ANYFUNC, getStream()); + + encodeULEB128(0, getStream()); // flags + encodeULEB128(TableElems.size(), getStream()); // initial + + endSection(Section); +} + +void WasmObjectWriter::writeMemorySection( + const SmallVector<char, 0> &DataBytes) { + // For now, always emit the memory section, since loads and stores are not + // valid without it. In the future, we could perhaps be more clever and omit + // it if there are no loads or stores. + SectionBookkeeping Section; + uint32_t NumPages = + (DataBytes.size() + wasm::WasmPageSize - 1) / wasm::WasmPageSize; + + startSection(Section, wasm::WASM_SEC_MEMORY); + encodeULEB128(1, getStream()); // number of memory spaces + + encodeULEB128(0, getStream()); // flags + encodeULEB128(NumPages, getStream()); // initial + + endSection(Section); +} + +void WasmObjectWriter::writeGlobalSection( + const SmallVector<WasmGlobal, 4> &Globals) { + if (Globals.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_GLOBAL); + + encodeULEB128(Globals.size(), getStream()); + for (const WasmGlobal &Global : Globals) { + writeValueType(Global.Type); + write8(Global.IsMutable); + + if (Global.HasImport) { + assert(Global.InitialValue == 0); + write8(wasm::WASM_OPCODE_GET_GLOBAL); + encodeULEB128(Global.ImportIndex, getStream()); + } else { + assert(Global.ImportIndex == 0); + write8(wasm::WASM_OPCODE_I32_CONST); + encodeSLEB128(Global.InitialValue, getStream()); // offset + } + write8(wasm::WASM_OPCODE_END); + } + + endSection(Section); +} + +void WasmObjectWriter::writeExportSection( + const SmallVector<WasmExport, 4> &Exports) { + if (Exports.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_EXPORT); + + encodeULEB128(Exports.size(), getStream()); + for (const WasmExport &Export : Exports) { + encodeULEB128(Export.FieldName.size(), getStream()); + writeBytes(Export.FieldName); + + encodeSLEB128(Export.Kind, getStream()); + + encodeULEB128(Export.Index, getStream()); + } + + endSection(Section); +} + +void WasmObjectWriter::writeElemSection( + const SmallVector<uint32_t, 4> &TableElems) { + if (TableElems.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_ELEM); + + encodeULEB128(1, getStream()); // number of "segments" + encodeULEB128(0, getStream()); // the table index + + // init expr for starting offset + write8(wasm::WASM_OPCODE_I32_CONST); + encodeSLEB128(0, getStream()); + write8(wasm::WASM_OPCODE_END); + + encodeULEB128(TableElems.size(), getStream()); + for (uint32_t Elem : TableElems) + encodeULEB128(Elem, getStream()); + + endSection(Section); +} + +void WasmObjectWriter::writeCodeSection( + const MCAssembler &Asm, const MCAsmLayout &Layout, + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices, + const SmallVector<WasmFunction, 4> &Functions) { + if (Functions.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_CODE); + + encodeULEB128(Functions.size(), getStream()); + + for (const WasmFunction &Func : Functions) { + MCSectionWasm &FuncSection = + static_cast<MCSectionWasm &>(Func.Sym->getSection()); + + if (Func.Sym->isVariable()) + report_fatal_error("weak symbols not supported yet"); + + if (Func.Sym->getOffset() != 0) + report_fatal_error("function sections must contain one function each"); + + if (!Func.Sym->getSize()) + report_fatal_error("function symbols must have a size set with .size"); + + int64_t Size = 0; + if (!Func.Sym->getSize()->evaluateAsAbsolute(Size, Layout)) + report_fatal_error(".size expression must be evaluatable"); + + encodeULEB128(Size, getStream()); + + FuncSection.setSectionOffset(getStream().tell() - + Section.ContentsOffset); + + Asm.writeSectionData(&FuncSection, Layout); + } + + // Apply the type index fixups for call_indirect etc. instructions. + for (size_t i = 0, e = TypeIndexFixups.size(); i < e; ++i) { + uint32_t Type = TypeIndexFixupTypes[i]; + unsigned Padding = PaddingFor5ByteULEB128(Type); + + const WasmRelocationEntry &Fixup = TypeIndexFixups[i]; + assert(Fixup.Addend == 0); + assert(Fixup.Type == wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB); + uint64_t Offset = Fixup.Offset + + Fixup.FixupSection->getSectionOffset(); + + uint8_t Buffer[16]; + unsigned SizeLen = encodeULEB128(Type, Buffer, Padding); + assert(SizeLen == 5); + getStream().pwrite((char *)Buffer, SizeLen, + Section.ContentsOffset + Offset); + } + + // Apply fixups. + ApplyRelocations(CodeRelocations, getStream(), SymbolIndices, + Section.ContentsOffset); + + endSection(Section); +} + +uint64_t WasmObjectWriter::writeDataSection( + const SmallVector<char, 0> &DataBytes, + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices) { + if (DataBytes.empty()) + return 0; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_DATA); + + encodeULEB128(1, getStream()); // count + encodeULEB128(0, getStream()); // memory index + write8(wasm::WASM_OPCODE_I32_CONST); + encodeSLEB128(0, getStream()); // offset + write8(wasm::WASM_OPCODE_END); + encodeULEB128(DataBytes.size(), getStream()); // size + uint32_t HeaderSize = getStream().tell() - Section.ContentsOffset; + writeBytes(DataBytes); // data + + // Apply fixups. + ApplyRelocations(DataRelocations, getStream(), SymbolIndices, + Section.ContentsOffset + HeaderSize); + + endSection(Section); + return HeaderSize; +} + +void WasmObjectWriter::writeNameSection( + const SmallVector<WasmFunction, 4> &Functions, + const SmallVector<WasmImport, 4> &Imports, + unsigned NumFuncImports) { + uint32_t TotalFunctions = NumFuncImports + Functions.size(); + if (TotalFunctions == 0) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_CUSTOM, "name"); + SectionBookkeeping SubSection; + startSection(SubSection, wasm::WASM_NAMES_FUNCTION); + + encodeULEB128(TotalFunctions, getStream()); + uint32_t Index = 0; + for (const WasmImport &Import : Imports) { + if (Import.Kind == wasm::WASM_EXTERNAL_FUNCTION) { + encodeULEB128(Index, getStream()); + encodeULEB128(Import.FieldName.size(), getStream()); + writeBytes(Import.FieldName); + ++Index; + } + } + for (const WasmFunction &Func : Functions) { + encodeULEB128(Index, getStream()); + encodeULEB128(Func.Sym->getName().size(), getStream()); + writeBytes(Func.Sym->getName()); + ++Index; + } + + endSection(SubSection); + endSection(Section); +} + +void WasmObjectWriter::writeCodeRelocSection( + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices) { + // See: https://github.com/WebAssembly/tool-conventions/blob/master/Linking.md + // for descriptions of the reloc sections. + + if (CodeRelocations.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_CUSTOM, "reloc.CODE"); + + encodeULEB128(wasm::WASM_SEC_CODE, getStream()); + encodeULEB128(CodeRelocations.size() + TypeIndexFixups.size(), getStream()); + + WriteRelocations(CodeRelocations, getStream(), SymbolIndices, 0); + WriteTypeRelocations(TypeIndexFixups, TypeIndexFixupTypes, getStream()); + + endSection(Section); +} + +void WasmObjectWriter::writeDataRelocSection( + DenseMap<const MCSymbolWasm *, uint32_t> &SymbolIndices, + uint64_t DataSectionHeaderSize) { + // See: https://github.com/WebAssembly/tool-conventions/blob/master/Linking.md + // for descriptions of the reloc sections. + + if (DataRelocations.empty()) + return; + + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_CUSTOM, "reloc.DATA"); + + encodeULEB128(wasm::WASM_SEC_DATA, getStream()); + encodeULEB128(DataRelocations.size(), getStream()); + + WriteRelocations(DataRelocations, getStream(), SymbolIndices, + DataSectionHeaderSize); + + endSection(Section); +} + +void WasmObjectWriter::writeLinkingMetaDataSection( + bool HasStackPointer, uint32_t StackPointerGlobal) { + if (!HasStackPointer) + return; + SectionBookkeeping Section; + startSection(Section, wasm::WASM_SEC_CUSTOM, "linking"); + + encodeULEB128(1, getStream()); // count + + encodeULEB128(wasm::WASM_STACK_POINTER, getStream()); // type + encodeULEB128(StackPointerGlobal, getStream()); // id + + endSection(Section); +} + void WasmObjectWriter::writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) { MCContext &Ctx = Asm.getContext(); @@ -730,16 +1110,21 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, if (IsAddressTaken.count(&WS)) TableElems.push_back(Index); } else { - if (WS.getOffset() != 0) - report_fatal_error("data sections must contain one variable each"); - if (!WS.getSize()) - report_fatal_error("data symbols must have a size set with .size"); - - int64_t Size = 0; - if (!WS.getSize()->evaluateAsAbsolute(Size, Layout)) - report_fatal_error(".size expression must be evaluatable"); + if (WS.isTemporary() && !WS.getSize()) + continue; if (WS.isDefined(false)) { + if (WS.getOffset() != 0) + report_fatal_error("data sections must contain one variable each: " + + WS.getName()); + if (!WS.getSize()) + report_fatal_error("data symbols must have a size set with .size: " + + WS.getName()); + + int64_t Size = 0; + if (!WS.getSize()->evaluateAsAbsolute(Size, Layout)) + report_fatal_error(".size expression must be evaluatable"); + MCSectionWasm &DataSection = static_cast<MCSectionWasm &>(WS.getSection()); @@ -827,322 +1212,23 @@ void WasmObjectWriter::writeObject(MCAssembler &Asm, // Write out the Wasm header. writeHeader(Asm); - SectionBookkeeping Section; - - // === Type Section ========================================================= - if (!FunctionTypes.empty()) { - startSection(Section, wasm::WASM_SEC_TYPE); - - encodeULEB128(FunctionTypes.size(), getStream()); - - for (WasmFunctionType &FuncTy : FunctionTypes) { - encodeSLEB128(wasm::WASM_TYPE_FUNC, getStream()); - encodeULEB128(FuncTy.Params.size(), getStream()); - for (wasm::ValType Ty : FuncTy.Params) - writeValueType(Ty); - encodeULEB128(FuncTy.Returns.size(), getStream()); - for (wasm::ValType Ty : FuncTy.Returns) - writeValueType(Ty); - } - - endSection(Section); - } - - // === Import Section ======================================================== - if (!Imports.empty()) { - startSection(Section, wasm::WASM_SEC_IMPORT); - - encodeULEB128(Imports.size(), getStream()); - for (const WasmImport &Import : Imports) { - StringRef ModuleName = Import.ModuleName; - encodeULEB128(ModuleName.size(), getStream()); - writeBytes(ModuleName); - - StringRef FieldName = Import.FieldName; - encodeULEB128(FieldName.size(), getStream()); - writeBytes(FieldName); - - encodeULEB128(Import.Kind, getStream()); - - switch (Import.Kind) { - case wasm::WASM_EXTERNAL_FUNCTION: - encodeULEB128(Import.Type, getStream()); - break; - case wasm::WASM_EXTERNAL_GLOBAL: - encodeSLEB128(int32_t(Import.Type), getStream()); - encodeULEB128(0, getStream()); // mutability - break; - default: - llvm_unreachable("unsupported import kind"); - } - } - - endSection(Section); - } - - // === Function Section ====================================================== - if (!Functions.empty()) { - startSection(Section, wasm::WASM_SEC_FUNCTION); - - encodeULEB128(Functions.size(), getStream()); - for (const WasmFunction &Func : Functions) - encodeULEB128(Func.Type, getStream()); - - endSection(Section); - } - - // === Table Section ========================================================= - // For now, always emit the table section, since indirect calls are not - // valid without it. In the future, we could perhaps be more clever and omit - // it if there are no indirect calls. - startSection(Section, wasm::WASM_SEC_TABLE); - - // The number of tables, fixed to 1 for now. - encodeULEB128(1, getStream()); - - encodeSLEB128(wasm::WASM_TYPE_ANYFUNC, getStream()); - - encodeULEB128(0, getStream()); // flags - encodeULEB128(TableElems.size(), getStream()); // initial - - endSection(Section); - - // === Memory Section ======================================================== - // For now, always emit the memory section, since loads and stores are not - // valid without it. In the future, we could perhaps be more clever and omit - // it if there are no loads or stores. - uint32_t NumPages = - (DataBytes.size() + wasm::WasmPageSize - 1) / wasm::WasmPageSize; - - startSection(Section, wasm::WASM_SEC_MEMORY); - encodeULEB128(1, getStream()); // number of memory spaces - - encodeULEB128(0, getStream()); // flags - encodeULEB128(NumPages, getStream()); // initial - - endSection(Section); - - // === Global Section ======================================================== - if (!Globals.empty()) { - startSection(Section, wasm::WASM_SEC_GLOBAL); - - encodeULEB128(Globals.size(), getStream()); - for (const WasmGlobal &Global : Globals) { - writeValueType(Global.Type); - write8(Global.IsMutable); - - if (Global.HasImport) { - assert(Global.InitialValue == 0); - write8(wasm::WASM_OPCODE_GET_GLOBAL); - encodeULEB128(Global.ImportIndex, getStream()); - } else { - assert(Global.ImportIndex == 0); - write8(wasm::WASM_OPCODE_I32_CONST); - encodeSLEB128(Global.InitialValue, getStream()); // offset - } - write8(wasm::WASM_OPCODE_END); - } - - endSection(Section); - } - - // === Export Section ======================================================== - if (!Exports.empty()) { - startSection(Section, wasm::WASM_SEC_EXPORT); - - encodeULEB128(Exports.size(), getStream()); - for (const WasmExport &Export : Exports) { - encodeULEB128(Export.FieldName.size(), getStream()); - writeBytes(Export.FieldName); - - encodeSLEB128(Export.Kind, getStream()); - - encodeULEB128(Export.Index, getStream()); - } - - endSection(Section); - } - -#if 0 // TODO: Start Section - if (HaveStartFunction) { - // === Start Section ========================================================= - startSection(Section, wasm::WASM_SEC_START); - - encodeSLEB128(StartFunction, getStream()); - - endSection(Section); - } -#endif - - // === Elem Section ========================================================== - if (!TableElems.empty()) { - startSection(Section, wasm::WASM_SEC_ELEM); - - encodeULEB128(1, getStream()); // number of "segments" - encodeULEB128(0, getStream()); // the table index - - // init expr for starting offset - write8(wasm::WASM_OPCODE_I32_CONST); - encodeSLEB128(0, getStream()); - write8(wasm::WASM_OPCODE_END); - - encodeULEB128(TableElems.size(), getStream()); - for (uint32_t Elem : TableElems) - encodeULEB128(Elem, getStream()); - - endSection(Section); - } - - // === Code Section ========================================================== - if (!Functions.empty()) { - startSection(Section, wasm::WASM_SEC_CODE); - - encodeULEB128(Functions.size(), getStream()); - - for (const WasmFunction &Func : Functions) { - MCSectionWasm &FuncSection = - static_cast<MCSectionWasm &>(Func.Sym->getSection()); - - if (Func.Sym->isVariable()) - report_fatal_error("weak symbols not supported yet"); - - if (Func.Sym->getOffset() != 0) - report_fatal_error("function sections must contain one function each"); - - if (!Func.Sym->getSize()) - report_fatal_error("function symbols must have a size set with .size"); - - int64_t Size = 0; - if (!Func.Sym->getSize()->evaluateAsAbsolute(Size, Layout)) - report_fatal_error(".size expression must be evaluatable"); - - encodeULEB128(Size, getStream()); - - FuncSection.setSectionOffset(getStream().tell() - - Section.ContentsOffset); - - Asm.writeSectionData(&FuncSection, Layout); - } - - // Apply the type index fixups for call_indirect etc. instructions. - for (size_t i = 0, e = TypeIndexFixups.size(); i < e; ++i) { - uint32_t Type = TypeIndexFixupTypes[i]; - unsigned Padding = PaddingFor5ByteULEB128(Type); - - const WasmRelocationEntry &Fixup = TypeIndexFixups[i]; - assert(Fixup.Addend == 0); - assert(Fixup.Type == wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB); - uint64_t Offset = Fixup.Offset + - Fixup.FixupSection->getSectionOffset(); - - uint8_t Buffer[16]; - unsigned SizeLen = encodeULEB128(Type, Buffer, Padding); - assert(SizeLen == 5); - getStream().pwrite((char *)Buffer, SizeLen, - Section.ContentsOffset + Offset); - } - - // Apply fixups. - ApplyRelocations(CodeRelocations, getStream(), SymbolIndices, - Section.ContentsOffset); - - endSection(Section); - } - - // === Data Section ========================================================== - uint32_t DataSectionHeaderSize = 0; - if (!DataBytes.empty()) { - startSection(Section, wasm::WASM_SEC_DATA); - - encodeULEB128(1, getStream()); // count - encodeULEB128(0, getStream()); // memory index - write8(wasm::WASM_OPCODE_I32_CONST); - encodeSLEB128(0, getStream()); // offset - write8(wasm::WASM_OPCODE_END); - encodeULEB128(DataBytes.size(), getStream()); // size - DataSectionHeaderSize = getStream().tell() - Section.ContentsOffset; - writeBytes(DataBytes); // data - - // Apply fixups. - ApplyRelocations(DataRelocations, getStream(), SymbolIndices, - Section.ContentsOffset + DataSectionHeaderSize); - - endSection(Section); - } - - // === Name Section ========================================================== - uint32_t TotalFunctions = NumFuncImports + Functions.size(); - if (TotalFunctions != 0) { - startSection(Section, wasm::WASM_SEC_CUSTOM, "name"); - SectionBookkeeping SubSection; - startSection(SubSection, wasm::WASM_NAMES_FUNCTION); - - encodeULEB128(TotalFunctions, getStream()); - uint32_t Index = 0; - for (const WasmImport &Import : Imports) { - if (Import.Kind == wasm::WASM_EXTERNAL_FUNCTION) { - encodeULEB128(Index, getStream()); - encodeULEB128(Import.FieldName.size(), getStream()); - writeBytes(Import.FieldName); - ++Index; - } - } - for (const WasmFunction &Func : Functions) { - encodeULEB128(Index, getStream()); - encodeULEB128(Func.Sym->getName().size(), getStream()); - writeBytes(Func.Sym->getName()); - ++Index; - } - - endSection(SubSection); - endSection(Section); - } - - // See: https://github.com/WebAssembly/tool-conventions/blob/master/Linking.md - // for descriptions of the reloc sections. - - // === Code Reloc Section ==================================================== - if (!CodeRelocations.empty()) { - startSection(Section, wasm::WASM_SEC_CUSTOM, "reloc.CODE"); - - encodeULEB128(wasm::WASM_SEC_CODE, getStream()); - - encodeULEB128(CodeRelocations.size() + TypeIndexFixups.size(), getStream()); - - WriteRelocations(CodeRelocations, getStream(), SymbolIndices, 0); - WriteTypeRelocations(TypeIndexFixups, TypeIndexFixupTypes, getStream()); - - endSection(Section); - } - - // === Data Reloc Section ==================================================== - if (!DataRelocations.empty()) { - startSection(Section, wasm::WASM_SEC_CUSTOM, "reloc.DATA"); - - encodeULEB128(wasm::WASM_SEC_DATA, getStream()); - - encodeULEB128(DataRelocations.size(), getStream()); - - WriteRelocations(DataRelocations, getStream(), SymbolIndices, - DataSectionHeaderSize); - - endSection(Section); - } - - // === Linking Metadata Section ============================================== - if (HasStackPointer) { - startSection(Section, wasm::WASM_SEC_CUSTOM, "linking"); - - encodeULEB128(1, getStream()); // count - - encodeULEB128(wasm::WASM_STACK_POINTER, getStream()); // type - encodeULEB128(StackPointerGlobal, getStream()); // id - - endSection(Section); - } + writeTypeSection(FunctionTypes); + writeImportSection(Imports); + writeFunctionSection(Functions); + writeTableSection(TableElems); + writeMemorySection(DataBytes); + writeGlobalSection(Globals); + writeExportSection(Exports); + // TODO: Start Section + writeElemSection(TableElems); + writeCodeSection(Asm, Layout, SymbolIndices, Functions); + uint64_t DataSectionHeaderSize = writeDataSection(DataBytes, SymbolIndices); + writeNameSection(Functions, Imports, NumFuncImports); + writeCodeRelocSection(SymbolIndices); + writeDataRelocSection(SymbolIndices, DataSectionHeaderSize); + writeLinkingMetaDataSection(HasStackPointer, StackPointerGlobal); // TODO: Translate the .comment section to the output. - // TODO: Translate debug sections to the output. } diff --git a/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp b/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp index f652ff57f30d..21d29835624e 100644 --- a/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp +++ b/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp @@ -17,6 +17,11 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h" #include "llvm/DebugInfo/CodeView/EnumTables.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" @@ -36,16 +41,80 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(InlineeInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(StringRef) LLVM_YAML_DECLARE_SCALAR_TRAITS(HexFormattedString, false) +LLVM_YAML_DECLARE_ENUM_TRAITS(DebugSubsectionKind) LLVM_YAML_DECLARE_ENUM_TRAITS(FileChecksumKind) LLVM_YAML_DECLARE_BITSET_TRAITS(LineFlags) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceLineEntry) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceColumnEntry) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceFileChecksumEntry) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceLineInfo) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::SourceLineBlock) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::InlineeInfo) -LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::InlineeSite) +LLVM_YAML_DECLARE_MAPPING_TRAITS(SourceLineEntry) +LLVM_YAML_DECLARE_MAPPING_TRAITS(SourceColumnEntry) +LLVM_YAML_DECLARE_MAPPING_TRAITS(SourceFileChecksumEntry) +LLVM_YAML_DECLARE_MAPPING_TRAITS(SourceLineBlock) +LLVM_YAML_DECLARE_MAPPING_TRAITS(InlineeSite) + +namespace llvm { +namespace CodeViewYAML { +namespace detail { +struct YAMLSubsectionBase { + explicit YAMLSubsectionBase(DebugSubsectionKind Kind) : Kind(Kind) {} + DebugSubsectionKind Kind; + virtual ~YAMLSubsectionBase() {} + + virtual void map(IO &IO) = 0; + virtual std::unique_ptr<DebugSubsection> + toCodeViewSubsection(DebugStringTableSubsection *UseStrings, + DebugChecksumsSubsection *UseChecksums) const = 0; +}; +} +} +} + +namespace { +struct YAMLChecksumsSubsection : public YAMLSubsectionBase { + YAMLChecksumsSubsection() + : YAMLSubsectionBase(DebugSubsectionKind::FileChecksums) {} + + void map(IO &IO) override; + std::unique_ptr<DebugSubsection> + toCodeViewSubsection(DebugStringTableSubsection *Strings, + DebugChecksumsSubsection *Checksums) const override; + static Expected<std::shared_ptr<YAMLChecksumsSubsection>> + fromCodeViewSubsection(const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &FC); + + std::vector<SourceFileChecksumEntry> Checksums; +}; + +struct YAMLLinesSubsection : public YAMLSubsectionBase { + YAMLLinesSubsection() : YAMLSubsectionBase(DebugSubsectionKind::Lines) {} + + void map(IO &IO) override; + std::unique_ptr<DebugSubsection> + toCodeViewSubsection(DebugStringTableSubsection *Strings, + DebugChecksumsSubsection *Checksums) const override; + static Expected<std::shared_ptr<YAMLLinesSubsection>> + fromCodeViewSubsection(const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, + const DebugLinesSubsectionRef &Lines); + + SourceLineInfo Lines; +}; + +struct YAMLInlineeLinesSubsection : public YAMLSubsectionBase { + YAMLInlineeLinesSubsection() + : YAMLSubsectionBase(DebugSubsectionKind::InlineeLines) {} + + void map(IO &IO) override; + std::unique_ptr<DebugSubsection> + toCodeViewSubsection(DebugStringTableSubsection *Strings, + DebugChecksumsSubsection *Checksums) const override; + static Expected<std::shared_ptr<YAMLInlineeLinesSubsection>> + fromCodeViewSubsection(const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, + const DebugInlineeLinesSubsectionRef &Lines); + + InlineeInfo InlineeLines; +}; +} void ScalarBitSetTraits<LineFlags>::bitset(IO &io, LineFlags &Flags) { io.bitSetCase(Flags, "HasColumnInfo", LF_HaveColumns); @@ -99,21 +168,6 @@ void MappingTraits<SourceFileChecksumEntry>::mapping( IO.mapRequired("Checksum", Obj.ChecksumBytes); } -void MappingTraits<SourceLineInfo>::mapping(IO &IO, SourceLineInfo &Obj) { - IO.mapRequired("CodeSize", Obj.CodeSize); - - IO.mapRequired("Flags", Obj.Flags); - IO.mapRequired("RelocOffset", Obj.RelocOffset); - IO.mapRequired("RelocSegment", Obj.RelocSegment); - IO.mapRequired("Blocks", Obj.Blocks); -} - -void MappingTraits<SourceFileInfo>::mapping(IO &IO, SourceFileInfo &Obj) { - IO.mapOptional("Checksums", Obj.FileChecksums); - IO.mapOptional("Lines", Obj.LineFragments); - IO.mapOptional("InlineeLines", Obj.Inlinees); -} - void MappingTraits<InlineeSite>::mapping(IO &IO, InlineeSite &Obj) { IO.mapRequired("FileName", Obj.FileName); IO.mapRequired("LineNum", Obj.SourceLineNum); @@ -121,7 +175,310 @@ void MappingTraits<InlineeSite>::mapping(IO &IO, InlineeSite &Obj) { IO.mapOptional("ExtraFiles", Obj.ExtraFiles); } -void MappingTraits<InlineeInfo>::mapping(IO &IO, InlineeInfo &Obj) { - IO.mapRequired("HasExtraFiles", Obj.HasExtraFiles); - IO.mapRequired("Sites", Obj.Sites); +void YAMLChecksumsSubsection::map(IO &IO) { + IO.mapTag("!FileChecksums", true); + IO.mapRequired("Checksums", Checksums); +} + +void YAMLLinesSubsection::map(IO &IO) { + IO.mapTag("!Lines", true); + IO.mapRequired("CodeSize", Lines.CodeSize); + + IO.mapRequired("Flags", Lines.Flags); + IO.mapRequired("RelocOffset", Lines.RelocOffset); + IO.mapRequired("RelocSegment", Lines.RelocSegment); + IO.mapRequired("Blocks", Lines.Blocks); +} + +void YAMLInlineeLinesSubsection::map(IO &IO) { + IO.mapTag("!InlineeLines", true); + IO.mapRequired("HasExtraFiles", InlineeLines.HasExtraFiles); + IO.mapRequired("Sites", InlineeLines.Sites); +} + +void MappingTraits<YAMLDebugSubsection>::mapping( + IO &IO, YAMLDebugSubsection &Subsection) { + if (!IO.outputting()) { + if (IO.mapTag("!FileChecksums")) { + auto SS = std::make_shared<YAMLChecksumsSubsection>(); + Subsection.Subsection = SS; + } else if (IO.mapTag("!Lines")) { + Subsection.Subsection = std::make_shared<YAMLLinesSubsection>(); + } else if (IO.mapTag("!InlineeLines")) { + Subsection.Subsection = std::make_shared<YAMLInlineeLinesSubsection>(); + } else { + llvm_unreachable("Unexpected subsection tag!"); + } + } + Subsection.Subsection->map(IO); +} + +static Expected<const YAMLChecksumsSubsection &> +findChecksums(ArrayRef<YAMLDebugSubsection> Subsections) { + for (const auto &SS : Subsections) { + if (SS.Subsection->Kind == DebugSubsectionKind::FileChecksums) { + return static_cast<const YAMLChecksumsSubsection &>(*SS.Subsection); + } + } + return make_error<CodeViewError>(cv_error_code::no_records); +} + +std::unique_ptr<DebugSubsection> YAMLChecksumsSubsection::toCodeViewSubsection( + DebugStringTableSubsection *UseStrings, + DebugChecksumsSubsection *UseChecksums) const { + assert(UseStrings && !UseChecksums); + auto Result = llvm::make_unique<DebugChecksumsSubsection>(*UseStrings); + for (const auto &CS : Checksums) { + Result->addChecksum(CS.FileName, CS.Kind, CS.ChecksumBytes.Bytes); + } + return std::move(Result); +} + +std::unique_ptr<DebugSubsection> YAMLLinesSubsection::toCodeViewSubsection( + DebugStringTableSubsection *UseStrings, + DebugChecksumsSubsection *UseChecksums) const { + assert(UseStrings && UseChecksums); + auto Result = + llvm::make_unique<DebugLinesSubsection>(*UseChecksums, *UseStrings); + Result->setCodeSize(Lines.CodeSize); + Result->setRelocationAddress(Lines.RelocSegment, Lines.RelocOffset); + Result->setFlags(Lines.Flags); + for (const auto &LC : Lines.Blocks) { + Result->createBlock(LC.FileName); + if (Result->hasColumnInfo()) { + for (const auto &Item : zip(LC.Lines, LC.Columns)) { + auto &L = std::get<0>(Item); + auto &C = std::get<1>(Item); + uint32_t LE = L.LineStart + L.EndDelta; + Result->addLineAndColumnInfo(L.Offset, + LineInfo(L.LineStart, LE, L.IsStatement), + C.StartColumn, C.EndColumn); + } + } else { + for (const auto &L : LC.Lines) { + uint32_t LE = L.LineStart + L.EndDelta; + Result->addLineInfo(L.Offset, LineInfo(L.LineStart, LE, L.IsStatement)); + } + } + } + return llvm::cast<DebugSubsection>(std::move(Result)); +} + +std::unique_ptr<DebugSubsection> +YAMLInlineeLinesSubsection::toCodeViewSubsection( + DebugStringTableSubsection *UseStrings, + DebugChecksumsSubsection *UseChecksums) const { + assert(UseChecksums); + auto Result = llvm::make_unique<DebugInlineeLinesSubsection>( + *UseChecksums, InlineeLines.HasExtraFiles); + + for (const auto &Site : InlineeLines.Sites) { + Result->addInlineSite(TypeIndex(Site.Inlinee), Site.FileName, + Site.SourceLineNum); + if (!InlineeLines.HasExtraFiles) + continue; + + for (auto EF : Site.ExtraFiles) { + Result->addExtraFile(EF); + } + } + return llvm::cast<DebugSubsection>(std::move(Result)); +} + +static Expected<SourceFileChecksumEntry> +convertOneChecksum(const DebugStringTableSubsectionRef &Strings, + const FileChecksumEntry &CS) { + auto ExpectedString = Strings.getString(CS.FileNameOffset); + if (!ExpectedString) + return ExpectedString.takeError(); + + SourceFileChecksumEntry Result; + Result.ChecksumBytes.Bytes = CS.Checksum; + Result.Kind = CS.Kind; + Result.FileName = *ExpectedString; + return Result; +} + +static Expected<StringRef> +getFileName(const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, uint32_t FileID) { + auto Iter = Checksums.getArray().at(FileID); + if (Iter == Checksums.getArray().end()) + return make_error<CodeViewError>(cv_error_code::no_records); + uint32_t Offset = Iter->FileNameOffset; + return Strings.getString(Offset); +} + +Expected<std::shared_ptr<YAMLChecksumsSubsection>> +YAMLChecksumsSubsection::fromCodeViewSubsection( + const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &FC) { + auto Result = std::make_shared<YAMLChecksumsSubsection>(); + + for (const auto &CS : FC) { + auto ConvertedCS = convertOneChecksum(Strings, CS); + if (!ConvertedCS) + return ConvertedCS.takeError(); + Result->Checksums.push_back(*ConvertedCS); + } + return Result; +} + +Expected<std::shared_ptr<YAMLLinesSubsection>> +YAMLLinesSubsection::fromCodeViewSubsection( + const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, + const DebugLinesSubsectionRef &Lines) { + auto Result = std::make_shared<YAMLLinesSubsection>(); + Result->Lines.CodeSize = Lines.header()->CodeSize; + Result->Lines.RelocOffset = Lines.header()->RelocOffset; + Result->Lines.RelocSegment = Lines.header()->RelocSegment; + Result->Lines.Flags = static_cast<LineFlags>(uint16_t(Lines.header()->Flags)); + for (const auto &L : Lines) { + SourceLineBlock Block; + auto EF = getFileName(Strings, Checksums, L.NameIndex); + if (!EF) + return EF.takeError(); + Block.FileName = *EF; + if (Lines.hasColumnInfo()) { + for (const auto &C : L.Columns) { + SourceColumnEntry SCE; + SCE.EndColumn = C.EndColumn; + SCE.StartColumn = C.StartColumn; + Block.Columns.push_back(SCE); + } + } + for (const auto &LN : L.LineNumbers) { + SourceLineEntry SLE; + LineInfo LI(LN.Flags); + SLE.Offset = LN.Offset; + SLE.LineStart = LI.getStartLine(); + SLE.EndDelta = LI.getLineDelta(); + SLE.IsStatement = LI.isStatement(); + Block.Lines.push_back(SLE); + } + Result->Lines.Blocks.push_back(Block); + } + return Result; +} + +Expected<std::shared_ptr<YAMLInlineeLinesSubsection>> +YAMLInlineeLinesSubsection::fromCodeViewSubsection( + const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, + const DebugInlineeLinesSubsectionRef &Lines) { + auto Result = std::make_shared<YAMLInlineeLinesSubsection>(); + + Result->InlineeLines.HasExtraFiles = Lines.hasExtraFiles(); + for (const auto &IL : Lines) { + InlineeSite Site; + auto ExpF = getFileName(Strings, Checksums, IL.Header->FileID); + if (!ExpF) + return ExpF.takeError(); + Site.FileName = *ExpF; + Site.Inlinee = IL.Header->Inlinee.getIndex(); + Site.SourceLineNum = IL.Header->SourceLineNum; + if (Lines.hasExtraFiles()) { + for (const auto EF : IL.ExtraFiles) { + auto ExpF2 = getFileName(Strings, Checksums, EF); + if (!ExpF2) + return ExpF2.takeError(); + Site.ExtraFiles.push_back(*ExpF2); + } + } + Result->InlineeLines.Sites.push_back(Site); + } + return Result; +} + +Expected<std::vector<std::unique_ptr<DebugSubsection>>> +llvm::CodeViewYAML::convertSubsectionList( + ArrayRef<YAMLDebugSubsection> Subsections, + DebugStringTableSubsection &Strings) { + std::vector<std::unique_ptr<DebugSubsection>> Result; + if (Subsections.empty()) + return std::move(Result); + + auto Checksums = findChecksums(Subsections); + if (!Checksums) + return Checksums.takeError(); + auto ChecksumsBase = Checksums->toCodeViewSubsection(&Strings, nullptr); + DebugChecksumsSubsection &CS = + llvm::cast<DebugChecksumsSubsection>(*ChecksumsBase); + for (const auto &SS : Subsections) { + // We've already converted the checksums subsection, don't do it + // twice. + std::unique_ptr<DebugSubsection> CVS; + if (SS.Subsection->Kind == DebugSubsectionKind::FileChecksums) + CVS = std::move(ChecksumsBase); + else + CVS = SS.Subsection->toCodeViewSubsection(&Strings, &CS); + Result.push_back(std::move(CVS)); + } + return std::move(Result); +} + +namespace { +struct SubsectionConversionVisitor : public DebugSubsectionVisitor { + explicit SubsectionConversionVisitor( + const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums) + : Strings(Strings), Checksums(Checksums) {} + + Error visitUnknown(DebugUnknownSubsectionRef &Unknown) override; + Error visitLines(DebugLinesSubsectionRef &Lines) override; + Error visitFileChecksums(DebugChecksumsSubsectionRef &Checksums) override; + Error visitInlineeLines(DebugInlineeLinesSubsectionRef &Inlinees) override; + + YAMLDebugSubsection Subsection; + +private: + const DebugStringTableSubsectionRef &Strings; + const DebugChecksumsSubsectionRef &Checksums; +}; + +Error SubsectionConversionVisitor::visitUnknown( + DebugUnknownSubsectionRef &Unknown) { + return make_error<CodeViewError>(cv_error_code::operation_unsupported); +} + +Error SubsectionConversionVisitor::visitLines(DebugLinesSubsectionRef &Lines) { + auto Result = + YAMLLinesSubsection::fromCodeViewSubsection(Strings, Checksums, Lines); + if (!Result) + return Result.takeError(); + Subsection.Subsection = *Result; + return Error::success(); +} + +Error SubsectionConversionVisitor::visitFileChecksums( + DebugChecksumsSubsectionRef &Checksums) { + auto Result = + YAMLChecksumsSubsection::fromCodeViewSubsection(Strings, Checksums); + if (!Result) + return Result.takeError(); + Subsection.Subsection = *Result; + return Error::success(); +} + +Error SubsectionConversionVisitor::visitInlineeLines( + DebugInlineeLinesSubsectionRef &Inlinees) { + auto Result = YAMLInlineeLinesSubsection::fromCodeViewSubsection( + Strings, Checksums, Inlinees); + if (!Result) + return Result.takeError(); + Subsection.Subsection = *Result; + return Error::success(); +} +} + +Expected<YAMLDebugSubsection> YAMLDebugSubsection::fromCodeViewSubection( + const DebugStringTableSubsectionRef &Strings, + const DebugChecksumsSubsectionRef &Checksums, + const DebugSubsectionRecord &SS) { + SubsectionConversionVisitor V(Strings, Checksums); + if (auto EC = visitDebugSubsection(SS, V)) + return std::move(EC); + + return V.Subsection; } diff --git a/lib/ObjectYAML/CodeViewYAMLSymbols.cpp b/lib/ObjectYAML/CodeViewYAMLSymbols.cpp index 6e8bb5c7372c..bd97af3a9323 100644 --- a/lib/ObjectYAML/CodeViewYAMLSymbols.cpp +++ b/lib/ObjectYAML/CodeViewYAMLSymbols.cpp @@ -148,7 +148,8 @@ struct SymbolRecordBase { virtual ~SymbolRecordBase() {} virtual void map(yaml::IO &io) = 0; virtual codeview::CVSymbol - toCodeViewSymbol(BumpPtrAllocator &Allocator) const = 0; + toCodeViewSymbol(BumpPtrAllocator &Allocator, + CodeViewContainer Container) const = 0; virtual Error fromCodeViewSymbol(codeview::CVSymbol Type) = 0; }; @@ -159,8 +160,9 @@ template <typename T> struct SymbolRecordImpl : public SymbolRecordBase { void map(yaml::IO &io) override; codeview::CVSymbol - toCodeViewSymbol(BumpPtrAllocator &Allocator) const override { - return SymbolSerializer::writeOneSymbol(Symbol, Allocator); + toCodeViewSymbol(BumpPtrAllocator &Allocator, + CodeViewContainer Container) const override { + return SymbolSerializer::writeOneSymbol(Symbol, Allocator, Container); } Error fromCodeViewSymbol(codeview::CVSymbol CVS) override { return SymbolDeserializer::deserializeAs<T>(CVS, Symbol); @@ -429,8 +431,8 @@ template <> void SymbolRecordImpl<ThreadLocalDataSym>::map(IO &IO) { } CVSymbol CodeViewYAML::SymbolRecord::toCodeViewSymbol( - BumpPtrAllocator &Allocator) const { - return Symbol->toCodeViewSymbol(Allocator); + BumpPtrAllocator &Allocator, CodeViewContainer Container) const { + return Symbol->toCodeViewSymbol(Allocator, Container); } namespace llvm { diff --git a/lib/Passes/PassBuilder.cpp b/lib/Passes/PassBuilder.cpp index eb81e58b9b0e..17c60348633c 100644 --- a/lib/Passes/PassBuilder.cpp +++ b/lib/Passes/PassBuilder.cpp @@ -310,6 +310,10 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, // Catch trivial redundancies FPM.addPass(EarlyCSEPass()); + // Hoisting of scalars and load expressions. + if (EnableGVNHoist) + FPM.addPass(GVNHoistPass()); + // Speculative execution if the target has divergent branches; otherwise nop. FPM.addPass(SpeculativeExecutionPass()); @@ -473,8 +477,6 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, EarlyFPM.addPass(SROA()); EarlyFPM.addPass(EarlyCSEPass()); EarlyFPM.addPass(LowerExpectIntrinsicPass()); - if (EnableGVNHoist) - EarlyFPM.addPass(GVNHoistPass()); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); // Interprocedural constant propagation now that basic cleanup has occured diff --git a/lib/Support/Triple.cpp b/lib/Support/Triple.cpp index 318e21da999d..f7b7ad89e959 100644 --- a/lib/Support/Triple.cpp +++ b/lib/Support/Triple.cpp @@ -649,12 +649,10 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) { case Triple::tce: case Triple::tcele: case Triple::thumbeb: - case Triple::xcore: - return Triple::ELF; - case Triple::wasm32: case Triple::wasm64: - return Triple::Wasm; + case Triple::xcore: + return Triple::ELF; case Triple::ppc: case Triple::ppc64: diff --git a/lib/Target/AArch64/AArch64PBQPRegAlloc.h b/lib/Target/AArch64/AArch64PBQPRegAlloc.h index 4f656f94ea12..b99c1d1d6b3e 100644 --- a/lib/Target/AArch64/AArch64PBQPRegAlloc.h +++ b/lib/Target/AArch64/AArch64PBQPRegAlloc.h @@ -1,4 +1,4 @@ -//===-- AArch64PBQPRegAlloc.h - AArch64 specific PBQP constraints -------===// +//==- AArch64PBQPRegAlloc.h - AArch64 specific PBQP constraints --*- C++ -*-==// // // The LLVM Compiler Infrastructure // @@ -15,6 +15,8 @@ namespace llvm { +class TargetRegisterInfo; + /// Add the accumulator chaining constraint to a PBQP graph class A57ChainingConstraint : public PBQPRAConstraint { public: @@ -33,6 +35,7 @@ private: // Add constraints between existing chains void addInterChainConstraint(PBQPRAGraph &G, unsigned Rd, unsigned Ra); }; -} + +} // end namespace llvm #endif // LLVM_LIB_TARGET_AARCH64_AARCH64PBQPREGALOC_H diff --git a/lib/Target/AArch64/AArch64SchedFalkorDetails.td b/lib/Target/AArch64/AArch64SchedFalkorDetails.td index d098cf7a5a37..7402bcf1346c 100644 --- a/lib/Target/AArch64/AArch64SchedFalkorDetails.td +++ b/lib/Target/AArch64/AArch64SchedFalkorDetails.td @@ -56,12 +56,14 @@ def FalkorWr_1Z_0cyc : SchedWriteRes<[FalkorUnitZ]> { let Latency = 0; } def FalkorWr_1ZB_0cyc : SchedWriteRes<[FalkorUnitZB]> { let Latency = 0; } def FalkorWr_1LD_3cyc : SchedWriteRes<[FalkorUnitLD]> { let Latency = 3; } def FalkorWr_1LD_4cyc : SchedWriteRes<[FalkorUnitLD]> { let Latency = 4; } +def FalkorWr_1XYZ_0cyc : SchedWriteRes<[FalkorUnitXYZ]> { let Latency = 0; } def FalkorWr_1XYZ_1cyc : SchedWriteRes<[FalkorUnitXYZ]> { let Latency = 1; } def FalkorWr_1XYZ_2cyc : SchedWriteRes<[FalkorUnitXYZ]> { let Latency = 2; } def FalkorWr_1XYZB_0cyc : SchedWriteRes<[FalkorUnitXYZB]>{ let Latency = 0; } def FalkorWr_1XYZB_1cyc : SchedWriteRes<[FalkorUnitXYZB]>{ let Latency = 1; } def FalkorWr_1none_0cyc : SchedWriteRes<[]> { let Latency = 0; } +def FalkorWr_1VXVY_0cyc : SchedWriteRes<[FalkorUnitVXVY]>{ let Latency = 0; } def FalkorWr_1VXVY_1cyc : SchedWriteRes<[FalkorUnitVXVY]>{ let Latency = 1; } def FalkorWr_1VXVY_2cyc : SchedWriteRes<[FalkorUnitVXVY]>{ let Latency = 2; } def FalkorWr_1VXVY_3cyc : SchedWriteRes<[FalkorUnitVXVY]>{ let Latency = 3; } @@ -76,6 +78,7 @@ def FalkorWr_1LD_0cyc : SchedWriteRes<[FalkorUnitLD]> { let Latency = 0; } def FalkorWr_1ST_0cyc : SchedWriteRes<[FalkorUnitST]> { let Latency = 0; } def FalkorWr_1ST_3cyc : SchedWriteRes<[FalkorUnitST]> { let Latency = 3; } +def FalkorWr_1GTOV_0cyc : SchedWriteRes<[FalkorUnitGTOV]>{ let Latency = 0; } def FalkorWr_1GTOV_1cyc : SchedWriteRes<[FalkorUnitGTOV]>{ let Latency = 1; } def FalkorWr_1GTOV_4cyc : SchedWriteRes<[FalkorUnitGTOV]>{ let Latency = 4; } def FalkorWr_1VTOG_1cyc : SchedWriteRes<[FalkorUnitVTOG]>{ let Latency = 1; } @@ -83,6 +86,10 @@ def FalkorWr_1VTOG_1cyc : SchedWriteRes<[FalkorUnitVTOG]>{ let Latency = 1; } //===----------------------------------------------------------------------===// // Define 2 micro-op types +def FalkorWr_2VXVY_0cyc : SchedWriteRes<[FalkorUnitVXVY, FalkorUnitVXVY]> { + let Latency = 0; + let NumMicroOps = 2; +} def FalkorWr_2VXVY_1cyc : SchedWriteRes<[FalkorUnitVXVY, FalkorUnitVXVY]> { let Latency = 1; let NumMicroOps = 2; @@ -476,17 +483,19 @@ def FalkorReadFMA64 : SchedReadAdvance<2, [FalkorWr_FMUL64_1VXVY_6cyc, FalkorWr // SchedPredicates and WriteVariants for Immediate Zero and LSLFast/ASRFast // ----------------------------------------------------------------------------- def FalkorImmZPred : SchedPredicate<[{MI->getOperand(1).getImm() == 0}]>; -def FalkorFMOVZrReg : SchedPredicate<[{MI->getOperand(1).getReg() == AArch64::WZR || +def FalkorOp1ZrReg : SchedPredicate<[{MI->getOperand(1).getReg() == AArch64::WZR || + MI->getOperand(1).getReg() == AArch64::XZR}]>; def FalkorShiftExtFastPred : SchedPredicate<[{TII->isFalkorShiftExtFast(*MI)}]>; def FalkorWr_FMOV : SchedWriteVariant<[ - SchedVar<FalkorFMOVZrReg, [FalkorWr_1none_0cyc]>, + SchedVar<FalkorOp1ZrReg, [FalkorWr_1none_0cyc]>, SchedVar<NoSchedPred, [FalkorWr_1GTOV_1cyc]>]>; def FalkorWr_MOVZ : SchedWriteVariant<[ SchedVar<FalkorImmZPred, [FalkorWr_1none_0cyc]>, - SchedVar<NoSchedPred, [FalkorWr_1XYZB_1cyc]>]>; + SchedVar<NoSchedPred, [FalkorWr_1XYZB_0cyc]>]>; // imm fwd + def FalkorWr_ADDSUBsx : SchedWriteVariant<[ SchedVar<FalkorShiftExtFastPred, [FalkorWr_1XYZ_1cyc]>, @@ -500,6 +509,10 @@ def FalkorWr_LDRSro : SchedWriteVariant<[ SchedVar<FalkorShiftExtFastPred, [FalkorWr_1LD_4cyc]>, SchedVar<NoSchedPred, [FalkorWr_1XYZ_1LD_5cyc]>]>; +def FalkorWr_ORRi : SchedWriteVariant<[ + SchedVar<FalkorOp1ZrReg, [FalkorWr_1XYZ_0cyc]>, // imm fwd + SchedVar<NoSchedPred, [FalkorWr_1XYZ_1cyc]>]>; + def FalkorWr_PRFMro : SchedWriteVariant<[ SchedVar<FalkorShiftExtFastPred, [FalkorWr_1ST_3cyc]>, SchedVar<NoSchedPred, [FalkorWr_1XYZ_1ST_4cyc]>]>; @@ -810,7 +823,8 @@ def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^BIC(S)?(W|X)r(r|s)$")>; def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^EON(W|X)r(r|s)$")>; def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^EOR(W|X)r(i|r|s)$")>; def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^ORN(W|X)r(r|s)$")>; -def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^ORR(W|X)r(i|r|s)$")>; +def : InstRW<[FalkorWr_ORRi], (instregex "^ORR(W|X)ri$")>; +def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^ORR(W|X)r(r|s)$")>; def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^SBC(S)?(W|X)r$")>; def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^SUB(S)?(W|X)r(r|i)$")>; def : InstRW<[FalkorWr_ADDSUBsx], (instregex "^ADD(S)?(W|X)r(s|x|x64)$")>; @@ -825,7 +839,7 @@ def : InstRW<[FalkorWr_1GTOV_1cyc], (instregex "^INSv(i8|i16)(gpr|lane)$")>; def : InstRW<[FalkorWr_1VTOG_1cyc], (instregex "^(S|U)MOVv.*$")>; def : InstRW<[FalkorWr_1VXVY_1cyc], (instregex "^(BIF|BIT|BSL)v8i8$")>; def : InstRW<[FalkorWr_1VXVY_1cyc], (instrs EXTv8i8)>; -def : InstRW<[FalkorWr_1VXVY_1cyc], (instregex "(MOVI|MVNI)(D|v8b_ns|v2i32|v4i16|v2s_msl)$")>; +def : InstRW<[FalkorWr_1VXVY_0cyc], (instregex "(MOVI|MVNI)(D|v8b_ns|v2i32|v4i16|v2s_msl)$")>; // imm fwd def : InstRW<[FalkorWr_1VXVY_1cyc], (instrs TBLv8i8One)>; def : InstRW<[FalkorWr_1VXVY_1cyc], (instrs NOTv8i8)>; def : InstRW<[FalkorWr_1VXVY_1cyc], (instregex "^REV(16|32|64)v.*$")>; @@ -849,7 +863,7 @@ def : InstRW<[FalkorWr_1GTOV_1VXVY_2cyc], def : InstRW<[FalkorWr_2GTOV_1cyc], (instregex "^DUP(v4i32|v2i64)(gpr|lane)$")>; def : InstRW<[FalkorWr_2VXVY_1cyc], (instregex "^(BIF|BIT|BSL)v16i8$")>; def : InstRW<[FalkorWr_2VXVY_1cyc], (instrs EXTv16i8)>; -def : InstRW<[FalkorWr_2VXVY_1cyc], (instregex "(MOVI|MVNI)(v2d_ns|v16b_ns|v4i32|v8i16|v4s_msl)$")>; +def : InstRW<[FalkorWr_2VXVY_0cyc], (instregex "(MOVI|MVNI)(v2d_ns|v16b_ns|v4i32|v8i16|v4s_msl)$")>; // imm fwd def : InstRW<[FalkorWr_2VXVY_1cyc], (instrs NOTv16i8)>; def : InstRW<[FalkorWr_2VXVY_1cyc], (instrs TBLv16i8One)>; @@ -1036,13 +1050,13 @@ def : InstRW<[FalkorWr_FMUL64_1VXVY_6cyc, ReadDefault, ReadDefault, FalkorReadFM // FP Miscellaneous Instructions // ----------------------------------------------------------------------------- def : InstRW<[FalkorWr_FMOV], (instregex "^FMOV(WS|XD|XDHigh)r$")>; -def : InstRW<[FalkorWr_1GTOV_1cyc], (instregex "^FMOV(S|D)i$")>; +def : InstRW<[FalkorWr_1GTOV_0cyc], (instregex "^FMOV(S|D)i$")>; // imm fwd def : InstRW<[FalkorWr_1VTOG_1cyc], (instregex "^FCVTZ(S|U)S(W|X)(D|S)ri$")>; def : InstRW<[FalkorWr_1VTOG_1cyc], (instregex "^FCVTZ(S|U)(d|s)$")>; def : InstRW<[FalkorWr_1VTOG_1cyc], (instregex "^FMOV(SW|DX|DXHigh)r$")>; -def : InstRW<[FalkorWr_1VXVY_1cyc], (instregex "^FMOV(Sr|Dr|v.*_ns)$")>; +def : InstRW<[FalkorWr_1VXVY_0cyc], (instregex "^FMOV(Sr|Dr|v.*_ns)$")>; // imm fwd // FIXME: We are currently generating movi v0.2d, #0 for these, which is worse than fmov wzr/xzr -def : InstRW<[FalkorWr_2VXVY_1cyc], (instrs FMOVD0, FMOVS0)>; +def : InstRW<[FalkorWr_2VXVY_0cyc], (instrs FMOVD0, FMOVS0)>; // imm fwd def : InstRW<[FalkorWr_1GTOV_4cyc], (instregex "^(S|U)CVTF(S|U)(W|X)(D|S)ri$")>; def : InstRW<[FalkorWr_1VXVY_4cyc], (instregex "^(S|U)CVTF(v1i32|v2i32|v1i64|v2f32|d|s)(_shift)?")>; @@ -1107,11 +1121,12 @@ def : InstRW<[FalkorWr_VMUL32_2VXVY_4cyc, FalkorReadVMA], // Move and Shift Instructions // ----------------------------------------------------------------------------- -def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^(LSLV|LSRV|ASRV|RORV|MOVK)(W|X).*")>; -def : InstRW<[FalkorWr_1XYZB_1cyc], (instregex "^ADRP?$")>; -def : InstRW<[FalkorWr_1XYZB_1cyc], (instregex "^MOVN(W|X)i$")>; +def : InstRW<[FalkorWr_1XYZ_1cyc], (instregex "^(LSLV|LSRV|ASRV|RORV)(W|X)r$")>; +def : InstRW<[FalkorWr_1XYZ_0cyc], (instregex "^MOVK(W|X)i$")>; // imm fwd +def : InstRW<[FalkorWr_1XYZB_0cyc], (instregex "^ADRP?$")>; // imm fwd +def : InstRW<[FalkorWr_1XYZB_0cyc], (instregex "^MOVN(W|X)i$")>; // imm fwd def : InstRW<[FalkorWr_MOVZ], (instregex "^MOVZ(W|X)i$")>; -def : InstRW<[FalkorWr_1XYZ_1cyc], (instrs MOVi32imm, MOVi64imm)>; +def : InstRW<[FalkorWr_1XYZ_0cyc], (instrs MOVi32imm, MOVi64imm)>; // imm fwd (approximation) def : InstRW<[WriteSequence<[FalkorWr_1XYZ_1cyc, FalkorWr_1XYZ_1cyc]>], (instrs MOVaddr, MOVaddrBA, MOVaddrCP, MOVaddrEXT, MOVaddrJT, MOVaddrTLS)>; def : InstRW<[WriteSequence<[FalkorWr_1LD_3cyc, FalkorWr_1XYZ_1cyc]>], diff --git a/lib/Target/AMDGPU/AMDGPU.h b/lib/Target/AMDGPU/AMDGPU.h index 78ff3bbe3d1a..55d18c3f3646 100644 --- a/lib/Target/AMDGPU/AMDGPU.h +++ b/lib/Target/AMDGPU/AMDGPU.h @@ -55,6 +55,8 @@ FunctionPass *createAMDGPUMachineCFGStructurizerPass(); void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&); extern char &AMDGPUMachineCFGStructurizerID; +void initializeAMDGPUAlwaysInlinePass(PassRegistry&); + ModulePass *createAMDGPUAnnotateKernelFeaturesPass(); void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &); extern char &AMDGPUAnnotateKernelFeaturesID; diff --git a/lib/Target/AMDGPU/AMDGPU.td b/lib/Target/AMDGPU/AMDGPU.td index e7ebb37a9d62..b50e8d1d659e 100644 --- a/lib/Target/AMDGPU/AMDGPU.td +++ b/lib/Target/AMDGPU/AMDGPU.td @@ -365,6 +365,13 @@ def FeatureFlatForGlobal : SubtargetFeature<"flat-for-global", "Force to generate flat instruction for global" >; +def FeatureAutoWaitcntBeforeBarrier : SubtargetFeature < + "auto-waitcnt-before-barrier", + "AutoWaitcntBeforeBarrier", + "true", + "Hardware automatically inserts waitcnt before barrier" +>; + // Dummy feature used to disable assembler instructions. def FeatureDisable : SubtargetFeature<"", "FeatureDisable","true", diff --git a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp index 1d03714874e2..8084d368c80f 100644 --- a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp +++ b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp @@ -22,18 +22,22 @@ using namespace llvm; namespace { class AMDGPUAlwaysInline : public ModulePass { - static char ID; - bool GlobalOpt; public: - AMDGPUAlwaysInline(bool GlobalOpt) : ModulePass(ID), GlobalOpt(GlobalOpt) { } + static char ID; + + AMDGPUAlwaysInline(bool GlobalOpt = false) : + ModulePass(ID), GlobalOpt(GlobalOpt) { } bool runOnModule(Module &M) override; StringRef getPassName() const override { return "AMDGPU Always Inline Pass"; } }; } // End anonymous namespace +INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline", + "AMDGPU Inline All Functions", false, false) + char AMDGPUAlwaysInline::ID = 0; bool AMDGPUAlwaysInline::runOnModule(Module &M) { diff --git a/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 57905be18813..267f4807a788 100644 --- a/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -28,11 +28,16 @@ using namespace llvm; AMDGPULegalizerInfo::AMDGPULegalizerInfo() { using namespace TargetOpcode; + const LLT S1= LLT::scalar(1); const LLT S32 = LLT::scalar(32); const LLT S64 = LLT::scalar(64); const LLT P1 = LLT::pointer(1, 64); const LLT P2 = LLT::pointer(2, 64); + // FIXME: i1 operands to intrinsics should always be legal, but other i1 + // values may not be legal. We need to figure out how to distinguish + // between these two scenarios. + setAction({G_CONSTANT, S1}, Legal); setAction({G_CONSTANT, S32}, Legal); setAction({G_CONSTANT, S64}, Legal); diff --git a/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp index 6e301b4ad527..8d157e2f98f2 100644 --- a/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -91,6 +91,7 @@ AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS, FPExceptions(false), DX10Clamp(false), FlatForGlobal(false), + AutoWaitcntBeforeBarrier(false), UnalignedScratchAccess(false), UnalignedBufferAccess(false), diff --git a/lib/Target/AMDGPU/AMDGPUSubtarget.h b/lib/Target/AMDGPU/AMDGPUSubtarget.h index 0582ce95693a..ed9cbb994fad 100644 --- a/lib/Target/AMDGPU/AMDGPUSubtarget.h +++ b/lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -110,6 +110,7 @@ protected: bool FPExceptions; bool DX10Clamp; bool FlatForGlobal; + bool AutoWaitcntBeforeBarrier; bool UnalignedScratchAccess; bool UnalignedBufferAccess; bool HasApertureRegs; @@ -195,7 +196,8 @@ public: } bool isOpenCLEnv() const { - return TargetTriple.getEnvironment() == Triple::OpenCL; + return TargetTriple.getEnvironment() == Triple::OpenCL || + TargetTriple.getEnvironmentName() == "amdgizcl"; } Generation getGeneration() const { @@ -363,6 +365,10 @@ public: return FlatForGlobal; } + bool hasAutoWaitcntBeforeBarrier() const { + return AutoWaitcntBeforeBarrier; + } + bool hasUnalignedBufferAccess() const { return UnalignedBufferAccess; } @@ -727,12 +733,6 @@ public: /// Return the maximum number of waves per SIMD for kernels using \p VGPRs VGPRs unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const; - /// \returns True if waitcnt instruction is needed before barrier instruction, - /// false otherwise. - bool needWaitcntBeforeBarrier() const { - return true; - } - /// \returns true if the flat_scratch register should be initialized with the /// pointer to the wave's scratch memory rather than a size and offset. bool flatScratchIsPointer() const { diff --git a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 596f02ae4a64..404598ff4738 100644 --- a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -116,7 +116,7 @@ static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden, static cl::opt<bool> EnableSIInsertWaitcntsPass( "enable-si-insert-waitcnts", cl::desc("Use new waitcnt insertion pass"), - cl::init(false)); + cl::init(true)); // Option to run late CFG structurizer static cl::opt<bool> LateCFGStructurize( @@ -139,6 +139,7 @@ extern "C" void LLVMInitializeAMDGPUTarget() { initializeSIShrinkInstructionsPass(*PR); initializeSIFixControlFlowLiveIntervalsPass(*PR); initializeSILoadStoreOptimizerPass(*PR); + initializeAMDGPUAlwaysInlinePass(*PR); initializeAMDGPUAnnotateKernelFeaturesPass(*PR); initializeAMDGPUAnnotateUniformValuesPass(*PR); initializeAMDGPULowerIntrinsicsPass(*PR); diff --git a/lib/Target/AMDGPU/SIFoldOperands.cpp b/lib/Target/AMDGPU/SIFoldOperands.cpp index f13629a3185f..dfac068d1f69 100644 --- a/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -35,9 +35,12 @@ struct FoldCandidate { }; unsigned char UseOpNo; MachineOperand::MachineOperandType Kind; + bool Commuted; - FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp) : - UseMI(MI), OpToFold(nullptr), UseOpNo(OpNo), Kind(FoldOp->getType()) { + FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp, + bool Commuted_ = false) : + UseMI(MI), OpToFold(nullptr), UseOpNo(OpNo), Kind(FoldOp->getType()), + Commuted(Commuted_) { if (FoldOp->isImm()) { ImmToFold = FoldOp->getImm(); } else if (FoldOp->isFI()) { @@ -59,6 +62,10 @@ struct FoldCandidate { bool isReg() const { return Kind == MachineOperand::MO_Register; } + + bool isCommuted() const { + return Commuted; + } }; class SIFoldOperands : public MachineFunctionPass { @@ -237,8 +244,13 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1)) return false; - if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) + if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) { + TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1); return false; + } + + FoldList.push_back(FoldCandidate(MI, OpNo, OpToFold, true)); + return true; } FoldList.push_back(FoldCandidate(MI, OpNo, OpToFold)); @@ -699,6 +711,9 @@ void SIFoldOperands::foldInstOperand(MachineInstr &MI, DEBUG(dbgs() << "Folded source from " << MI << " into OpNo " << static_cast<int>(Fold.UseOpNo) << " of " << *Fold.UseMI << '\n'); tryFoldInst(TII, Fold.UseMI); + } else if (Fold.isCommuted()) { + // Restoring instruction's original operand order if fold has failed. + TII->commuteInstruction(*Fold.UseMI, false); } } } diff --git a/lib/Target/AMDGPU/SIISelLowering.cpp b/lib/Target/AMDGPU/SIISelLowering.cpp index 76c2644867aa..b48b23911105 100644 --- a/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/lib/Target/AMDGPU/SIISelLowering.cpp @@ -3571,7 +3571,7 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { } if (AS == AMDGPUASI.CONSTANT_ADDRESS || AS == AMDGPUASI.GLOBAL_ADDRESS) { if (Subtarget->getScalarizeGlobalBehavior() && isMemOpUniform(Load) && - isMemOpHasNoClobberedMemOperand(Load)) + !Load->isVolatile() && isMemOpHasNoClobberedMemOperand(Load)) return SDValue(); // Non-uniform loads will be selected to MUBUF instructions, so they // have the same legalization requirements as global and private diff --git a/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/lib/Target/AMDGPU/SIInsertWaitcnts.cpp index e22166d03e9a..c10badba88f3 100644 --- a/lib/Target/AMDGPU/SIInsertWaitcnts.cpp +++ b/lib/Target/AMDGPU/SIInsertWaitcnts.cpp @@ -1009,7 +1009,8 @@ MachineInstr *SIInsertWaitcnts::generateSWaitCntInstBefore( // occurs before the instruction. Doing it here prevents any additional // S_WAITCNTs from being emitted if the instruction was marked as // requiring a WAITCNT beforehand. - if (MI.getOpcode() == AMDGPU::S_BARRIER && ST->needWaitcntBeforeBarrier()) { + if (MI.getOpcode() == AMDGPU::S_BARRIER && + !ST->hasAutoWaitcntBeforeBarrier()) { EmitSwaitcnt |= ScoreBrackets->updateByWait(VM_CNT, ScoreBrackets->getScoreUB(VM_CNT)); EmitSwaitcnt |= ScoreBrackets->updateByWait( diff --git a/lib/Target/AMDGPU/SIInsertWaits.cpp b/lib/Target/AMDGPU/SIInsertWaits.cpp index 9f32ecfa52ff..bc86515d8b1f 100644 --- a/lib/Target/AMDGPU/SIInsertWaits.cpp +++ b/lib/Target/AMDGPU/SIInsertWaits.cpp @@ -630,7 +630,7 @@ bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) { // but we also want to wait for any other outstanding transfers before // signalling other hardware blocks if ((I->getOpcode() == AMDGPU::S_BARRIER && - ST->needWaitcntBeforeBarrier()) || + !ST->hasAutoWaitcntBeforeBarrier()) || I->getOpcode() == AMDGPU::S_SENDMSG || I->getOpcode() == AMDGPU::S_SENDMSGHALT) Required = LastIssued; diff --git a/lib/Target/AMDGPU/SMInstructions.td b/lib/Target/AMDGPU/SMInstructions.td index 5b840a14dbc3..73dd8b7daa4e 100644 --- a/lib/Target/AMDGPU/SMInstructions.td +++ b/lib/Target/AMDGPU/SMInstructions.td @@ -229,6 +229,7 @@ def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ ((Ld->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS && static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpUniform(N)) || (Subtarget->getScalarizeGlobalBehavior() && Ld->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS && + !Ld->isVolatile() && static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpUniform(N) && static_cast<const SITargetLowering *>(getTargetLowering())->isMemOpHasNoClobberedMemOperand(N))); }]>; diff --git a/lib/Target/AMDGPU/VOP3Instructions.td b/lib/Target/AMDGPU/VOP3Instructions.td index 001fc960b228..77fc9551cff9 100644 --- a/lib/Target/AMDGPU/VOP3Instructions.td +++ b/lib/Target/AMDGPU/VOP3Instructions.td @@ -245,9 +245,10 @@ def V_MAD_I64_I32 : VOP3Inst <"v_mad_i64_i32", VOP3b_I64_I1_I32_I32_I64>; let SubtargetPredicate = Has16BitInsts in { +def V_DIV_FIXUP_F16 : VOP3Inst <"v_div_fixup_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, AMDGPUdiv_fixup>; + let isCommutable = 1 in { -def V_DIV_FIXUP_F16 : VOP3Inst <"v_div_fixup_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, AMDGPUdiv_fixup>; def V_FMA_F16 : VOP3Inst <"v_fma_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fma>; def V_INTERP_P1LL_F16 : VOP3Inst <"v_interp_p1ll_f16", VOP3_Profile<VOP_F32_F32_F16>>; def V_INTERP_P1LV_F16 : VOP3Inst <"v_interp_p1lv_f16", VOP3_Profile<VOP_F32_F32_F16_F16>>; diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td index 46fd1f70ee99..ca68f5d42c32 100644 --- a/lib/Target/ARM/ARM.td +++ b/lib/Target/ARM/ARM.td @@ -205,6 +205,13 @@ def FeatureAvoidPartialCPSR : SubtargetFeature<"avoid-partial-cpsr", "AvoidCPSRPartialUpdate", "true", "Avoid CPSR partial update for OOO execution">; +/// Disable +1 predication cost for instructions updating CPSR. +/// Enabled for Cortex-A57. +def FeatureCheapPredicableCPSR : SubtargetFeature<"cheap-predicable-cpsr", + "CheapPredicableCPSRDef", + "true", + "Disable +1 predication cost for instructions updating CPSR">; + def FeatureAvoidMOVsShOp : SubtargetFeature<"avoid-movs-shop", "AvoidMOVsShifterOperand", "true", "Avoid movs instructions with shifter operand">; @@ -788,12 +795,14 @@ def : ProcNoItin<"cortex-a53", [ARMv8a, ProcA53, FeatureCRC, FeatureFPAO]>; -def : ProcNoItin<"cortex-a57", [ARMv8a, ProcA57, - FeatureHWDivThumb, - FeatureHWDivARM, - FeatureCrypto, - FeatureCRC, - FeatureFPAO]>; +def : ProcessorModel<"cortex-a57", CortexA57Model, [ARMv8a, ProcA57, + FeatureHWDivThumb, + FeatureHWDivARM, + FeatureCrypto, + FeatureCRC, + FeatureFPAO, + FeatureAvoidPartialCPSR, + FeatureCheapPredicableCPSR]>; def : ProcNoItin<"cortex-a72", [ARMv8a, ProcA72, FeatureHWDivThumb, diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp index 5c9d589e2625..f8b65573f9cd 100644 --- a/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -558,13 +558,68 @@ bool ARMBaseInstrInfo::DefinesPredicate( return Found; } -static bool isCPSRDefined(const MachineInstr *MI) { - for (const auto &MO : MI->operands()) +bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) { + for (const auto &MO : MI.operands()) if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead()) return true; return false; } +bool ARMBaseInstrInfo::isAddrMode3OpImm(const MachineInstr &MI, + unsigned Op) const { + const MachineOperand &Offset = MI.getOperand(Op + 1); + return Offset.getReg() != 0; +} + +// Load with negative register offset requires additional 1cyc and +I unit +// for Cortex A57 +bool ARMBaseInstrInfo::isAddrMode3OpMinusReg(const MachineInstr &MI, + unsigned Op) const { + const MachineOperand &Offset = MI.getOperand(Op + 1); + const MachineOperand &Opc = MI.getOperand(Op + 2); + assert(Opc.isImm()); + assert(Offset.isReg()); + int64_t OpcImm = Opc.getImm(); + + bool isSub = ARM_AM::getAM3Op(OpcImm) == ARM_AM::sub; + return (isSub && Offset.getReg() != 0); +} + +bool ARMBaseInstrInfo::isLdstScaledReg(const MachineInstr &MI, + unsigned Op) const { + const MachineOperand &Opc = MI.getOperand(Op + 2); + unsigned OffImm = Opc.getImm(); + return ARM_AM::getAM2ShiftOpc(OffImm) != ARM_AM::no_shift; +} + +// Load, scaled register offset, not plus LSL2 +bool ARMBaseInstrInfo::isLdstScaledRegNotPlusLsl2(const MachineInstr &MI, + unsigned Op) const { + const MachineOperand &Opc = MI.getOperand(Op + 2); + unsigned OffImm = Opc.getImm(); + + bool isAdd = ARM_AM::getAM2Op(OffImm) == ARM_AM::add; + unsigned Amt = ARM_AM::getAM2Offset(OffImm); + ARM_AM::ShiftOpc ShiftOpc = ARM_AM::getAM2ShiftOpc(OffImm); + if (ShiftOpc == ARM_AM::no_shift) return false; // not scaled + bool SimpleScaled = (isAdd && ShiftOpc == ARM_AM::lsl && Amt == 2); + return !SimpleScaled; +} + +// Minus reg for ldstso addr mode +bool ARMBaseInstrInfo::isLdstSoMinusReg(const MachineInstr &MI, + unsigned Op) const { + unsigned OffImm = MI.getOperand(Op + 2).getImm(); + return ARM_AM::getAM2Op(OffImm) == ARM_AM::sub; +} + +// Load, scaled register offset +bool ARMBaseInstrInfo::isAm2ScaledReg(const MachineInstr &MI, + unsigned Op) const { + unsigned OffImm = MI.getOperand(Op + 2).getImm(); + return ARM_AM::getAM2ShiftOpc(OffImm) != ARM_AM::no_shift; +} + static bool isEligibleForITBlock(const MachineInstr *MI) { switch (MI->getOpcode()) { default: return true; @@ -590,7 +645,7 @@ static bool isEligibleForITBlock(const MachineInstr *MI) { case ARM::tSUBi3: // SUB (immediate) T1 case ARM::tSUBi8: // SUB (immediate) T2 case ARM::tSUBrr: // SUB (register) T1 - return !isCPSRDefined(MI); + return !ARMBaseInstrInfo::isCPSRDefined(*MI); } } @@ -3349,6 +3404,22 @@ ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData, return DefCycle; } +bool ARMBaseInstrInfo::isLDMBaseRegInList(const MachineInstr &MI) const { + unsigned BaseReg = MI.getOperand(0).getReg(); + for (unsigned i = 1, sz = MI.getNumOperands(); i < sz; ++i) { + const auto &Op = MI.getOperand(i); + if (Op.isReg() && Op.getReg() == BaseReg) + return true; + } + return false; +} +unsigned +ARMBaseInstrInfo::getLDMVariableDefsSize(const MachineInstr &MI) const { + // ins GPR:$Rn, pred:$p (2xOp), reglist:$regs, variable_ops + // (outs GPR:$wb), (ins GPR:$Rn, pred:$p (2xOp), reglist:$regs, variable_ops) + return MI.getNumOperands() + 1 - MI.getDesc().getNumOperands(); +} + int ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData, const MCInstrDesc &DefMCID, @@ -4119,7 +4190,8 @@ unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const { const MCInstrDesc &MCID = MI.getDesc(); - if (MCID.isCall() || MCID.hasImplicitDefOfPhysReg(ARM::CPSR)) { + if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) && + !Subtarget.cheapPredicableCPSRDef())) { // When predicated, CPSR is an additional source operand for CPSR updating // instructions, this apparently increases their latencies. return 1; @@ -4148,7 +4220,8 @@ unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, } const MCInstrDesc &MCID = MI.getDesc(); - if (PredCost && (MCID.isCall() || MCID.hasImplicitDefOfPhysReg(ARM::CPSR))) { + if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) && + !Subtarget.cheapPredicableCPSRDef()))) { // When predicated, CPSR is an additional source operand for CPSR updating // instructions, this apparently increases their latencies. *PredCost = 1; diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h index dd7fe871345a..c52e572786d4 100644 --- a/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/lib/Target/ARM/ARMBaseInstrInfo.h @@ -159,6 +159,24 @@ public: bool isPredicable(const MachineInstr &MI) const override; + // CPSR defined in instruction + static bool isCPSRDefined(const MachineInstr &MI); + bool isAddrMode3OpImm(const MachineInstr &MI, unsigned Op) const; + bool isAddrMode3OpMinusReg(const MachineInstr &MI, unsigned Op) const; + + // Load, scaled register offset + bool isLdstScaledReg(const MachineInstr &MI, unsigned Op) const; + // Load, scaled register offset, not plus LSL2 + bool isLdstScaledRegNotPlusLsl2(const MachineInstr &MI, unsigned Op) const; + // Minus reg for ldstso addr mode + bool isLdstSoMinusReg(const MachineInstr &MI, unsigned Op) const; + // Scaled register offset in address mode 2 + bool isAm2ScaledReg(const MachineInstr &MI, unsigned Op) const; + // Load multiple, base reg in list + bool isLDMBaseRegInList(const MachineInstr &MI) const; + // get LDM variable defs size + unsigned getLDMVariableDefsSize(const MachineInstr &MI) const; + /// GetInstSize - Returns the size of the specified MachineInstr. /// unsigned getInstSizeInBytes(const MachineInstr &MI) const override; diff --git a/lib/Target/ARM/ARMCallLowering.cpp b/lib/Target/ARM/ARMCallLowering.cpp index 31a2f499a9a7..a33d025d114e 100644 --- a/lib/Target/ARM/ARMCallLowering.cpp +++ b/lib/Target/ARM/ARMCallLowering.cpp @@ -34,7 +34,7 @@ ARMCallLowering::ARMCallLowering(const ARMTargetLowering &TLI) static bool isSupportedType(const DataLayout &DL, const ARMTargetLowering &TLI, Type *T) { - if (T->isArrayTy()) + if (T->isArrayTy() || T->isStructTy()) return true; EVT VT = TLI.getValueType(DL, T, true); @@ -167,8 +167,11 @@ void ARMCallLowering::splitToValueTypes( if (SplitVTs.size() == 1) { // Even if there is no splitting to do, we still want to replace the // original type (e.g. pointer type -> integer). - SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), - OrigArg.Flags, OrigArg.IsFixed); + auto Flags = OrigArg.Flags; + unsigned OriginalAlignment = DL.getABITypeAlignment(OrigArg.Ty); + Flags.setOrigAlign(OriginalAlignment); + SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), Flags, + OrigArg.IsFixed); return; } @@ -177,6 +180,10 @@ void ARMCallLowering::splitToValueTypes( EVT SplitVT = SplitVTs[i]; Type *SplitTy = SplitVT.getTypeForEVT(Ctx); auto Flags = OrigArg.Flags; + + unsigned OriginalAlignment = DL.getABITypeAlignment(SplitTy); + Flags.setOrigAlign(OriginalAlignment); + bool NeedsConsecutiveRegisters = TLI.functionArgumentNeedsConsecutiveRegisters( SplitTy, F->getCallingConv(), F->isVarArg()); @@ -185,6 +192,7 @@ void ARMCallLowering::splitToValueTypes( if (i == e - 1) Flags.setInConsecutiveRegsLast(); } + SplitArgs.push_back( ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)), SplitTy, Flags, OrigArg.IsFixed}); diff --git a/lib/Target/ARM/ARMSchedule.td b/lib/Target/ARM/ARMSchedule.td index ec5b97cba8cd..1c7902520f2d 100644 --- a/lib/Target/ARM/ARMSchedule.td +++ b/lib/Target/ARM/ARMSchedule.td @@ -147,6 +147,9 @@ def : PredicateProlog<[{ const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo*>(SchedModel->getInstrInfo()); (void)TII; + const ARMSubtarget *STI = + static_cast<const ARMSubtarget*>(SchedModel->getSubtargetInfo()); + (void)STI; }]>; def IsPredicatedPred : SchedPredicate<[{TII->isPredicated(*MI)}]>; @@ -420,3 +423,4 @@ include "ARMScheduleA8.td" include "ARMScheduleA9.td" include "ARMScheduleSwift.td" include "ARMScheduleR52.td" +include "ARMScheduleA57.td" diff --git a/lib/Target/ARM/ARMScheduleA57.td b/lib/Target/ARM/ARMScheduleA57.td new file mode 100644 index 000000000000..525079d12d51 --- /dev/null +++ b/lib/Target/ARM/ARMScheduleA57.td @@ -0,0 +1,1471 @@ +//=- ARMScheduleA57.td - ARM Cortex-A57 Scheduling Defs -----*- tablegen -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the machine model for ARM Cortex-A57 to support +// instruction scheduling and other instruction cost heuristics. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// *** Common description and scheduling model parameters taken from AArch64 *** +// The Cortex-A57 is a traditional superscalar microprocessor with a +// conservative 3-wide in-order stage for decode and dispatch. Combined with the +// much wider out-of-order issue stage, this produced a need to carefully +// schedule micro-ops so that all three decoded each cycle are successfully +// issued as the reservation station(s) simply don't stay occupied for long. +// Therefore, IssueWidth is set to the narrower of the two at three, while still +// modeling the machine as out-of-order. + +def IsCPSRDefinedPred : SchedPredicate<[{TII->isCPSRDefined(*MI)}]>; +def IsCPSRDefinedAndPredicatedPred : + SchedPredicate<[{TII->isCPSRDefined(*MI) && TII->isPredicated(*MI)}]>; + +// Cortex A57 rev. r1p0 or later (false = r0px) +def IsR1P0AndLaterPred : SchedPredicate<[{false}]>; + +// If Addrmode3 contains register offset (not immediate) +def IsLdrAm3RegOffPred : + SchedPredicate<[{!TII->isAddrMode3OpImm(*MI, 1)}]>; +// The same predicate with operand offset 2 and 3: +def IsLdrAm3RegOffPredX2 : + SchedPredicate<[{!TII->isAddrMode3OpImm(*MI, 2)}]>; +def IsLdrAm3RegOffPredX3 : + SchedPredicate<[{!TII->isAddrMode3OpImm(*MI, 3)}]>; + +// If Addrmode3 contains "minus register" +def IsLdrAm3NegRegOffPred : + SchedPredicate<[{TII->isAddrMode3OpMinusReg(*MI, 1)}]>; +// The same predicate with operand offset 2 and 3: +def IsLdrAm3NegRegOffPredX2 : + SchedPredicate<[{TII->isAddrMode3OpMinusReg(*MI, 2)}]>; +def IsLdrAm3NegRegOffPredX3 : + SchedPredicate<[{TII->isAddrMode3OpMinusReg(*MI, 3)}]>; + +// Load, scaled register offset, not plus LSL2 +def IsLdstsoScaledNotOptimalPredX0 : + SchedPredicate<[{TII->isLdstScaledRegNotPlusLsl2(*MI, 0)}]>; +def IsLdstsoScaledNotOptimalPred : + SchedPredicate<[{TII->isLdstScaledRegNotPlusLsl2(*MI, 1)}]>; +def IsLdstsoScaledNotOptimalPredX2 : + SchedPredicate<[{TII->isLdstScaledRegNotPlusLsl2(*MI, 2)}]>; + +// Load, scaled register offset +def IsLdstsoScaledPred : + SchedPredicate<[{TII->isLdstScaledReg(*MI, 1)}]>; +def IsLdstsoScaledPredX2 : + SchedPredicate<[{TII->isLdstScaledReg(*MI, 2)}]>; + +def IsLdstsoMinusRegPredX0 : + SchedPredicate<[{TII->isLdstSoMinusReg(*MI, 0)}]>; +def IsLdstsoMinusRegPred : + SchedPredicate<[{TII->isLdstSoMinusReg(*MI, 1)}]>; +def IsLdstsoMinusRegPredX2 : + SchedPredicate<[{TII->isLdstSoMinusReg(*MI, 2)}]>; + +// Load, scaled register offset +def IsLdrAm2ScaledPred : + SchedPredicate<[{TII->isAm2ScaledReg(*MI, 1)}]>; + +// LDM, base reg in list +def IsLdmBaseRegInList : + SchedPredicate<[{TII->isLDMBaseRegInList(*MI)}]>; + +class A57WriteLMOpsListType<list<SchedWriteRes> writes> { + list <SchedWriteRes> Writes = writes; + SchedMachineModel SchedModel = ?; +} + +// *** Common description and scheduling model parameters taken from AArch64 *** +// (AArch64SchedA57.td) +def CortexA57Model : SchedMachineModel { + let IssueWidth = 3; // 3-way decode and dispatch + let MicroOpBufferSize = 128; // 128 micro-op re-order buffer + let LoadLatency = 4; // Optimistic load latency + let MispredictPenalty = 16; // Fetch + Decode/Rename/Dispatch + Branch + + // Enable partial & runtime unrolling. + let LoopMicroOpBufferSize = 16; + let CompleteModel = 1; +} + +//===----------------------------------------------------------------------===// +// Define each kind of processor resource and number available on Cortex-A57. +// Cortex A-57 has 8 pipelines that each has its own 8-entry queue where +// micro-ops wait for their operands and then issue out-of-order. + +def A57UnitB : ProcResource<1>; // Type B micro-ops +def A57UnitI : ProcResource<2>; // Type I micro-ops +def A57UnitM : ProcResource<1>; // Type M micro-ops +def A57UnitL : ProcResource<1>; // Type L micro-ops +def A57UnitS : ProcResource<1>; // Type S micro-ops + +def A57UnitX : ProcResource<1>; // Type X micro-ops (F1) +def A57UnitW : ProcResource<1>; // Type W micro-ops (F0) + +let SchedModel = CortexA57Model in { + def A57UnitV : ProcResGroup<[A57UnitX, A57UnitW]>; // Type V micro-ops +} + +let SchedModel = CortexA57Model in { + +//===----------------------------------------------------------------------===// +// Define customized scheduler read/write types specific to the Cortex-A57. + +include "ARMScheduleA57WriteRes.td" + +// To have "CompleteModel = 1", support of pseudos and special instructions +def : InstRW<[WriteNoop], (instregex "(t)?BKPT$", "(t2)?CDP(2)?$", + "(t2)?CLREX$", "CONSTPOOL_ENTRY$", "COPY_STRUCT_BYVAL_I32$", + "(t2)?CPS[123]p$", "(t2)?DBG$", "(t2)?DMB$", "(t2)?DSB$", "ERET$", + "(t2|t)?HINT$", "(t)?HLT$", "(t2)?HVC$", "(t2)?ISB$", "ITasm$", + "(t2)?RFE(DA|DB|IA|IB)", "(t)?SETEND", "(t2)?SETPAN", "(t2)?SMC", "SPACE", + "(t2)?SRS(DA|DB|IA|IB)", "SWP(B)?", "t?TRAP", "UDF$", "t2DCPS", "t2SG", + "t2TT", "tCPS", "CMP_SWAP", "t?SVC", "t2IT", "CompilerBarrier")>; + +def : InstRW<[WriteNoop], (instregex "VMRS", "VMSR", "FMSTAT")>; + +// Specific memory instrs +def : InstRW<[WriteNoop, WriteNoop], (instregex "(t2)?LDA", "(t2)?LDC", "(t2)?STC", + "(t2)?STL", "(t2)?LDREX", "(t2)?STREX", "MEMCPY")>; + +// coprocessor moves +def : InstRW<[WriteNoop, WriteNoop], (instregex + "(t2)?MCR(2|R|R2)?$", "(t2)?MRC(2)?$", + "(t2)?MRRC(2)?$", "(t2)?MRS(banked|sys|_AR|_M|sys_AR)?$", + "(t2)?MSR(banked|i|_AR|_M)?$")>; + +// Deprecated instructions +def : InstRW<[WriteNoop], (instregex "FLDM", "FSTM")>; + +// Pseudos +def : InstRW<[WriteNoop], (instregex "(t2)?ABS$", + "(t)?ADJCALLSTACKDOWN$", "(t)?ADJCALLSTACKUP$", "(t2|t)?Int_eh_sjlj", + "tLDRpci_pic", "t2SUBS_PC_LR", + "JUMPTABLE", "tInt_WIN_eh_sjlj_longjmp", + "VLD(1|2)LN(d|q)(WB_fixed_|WB_register_)?Asm", + "VLD(3|4)(DUP|LN)?(d|q)(WB_fixed_|WB_register_)?Asm", + "VST(1|2)LN(d|q)(WB_fixed_|WB_register_)?Asm", + "VST(3|4)(DUP|LN)?(d|q)(WB_fixed_|WB_register_)?Asm", + "WIN__CHKSTK", "WIN__DBZCHK")>; + +// Miscellaneous +// ----------------------------------------------------------------------------- + +def : InstRW<[A57Write_1cyc_1I], (instrs COPY)>; + +// --- 3.2 Branch Instructions --- +// B, BX, BL, BLX (imm, reg != LR, reg == LR), CBZ, CBNZ + +def : InstRW<[A57Write_1cyc_1B], (instregex "(t2|t)?B$", "t?BX", "(t2|t)?Bcc$", + "t?TAILJMP(d|r)", "TCRETURN(d|r)i", "tBfar", "tCBN?Z")>; +def : InstRW<[A57Write_1cyc_1B_1I], + (instregex "t?BL$", "BL_pred$", "t?BLXi", "t?TPsoft")>; +def : InstRW<[A57Write_2cyc_1B_1I], (instregex "BLX", "tBLX(NS)?r")>; +// Pseudos +def : InstRW<[A57Write_2cyc_1B_1I], (instregex "BCCi64", "BCCZi64")>; +def : InstRW<[A57Write_3cyc_1B_1I], (instregex "BR_JTadd", "t?BR_JTr", + "t2BR_JT", "t2BXJ", "(t2)?TB(B|H)(_JT)?$", "tBRIND")>; +def : InstRW<[A57Write_6cyc_1B_1L], (instregex "BR_JTm")>; + +// --- 3.3 Arithmetic and Logical Instructions --- +// ADD{S}, ADC{S}, ADR, AND{S}, BIC{S}, CMN, CMP, EOR{S}, ORN{S}, ORR{S}, +// RSB{S}, RSC{S}, SUB{S}, SBC{S}, TEQ, TST + +def : InstRW<[A57Write_1cyc_1I], (instregex "tADDframe")>; + +// shift by register, conditional or unconditional +// TODO: according to the doc, conditional uses I0/I1, unconditional uses M +// Why more complex instruction uses more simple pipeline? +// May be an error in doc. +def A57WriteALUsi : SchedWriteVariant<[ + // lsl #2, lsl #1, or lsr #1. + SchedVar<IsPredicatedPred, [A57Write_2cyc_1M]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1M]> +]>; +def A57WriteALUsr : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1M]> +]>; +def A57WriteALUSsr : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1M]> +]>; +def A57ReadALUsr : SchedReadVariant<[ + SchedVar<IsPredicatedPred, [ReadDefault]>, + SchedVar<NoSchedPred, [ReadDefault]> +]>; +def : SchedAlias<WriteALUsi, A57WriteALUsi>; +def : SchedAlias<WriteALUsr, A57WriteALUsr>; +def : SchedAlias<WriteALUSsr, A57WriteALUSsr>; +def : SchedAlias<ReadALUsr, A57ReadALUsr>; + +def A57WriteCMPsr : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1M]> +]>; +def : SchedAlias<WriteCMP, A57Write_1cyc_1I>; +def : SchedAlias<WriteCMPsi, A57Write_2cyc_1M>; +def : SchedAlias<WriteCMPsr, A57WriteCMPsr>; + +// --- 3.4 Move and Shift Instructions --- +// Move, basic +// MOV{S}, MOVW, MVN{S} +def : InstRW<[A57Write_1cyc_1I], (instregex "MOV(r|i|i16|r_TC)", + "(t2)?MVN(CC)?(r|i)", "BMOVPCB_CALL", "BMOVPCRX_CALL", + "MOVCC(r|i|i16|i32imm)", "tMOV", "tMVN")>; + +// Move, shift by immed, setflags/no setflags +// (ASR, LSL, LSR, ROR, RRX)=MOVsi, MVN +// setflags = isCPSRDefined +def A57WriteMOVsi : SchedWriteVariant<[ + SchedVar<IsCPSRDefinedPred, [A57Write_2cyc_1M]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1I]> +]>; +def : InstRW<[A57WriteMOVsi], (instregex "MOV(CC)?si", "MVNsi", + "ASRi", "(t2|t)ASRri", "LSRi", "(t2|t)LSRri", "LSLi", "(t2|t)LSLri", "RORi", + "(t2|t)RORri", "(t2)?RRX", "t2MOV", "tROR")>; + +// shift by register, conditional or unconditional, setflags/no setflags +def A57WriteMOVsr : SchedWriteVariant<[ + SchedVar<IsCPSRDefinedAndPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<IsCPSRDefinedPred, [A57Write_2cyc_1M]>, + SchedVar<IsPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1I]> +]>; +def : InstRW<[A57WriteMOVsr], (instregex "MOV(CC)?sr", "MVNsr", "t2MVNs", + "ASRr", "(t2|t)ASRrr", "LSRr", "(t2|t)LSRrr", "LSLr", "(t2|t)?LSLrr", "RORr", + "(t2|t)RORrr")>; + +// Move, top +// MOVT - A57Write_2cyc_1M for r0px, A57Write_1cyc_1I for r1p0 and later +def A57WriteMOVT : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_1cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1M]> +]>; +def : InstRW<[A57WriteMOVT], (instregex "MOVTi16")>; + +def A57WriteI2pc : + WriteSequence<[A57Write_1cyc_1I, A57Write_1cyc_1I, A57Write_1cyc_1I]>; +def A57WriteI2ld : + WriteSequence<[A57Write_1cyc_1I, A57Write_1cyc_1I, A57Write_4cyc_1L]>; +def : InstRW< [A57WriteI2pc], (instregex "MOV_ga_pcrel")>; +def : InstRW< [A57WriteI2ld], (instregex "MOV_ga_pcrel_ldr")>; + +// +2cyc for branch forms +def : InstRW<[A57Write_3cyc_1I], (instregex "MOVPC(LR|RX)")>; + +// --- 3.5 Divide and Multiply Instructions --- +// Divide: SDIV, UDIV +// latency from documentration: 4 ‐ 20, maximum taken +def : SchedAlias<WriteDIV, A57Write_20cyc_1M>; +// Multiply: tMul not bound to common WriteRes types +def : InstRW<[A57Write_3cyc_1M], (instregex "tMUL")>; +def : SchedAlias<WriteMUL16, A57Write_3cyc_1M>; +def : SchedAlias<WriteMUL32, A57Write_3cyc_1M>; +def : ReadAdvance<ReadMUL, 0>; + +// Multiply accumulate: MLA, MLS, SMLABB, SMLABT, SMLATB, SMLATT, SMLAWB, +// SMLAWT, SMLAD{X}, SMLSD{X}, SMMLA{R}, SMMLS{R} +// Multiply-accumulate pipelines support late-forwarding of accumulate operands +// from similar μops, allowing a typical sequence of multiply-accumulate μops +// to issue one every 1 cycle (sched advance = 2). +def A57WriteMLA : SchedWriteRes<[A57UnitM]> { let Latency = 3; } +def A57WriteMLAL : SchedWriteRes<[A57UnitM]> { let Latency = 4; } +def A57ReadMLA : SchedReadAdvance<2, [A57WriteMLA, A57WriteMLAL]>; + +def : SchedAlias<WriteMAC16, A57WriteMLA>; +def : SchedAlias<WriteMAC32, A57WriteMLA>; +def : SchedAlias<ReadMAC, A57ReadMLA>; + +def : SchedAlias<WriteMAC64Lo, A57WriteMLAL>; +def : SchedAlias<WriteMAC64Hi, A57WriteMLAL>; + +// Multiply long: SMULL, UMULL +def : SchedAlias<WriteMUL64Lo, A57Write_4cyc_1M>; +def : SchedAlias<WriteMUL64Hi, A57Write_4cyc_1M>; + +// --- 3.6 Saturating and Parallel Arithmetic Instructions --- +// Parallel arith +// SADD16, SADD8, SSUB16, SSUB8, UADD16, UADD8, USUB16, USUB8 +// Conditional GE-setting instructions require three extra μops +// and two additional cycles to conditionally update the GE field. +def A57WriteParArith : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_4cyc_1I_1M]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1I_1M]> +]>; +def : InstRW< [A57WriteParArith], (instregex + "(t2)?SADD(16|8)", "(t2)?SSUB(16|8)", + "(t2)?UADD(16|8)", "(t2)?USUB(16|8)")>; + +// Parallel arith with exchange: SASX, SSAX, UASX, USAX +def A57WriteParArithExch : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_5cyc_1I_1M]>, + SchedVar<NoSchedPred, [A57Write_3cyc_1I_1M]> +]>; +def : InstRW<[A57WriteParArithExch], + (instregex "(t2)?SASX", "(t2)?SSAX", "(t2)?UASX", "(t2)?USAX")>; + +// Parallel halving arith +// SHADD16, SHADD8, SHSUB16, SHSUB8, UHADD16, UHADD8, UHSUB16, UHSUB8 +def : InstRW<[A57Write_2cyc_1M], (instregex + "(t2)?SHADD(16|8)", "(t2)?SHSUB(16|8)", + "(t2)?UHADD(16|8)", "(t2)?UHSUB(16|8)")>; + +// Parallel halving arith with exchange +// SHASX, SHSAX, UHASX, UHSAX +def : InstRW<[A57Write_3cyc_1I_1M], (instregex "(t2)?SHASX", "(t2)?SHSAX", + "(t2)?UHASX", "(t2)?UHSAX")>; + +// Parallel saturating arith +// QADD16, QADD8, QSUB16, QSUB8, UQADD16, UQADD8, UQSUB16, UQSUB8 +def : InstRW<[A57Write_2cyc_1M], (instregex "QADD(16|8)", "QSUB(16|8)", + "UQADD(16|8)", "UQSUB(16|8)", "t2(U?)QADD", "t2(U?)QSUB")>; + +// Parallel saturating arith with exchange +// QASX, QSAX, UQASX, UQSAX +def : InstRW<[A57Write_3cyc_1I_1M], (instregex "(t2)?QASX", "(t2)?QSAX", + "(t2)?UQASX", "(t2)?UQSAX")>; + +// Saturate: SSAT, SSAT16, USAT, USAT16 +def : InstRW<[A57Write_2cyc_1M], + (instregex "(t2)?SSAT(16)?", "(t2)?USAT(16)?")>; + +// Saturating arith: QADD, QSUB +def : InstRW<[A57Write_2cyc_1M], (instregex "QADD$", "QSUB$")>; + +// Saturating doubling arith: QDADD, QDSUB +def : InstRW<[A57Write_3cyc_1I_1M], (instregex "(t2)?QDADD", "(t2)?QDSUB")>; + +// --- 3.7 Miscellaneous Data-Processing Instructions --- +// Bit field extract: SBFX, UBFX +def : InstRW<[A57Write_1cyc_1I], (instregex "(t2)?SBFX", "(t2)?UBFX")>; + +// Bit field insert/clear: BFI, BFC +def : InstRW<[A57Write_2cyc_1M], (instregex "(t2)?BFI", "(t2)?BFC")>; + +// Select bytes, conditional/unconditional +def A57WriteSEL : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_2cyc_1I]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1I]> +]>; +def : InstRW<[A57WriteSEL], (instregex "(t2)?SEL")>; + +// Sign/zero extend, normal: SXTB, SXTH, UXTB, UXTH +def : InstRW<[A57Write_1cyc_1I], + (instregex "(t2|t)?SXT(B|H)$", "(t2|t)?UXT(B|H)$")>; + +// Sign/zero extend and add, normal: SXTAB, SXTAH, UXTAB, UXTAH +def : InstRW<[A57Write_2cyc_1M], + (instregex "(t2)?SXTA(B|H)$", "(t2)?UXTA(B|H)$")>; + +// Sign/zero extend and add, parallel: SXTAB16, UXTAB16 +def : InstRW<[A57Write_4cyc_1M], (instregex "(t2)?SXTAB16", "(t2)?UXTAB16")>; + +// Sum of absolute differences: USAD8, USADA8 +def : InstRW<[A57Write_3cyc_1M], (instregex "(t2)?USAD8", "(t2)?USADA8")>; + +// --- 3.8 Load Instructions --- + +// Load, immed offset +// LDR and LDRB have LDRi12 and LDRBi12 forms for immediate +def : InstRW<[A57Write_4cyc_1L], (instregex "LDRi12", "LDRBi12", + "LDRcp", "(t2|t)?LDRConstPool", "LDRLIT_ga_(pcrel|abs)", + "PICLDR", "tLDR")>; + +def : InstRW<[A57Write_4cyc_1L], + (instregex "t2LDRS?(B|H)?(pcrel|T|i8|i12|pci|pci_pic|s)?$")>; + +// For "Load, register offset, minus" we need +1cyc, +1I +def A57WriteLdrAm3 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPred, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L]> +]>; +def : InstRW<[A57WriteLdrAm3], (instregex "LDR(H|SH|SB)$")>; +def A57WriteLdrAm3X2 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX2, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L]> +]>; +def : InstRW<[A57WriteLdrAm3X2, A57WriteLdrAm3X2], (instregex "LDRD$")>; +def : InstRW<[A57Write_4cyc_1L, A57Write_4cyc_1L], (instregex "t2LDRDi8")>; + +def A57WriteLdrAmLDSTSO : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledNotOptimalPred, [A57Write_5cyc_1I_1L]>, + SchedVar<IsLdstsoMinusRegPred, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L]> +]>; +def : InstRW<[A57WriteLdrAmLDSTSO], (instregex "LDRrs", "LDRBrs")>; + +def A57WrBackOne : SchedWriteRes<[]> { + let Latency = 1; + let NumMicroOps = 0; +} +def A57WrBackTwo : SchedWriteRes<[]> { + let Latency = 2; + let NumMicroOps = 0; +} +def A57WrBackThree : SchedWriteRes<[]> { + let Latency = 3; + let NumMicroOps = 0; +} + +// --- LDR pre-indexed --- +// Load, immed pre-indexed (4 cyc for load result, 1 cyc for Base update) +def : InstRW<[A57Write_4cyc_1L_1I, A57WrBackOne], (instregex "LDR_PRE_IMM", + "LDRB_PRE_IMM", "t2LDRB_PRE")>; + +// Load, register pre-indexed (4 cyc for load result, 2 cyc for Base update) +// (5 cyc load result for not-lsl2 scaled) +def A57WriteLdrAmLDSTSOPre : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledNotOptimalPredX2, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L_1I]> +]>; +def : InstRW<[A57WriteLdrAmLDSTSOPre, A57WrBackTwo], + (instregex "LDR_PRE_REG", "LDRB_PRE_REG")>; + +def A57WriteLdrAm3PreWrBack : SchedWriteVariant<[ + SchedVar<IsLdrAm3RegOffPredX2, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57Write_4cyc_1L, A57WriteLdrAm3PreWrBack], + (instregex "LDR(H|SH|SB)_PRE")>; +def : InstRW<[A57Write_4cyc_1L, A57WrBackOne], + (instregex "t2LDR(H|SH|SB)?_PRE")>; + +// LDRD pre-indexed: 5(2) cyc for reg, 4(1) cyc for imm. +def A57WriteLdrDAm3Pre : SchedWriteVariant<[ + SchedVar<IsLdrAm3RegOffPredX3, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L_1I]> +]>; +def A57WriteLdrDAm3PreWrBack : SchedWriteVariant<[ + SchedVar<IsLdrAm3RegOffPredX3, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57WriteLdrDAm3Pre, A57WriteLdrDAm3Pre, A57WriteLdrDAm3PreWrBack], + (instregex "LDRD_PRE")>; +def : InstRW<[A57Write_4cyc_1L_1I, A57Write_4cyc_1L_1I, A57WrBackOne], + (instregex "t2LDRD_PRE")>; + +// --- LDR post-indexed --- +def : InstRW<[A57Write_4cyc_1L_1I, A57WrBackOne], (instregex "LDR(T?)_POST_IMM", + "LDRB(T?)_POST_IMM", "LDR(SB|H|SH)Ti", "t2LDRB_POST")>; + +def A57WriteLdrAm3PostWrBack : SchedWriteVariant<[ + SchedVar<IsLdrAm3RegOffPred, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57Write_4cyc_1L_1I, A57WriteLdrAm3PostWrBack], + (instregex "LDR(H|SH|SB)_POST")>; +def : InstRW<[A57Write_4cyc_1L, A57WrBackOne], + (instregex "t2LDR(H|SH|SB)?_POST")>; + +def : InstRW<[A57Write_4cyc_1L_1I, A57WrBackTwo], (instregex "LDR_POST_REG", + "LDRB_POST_REG", "LDR(B?)T_POST$")>; + +def A57WriteLdrTRegPost : SchedWriteVariant<[ + SchedVar<IsLdrAm2ScaledPred, [A57Write_4cyc_1I_1L_1M]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L_1I]> +]>; +def A57WriteLdrTRegPostWrBack : SchedWriteVariant<[ + SchedVar<IsLdrAm2ScaledPred, [A57WrBackThree]>, + SchedVar<NoSchedPred, [A57WrBackTwo]> +]>; +// 4(3) "I0/I1,L,M" for scaled register, otherwise 4(2) "I0/I1,L" +def : InstRW<[A57WriteLdrTRegPost, A57WriteLdrTRegPostWrBack], + (instregex "LDRT_POST_REG", "LDRBT_POST_REG")>; + +def : InstRW<[A57Write_4cyc_1L_1I, A57WrBackTwo], (instregex "LDR(SB|H|SH)Tr")>; + +def A57WriteLdrAm3PostWrBackX3 : SchedWriteVariant<[ + SchedVar<IsLdrAm3RegOffPredX3, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +// LDRD post-indexed: 4(2) cyc for reg, 4(1) cyc for imm. +def : InstRW<[A57Write_4cyc_1L_1I, A57Write_4cyc_1L_1I, + A57WriteLdrAm3PostWrBackX3], (instregex "LDRD_POST")>; +def : InstRW<[A57Write_4cyc_1L_1I, A57Write_4cyc_1L_1I, A57WrBackOne], + (instregex "t2LDRD_POST")>; + +// --- Preload instructions --- +// Preload, immed offset +def : InstRW<[A57Write_4cyc_1L], (instregex "(t2)?PLDi12", "(t2)?PLDWi12", + "t2PLDW?(i8|pci|s)", "(t2)?PLI")>; + +// Preload, register offset, +// 5cyc "I0/I1,L" for minus reg or scaled not plus lsl2 +// otherwise 4cyc "L" +def A57WritePLD : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledNotOptimalPredX0, [A57Write_5cyc_1I_1L]>, + SchedVar<IsLdstsoMinusRegPredX0, [A57Write_5cyc_1I_1L]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1L]> +]>; +def : InstRW<[A57WritePLD], (instregex "PLDrs", "PLDWrs")>; + +// --- Load multiple instructions --- +foreach NumAddr = 1-8 in { + def A57LMAddrPred#NumAddr : + SchedPredicate<"(TII->getLDMVariableDefsSize(*MI)+1)/2 == "#NumAddr>; +} + +def A57LDMOpsListNoregin : A57WriteLMOpsListType< + [A57Write_3cyc_1L, A57Write_3cyc_1L, + A57Write_4cyc_1L, A57Write_4cyc_1L, + A57Write_5cyc_1L, A57Write_5cyc_1L, + A57Write_6cyc_1L, A57Write_6cyc_1L, + A57Write_7cyc_1L, A57Write_7cyc_1L, + A57Write_8cyc_1L, A57Write_8cyc_1L, + A57Write_9cyc_1L, A57Write_9cyc_1L, + A57Write_10cyc_1L, A57Write_10cyc_1L]>; +def A57WriteLDMnoreginlist : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57LDMOpsListNoregin.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57LDMOpsListNoregin.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57LDMOpsListNoregin.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57LDMOpsListNoregin.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57LDMOpsListNoregin.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57LDMOpsListNoregin.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57LDMOpsListNoregin.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57LDMOpsListNoregin.Writes[0-15]>, + SchedVar<NoSchedPred, A57LDMOpsListNoregin.Writes[0-15]> +]> { let Variadic=1; } + +def A57LDMOpsListRegin : A57WriteLMOpsListType< + [A57Write_4cyc_1L_1I, A57Write_4cyc_1L_1I, + A57Write_5cyc_1L_1I, A57Write_5cyc_1L_1I, + A57Write_6cyc_1L_1I, A57Write_6cyc_1L_1I, + A57Write_7cyc_1L_1I, A57Write_7cyc_1L_1I, + A57Write_8cyc_1L_1I, A57Write_8cyc_1L_1I, + A57Write_9cyc_1L_1I, A57Write_9cyc_1L_1I, + A57Write_10cyc_1L_1I, A57Write_10cyc_1L_1I, + A57Write_11cyc_1L_1I, A57Write_11cyc_1L_1I]>; +def A57WriteLDMreginlist : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57LDMOpsListRegin.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57LDMOpsListRegin.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57LDMOpsListRegin.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57LDMOpsListRegin.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57LDMOpsListRegin.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57LDMOpsListRegin.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57LDMOpsListRegin.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57LDMOpsListRegin.Writes[0-15]>, + SchedVar<NoSchedPred, A57LDMOpsListRegin.Writes[0-15]> +]> { let Variadic=1; } + +def A57LDMOpsList_Upd : A57WriteLMOpsListType< + [A57WrBackOne, + A57Write_3cyc_1L_1I, A57Write_3cyc_1L_1I, + A57Write_4cyc_1L_1I, A57Write_4cyc_1L_1I, + A57Write_5cyc_1L_1I, A57Write_5cyc_1L_1I, + A57Write_6cyc_1L_1I, A57Write_6cyc_1L_1I, + A57Write_7cyc_1L_1I, A57Write_7cyc_1L_1I, + A57Write_8cyc_1L_1I, A57Write_8cyc_1L_1I, + A57Write_9cyc_1L_1I, A57Write_9cyc_1L_1I, + A57Write_10cyc_1L_1I, A57Write_10cyc_1L_1I]>; +def A57WriteLDM_Upd : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57LDMOpsList_Upd.Writes[0-2]>, + SchedVar<A57LMAddrPred2, A57LDMOpsList_Upd.Writes[0-4]>, + SchedVar<A57LMAddrPred3, A57LDMOpsList_Upd.Writes[0-6]>, + SchedVar<A57LMAddrPred4, A57LDMOpsList_Upd.Writes[0-8]>, + SchedVar<A57LMAddrPred5, A57LDMOpsList_Upd.Writes[0-10]>, + SchedVar<A57LMAddrPred6, A57LDMOpsList_Upd.Writes[0-12]>, + SchedVar<A57LMAddrPred7, A57LDMOpsList_Upd.Writes[0-14]>, + SchedVar<A57LMAddrPred8, A57LDMOpsList_Upd.Writes[0-16]>, + SchedVar<NoSchedPred, A57LDMOpsList_Upd.Writes[0-16]> +]> { let Variadic=1; } + +def A57WriteLDM : SchedWriteVariant<[ + SchedVar<IsLdmBaseRegInList, [A57WriteLDMreginlist]>, + SchedVar<NoSchedPred, [A57WriteLDMnoreginlist]> +]> { let Variadic=1; } + +def : InstRW<[A57WriteLDM], (instregex "(t|t2|sys)?LDM(IA|DA|DB|IB)$")>; + +// TODO: no writeback latency defined in documentation (implemented as 1 cyc) +def : InstRW<[A57WriteLDM_Upd], + (instregex "(t|t2|sys)?LDM(IA_UPD|DA_UPD|DB_UPD|IB_UPD|IA_RET)", "tPOP")>; + +// --- 3.9 Store Instructions --- + +// Store, immed offset +def : InstRW<[A57Write_1cyc_1S], (instregex "STRi12", "STRBi12", "PICSTR", + "t2STR(B?)(T|i12|i8|s)", "t2STRDi8", "t2STRH(i12|i8|s)", "tSTR")>; + +// Store, register offset +// For minus or for not plus lsl2 scaled we need 3cyc "I0/I1, S", +// otherwise 1cyc S. +def A57WriteStrAmLDSTSO : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledNotOptimalPred, [A57Write_3cyc_1I_1S]>, + SchedVar<IsLdstsoMinusRegPred, [A57Write_3cyc_1I_1S]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S]> +]>; +def : InstRW<[A57WriteStrAmLDSTSO], (instregex "STRrs", "STRBrs")>; + +// STRH,STRD: 3cyc "I0/I1, S" for minus reg, 1cyc S for imm or for plus reg. +def A57WriteStrAm3 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPred, [A57Write_3cyc_1I_1S]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S]> +]>; +def : InstRW<[A57WriteStrAm3], (instregex "STRH$")>; +def A57WriteStrAm3X2 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX2, [A57Write_3cyc_1I_1S]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S]> +]>; +def : InstRW<[A57WriteStrAm3X2], (instregex "STRD$")>; + +// Store, immed pre-indexed (1cyc "S, I0/I1", 1cyc writeback) +def : InstRW<[A57WrBackOne, A57Write_1cyc_1S_1I], (instregex "STR_PRE_IMM", + "STRB_PRE_IMM", "STR(B)?(r|i)_preidx", "(t2)?STRH_(preidx|PRE)", + "t2STR(B?)_(PRE|preidx)", "t2STRD_PRE")>; + +// Store, register pre-indexed: +// 1(1) "S, I0/I1" for plus reg +// 3(2) "I0/I1, S" for minus reg +// 1(2) "S, M" for scaled plus lsl2 +// 3(2) "I0/I1, S" for other scaled +def A57WriteStrAmLDSTSOPre : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledNotOptimalPredX2, [A57Write_3cyc_1I_1S]>, + SchedVar<IsLdstsoMinusRegPredX2, [A57Write_3cyc_1I_1S]>, + SchedVar<IsLdstsoScaledPredX2, [A57Write_1cyc_1S_1M]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S_1I]> +]>; +def A57WriteStrAmLDSTSOPreWrBack : SchedWriteVariant<[ + SchedVar<IsLdstsoScaledPredX2, [A57WrBackTwo]>, + SchedVar<IsLdstsoMinusRegPredX2, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57WriteStrAmLDSTSOPreWrBack, A57WriteStrAmLDSTSOPre], + (instregex "STR_PRE_REG", "STRB_PRE_REG")>; + +// pre-indexed STRH/STRD (STRH_PRE, STRD_PRE) +// 1(1) "S, I0/I1" for imm or reg plus +// 3(2) "I0/I1, S" for reg minus +def A57WriteStrAm3PreX2 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX2, [A57Write_3cyc_1I_1S]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S_1I]> +]>; +def A57WriteStrAm3PreWrBackX2 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX2, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57WriteStrAm3PreWrBackX2, A57WriteStrAm3PreX2], + (instregex "STRH_PRE")>; + +def A57WriteStrAm3PreX3 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX3, [A57Write_3cyc_1I_1S]>, + SchedVar<NoSchedPred, [A57Write_1cyc_1S_1I]> +]>; +def A57WriteStrAm3PreWrBackX3 : SchedWriteVariant<[ + SchedVar<IsLdrAm3NegRegOffPredX3, [A57WrBackTwo]>, + SchedVar<NoSchedPred, [A57WrBackOne]> +]>; +def : InstRW<[A57WriteStrAm3PreWrBackX3, A57WriteStrAm3PreX3], + (instregex "STRD_PRE")>; + +def : InstRW<[A57WrBackOne, A57Write_1cyc_1S_1I], (instregex "STR(T?)_POST_IMM", + "STRB(T?)_POST_IMM", "t2STR(B?)_POST")>; + +// 1(2) "S, M" for STR/STRB register post-indexed (both scaled or not) +def : InstRW<[A57WrBackTwo, A57Write_1cyc_1S_1M], (instregex "STR(T?)_POST_REG", + "STRB(T?)_POST_REG", "STR(B?)T_POST$")>; + +// post-indexed STRH/STRD(STRH_POST, STRD_POST), STRHTi, STRHTr +// 1(1) "S, I0/I1" both for reg or imm +def : InstRW<[A57WrBackOne, A57Write_1cyc_1S_1I], + (instregex "(t2)?STR(H|D)_POST", "STRHT(i|r)", "t2STRHT")>; + +// --- Store multiple instructions --- +// TODO: no writeback latency defined in documentation +def A57WriteSTM : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_1cyc_1S]>, + SchedVar<A57LMAddrPred2, [A57Write_2cyc_1S]>, + SchedVar<A57LMAddrPred3, [A57Write_3cyc_1S]>, + SchedVar<A57LMAddrPred4, [A57Write_4cyc_1S]>, + SchedVar<A57LMAddrPred5, [A57Write_5cyc_1S]>, + SchedVar<A57LMAddrPred6, [A57Write_6cyc_1S]>, + SchedVar<A57LMAddrPred7, [A57Write_7cyc_1S]>, + SchedVar<A57LMAddrPred8, [A57Write_8cyc_1S]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1S]> +]>; +def A57WriteSTM_Upd : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_1cyc_1S_1I]>, + SchedVar<A57LMAddrPred2, [A57Write_2cyc_1S_1I]>, + SchedVar<A57LMAddrPred3, [A57Write_3cyc_1S_1I]>, + SchedVar<A57LMAddrPred4, [A57Write_4cyc_1S_1I]>, + SchedVar<A57LMAddrPred5, [A57Write_5cyc_1S_1I]>, + SchedVar<A57LMAddrPred6, [A57Write_6cyc_1S_1I]>, + SchedVar<A57LMAddrPred7, [A57Write_7cyc_1S_1I]>, + SchedVar<A57LMAddrPred8, [A57Write_8cyc_1S_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1S_1I]> +]>; + +def : InstRW<[A57WriteSTM], (instregex "(t2|sys|t)?STM(IA|DA|DB|IB)$")>; +def : InstRW<[A57WrBackOne, A57WriteSTM_Upd], + (instregex "(t2|sys|t)?STM(IA_UPD|DA_UPD|DB_UPD|IB_UPD)", "tPUSH")>; + +// --- 3.10 FP Data Processing Instructions --- +def : SchedAlias<WriteFPALU32, A57Write_5cyc_1V>; +def : SchedAlias<WriteFPALU64, A57Write_5cyc_1V>; + +def : InstRW<[A57Write_3cyc_1V], (instregex "VABS(S|D|H)")>; + +// fp compare - 3cyc F1 for unconditional, 6cyc "F0/F1, F1" for conditional +def A57WriteVcmp : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57Write_6cyc_1V_1X]>, + SchedVar<NoSchedPred, [A57Write_3cyc_1X]> +]>; +def : InstRW<[A57WriteVcmp], + (instregex "VCMP(D|S|H|ZD|ZS|ZH)$", "VCMPE(D|S|H|ZD|ZS|ZH)")>; + +// fp convert +def : InstRW<[A57Write_5cyc_1V], (instregex + "VCVT(A|N|P|M)(SH|UH|SS|US|SD|UD)", "VCVT(BDH|THD|TDH)")>; + +def : SchedAlias<WriteFPCVT, A57Write_5cyc_1V>; + +// FP round to integral +def : InstRW<[A57Write_5cyc_1V], (instregex "VRINT(A|N|P|M|Z|R|X)(H|S|D)$")>; + +// FP divide, FP square root +def : SchedAlias<WriteFPDIV32, A57Write_17cyc_1W>; +def : SchedAlias<WriteFPDIV64, A57Write_32cyc_1W>; +def : SchedAlias<WriteFPSQRT32, A57Write_17cyc_1W>; +def : SchedAlias<WriteFPSQRT64, A57Write_32cyc_1W>; + +// FP max/min +def : InstRW<[A57Write_5cyc_1V], (instregex "VMAX", "VMIN")>; + +// FP multiply-accumulate pipelines support late forwarding of the result +// from FP multiply μops to the accumulate operands of an +// FP multiply-accumulate μop. The latter can potentially be issued 1 cycle +// after the FP multiply μop has been issued +// FP multiply, FZ +def A57WriteVMUL : SchedWriteRes<[A57UnitV]> { let Latency = 5; } + +def : SchedAlias<WriteFPMUL32, A57WriteVMUL>; +def : SchedAlias<WriteFPMUL64, A57WriteVMUL>; +def : ReadAdvance<ReadFPMUL, 0>; + +// FP multiply accumulate, FZ: 9cyc "F0/F1" or 4 cyc for sequenced accumulate +// VFMA, VFMS, VFNMA, VFNMS, VMLA, VMLS, VNMLA, VNMLS +def A57WriteVFMA : SchedWriteRes<[A57UnitV]> { let Latency = 9; } + +// VFMA takes 9 cyc for common case and 4 cyc for VFMA->VFMA chain (5 read adv.) +// VMUL takes 5 cyc for common case and 1 cyc for VMUL->VFMA chain (4 read adv.) +// Currently, there is no way to define different read advances for VFMA operand +// from VFMA or from VMUL, so there will be 5 read advance. +// Zero latency (instead of one) for VMUL->VFMA shouldn't break something. +// The same situation with ASIMD VMUL/VFMA instructions +// def A57ReadVFMA : SchedRead; +// def : ReadAdvance<A57ReadVFMA, 5, [A57WriteVFMA]>; +// def : ReadAdvance<A57ReadVFMA, 4, [A57WriteVMUL]>; +def A57ReadVFMA5 : SchedReadAdvance<5, [A57WriteVFMA, A57WriteVMUL]>; + +def : SchedAlias<WriteFPMAC32, A57WriteVFMA>; +def : SchedAlias<WriteFPMAC64, A57WriteVFMA>; +def : SchedAlias<ReadFPMAC, A57ReadVFMA5>; + +def : InstRW<[A57Write_3cyc_1V], (instregex "VNEG")>; +def : InstRW<[A57Write_3cyc_1V], (instregex "VSEL")>; + +// --- 3.11 FP Miscellaneous Instructions --- +// VMOV: 3cyc "F0/F1" for imm/reg +def : InstRW<[A57Write_3cyc_1V], (instregex "FCONST(D|S|H)")>; +def : InstRW<[A57Write_3cyc_1V], (instregex "VMOV(D|S|H)(cc)?$")>; + +// 5cyc L for FP transfer, vfp to core reg, +// 5cyc L for FP transfer, core reg to vfp +def : SchedAlias<WriteFPMOV, A57Write_5cyc_1L>; +// VMOVRRS/VMOVRRD in common code declared with one WriteFPMOV (instead of 2). +def : InstRW<[A57Write_5cyc_1L, A57Write_5cyc_1L], (instregex "VMOV(RRS|RRD)")>; + +// 8cyc "L,F0/F1" for FP transfer, core reg to upper or lower half of vfp D-reg +def : InstRW<[A57Write_8cyc_1L_1I], (instregex "VMOVDRR")>; + +// --- 3.12 FP Load Instructions --- +def : InstRW<[A57Write_5cyc_1L], (instregex "VLDR(D|S|H)")>; + +def : InstRW<[A57Write_5cyc_1L], (instregex "VLDMQIA$")>; + +// FP load multiple (VLDM) + +def A57VLDMOpsListUncond : A57WriteLMOpsListType< + [A57Write_5cyc_1L, A57Write_5cyc_1L, + A57Write_6cyc_1L, A57Write_6cyc_1L, + A57Write_7cyc_1L, A57Write_7cyc_1L, + A57Write_8cyc_1L, A57Write_8cyc_1L, + A57Write_9cyc_1L, A57Write_9cyc_1L, + A57Write_10cyc_1L, A57Write_10cyc_1L, + A57Write_11cyc_1L, A57Write_11cyc_1L, + A57Write_12cyc_1L, A57Write_12cyc_1L]>; +def A57WriteVLDMuncond : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57VLDMOpsListUncond.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57VLDMOpsListUncond.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57VLDMOpsListUncond.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57VLDMOpsListUncond.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57VLDMOpsListUncond.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57VLDMOpsListUncond.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57VLDMOpsListUncond.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57VLDMOpsListUncond.Writes[0-15]>, + SchedVar<NoSchedPred, A57VLDMOpsListUncond.Writes[0-15]> +]> { let Variadic=1; } + +def A57VLDMOpsListCond : A57WriteLMOpsListType< + [A57Write_5cyc_1L, A57Write_6cyc_1L, + A57Write_7cyc_1L, A57Write_8cyc_1L, + A57Write_9cyc_1L, A57Write_10cyc_1L, + A57Write_11cyc_1L, A57Write_12cyc_1L, + A57Write_13cyc_1L, A57Write_14cyc_1L, + A57Write_15cyc_1L, A57Write_16cyc_1L, + A57Write_17cyc_1L, A57Write_18cyc_1L, + A57Write_19cyc_1L, A57Write_20cyc_1L]>; +def A57WriteVLDMcond : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57VLDMOpsListCond.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57VLDMOpsListCond.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57VLDMOpsListCond.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57VLDMOpsListCond.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57VLDMOpsListCond.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57VLDMOpsListCond.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57VLDMOpsListCond.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57VLDMOpsListCond.Writes[0-15]>, + SchedVar<NoSchedPred, A57VLDMOpsListCond.Writes[0-15]> +]> { let Variadic=1; } + +def A57WriteVLDM : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57WriteVLDMcond]>, + SchedVar<NoSchedPred, [A57WriteVLDMuncond]> +]> { let Variadic=1; } + +def : InstRW<[A57WriteVLDM], (instregex "VLDM(DIA|SIA)$")>; + +def A57VLDMOpsListUncond_Upd : A57WriteLMOpsListType< + [A57Write_5cyc_1L_1I, A57Write_5cyc_1L_1I, + A57Write_6cyc_1L_1I, A57Write_6cyc_1L_1I, + A57Write_7cyc_1L_1I, A57Write_7cyc_1L_1I, + A57Write_8cyc_1L_1I, A57Write_8cyc_1L_1I, + A57Write_9cyc_1L_1I, A57Write_9cyc_1L_1I, + A57Write_10cyc_1L_1I, A57Write_10cyc_1L_1I, + A57Write_11cyc_1L_1I, A57Write_11cyc_1L_1I, + A57Write_12cyc_1L_1I, A57Write_12cyc_1L_1I]>; +def A57WriteVLDMuncond_UPD : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57VLDMOpsListUncond_Upd.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57VLDMOpsListUncond_Upd.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57VLDMOpsListUncond_Upd.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57VLDMOpsListUncond_Upd.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57VLDMOpsListUncond_Upd.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57VLDMOpsListUncond_Upd.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57VLDMOpsListUncond_Upd.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57VLDMOpsListUncond_Upd.Writes[0-15]>, + SchedVar<NoSchedPred, A57VLDMOpsListUncond_Upd.Writes[0-15]> +]> { let Variadic=1; } + +def A57VLDMOpsListCond_Upd : A57WriteLMOpsListType< + [A57Write_5cyc_1L_1I, A57Write_6cyc_1L_1I, + A57Write_7cyc_1L_1I, A57Write_8cyc_1L_1I, + A57Write_9cyc_1L_1I, A57Write_10cyc_1L_1I, + A57Write_11cyc_1L_1I, A57Write_12cyc_1L_1I, + A57Write_13cyc_1L_1I, A57Write_14cyc_1L_1I, + A57Write_15cyc_1L_1I, A57Write_16cyc_1L_1I, + A57Write_17cyc_1L_1I, A57Write_18cyc_1L_1I, + A57Write_19cyc_1L_1I, A57Write_20cyc_1L_1I]>; +def A57WriteVLDMcond_UPD : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, A57VLDMOpsListCond_Upd.Writes[0-1]>, + SchedVar<A57LMAddrPred2, A57VLDMOpsListCond_Upd.Writes[0-3]>, + SchedVar<A57LMAddrPred3, A57VLDMOpsListCond_Upd.Writes[0-5]>, + SchedVar<A57LMAddrPred4, A57VLDMOpsListCond_Upd.Writes[0-7]>, + SchedVar<A57LMAddrPred5, A57VLDMOpsListCond_Upd.Writes[0-9]>, + SchedVar<A57LMAddrPred6, A57VLDMOpsListCond_Upd.Writes[0-11]>, + SchedVar<A57LMAddrPred7, A57VLDMOpsListCond_Upd.Writes[0-13]>, + SchedVar<A57LMAddrPred8, A57VLDMOpsListCond_Upd.Writes[0-15]>, + SchedVar<NoSchedPred, A57VLDMOpsListCond_Upd.Writes[0-15]> +]> { let Variadic=1; } + +def A57WriteVLDM_UPD : SchedWriteVariant<[ + SchedVar<IsPredicatedPred, [A57WriteVLDMcond_UPD]>, + SchedVar<NoSchedPred, [A57WriteVLDMuncond_UPD]> +]> { let Variadic=1; } + +def : InstRW<[A57WrBackOne, A57WriteVLDM_UPD], + (instregex "VLDM(DIA_UPD|DDB_UPD|SIA_UPD|SDB_UPD)")>; + +// --- 3.13 FP Store Instructions --- +def : InstRW<[A57Write_1cyc_1S], (instregex "VSTR(D|S|H)")>; + +def : InstRW<[A57Write_2cyc_1S], (instregex "VSTMQIA$")>; + +def A57WriteVSTMs : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_1cyc_1S]>, + SchedVar<A57LMAddrPred2, [A57Write_2cyc_1S]>, + SchedVar<A57LMAddrPred3, [A57Write_3cyc_1S]>, + SchedVar<A57LMAddrPred4, [A57Write_4cyc_1S]>, + SchedVar<A57LMAddrPred5, [A57Write_5cyc_1S]>, + SchedVar<A57LMAddrPred6, [A57Write_6cyc_1S]>, + SchedVar<A57LMAddrPred7, [A57Write_7cyc_1S]>, + SchedVar<A57LMAddrPred8, [A57Write_8cyc_1S]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1S]> +]>; +def A57WriteVSTMd : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_2cyc_1S]>, + SchedVar<A57LMAddrPred2, [A57Write_4cyc_1S]>, + SchedVar<A57LMAddrPred3, [A57Write_6cyc_1S]>, + SchedVar<A57LMAddrPred4, [A57Write_8cyc_1S]>, + SchedVar<A57LMAddrPred5, [A57Write_10cyc_1S]>, + SchedVar<A57LMAddrPred6, [A57Write_12cyc_1S]>, + SchedVar<A57LMAddrPred7, [A57Write_14cyc_1S]>, + SchedVar<A57LMAddrPred8, [A57Write_16cyc_1S]>, + SchedVar<NoSchedPred, [A57Write_4cyc_1S]> +]>; +def A57WriteVSTMs_Upd : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_1cyc_1S_1I]>, + SchedVar<A57LMAddrPred2, [A57Write_2cyc_1S_1I]>, + SchedVar<A57LMAddrPred3, [A57Write_3cyc_1S_1I]>, + SchedVar<A57LMAddrPred4, [A57Write_4cyc_1S_1I]>, + SchedVar<A57LMAddrPred5, [A57Write_5cyc_1S_1I]>, + SchedVar<A57LMAddrPred6, [A57Write_6cyc_1S_1I]>, + SchedVar<A57LMAddrPred7, [A57Write_7cyc_1S_1I]>, + SchedVar<A57LMAddrPred8, [A57Write_8cyc_1S_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1S_1I]> +]>; +def A57WriteVSTMd_Upd : SchedWriteVariant<[ + SchedVar<A57LMAddrPred1, [A57Write_2cyc_1S_1I]>, + SchedVar<A57LMAddrPred2, [A57Write_4cyc_1S_1I]>, + SchedVar<A57LMAddrPred3, [A57Write_6cyc_1S_1I]>, + SchedVar<A57LMAddrPred4, [A57Write_8cyc_1S_1I]>, + SchedVar<A57LMAddrPred5, [A57Write_10cyc_1S_1I]>, + SchedVar<A57LMAddrPred6, [A57Write_12cyc_1S_1I]>, + SchedVar<A57LMAddrPred7, [A57Write_14cyc_1S_1I]>, + SchedVar<A57LMAddrPred8, [A57Write_16cyc_1S_1I]>, + SchedVar<NoSchedPred, [A57Write_2cyc_1S_1I]> +]>; + +def : InstRW<[A57WriteVSTMs], (instregex "VSTMSIA$")>; +def : InstRW<[A57WriteVSTMd], (instregex "VSTMDIA$")>; +def : InstRW<[A57WrBackOne, A57WriteVSTMs_Upd], + (instregex "VSTM(SIA_UPD|SDB_UPD)")>; +def : InstRW<[A57WrBackOne, A57WriteVSTMd_Upd], + (instregex "VSTM(DIA_UPD|DDB_UPD)")>; + +// --- 3.14 ASIMD Integer Instructions --- + +// ASIMD absolute diff, 3cyc F0/F1 for integer VABD +def : InstRW<[A57Write_3cyc_1V], (instregex "VABD(s|u)")>; + +// ASIMD absolute diff accum: 4(1) F1 for D-form, 5(2) F1 for Q-form +def A57WriteVABAD : SchedWriteRes<[A57UnitX]> { let Latency = 4; } +def A57ReadVABAD : SchedReadAdvance<3, [A57WriteVABAD]>; +def : InstRW<[A57WriteVABAD, A57ReadVABAD], + (instregex "VABA(s|u)(v8i8|v4i16|v2i32)")>; +def A57WriteVABAQ : SchedWriteRes<[A57UnitX]> { let Latency = 5; } +def A57ReadVABAQ : SchedReadAdvance<3, [A57WriteVABAQ]>; +def : InstRW<[A57WriteVABAQ, A57ReadVABAQ], + (instregex "VABA(s|u)(v16i8|v8i16|v4i32)")>; + +// ASIMD absolute diff accum long: 4(1) F1 for VABAL +def A57WriteVABAL : SchedWriteRes<[A57UnitX]> { let Latency = 4; } +def A57ReadVABAL : SchedReadAdvance<3, [A57WriteVABAL]>; +def : InstRW<[A57WriteVABAL, A57ReadVABAL], (instregex "VABAL(s|u)")>; + +// ASIMD absolute diff long: 3cyc F0/F1 for VABDL +def : InstRW<[A57Write_3cyc_1V], (instregex "VABDL(s|u)")>; + +// ASIMD arith, basic +def : InstRW<[A57Write_3cyc_1V], (instregex "VADD", "VADDL", "VADDW", + "VNEG(s8d|s16d|s32d|s8q|s16q|s32q|d|q)", + "VPADDi", "VPADDL", "VSUB", "VSUBL", "VSUBW")>; + +// ASIMD arith, complex +def : InstRW<[A57Write_3cyc_1V], (instregex "VABS", "VADDHN", "VHADD", "VHSUB", + "VQABS", "VQADD", "VQNEG", "VQSUB", + "VRADDHN", "VRHADD", "VRSUBHN", "VSUBHN")>; + +// ASIMD compare +def : InstRW<[A57Write_3cyc_1V], + (instregex "VCEQ", "VCGE", "VCGT", "VCLE", "VTST", "VCLT")>; + +// ASIMD logical +def : InstRW<[A57Write_3cyc_1V], + (instregex "VAND", "VBIC", "VMVN", "VORR", "VORN", "VEOR")>; + +// ASIMD max/min +def : InstRW<[A57Write_3cyc_1V], + (instregex "(VMAX|VMIN)(s|u)", "(VPMAX|VPMIN)(s8|s16|s32|u8|u16|u32)")>; + +// ASIMD multiply, D-form: 5cyc F0 for r0px, 4cyc F0 for r1p0 and later +// Cortex-A57 r1p0 and later reduce the latency of ASIMD multiply +// and multiply-with-accumulate instructions relative to r0pX. +def A57WriteVMULD_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_5cyc_1W]>]>; +def : InstRW<[A57WriteVMULD_VecInt], (instregex + "VMUL(v8i8|v4i16|v2i32|pd)", "VMULsl(v4i16|v2i32)", + "VQDMULH(sl)?(v4i16|v2i32)", "VQRDMULH(sl)?(v4i16|v2i32)")>; + +// ASIMD multiply, Q-form: 6cyc F0 for r0px, 5cyc F0 for r1p0 and later +def A57WriteVMULQ_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_5cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_6cyc_1W]>]>; +def : InstRW<[A57WriteVMULQ_VecInt], (instregex + "VMUL(v16i8|v8i16|v4i32|pq)", "VMULsl(v8i16|v4i32)", + "VQDMULH(sl)?(v8i16|v4i32)", "VQRDMULH(sl)?(v8i16|v4i32)")>; + +// ASIMD multiply accumulate, D-form +// 5cyc F0 for r0px, 4cyc F0 for r1p0 and later, 1cyc for accumulate sequence +// (4 or 3 ReadAdvance) +def A57WriteVMLAD_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_5cyc_1W]>]>; +def A57ReadVMLAD_VecInt : SchedReadVariant<[ + SchedVar<IsR1P0AndLaterPred, [SchedReadAdvance<3, [A57WriteVMLAD_VecInt]>]>, + SchedVar<NoSchedPred, [SchedReadAdvance<4, [A57WriteVMLAD_VecInt]>]> +]>; +def : InstRW<[A57WriteVMLAD_VecInt, A57ReadVMLAD_VecInt], + (instregex "VMLA(sl)?(v8i8|v4i16|v2i32)", "VMLS(sl)?(v8i8|v4i16|v2i32)")>; + +// ASIMD multiply accumulate, Q-form +// 6cyc F0 for r0px, 5cyc F0 for r1p0 and later, 2cyc for accumulate sequence +// (4 or 3 ReadAdvance) +def A57WriteVMLAQ_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_5cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_6cyc_1W]>]>; +def A57ReadVMLAQ_VecInt : SchedReadVariant<[ + SchedVar<IsR1P0AndLaterPred, [SchedReadAdvance<3, [A57WriteVMLAQ_VecInt]>]>, + SchedVar<NoSchedPred, [SchedReadAdvance<4, [A57WriteVMLAQ_VecInt]>]> +]>; +def : InstRW<[A57WriteVMLAQ_VecInt, A57ReadVMLAQ_VecInt], + (instregex "VMLA(sl)?(v16i8|v8i16|v4i32)", "VMLS(sl)?(v16i8|v8i16|v4i32)")>; + +// ASIMD multiply accumulate long +// 5cyc F0 for r0px, 4cyc F0 for r1p0 and later, 1cyc for accumulate sequence +// (4 or 3 ReadAdvance) +def A57WriteVMLAL_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_5cyc_1W]>]>; +def A57ReadVMLAL_VecInt : SchedReadVariant<[ + SchedVar<IsR1P0AndLaterPred, [SchedReadAdvance<3, [A57WriteVMLAL_VecInt]>]>, + SchedVar<NoSchedPred, [SchedReadAdvance<4, [A57WriteVMLAL_VecInt]>]> +]>; +def : InstRW<[A57WriteVMLAL_VecInt, A57ReadVMLAL_VecInt], + (instregex "VMLAL(s|u)", "VMLSL(s|u)")>; + +// ASIMD multiply accumulate saturating long +// 5cyc F0 for r0px, 4cyc F0 for r1p0 and later, 2cyc for accumulate sequence +// (3 or 2 ReadAdvance) +def A57WriteVQDMLAL_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_5cyc_1W]>]>; +def A57ReadVQDMLAL_VecInt : SchedReadVariant<[ + SchedVar<IsR1P0AndLaterPred, [SchedReadAdvance<2, [A57WriteVQDMLAL_VecInt]>]>, + SchedVar<NoSchedPred, [SchedReadAdvance<3, [A57WriteVQDMLAL_VecInt]>]> +]>; +def : InstRW<[A57WriteVQDMLAL_VecInt, A57ReadVQDMLAL_VecInt], + (instregex "VQDMLAL", "VQDMLSL")>; + +// ASIMD multiply long +// 5cyc F0 for r0px, 4cyc F0 for r1p0 and later +def A57WriteVMULL_VecInt : SchedWriteVariant<[ + SchedVar<IsR1P0AndLaterPred, [A57Write_4cyc_1W]>, + SchedVar<NoSchedPred, [A57Write_5cyc_1W]>]>; +def : InstRW<[A57WriteVMULL_VecInt], + (instregex "VMULL(s|u|p8|sls|slu)", "VQDMULL")>; + +// ASIMD pairwise add and accumulate +// 4cyc F1, 1cyc for accumulate sequence (3cyc ReadAdvance) +def A57WriteVPADAL : SchedWriteRes<[A57UnitX]> { let Latency = 4; } +def A57ReadVPADAL : SchedReadAdvance<3, [A57WriteVPADAL]>; +def : InstRW<[A57WriteVPADAL, A57ReadVPADAL], (instregex "VPADAL(s|u)")>; + +// ASIMD shift accumulate +// 4cyc F1, 1cyc for accumulate sequence (3cyc ReadAdvance) +def A57WriteVSRA : SchedWriteRes<[A57UnitX]> { let Latency = 4; } +def A57ReadVSRA : SchedReadAdvance<3, [A57WriteVSRA]>; +def : InstRW<[A57WriteVSRA, A57ReadVSRA], (instregex "VSRA", "VRSRA")>; + +// ASIMD shift by immed, basic +def : InstRW<[A57Write_3cyc_1X], + (instregex "VMOVL", "VSHLi", "VSHLL", "VSHR(s|u)", "VSHRN")>; + +// ASIMD shift by immed, complex +def : InstRW<[A57Write_4cyc_1X], (instregex + "VQRSHRN", "VQRSHRUN", "VQSHL(si|ui|su)", "VQSHRN", "VQSHRUN", "VRSHR(s|u)", + "VRSHRN")>; + +// ASIMD shift by immed and insert, basic, D-form +def : InstRW<[A57Write_4cyc_1X], (instregex + "VSLI(v8i8|v4i16|v2i32|v1i64)", "VSRI(v8i8|v4i16|v2i32|v1i64)")>; + +// ASIMD shift by immed and insert, basic, Q-form +def : InstRW<[A57Write_5cyc_1X], (instregex + "VSLI(v16i8|v8i16|v4i32|v2i64)", "VSRI(v16i8|v8i16|v4i32|v2i64)")>; + +// ASIMD shift by register, basic, D-form +def : InstRW<[A57Write_3cyc_1X], (instregex + "VSHL(s|u)(v8i8|v4i16|v2i32|v1i64)")>; + +// ASIMD shift by register, basic, Q-form +def : InstRW<[A57Write_4cyc_1X], (instregex + "VSHL(s|u)(v16i8|v8i16|v4i32|v2i64)")>; + +// ASIMD shift by register, complex, D-form +// VQRSHL, VQSHL, VRSHL +def : InstRW<[A57Write_4cyc_1X], (instregex + "VQRSHL(s|u)(v8i8|v4i16|v2i32|v1i64)", "VQSHL(s|u)(v8i8|v4i16|v2i32|v1i64)", + "VRSHL(s|u)(v8i8|v4i16|v2i32|v1i64)")>; + +// ASIMD shift by register, complex, Q-form +def : InstRW<[A57Write_5cyc_1X], (instregex + "VQRSHL(s|u)(v16i8|v8i16|v4i32|v2i64)", "VQSHL(s|u)(v16i8|v8i16|v4i32|v2i64)", + "VRSHL(s|u)(v16i8|v8i16|v4i32|v2i64)")>; + +// --- 3.15 ASIMD Floating-Point Instructions --- +// ASIMD FP absolute value +def : InstRW<[A57Write_3cyc_1V], (instregex "VABS(fd|fq|hd|hq)")>; + +// ASIMD FP arith +def : InstRW<[A57Write_5cyc_1V], (instregex "VABD(fd|fq|hd|hq)", + "VADD(fd|fq|hd|hq)", "VPADD(f|h)", "VSUB(fd|fq|hd|hq)")>; + +// ASIMD FP compare +def : InstRW<[A57Write_5cyc_1V], (instregex "VAC(GE|GT|LE|LT)", + "VC(EQ|GE|GT|LE)(fd|fq|hd|hq)")>; + +// ASIMD FP convert, integer +def : InstRW<[A57Write_5cyc_1V], (instregex + "VCVT(f2sd|f2ud|s2fd|u2fd|f2sq|f2uq|s2fq|u2fq|f2xsd|f2xud|xs2fd|xu2fd)", + "VCVT(f2xsq|f2xuq|xs2fq|xu2fq)", + "VCVT(AN|MN|NN|PN)(SDf|SQf|UDf|UQf|SDh|SQh|UDh|UQh)")>; + +// ASIMD FP convert, half-precision: 8cyc F0/F1 +def : InstRW<[A57Write_8cyc_1V], (instregex + "VCVT(h2sd|h2ud|s2hd|u2hd|h2sq|h2uq|s2hq|u2hq|h2xsd|h2xud|xs2hd|xu2hd)", + "VCVT(h2xsq|h2xuq|xs2hq|xu2hq)", + "VCVT(f2h|h2f)")>; + +// ASIMD FP max/min +def : InstRW<[A57Write_5cyc_1V], (instregex + "(VMAX|VMIN)(fd|fq|hd|hq)", "(VPMAX|VPMIN)(f|h)", "VMAXNM", "VMINNM")>; + +// ASIMD FP multiply +def A57WriteVMUL_VecFP : SchedWriteRes<[A57UnitV]> { let Latency = 5; } +def : InstRW<[A57WriteVMUL_VecFP], (instregex "VMUL(sl)?(fd|fq|hd|hq)")>; + +// ASIMD FP multiply accumulate: 9cyc F0/F1, 4cyc for accumulate sequence +def A57WriteVMLA_VecFP : SchedWriteRes<[A57UnitV]> { let Latency = 9; } +def A57ReadVMLA_VecFP : + SchedReadAdvance<5, [A57WriteVMLA_VecFP, A57WriteVMUL_VecFP]>; +def : InstRW<[A57WriteVMLA_VecFP, A57ReadVMLA_VecFP], + (instregex "(VMLA|VMLS)(sl)?(fd|fq|hd|hq)", "(VFMA|VFMS)(fd|fq|hd|hq)")>; + +// ASIMD FP negate +def : InstRW<[A57Write_3cyc_1V], (instregex "VNEG(fd|f32q|hd|hq)")>; + +// ASIMD FP round to integral +def : InstRW<[A57Write_5cyc_1V], (instregex + "VRINT(AN|MN|NN|PN|XN|ZN)(Df|Qf|Dh|Qh)")>; + +// --- 3.16 ASIMD Miscellaneous Instructions --- + +// ASIMD bitwise insert +def : InstRW<[A57Write_3cyc_1V], (instregex "VBIF", "VBIT", "VBSL")>; + +// ASIMD count +def : InstRW<[A57Write_3cyc_1V], (instregex "VCLS", "VCLZ", "VCNT")>; + +// ASIMD duplicate, core reg: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V], (instregex "VDUP(8|16|32)(d|q)")>; + +// ASIMD duplicate, scalar: 3cyc "F0/F1" +def : InstRW<[A57Write_3cyc_1V], (instregex "VDUPLN(8|16|32)(d|q)")>; + +// ASIMD extract +def : InstRW<[A57Write_3cyc_1V], (instregex "VEXT(d|q)(8|16|32|64)")>; + +// ASIMD move, immed +def : InstRW<[A57Write_3cyc_1V], (instregex + "VMOV(v8i8|v16i8|v4i16|v8i16|v2i32|v4i32|v1i64|v2i64|v2f32|v4f32)", + "VMOVQ0")>; + +// ASIMD move, narrowing +def : InstRW<[A57Write_3cyc_1V], (instregex "VMOVN")>; + +// ASIMD move, saturating +def : InstRW<[A57Write_4cyc_1X], (instregex "VQMOVN")>; + +// ASIMD reciprocal estimate +def : InstRW<[A57Write_5cyc_1V], (instregex "VRECPE", "VRSQRTE")>; + +// ASIMD reciprocal step, FZ +def : InstRW<[A57Write_9cyc_1V], (instregex "VRECPS", "VRSQRTS")>; + +// ASIMD reverse, swap, table lookup (1-2 reg) +def : InstRW<[A57Write_3cyc_1V], (instregex "VREV", "VSWP", "VTB(L|X)(1|2)")>; + +// ASIMD table lookup (3-4 reg) +def : InstRW<[A57Write_6cyc_1V], (instregex "VTBL(3|4)", "VTBX(3|4)")>; + +// ASIMD transfer, scalar to core reg: 6cyc "L, I0/I1" +def : InstRW<[A57Write_6cyc_1L_1I], (instregex "VGETLN")>; + +// ASIMD transfer, core reg to scalar: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V], (instregex "VSETLN")>; + +// ASIMD transpose +def : InstRW<[A57Write_3cyc_1V, A57Write_3cyc_1V], (instregex "VTRN")>; + +// ASIMD unzip/zip, D-form +def : InstRW<[A57Write_3cyc_1V, A57Write_3cyc_1V], + (instregex "VUZPd", "VZIPd")>; + +// ASIMD unzip/zip, Q-form +def : InstRW<[A57Write_6cyc_1V, A57Write_6cyc_1V], + (instregex "VUZPq", "VZIPq")>; + +// --- 3.17 ASIMD Load Instructions --- + +// Overriden via InstRW for this processor. +def : WriteRes<WriteVLD1, []>; +def : WriteRes<WriteVLD2, []>; +def : WriteRes<WriteVLD3, []>; +def : WriteRes<WriteVLD4, []>; +def : WriteRes<WriteVST1, []>; +def : WriteRes<WriteVST2, []>; +def : WriteRes<WriteVST3, []>; +def : WriteRes<WriteVST4, []>; + +// 1-2 reg: 5cyc L, +I for writeback, 1 cyc wb latency +def : InstRW<[A57Write_5cyc_1L], (instregex "VLD1(d|q)(8|16|32|64)$")>; +def : InstRW<[A57Write_5cyc_1L_1I, A57WrBackOne], + (instregex "VLD1(d|q)(8|16|32|64)wb")>; + +// 3-4 reg: 6cyc L, +I for writeback, 1 cyc wb latency +def : InstRW<[A57Write_6cyc_1L], + (instregex "VLD1(d|q)(8|16|32|64)(T|Q)$", "VLD1d64(T|Q)Pseudo")>; + +def : InstRW<[A57Write_6cyc_1L_1I, A57WrBackOne], + (instregex "VLD1(d|q)(8|16|32|64)(T|Q)wb")>; + +// ASIMD load, 1 element, one lane and all lanes: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V], (instregex + "VLD1(LN|DUP)(d|q)(8|16|32)$", "VLD1(LN|DUP)(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], (instregex + "VLD1(LN|DUP)(d|q)(8|16|32)(wb|_UPD)", "VLD1LNq(8|16|32)Pseudo_UPD")>; + +// ASIMD load, 2 element, multiple, 2 reg: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V], + (instregex "VLD2(d|q)(8|16|32)$", "VLD2q(8|16|32)Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD2(d|q)(8|16|32)wb", "VLD2q(8|16|32)PseudoWB")>; + +// ASIMD load, 2 element, multiple, 4 reg: 9cyc "L, F0/F1" +def : InstRW<[A57Write_9cyc_1L_1V], (instregex "VLD2b(8|16|32)$")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD2b(8|16|32)wb")>; + +// ASIMD load, 2 element, one lane and all lanes: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V], + (instregex "VLD2(DUP|LN)(d|q)(8|16|32|8x2|16x2|32x2)$", + "VLD2LN(d|q)(8|16|32)Pseudo$")>; +// 2 results + wb result +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V, A57WrBackOne], + (instregex "VLD2LN(d|q)(8|16|32)_UPD$")>; +// 1 result + wb result +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD2DUPd(8|16|32|8x2|16x2|32x2)wb", + "VLD2LN(d|q)(8|16|32)Pseudo_UPD")>; + +// ASIMD load, 3 element, multiple, 3 reg: 9cyc "L, F0/F1" +// 3 results +def : InstRW<[A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V], + (instregex "VLD3(d|q)(8|16|32)$")>; +// 1 result +def : InstRW<[A57Write_9cyc_1L_1V], + (instregex "VLD3(d|q)(8|16|32)(oddP|P)seudo$")>; +// 3 results + wb +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, + A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3(d|q)(8|16|32)_UPD$")>; +// 1 result + wb +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3(d|q)(8|16|32)(oddP|P)seudo_UPD")>; + +// ASIMD load, 3 element, one lane, size 32: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V], + (instregex "VLD3LN(d|q)32$", + "VLD3LN(d|q)32Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3LN(d|q)32_UPD")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3LN(d|q)32Pseudo_UPD")>; + +// ASIMD load, 3 element, one lane, size 8/16: 9cyc "L, F0/F1" +def : InstRW<[A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V], + (instregex "VLD3LN(d|q)(8|16)$", + "VLD3LN(d|q)(8|16)Pseudo$")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, + A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3LN(d|q)(8|16)_UPD")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3LN(d|q)(8|16)Pseudo_UPD")>; + +// ASIMD load, 3 element, all lanes: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V], + (instregex "VLD3DUP(d|q)(8|16|32)$", + "VLD3DUP(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3DUP(d|q)(8|16|32)_UPD")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD3DUP(d|q)(8|16|32)Pseudo_UPD")>; + +// ASIMD load, 4 element, multiple, 4 reg: 9cyc "L, F0/F1" +def : InstRW<[A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, + A57Write_9cyc_1L_1V], + (instregex "VLD4(d|q)(8|16|32)$")>; +def : InstRW<[A57Write_9cyc_1L_1V], + (instregex "VLD4(d|q)(8|16|32)(oddP|P)seudo$")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, + A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD4(d|q)(8|16|32)_UPD")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD4(d|q)(8|16|32)(oddP|P)seudo_UPD")>; + +// ASIMD load, 4 element, one lane, size 32: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, + A57Write_8cyc_1L_1V], + (instregex "VLD4LN(d|q)32$", + "VLD4LN(d|q)32Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57WrBackOne], + (instregex "VLD4LN(d|q)32_UPD")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD4LN(d|q)32Pseudo_UPD")>; + +// ASIMD load, 4 element, one lane, size 8/16: 9cyc "L, F0/F1" +def : InstRW<[A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, A57Write_9cyc_1L_1V, + A57Write_9cyc_1L_1V], + (instregex "VLD4LN(d|q)(8|16)$", + "VLD4LN(d|q)(8|16)Pseudo$")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, + A57Write_9cyc_1L_1V_1I, A57Write_9cyc_1L_1V_1I, + A57WrBackOne], + (instregex "VLD4LN(d|q)(8|16)_UPD")>; +def : InstRW<[A57Write_9cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD4LN(d|q)(8|16)Pseudo_UPD")>; + +// ASIMD load, 4 element, all lanes: 8cyc "L, F0/F1" +def : InstRW<[A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, A57Write_8cyc_1L_1V, + A57Write_8cyc_1L_1V], + (instregex "VLD4DUP(d|q)(8|16|32)$", + "VLD4DUP(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57Write_8cyc_1L_1V_1I, A57Write_8cyc_1L_1V_1I, + A57WrBackOne], + (instregex "VLD4DUP(d|q)(8|16|32)_UPD")>; +def : InstRW<[A57Write_8cyc_1L_1V_1I, A57WrBackOne], + (instregex "VLD4DUP(d|q)(8|16|32)Pseudo_UPD")>; + +// --- 3.18 ASIMD Store Instructions --- + +// ASIMD store, 1 element, multiple, 1 reg: 1cyc S +def : InstRW<[A57Write_1cyc_1S], (instregex "VST1d(8|16|32|64)$")>; +def : InstRW<[A57WrBackOne, A57Write_1cyc_1S_1I], + (instregex "VST1d(8|16|32|64)wb")>; +// ASIMD store, 1 element, multiple, 2 reg: 2cyc S +def : InstRW<[A57Write_2cyc_1S], (instregex "VST1q(8|16|32|64)$")>; +def : InstRW<[A57WrBackOne, A57Write_2cyc_1S_1I], + (instregex "VST1q(8|16|32|64)wb")>; +// ASIMD store, 1 element, multiple, 3 reg: 3cyc S +def : InstRW<[A57Write_3cyc_1S], + (instregex "VST1d(8|16|32|64)T$", "VST1d64TPseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1I], + (instregex "VST1d(8|16|32|64)Twb", "VST1d64TPseudoWB")>; +// ASIMD store, 1 element, multiple, 4 reg: 4cyc S +def : InstRW<[A57Write_4cyc_1S], + (instregex "VST1d(8|16|32|64)(Q|QPseudo)$")>; +def : InstRW<[A57WrBackOne, A57Write_4cyc_1S_1I], + (instregex "VST1d(8|16|32|64)(Qwb|QPseudoWB)")>; +// ASIMD store, 1 element, one lane: 3cyc "F0/F1, S" +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST1LNd(8|16|32)$", "VST1LNq(8|16|32)Pseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST1LNd(8|16|32)_UPD", "VST1LNq(8|16|32)Pseudo_UPD")>; +// ASIMD store, 2 element, multiple, 2 reg: 3cyc "F0/F1, S" +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST2(d|b)(8|16|32)$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST2(b|d)(8|16|32)wb")>; +// ASIMD store, 2 element, multiple, 4 reg: 4cyc "F0/F1, S" +def : InstRW<[A57Write_4cyc_1S_1V], + (instregex "VST2q(8|16|32)$", "VST2q(8|16|32)Pseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_4cyc_1S_1V_1I], + (instregex "VST2q(8|16|32)wb", "VST2q(8|16|32)PseudoWB")>; +// ASIMD store, 2 element, one lane: 3cyc "F0/F1, S" +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST2LN(d|q)(8|16|32)$", "VST2LN(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST2LN(d|q)(8|16|32)_UPD", + "VST2LN(d|q)(8|16|32)Pseudo_UPD")>; +// ASIMD store, 3 element, multiple, 3 reg +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST3(d|q)(8|16|32)$", "VST3(d|q)(8|16|32)(oddP|P)seudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST3(d|q)(8|16|32)_UPD", + "VST3(d|q)(8|16|32)(oddP|P)seudo_UPD$")>; +// ASIMD store, 3 element, one lane +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST3LN(d|q)(8|16|32)$", "VST3LN(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST3LN(d|q)(8|16|32)_UPD", + "VST3LN(d|q)(8|16|32)Pseudo_UPD")>; +// ASIMD store, 4 element, multiple, 4 reg +def : InstRW<[A57Write_4cyc_1S_1V], + (instregex "VST4(d|q)(8|16|32)$", "VST4(d|q)(8|16|32)(oddP|P)seudo$")>; +def : InstRW<[A57WrBackOne, A57Write_4cyc_1S_1V_1I], + (instregex "VST4(d|q)(8|16|32)_UPD", + "VST4(d|q)(8|16|32)(oddP|P)seudo_UPD$")>; +// ASIMD store, 4 element, one lane +def : InstRW<[A57Write_3cyc_1S_1V], + (instregex "VST4LN(d|q)(8|16|32)$", "VST4LN(d|q)(8|16|32)Pseudo$")>; +def : InstRW<[A57WrBackOne, A57Write_3cyc_1S_1V_1I], + (instregex "VST4LN(d|q)(8|16|32)_UPD", + "VST4LN(d|q)(8|16|32)Pseudo_UPD")>; + +// --- 3.19 Cryptography Extensions --- +// Crypto AES ops +// AESD, AESE, AESIMC, AESMC: 3cyc F0 +def : InstRW<[A57Write_3cyc_1W], (instregex "^AES")>; +// Crypto polynomial (64x64) multiply long (VMULL.P64): 3cyc F0 +def : InstRW<[A57Write_3cyc_1W], (instregex "^VMULLp64")>; +// Crypto SHA1 xor ops: 6cyc F0/F1 +def : InstRW<[A57Write_6cyc_2V], (instregex "^SHA1SU0")>; +// Crypto SHA1 fast ops: 3cyc F0 +def : InstRW<[A57Write_3cyc_1W], (instregex "^SHA1(H|SU1)")>; +// Crypto SHA1 slow ops: 6cyc F0 +def : InstRW<[A57Write_6cyc_2W], (instregex "^SHA1[CMP]")>; +// Crypto SHA256 fast ops: 3cyc F0 +def : InstRW<[A57Write_3cyc_1W], (instregex "^SHA256SU0")>; +// Crypto SHA256 slow ops: 6cyc F0 +def : InstRW<[A57Write_6cyc_2W], (instregex "^SHA256(H|H2|SU1)")>; + +// --- 3.20 CRC --- +def : InstRW<[A57Write_3cyc_1W], (instregex "^(t2)?CRC32")>; + +// ----------------------------------------------------------------------------- +// Common definitions +def : WriteRes<WriteNoop, []> { let Latency = 0; let NumMicroOps = 0; } +def : SchedAlias<WriteALU, A57Write_1cyc_1I>; + +def : SchedAlias<WriteBr, A57Write_1cyc_1B>; +def : SchedAlias<WriteBrL, A57Write_1cyc_1B_1I>; +def : SchedAlias<WriteBrTbl, A57Write_1cyc_1B_1I>; +def : SchedAlias<WritePreLd, A57Write_4cyc_1L>; + +def : SchedAlias<WriteLd, A57Write_4cyc_1L>; +def : SchedAlias<WriteST, A57Write_1cyc_1S>; +def : ReadAdvance<ReadALU, 0>; + +} // SchedModel = CortexA57Model + diff --git a/lib/Target/ARM/ARMScheduleA57WriteRes.td b/lib/Target/ARM/ARMScheduleA57WriteRes.td new file mode 100644 index 000000000000..670717dc7c13 --- /dev/null +++ b/lib/Target/ARM/ARMScheduleA57WriteRes.td @@ -0,0 +1,323 @@ +//=- ARMScheduleA57WriteRes.td - ARM Cortex-A57 Write Res ---*- tablegen -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Contains all of the Cortex-A57 specific SchedWriteRes types. The approach +// below is to define a generic SchedWriteRes for every combination of +// latency and microOps. The naming conventions is to use a prefix, one field +// for latency, and one or more microOp count/type designators. +// Prefix: A57Write +// Latency: #cyc +// MicroOp Count/Types: #(B|I|M|L|S|X|W|V) +// +// e.g. A57Write_6cyc_1I_6S_4V means the total latency is 6 and there are +// 11 micro-ops to be issued as follows: one to I pipe, six to S pipes and +// four to V pipes. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Define Generic 1 micro-op types + +def A57Write_5cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 5; } +def A57Write_5cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 5; } +def A57Write_5cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 5; } +def A57Write_10cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 10; } +def A57Write_17cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 17; + let ResourceCycles = [17]; } +def A57Write_18cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 18; + let ResourceCycles = [18]; } +def A57Write_19cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 19; + let ResourceCycles = [19]; } +def A57Write_20cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 20; + let ResourceCycles = [20]; } +def A57Write_1cyc_1B : SchedWriteRes<[A57UnitB]> { let Latency = 1; } +def A57Write_1cyc_1I : SchedWriteRes<[A57UnitI]> { let Latency = 1; } +def A57Write_2cyc_1I : SchedWriteRes<[A57UnitI]> { let Latency = 2; } +def A57Write_3cyc_1I : SchedWriteRes<[A57UnitI]> { let Latency = 3; } +def A57Write_1cyc_1S : SchedWriteRes<[A57UnitS]> { let Latency = 1; } +def A57Write_2cyc_1S : SchedWriteRes<[A57UnitS]> { let Latency = 2; } +def A57Write_3cyc_1S : SchedWriteRes<[A57UnitS]> { let Latency = 3; } +def A57Write_2cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 2; } +def A57Write_32cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 32; + let ResourceCycles = [32]; } +def A57Write_32cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 32; + let ResourceCycles = [32]; } +def A57Write_35cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 35; + let ResourceCycles = [35]; } +def A57Write_3cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 3; } +def A57Write_3cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 3; } +def A57Write_3cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 3; } +def A57Write_3cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 3; } + +// A57Write_3cyc_1L - A57Write_20cyc_1L +foreach Lat = 3-20 in { + def A57Write_#Lat#cyc_1L : SchedWriteRes<[A57UnitL]> { + let Latency = Lat; + } +} + +// A57Write_4cyc_1S - A57Write_16cyc_1S +foreach Lat = 4-16 in { + def A57Write_#Lat#cyc_1S : SchedWriteRes<[A57UnitS]> { + let Latency = Lat; + } +} + +def A57Write_4cyc_1M : SchedWriteRes<[A57UnitL]> { let Latency = 4; } +def A57Write_4cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 4; } +def A57Write_4cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 4; } +def A57Write_5cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 5; } +def A57Write_6cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 6; } +def A57Write_6cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 6; } +def A57Write_8cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 8; } +def A57Write_9cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 9; } +def A57Write_6cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 6; } +def A57Write_6cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 6; } + + +//===----------------------------------------------------------------------===// +// Define Generic 2 micro-op types + +def A57Write_64cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { + let Latency = 64; + let NumMicroOps = 2; + let ResourceCycles = [32, 32]; +} +def A57Write_6cyc_1I_1L : SchedWriteRes<[A57UnitI, + A57UnitL]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_6cyc_1V_1X : SchedWriteRes<[A57UnitV, + A57UnitX]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_7cyc_1V_1X : SchedWriteRes<[A57UnitV, + A57UnitX]> { + let Latency = 7; + let NumMicroOps = 2; +} +def A57Write_8cyc_1L_1V : SchedWriteRes<[A57UnitL, + A57UnitV]> { + let Latency = 8; + let NumMicroOps = 2; +} +def A57Write_9cyc_1L_1V : SchedWriteRes<[A57UnitL, + A57UnitV]> { + let Latency = 9; + let NumMicroOps = 2; +} +def A57Write_9cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 9; + let NumMicroOps = 2; +} +def A57Write_8cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { + let Latency = 8; + let NumMicroOps = 2; +} +def A57Write_6cyc_2L : SchedWriteRes<[A57UnitL, A57UnitL]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_6cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_6cyc_2W : SchedWriteRes<[A57UnitW, A57UnitW]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_5cyc_1I_1L : SchedWriteRes<[A57UnitI, + A57UnitL]> { + let Latency = 5; + let NumMicroOps = 2; +} +def A57Write_5cyc_1I_1M : SchedWriteRes<[A57UnitI, + A57UnitM]> { + let Latency = 5; + let NumMicroOps = 2; +} +def A57Write_5cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 5; + let NumMicroOps = 2; +} +def A57Write_5cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { + let Latency = 5; + let NumMicroOps = 2; +} +def A57Write_10cyc_1L_1V : SchedWriteRes<[A57UnitL, + A57UnitV]> { + let Latency = 10; + let NumMicroOps = 2; +} +def A57Write_10cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 10; + let NumMicroOps = 2; +} +def A57Write_1cyc_1B_1I : SchedWriteRes<[A57UnitB, + A57UnitI]> { + let Latency = 1; + let NumMicroOps = 2; +} +def A57Write_1cyc_1I_1S : SchedWriteRes<[A57UnitI, + A57UnitS]> { + let Latency = 1; + let NumMicroOps = 2; +} +def A57Write_1cyc_1S_1I : SchedWriteRes<[A57UnitS, + A57UnitI]> { + let Latency = 1; + let NumMicroOps = 2; +} +def A57Write_2cyc_1S_1I : SchedWriteRes<[A57UnitS, + A57UnitI]> { + let Latency = 2; + let NumMicroOps = 2; +} +def A57Write_3cyc_1S_1I : SchedWriteRes<[A57UnitS, + A57UnitI]> { + let Latency = 3; + let NumMicroOps = 2; +} +def A57Write_1cyc_1S_1M : SchedWriteRes<[A57UnitS, + A57UnitM]> { + let Latency = 1; + let NumMicroOps = 2; +} +def A57Write_2cyc_1B_1I : SchedWriteRes<[A57UnitB, + A57UnitI]> { + let Latency = 2; + let NumMicroOps = 2; +} +def A57Write_3cyc_1B_1I : SchedWriteRes<[A57UnitB, + A57UnitI]> { + let Latency = 3; + let NumMicroOps = 2; +} +def A57Write_6cyc_1B_1L : SchedWriteRes<[A57UnitB, + A57UnitI]> { + let Latency = 6; + let NumMicroOps = 2; +} +def A57Write_2cyc_1I_1M : SchedWriteRes<[A57UnitI, + A57UnitM]> { + let Latency = 2; + let NumMicroOps = 2; +} +def A57Write_2cyc_2S : SchedWriteRes<[A57UnitS, A57UnitS]> { + let Latency = 2; + let NumMicroOps = 2; +} +def A57Write_2cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 2; + let NumMicroOps = 2; +} +def A57Write_36cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { + let Latency = 36; + let NumMicroOps = 2; + let ResourceCycles = [18, 18]; +} +def A57Write_3cyc_1I_1M : SchedWriteRes<[A57UnitI, + A57UnitM]> { + let Latency = 3; + let NumMicroOps = 2; +} +def A57Write_4cyc_1I_1M : SchedWriteRes<[A57UnitI, + A57UnitM]> { + let Latency = 4; + let NumMicroOps = 2; +} + +// A57Write_3cyc_1L_1I - A57Write_20cyc_1L_1I +foreach Lat = 3-20 in { + def A57Write_#Lat#cyc_1L_1I : SchedWriteRes<[A57UnitL, A57UnitI]> { + let Latency = Lat; let NumMicroOps = 2; + } +} + +def A57Write_3cyc_1I_1S : SchedWriteRes<[A57UnitI, + A57UnitS]> { + let Latency = 3; + let NumMicroOps = 2; +} +def A57Write_3cyc_1S_1V : SchedWriteRes<[A57UnitS, + A57UnitV]> { + let Latency = 3; + let NumMicroOps = 2; +} +def A57Write_4cyc_1S_1V : SchedWriteRes<[A57UnitS, + A57UnitV]> { + let Latency = 4; + let NumMicroOps = 2; +} +def A57Write_3cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { + let Latency = 3; + let NumMicroOps = 2; +} + +// A57Write_4cyc_1S_1I - A57Write_16cyc_1S_1I +foreach Lat = 4-16 in { + def A57Write_#Lat#cyc_1S_1I : SchedWriteRes<[A57UnitS, A57UnitI]> { + let Latency = Lat; let NumMicroOps = 2; + } +} + +def A57Write_4cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { + let Latency = 4; + let NumMicroOps = 2; +} + + +//===----------------------------------------------------------------------===// +// Define Generic 3 micro-op types + +def A57Write_10cyc_3V : SchedWriteRes<[A57UnitV, A57UnitV, A57UnitV]> { + let Latency = 10; + let NumMicroOps = 3; +} +def A57Write_2cyc_1I_2S : SchedWriteRes<[A57UnitI, + A57UnitS, A57UnitS]> { + let Latency = 2; + let NumMicroOps = 3; +} +def A57Write_3cyc_1I_1S_1V : SchedWriteRes<[A57UnitI, + A57UnitS, + A57UnitV]> { + let Latency = 3; + let NumMicroOps = 3; +} +def A57Write_3cyc_1S_1V_1I : SchedWriteRes<[A57UnitS, + A57UnitV, + A57UnitI]> { + let Latency = 3; + let NumMicroOps = 3; +} +def A57Write_4cyc_1S_1V_1I : SchedWriteRes<[A57UnitS, + A57UnitV, + A57UnitI]> { + let Latency = 4; + let NumMicroOps = 3; +} +def A57Write_4cyc_1I_1L_1M : SchedWriteRes<[A57UnitI, A57UnitL, A57UnitM]> { + let Latency = 4; + let NumMicroOps = 3; +} +def A57Write_8cyc_1L_1V_1I : SchedWriteRes<[A57UnitL, + A57UnitV, + A57UnitI]> { + let Latency = 8; + let NumMicroOps = 3; +} +def A57Write_9cyc_1L_1V_1I : SchedWriteRes<[A57UnitL, + A57UnitV, + A57UnitI]> { + let Latency = 9; + let NumMicroOps = 3; +} diff --git a/lib/Target/ARM/ARMSubtarget.h b/lib/Target/ARM/ARMSubtarget.h index d2630685d91b..af682dd8321c 100644 --- a/lib/Target/ARM/ARMSubtarget.h +++ b/lib/Target/ARM/ARMSubtarget.h @@ -234,6 +234,10 @@ protected: /// CPSR setting instruction. bool AvoidCPSRPartialUpdate = false; + /// CheapPredicableCPSRDef - If true, disable +1 predication cost + /// for instructions updating CPSR. Enabled for Cortex-A57. + bool CheapPredicableCPSRDef = false; + /// AvoidMOVsShifterOperand - If true, codegen should avoid using flag setting /// movs with shifter operand (i.e. asr, lsl, lsr). bool AvoidMOVsShifterOperand = false; @@ -543,6 +547,7 @@ public: bool nonpipelinedVFP() const { return NonpipelinedVFP; } bool prefers32BitThumb() const { return Pref32BitThumb; } bool avoidCPSRPartialUpdate() const { return AvoidCPSRPartialUpdate; } + bool cheapPredicableCPSRDef() const { return CheapPredicableCPSRDef; } bool avoidMOVsShifterOperand() const { return AvoidMOVsShifterOperand; } bool hasRetAddrStack() const { return HasRetAddrStack; } bool hasMPExtension() const { return HasMPExtension; } diff --git a/lib/Target/Hexagon/HexagonInstrInfo.cpp b/lib/Target/Hexagon/HexagonInstrInfo.cpp index 0fef91ec4d3e..b76da727237c 100644 --- a/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -3419,9 +3419,7 @@ int HexagonInstrInfo::getDotNewPredOp(const MachineInstr &MI, int NewOpcode = Hexagon::getPredNewOpcode(MI.getOpcode()); if (NewOpcode >= 0) return NewOpcode; - - dbgs() << "Cannot convert to .new: " << getName(MI.getOpcode()) << '\n'; - llvm_unreachable(nullptr); + return 0; } int HexagonInstrInfo::getDotOldOp(const MachineInstr &MI) const { diff --git a/lib/Target/Mips/MicroMipsSizeReduction.cpp b/lib/Target/Mips/MicroMipsSizeReduction.cpp index 4593fc92ca6f..35948e36ad91 100644 --- a/lib/Target/Mips/MicroMipsSizeReduction.cpp +++ b/lib/Target/Mips/MicroMipsSizeReduction.cpp @@ -135,6 +135,14 @@ private: // returns true on success. static bool ReduceXWtoXWSP(MachineInstr *MI, const ReduceEntry &Entry); + // Attempts to reduce LBU/LHU instruction into LBU16/LHU16, + // returns true on success. + static bool ReduceLXUtoLXU16(MachineInstr *MI, const ReduceEntry &Entry); + + // Attempts to reduce SB/SH instruction into SB16/SH16, + // returns true on success. + static bool ReduceSXtoSX16(MachineInstr *MI, const ReduceEntry &Entry); + // Attempts to reduce arithmetic instructions, returns true on success static bool ReduceArithmeticInstructions(MachineInstr *MI, const ReduceEntry &Entry); @@ -162,10 +170,26 @@ llvm::SmallVector<ReduceEntry, 16> MicroMipsSizeReduce::ReduceTable = { {RT_OneInstr, OpCodes(Mips::ADDu_MM, Mips::ADDU16_MM), ReduceArithmeticInstructions, OpInfo(OT_OperandsAll), ImmField(0, 0, 0, -1)}, + {RT_OneInstr, OpCodes(Mips::LBu, Mips::LBU16_MM), ReduceLXUtoLXU16, + OpInfo(OT_OperandsAll), ImmField(0, -1, 15, 2)}, + {RT_OneInstr, OpCodes(Mips::LBu_MM, Mips::LBU16_MM), ReduceLXUtoLXU16, + OpInfo(OT_OperandsAll), ImmField(0, -1, 15, 2)}, + {RT_OneInstr, OpCodes(Mips::LHu, Mips::LHU16_MM), ReduceLXUtoLXU16, + OpInfo(OT_OperandsAll), ImmField(1, 0, 16, 2)}, + {RT_OneInstr, OpCodes(Mips::LHu_MM, Mips::LHU16_MM), ReduceLXUtoLXU16, + OpInfo(OT_OperandsAll), ImmField(1, 0, 16, 2)}, {RT_OneInstr, OpCodes(Mips::LW, Mips::LWSP_MM), ReduceXWtoXWSP, OpInfo(OT_OperandsAll), ImmField(2, 0, 32, 2)}, {RT_OneInstr, OpCodes(Mips::LW_MM, Mips::LWSP_MM), ReduceXWtoXWSP, OpInfo(OT_OperandsAll), ImmField(2, 0, 32, 2)}, + {RT_OneInstr, OpCodes(Mips::SB, Mips::SB16_MM), ReduceSXtoSX16, + OpInfo(OT_OperandsAll), ImmField(0, 0, 16, 2)}, + {RT_OneInstr, OpCodes(Mips::SB_MM, Mips::SB16_MM), ReduceSXtoSX16, + OpInfo(OT_OperandsAll), ImmField(0, 0, 16, 2)}, + {RT_OneInstr, OpCodes(Mips::SH, Mips::SH16_MM), ReduceSXtoSX16, + OpInfo(OT_OperandsAll), ImmField(1, 0, 16, 2)}, + {RT_OneInstr, OpCodes(Mips::SH_MM, Mips::SH16_MM), ReduceSXtoSX16, + OpInfo(OT_OperandsAll), ImmField(1, 0, 16, 2)}, {RT_OneInstr, OpCodes(Mips::SUBu, Mips::SUBU16_MM), ReduceArithmeticInstructions, OpInfo(OT_OperandsAll), ImmField(0, 0, 0, -1)}, @@ -193,6 +217,13 @@ static bool isMMThreeBitGPRegister(const MachineOperand &MO) { return false; } +// Returns true if the machine operand MO is register $0, $17, or $2-$7. +static bool isMMSourceRegister(const MachineOperand &MO) { + if (MO.isReg() && Mips::GPRMM16ZeroRegClass.contains(MO.getReg())) + return true; + return false; +} + // Returns true if the operand Op is an immediate value // and writes the immediate value into variable Imm static bool GetImm(MachineInstr *MI, unsigned Op, int64_t &Imm) { @@ -279,6 +310,32 @@ bool MicroMipsSizeReduce::ReduceArithmeticInstructions( return ReplaceInstruction(MI, Entry); } +bool MicroMipsSizeReduce::ReduceLXUtoLXU16(MachineInstr *MI, + const ReduceEntry &Entry) { + + if (!ImmInRange(MI, Entry)) + return false; + + if (!isMMThreeBitGPRegister(MI->getOperand(0)) || + !isMMThreeBitGPRegister(MI->getOperand(1))) + return false; + + return ReplaceInstruction(MI, Entry); +} + +bool MicroMipsSizeReduce::ReduceSXtoSX16(MachineInstr *MI, + const ReduceEntry &Entry) { + + if (!ImmInRange(MI, Entry)) + return false; + + if (!isMMSourceRegister(MI->getOperand(0)) || + !isMMThreeBitGPRegister(MI->getOperand(1))) + return false; + + return ReplaceInstruction(MI, Entry); +} + bool MicroMipsSizeReduce::ReduceMBB(MachineBasicBlock &MBB) { bool Modified = false; MachineBasicBlock::instr_iterator MII = MBB.instr_begin(), diff --git a/lib/Target/WebAssembly/known_gcc_test_failures.txt b/lib/Target/WebAssembly/known_gcc_test_failures.txt index 54619589c341..35a67134775a 100644 --- a/lib/Target/WebAssembly/known_gcc_test_failures.txt +++ b/lib/Target/WebAssembly/known_gcc_test_failures.txt @@ -88,6 +88,3 @@ pr45695.c wasm-o pr49279.c wasm-o pr49390.c wasm-o pr52286.c wasm-o - -# fatal error: error in backend: data symbols must have a size set with .size -921110-1.c wasm-o diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 0a41f35f9320..5303d7a406ad 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -4753,7 +4753,7 @@ static void scaleShuffleMask(int Scale, ArrayRef<int> Mask, SmallVectorImpl<int> &ScaledMask) { assert(0 < Scale && "Unexpected scaling factor"); int NumElts = Mask.size(); - ScaledMask.assign(NumElts * Scale, -1); + ScaledMask.assign(static_cast<size_t>(NumElts * Scale), -1); for (int i = 0; i != NumElts; ++i) { int M = Mask[i]; @@ -5848,17 +5848,39 @@ static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask, return true; } case ISD::SCALAR_TO_VECTOR: { - // Match against a scalar_to_vector of an extract from a similar vector. + // Match against a scalar_to_vector of an extract from a vector, + // for PEXTRW/PEXTRB we must handle the implicit zext of the scalar. SDValue N0 = N.getOperand(0); - if (N0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - N0.getOperand(0).getValueType() != VT || - !isa<ConstantSDNode>(N0.getOperand(1)) || - NumElts <= N0.getConstantOperandVal(1) || - !N->isOnlyUserOf(N0.getNode())) + SDValue SrcExtract; + + if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && + N0.getOperand(0).getValueType() == VT) { + SrcExtract = N0; + } else if (N0.getOpcode() == ISD::AssertZext && + N0.getOperand(0).getOpcode() == X86ISD::PEXTRW && + cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i16) { + SrcExtract = N0.getOperand(0); + assert(SrcExtract.getOperand(0).getValueType() == MVT::v8i16); + } else if (N0.getOpcode() == ISD::AssertZext && + N0.getOperand(0).getOpcode() == X86ISD::PEXTRB && + cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i8) { + SrcExtract = N0.getOperand(0); + assert(SrcExtract.getOperand(0).getValueType() == MVT::v16i8); + } + + if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)) || + NumElts <= SrcExtract.getConstantOperandVal(1)) return false; - Ops.push_back(N0.getOperand(0)); - Mask.push_back(N0.getConstantOperandVal(1)); - Mask.append(NumElts - 1, SM_SentinelUndef); + + SDValue SrcVec = SrcExtract.getOperand(0); + EVT SrcVT = SrcVec.getValueType(); + unsigned NumSrcElts = SrcVT.getVectorNumElements(); + unsigned NumZeros = (NumBitsPerElt / SrcVT.getScalarSizeInBits()) - 1; + + Ops.push_back(SrcVec); + Mask.push_back(SrcExtract.getConstantOperandVal(1)); + Mask.append(NumZeros, SM_SentinelZero); + Mask.append(NumSrcElts - Mask.size(), SM_SentinelUndef); return true; } case X86ISD::PINSRB: @@ -6542,12 +6564,12 @@ static Constant *getConstantVector(MVT VT, const APInt &SplatValue, APInt Val = SplatValue.extractBits(ScalarSize, ScalarSize * i); Constant *Const; if (VT.isFloatingPoint()) { - assert((ScalarSize == 32 || ScalarSize == 64) && - "Unsupported floating point scalar size"); - if (ScalarSize == 32) - Const = ConstantFP::get(Type::getFloatTy(C), Val.bitsToFloat()); - else - Const = ConstantFP::get(Type::getDoubleTy(C), Val.bitsToDouble()); + if (ScalarSize == 32) { + Const = ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val)); + } else { + assert(ScalarSize == 64 && "Unsupported floating point scalar size"); + Const = ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val)); + } } else Const = Constant::getIntegerValue(Type::getIntNTy(C, ScalarSize), Val); ConstantVec.push_back(Const); @@ -6633,11 +6655,13 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, // AVX have support for 32 and 64 bit broadcast for floats only. // No 64bit integer in 32bit subtarget. MVT CVT = MVT::getFloatingPointVT(SplatBitSize); - Constant *C = SplatBitSize == 32 - ? ConstantFP::get(Type::getFloatTy(*Ctx), - SplatValue.bitsToFloat()) - : ConstantFP::get(Type::getDoubleTy(*Ctx), - SplatValue.bitsToDouble()); + // Lower the splat via APFloat directly, to avoid any conversion. + Constant *C = + SplatBitSize == 32 + ? ConstantFP::get(*Ctx, + APFloat(APFloat::IEEEsingle(), SplatValue)) + : ConstantFP::get(*Ctx, + APFloat(APFloat::IEEEdouble(), SplatValue)); SDValue CP = DAG.getConstantPool(C, PVT); unsigned Repeat = VT.getSizeInBits() / SplatBitSize; @@ -8003,7 +8027,7 @@ static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, ArrayRef<int> Mask, SmallVectorImpl<int> &RepeatedMask) { - int LaneSize = LaneSizeInBits / VT.getScalarSizeInBits(); + auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits(); RepeatedMask.assign(LaneSize, -1); int Size = Mask.size(); for (int i = 0; i < Size; ++i) { @@ -16997,7 +17021,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, SDValue Op1 = Op.getOperand(1); SDValue CC = Op.getOperand(2); MVT VT = Op.getSimpleValueType(); - ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); + ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get(); bool isFP = Op.getOperand(1).getSimpleValueType().isFloatingPoint(); SDLoc dl(Op); @@ -17024,18 +17048,18 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, // TODO: This can be avoided if Intel (and only Intel as of 2016) AVX is // available. SDValue Cmp; - unsigned SSECC = translateX86FSETCC(SetCCOpcode, Op0, Op1); + unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1); if (SSECC == 8) { // LLVM predicate is SETUEQ or SETONE. unsigned CC0, CC1; unsigned CombineOpc; - if (SetCCOpcode == ISD::SETUEQ) { + if (Cond == ISD::SETUEQ) { CC0 = 3; // UNORD CC1 = 0; // EQ CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FOR) : static_cast<unsigned>(ISD::OR); } else { - assert(SetCCOpcode == ISD::SETONE); + assert(Cond == ISD::SETONE); CC0 = 7; // ORD CC1 = 4; // NEQ CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FAND) : @@ -17082,7 +17106,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, // 2. The original operand type has been promoted to a 256-bit vector. // // Note that condition 2. only applies for AVX targets. - SDValue NewOp = DAG.getSetCC(dl, VTOp0, Op0, Op1, SetCCOpcode); + SDValue NewOp = DAG.getSetCC(dl, VTOp0, Op0, Op1, Cond); return DAG.getZExtOrTrunc(NewOp, dl, VT); } @@ -17122,7 +17146,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, VT == MVT::v4i32 || VT == MVT::v2i64) && Subtarget.hasXOP()) { // Translate compare code to XOP PCOM compare mode. unsigned CmpMode = 0; - switch (SetCCOpcode) { + switch (Cond) { default: llvm_unreachable("Unexpected SETCC condition"); case ISD::SETULT: case ISD::SETLT: CmpMode = 0x00; break; @@ -17137,60 +17161,49 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, } // Are we comparing unsigned or signed integers? - unsigned Opc = ISD::isUnsignedIntSetCC(SetCCOpcode) - ? X86ISD::VPCOMU : X86ISD::VPCOM; + unsigned Opc = + ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM; return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(CmpMode, dl, MVT::i8)); } - // We are handling one of the integer comparisons here. Since SSE only has + // We are handling one of the integer comparisons here. Since SSE only has // GT and EQ comparisons for integer, swapping operands and multiple // operations may be required for some comparisons. - unsigned Opc; - bool Swap = false, Invert = false, FlipSigns = false, MinMax = false; - bool Subus = false; - - switch (SetCCOpcode) { - default: llvm_unreachable("Unexpected SETCC condition"); - case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH; - case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break; - case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH; - case ISD::SETGT: Opc = X86ISD::PCMPGT; break; - case ISD::SETGE: Swap = true; LLVM_FALLTHROUGH; - case ISD::SETLE: Opc = X86ISD::PCMPGT; - Invert = true; break; - case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH; - case ISD::SETUGT: Opc = X86ISD::PCMPGT; - FlipSigns = true; break; - case ISD::SETUGE: Swap = true; LLVM_FALLTHROUGH; - case ISD::SETULE: Opc = X86ISD::PCMPGT; - FlipSigns = true; Invert = true; break; - } + unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ + : X86ISD::PCMPGT; + bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT || + Cond == ISD::SETGE || Cond == ISD::SETUGE; + bool Invert = Cond == ISD::SETNE || + (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond)); + bool FlipSigns = ISD::isUnsignedIntSetCC(Cond); // Special case: Use min/max operations for SETULE/SETUGE MVT VET = VT.getVectorElementType(); - bool hasMinMax = - (Subtarget.hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) - || (Subtarget.hasSSE2() && (VET == MVT::i8)); - - if (hasMinMax) { - switch (SetCCOpcode) { + bool HasMinMax = + (Subtarget.hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) || + (Subtarget.hasSSE2() && (VET == MVT::i8)); + bool MinMax = false; + if (HasMinMax) { + switch (Cond) { default: break; case ISD::SETULE: Opc = ISD::UMIN; MinMax = true; break; case ISD::SETUGE: Opc = ISD::UMAX; MinMax = true; break; } - if (MinMax) { Swap = false; Invert = false; FlipSigns = false; } + if (MinMax) + Swap = Invert = FlipSigns = false; } - bool hasSubus = Subtarget.hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); - if (!MinMax && hasSubus) { + bool HasSubus = Subtarget.hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); + bool Subus = false; + if (!MinMax && HasSubus) { // As another special case, use PSUBUS[BW] when it's profitable. E.g. for // Op0 u<= Op1: // t = psubus Op0, Op1 // pcmpeq t, <0..0> - switch (SetCCOpcode) { + switch (Cond) { default: break; case ISD::SETULT: { // If the comparison is against a constant we can turn this into a diff --git a/lib/Transforms/Coroutines/CoroSplit.cpp b/lib/Transforms/Coroutines/CoroSplit.cpp index 613b4a7f03e9..626a891f65c6 100644 --- a/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/lib/Transforms/Coroutines/CoroSplit.cpp @@ -228,7 +228,7 @@ static Function *createClone(Function &F, Twine Suffix, coro::Shape &Shape, SmallVector<ReturnInst *, 4> Returns; - CloneFunctionInto(NewF, &F, VMap, /*ModuleLevelChanges=*/false, Returns); + CloneFunctionInto(NewF, &F, VMap, /*ModuleLevelChanges=*/true, Returns); // Remove old returns. for (ReturnInst *Return : Returns) diff --git a/lib/Transforms/Coroutines/Coroutines.cpp b/lib/Transforms/Coroutines/Coroutines.cpp index ea48043f9381..44e1f9b404ed 100644 --- a/lib/Transforms/Coroutines/Coroutines.cpp +++ b/lib/Transforms/Coroutines/Coroutines.cpp @@ -218,6 +218,8 @@ void coro::Shape::buildFrom(Function &F) { size_t FinalSuspendIndex = 0; clear(*this); SmallVector<CoroFrameInst *, 8> CoroFrames; + SmallVector<CoroSaveInst *, 2> UnusedCoroSaves; + for (Instruction &I : instructions(F)) { if (auto II = dyn_cast<IntrinsicInst>(&I)) { switch (II->getIntrinsicID()) { @@ -229,6 +231,12 @@ void coro::Shape::buildFrom(Function &F) { case Intrinsic::coro_frame: CoroFrames.push_back(cast<CoroFrameInst>(II)); break; + case Intrinsic::coro_save: + // After optimizations, coro_suspends using this coro_save might have + // been removed, remember orphaned coro_saves to remove them later. + if (II->use_empty()) + UnusedCoroSaves.push_back(cast<CoroSaveInst>(II)); + break; case Intrinsic::coro_suspend: CoroSuspends.push_back(cast<CoroSuspendInst>(II)); if (CoroSuspends.back()->isFinal()) { @@ -311,4 +319,8 @@ void coro::Shape::buildFrom(Function &F) { if (HasFinalSuspend && FinalSuspendIndex != CoroSuspends.size() - 1) std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back()); + + // Remove orphaned coro.saves. + for (CoroSaveInst *CoroSave : UnusedCoroSaves) + CoroSave->eraseFromParent(); } diff --git a/lib/Transforms/IPO/FunctionAttrs.cpp b/lib/Transforms/IPO/FunctionAttrs.cpp index 28cc81c76d4f..5cc29a493798 100644 --- a/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/lib/Transforms/IPO/FunctionAttrs.cpp @@ -1188,6 +1188,10 @@ static bool runImpl(CallGraphSCC &SCC, AARGetterT AARGetter) { SCCNodes.insert(F); } + // Skip it if the SCC only contains optnone functions. + if (SCCNodes.empty()) + return Changed; + Changed |= addArgumentReturnedAttrs(SCCNodes); Changed |= addReadAttrs(SCCNodes, AARGetter); Changed |= addArgumentAttrs(SCCNodes); diff --git a/lib/Transforms/IPO/FunctionImport.cpp b/lib/Transforms/IPO/FunctionImport.cpp index 231487923fad..6d34ab8b0d96 100644 --- a/lib/Transforms/IPO/FunctionImport.cpp +++ b/lib/Transforms/IPO/FunctionImport.cpp @@ -292,8 +292,7 @@ static void computeImportForFunction( static void ComputeImportForModule( const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList, - StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr, - const DenseSet<GlobalValue::GUID> *DeadSymbols = nullptr) { + StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) { // Worklist contains the list of function imported in this module, for which // we will analyse the callees and may import further down the callgraph. SmallVector<EdgeInfo, 128> Worklist; @@ -301,7 +300,7 @@ static void ComputeImportForModule( // Populate the worklist with the import for the functions in the current // module for (auto &GVSummary : DefinedGVSummaries) { - if (DeadSymbols && DeadSymbols->count(GVSummary.first)) { + if (!Index.isGlobalValueLive(GVSummary.second)) { DEBUG(dbgs() << "Ignores Dead GUID: " << GVSummary.first << "\n"); continue; } @@ -344,15 +343,14 @@ void llvm::ComputeCrossModuleImport( const ModuleSummaryIndex &Index, const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, StringMap<FunctionImporter::ImportMapTy> &ImportLists, - StringMap<FunctionImporter::ExportSetTy> &ExportLists, - const DenseSet<GlobalValue::GUID> *DeadSymbols) { + StringMap<FunctionImporter::ExportSetTy> &ExportLists) { // For each module that has function defined, compute the import/export lists. for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) { auto &ImportList = ImportLists[DefinedGVSummaries.first()]; DEBUG(dbgs() << "Computing import for Module '" << DefinedGVSummaries.first() << "'\n"); ComputeImportForModule(DefinedGVSummaries.second, Index, ImportList, - &ExportLists, DeadSymbols); + &ExportLists); } // When computing imports we added all GUIDs referenced by anything @@ -414,82 +412,71 @@ void llvm::ComputeCrossModuleImportForModule( #endif } -DenseSet<GlobalValue::GUID> llvm::computeDeadSymbols( - const ModuleSummaryIndex &Index, +void llvm::computeDeadSymbols( + ModuleSummaryIndex &Index, const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { + assert(!Index.withGlobalValueDeadStripping()); if (!ComputeDead) - return DenseSet<GlobalValue::GUID>(); + return; if (GUIDPreservedSymbols.empty()) // Don't do anything when nothing is live, this is friendly with tests. - return DenseSet<GlobalValue::GUID>(); - DenseSet<ValueInfo> LiveSymbols; + return; + unsigned LiveSymbols = 0; SmallVector<ValueInfo, 128> Worklist; Worklist.reserve(GUIDPreservedSymbols.size() * 2); for (auto GUID : GUIDPreservedSymbols) { ValueInfo VI = Index.getValueInfo(GUID); if (!VI) continue; - DEBUG(dbgs() << "Live root: " << VI.getGUID() << "\n"); - LiveSymbols.insert(VI); - Worklist.push_back(VI); + for (auto &S : VI.getSummaryList()) + S->setLive(true); } + // Add values flagged in the index as live roots to the worklist. - for (const auto &Entry : Index) { - bool IsLiveRoot = llvm::any_of( - Entry.second.SummaryList, - [&](const std::unique_ptr<llvm::GlobalValueSummary> &Summary) { - return Summary->liveRoot(); - }); - if (!IsLiveRoot) - continue; - DEBUG(dbgs() << "Live root (summary): " << Entry.first << "\n"); - Worklist.push_back(ValueInfo(&Entry)); - } + for (const auto &Entry : Index) + for (auto &S : Entry.second.SummaryList) + if (S->isLive()) { + DEBUG(dbgs() << "Live root: " << Entry.first << "\n"); + Worklist.push_back(ValueInfo(&Entry)); + ++LiveSymbols; + break; + } + + // Make value live and add it to the worklist if it was not live before. + // FIXME: we should only make the prevailing copy live here + auto visit = [&](ValueInfo VI) { + for (auto &S : VI.getSummaryList()) + if (S->isLive()) + return; + for (auto &S : VI.getSummaryList()) + S->setLive(true); + ++LiveSymbols; + Worklist.push_back(VI); + }; while (!Worklist.empty()) { auto VI = Worklist.pop_back_val(); - - // FIXME: we should only make the prevailing copy live here for (auto &Summary : VI.getSummaryList()) { - for (auto Ref : Summary->refs()) { - if (LiveSymbols.insert(Ref).second) { - DEBUG(dbgs() << "Marking live (ref): " << Ref.getGUID() << "\n"); - Worklist.push_back(Ref); - } - } - if (auto *FS = dyn_cast<FunctionSummary>(Summary.get())) { - for (auto Call : FS->calls()) { - if (LiveSymbols.insert(Call.first).second) { - DEBUG(dbgs() << "Marking live (call): " << Call.first.getGUID() - << "\n"); - Worklist.push_back(Call.first); - } - } - } + for (auto Ref : Summary->refs()) + visit(Ref); + if (auto *FS = dyn_cast<FunctionSummary>(Summary.get())) + for (auto Call : FS->calls()) + visit(Call.first); if (auto *AS = dyn_cast<AliasSummary>(Summary.get())) { auto AliaseeGUID = AS->getAliasee().getOriginalName(); ValueInfo AliaseeVI = Index.getValueInfo(AliaseeGUID); - if (AliaseeVI && LiveSymbols.insert(AliaseeVI).second) { - DEBUG(dbgs() << "Marking live (alias): " << AliaseeGUID << "\n"); - Worklist.push_back(AliaseeVI); - } + if (AliaseeVI) + visit(AliaseeVI); } } } - DenseSet<GlobalValue::GUID> DeadSymbols; - DeadSymbols.reserve( - std::min(Index.size(), Index.size() - LiveSymbols.size())); - for (auto &Entry : Index) { - if (!LiveSymbols.count(ValueInfo(&Entry))) { - DEBUG(dbgs() << "Marking dead: " << Entry.first << "\n"); - DeadSymbols.insert(Entry.first); - } - } - DEBUG(dbgs() << LiveSymbols.size() << " symbols Live, and " - << DeadSymbols.size() << " symbols Dead \n"); - NumDeadSymbols += DeadSymbols.size(); - NumLiveSymbols += LiveSymbols.size(); - return DeadSymbols; + Index.setWithGlobalValueDeadStripping(); + + unsigned DeadSymbols = Index.size() - LiveSymbols; + DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols + << " symbols Dead \n"); + NumDeadSymbols += DeadSymbols; + NumLiveSymbols += LiveSymbols; } /// Compute the set of summaries needed for a ThinLTO backend compilation of diff --git a/lib/Transforms/IPO/LowerTypeTests.cpp b/lib/Transforms/IPO/LowerTypeTests.cpp index ca4ee92f971a..7bec50d9d25f 100644 --- a/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/lib/Transforms/IPO/LowerTypeTests.cpp @@ -1442,9 +1442,8 @@ bool LowerTypeTestsModule::lower() { for (auto &P : *ExportSummary) { for (auto &S : P.second.SummaryList) { auto *FS = dyn_cast<FunctionSummary>(S.get()); - if (!FS) + if (!FS || !ExportSummary->isGlobalValueLive(FS)) continue; - // FIXME: Only add live functions. for (GlobalValue::GUID G : FS->type_tests()) for (Metadata *MD : MetadataByGUID[G]) AddTypeIdUse(MD).IsExported = true; diff --git a/lib/Transforms/IPO/PartialInlining.cpp b/lib/Transforms/IPO/PartialInlining.cpp index bc0967448cdd..ea805efc66b7 100644 --- a/lib/Transforms/IPO/PartialInlining.cpp +++ b/lib/Transforms/IPO/PartialInlining.cpp @@ -68,6 +68,10 @@ static cl::opt<int> cl::desc("Relative frequency of outline region to " "the entry block")); +static cl::opt<unsigned> ExtraOutliningPenalty( + "partial-inlining-extra-penalty", cl::init(0), cl::Hidden, + cl::desc("A debug option to add additional penalty to the computed one.")); + namespace { struct FunctionOutliningInfo { @@ -83,7 +87,7 @@ struct FunctionOutliningInfo { SmallVector<BasicBlock *, 4> Entries; // The return block that is not included in the outlined region. BasicBlock *ReturnBlock; - // The dominating block of the region ot be outlined. + // The dominating block of the region to be outlined. BasicBlock *NonReturnBlock; // The set of blocks in Entries that that are predecessors to ReturnBlock SmallVector<BasicBlock *, 4> ReturnBlockPreds; @@ -407,11 +411,23 @@ BranchProbability PartialInlinerImpl::getOutliningCallBBRelativeFreq( if (hasProfileData(F, OI)) return OutlineRegionRelFreq; - // When profile data is not available, we need to be very - // conservative in estimating the overall savings. We need to make sure - // the outline region relative frequency is not below the threshold - // specified by the option. - OutlineRegionRelFreq = std::max(OutlineRegionRelFreq, BranchProbability(OutlineRegionFreqPercent, 100)); + // When profile data is not available, we need to be conservative in + // estimating the overall savings. Static branch prediction can usually + // guess the branch direction right (taken/non-taken), but the guessed + // branch probability is usually not biased enough. In case when the + // outlined region is predicted to be likely, its probability needs + // to be made higher (more biased) to not under-estimate the cost of + // function outlining. On the other hand, if the outlined region + // is predicted to be less likely, the predicted probablity is usually + // higher than the actual. For instance, the actual probability of the + // less likely target is only 5%, but the guessed probablity can be + // 40%. In the latter case, there is no need for further adjustement. + // FIXME: add an option for this. + if (OutlineRegionRelFreq < BranchProbability(45, 100)) + return OutlineRegionRelFreq; + + OutlineRegionRelFreq = std::max( + OutlineRegionRelFreq, BranchProbability(OutlineRegionFreqPercent, 100)); return OutlineRegionRelFreq; } @@ -496,6 +512,26 @@ int PartialInlinerImpl::computeBBInlineCost(BasicBlock *BB) { if (isa<DbgInfoIntrinsic>(I)) continue; + switch (I->getOpcode()) { + case Instruction::BitCast: + case Instruction::PtrToInt: + case Instruction::IntToPtr: + case Instruction::Alloca: + continue; + case Instruction::GetElementPtr: + if (cast<GetElementPtrInst>(I)->hasAllZeroIndices()) + continue; + default: + break; + } + + IntrinsicInst *IntrInst = dyn_cast<IntrinsicInst>(I); + if (IntrInst) { + if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_start || + IntrInst->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + } + if (CallInst *CI = dyn_cast<CallInst>(I)) { InlineCost += getCallsiteCost(CallSite(CI), DL); continue; @@ -519,7 +555,13 @@ std::tuple<int, int, int> PartialInlinerImpl::computeOutliningCosts( Function *F, const FunctionOutliningInfo *OI, Function *OutlinedFunction, BasicBlock *OutliningCallBB) { // First compute the cost of the outlined region 'OI' in the original - // function 'F': + // function 'F'. + // FIXME: The code extractor (outliner) can now do code sinking/hoisting + // to reduce outlining cost. The hoisted/sunk code currently do not + // incur any runtime cost so it is still OK to compare the outlined + // function cost with the outlined region in the original function. + // If this ever changes, we will need to introduce new extractor api + // to pass the information. int OutlinedRegionCost = 0; for (BasicBlock &BB : *F) { if (&BB != OI->ReturnBlock && @@ -542,8 +584,14 @@ std::tuple<int, int, int> PartialInlinerImpl::computeOutliningCosts( assert(OutlinedFunctionCost >= OutlinedRegionCost && "Outlined function cost should be no less than the outlined region"); - int OutliningRuntimeOverhead = - OutliningFuncCallCost + (OutlinedFunctionCost - OutlinedRegionCost); + // The code extractor introduces a new root and exit stub blocks with + // additional unconditional branches. Those branches will be eliminated + // later with bb layout. The cost should be adjusted accordingly: + OutlinedFunctionCost -= 2 * InlineConstants::InstrCost; + + int OutliningRuntimeOverhead = OutliningFuncCallCost + + (OutlinedFunctionCost - OutlinedRegionCost) + + ExtraOutliningPenalty; return std::make_tuple(OutliningFuncCallCost, OutliningRuntimeOverhead, OutlinedRegionCost); diff --git a/lib/Transforms/IPO/PassManagerBuilder.cpp b/lib/Transforms/IPO/PassManagerBuilder.cpp index 9fd3a9021a27..16fba32e9805 100644 --- a/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -157,7 +157,7 @@ static cl::opt<bool> static cl::opt<bool> EnableGVNSink( "enable-gvn-sink", cl::init(false), cl::Hidden, - cl::desc("Enable the GVN sinking pass (default = on)")); + cl::desc("Enable the GVN sinking pass (default = off)")); PassManagerBuilder::PassManagerBuilder() { OptLevel = 2; diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp index 2c2b7317a1c0..c0798e164c39 100644 --- a/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -4508,13 +4508,16 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) { Builder->CreateAnd(A, B), Op1); - // ~x < ~y --> y < x - // ~x < cst --> ~cst < x + // ~X < ~Y --> Y < X + // ~X < C --> X > ~C if (match(Op0, m_Not(m_Value(A)))) { if (match(Op1, m_Not(m_Value(B)))) return new ICmpInst(I.getPredicate(), B, A); - if (ConstantInt *RHSC = dyn_cast<ConstantInt>(Op1)) - return new ICmpInst(I.getPredicate(), ConstantExpr::getNot(RHSC), A); + + const APInt *C; + if (match(Op1, m_APInt(C))) + return new ICmpInst(I.getSwappedPredicate(), A, + ConstantInt::get(Op1->getType(), ~(*C))); } Instruction *AddI = nullptr; diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp index ff753c20a94a..df4ee9969c02 100644 --- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2087,6 +2087,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { switch (I.getNumArgOperands()) { case 3: assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode"); + LLVM_FALLTHROUGH; case 2: CopyOp = I.getArgOperand(0); ConvertOp = I.getArgOperand(1); diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index 325b64cd8b43..8aa40d1759de 100644 --- a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -57,6 +57,11 @@ static const char *const SanCovTracePCGuardName = "__sanitizer_cov_trace_pc_guard"; static const char *const SanCovTracePCGuardInitName = "__sanitizer_cov_trace_pc_guard_init"; +static const char *const SanCov8bitCountersInitName = + "__sanitizer_cov_8bit_counters_init"; + +static const char *const SanCovGuardsSectionName = "sancov_guards"; +static const char *const SanCovCountersSectionName = "sancov_counters"; static cl::opt<int> ClCoverageLevel( "sanitizer-coverage-level", @@ -64,14 +69,18 @@ static cl::opt<int> ClCoverageLevel( "3: all blocks and critical edges"), cl::Hidden, cl::init(0)); -static cl::opt<bool> ClExperimentalTracePC("sanitizer-coverage-trace-pc", - cl::desc("Experimental pc tracing"), - cl::Hidden, cl::init(false)); +static cl::opt<bool> ClTracePC("sanitizer-coverage-trace-pc", + cl::desc("Experimental pc tracing"), cl::Hidden, + cl::init(false)); static cl::opt<bool> ClTracePCGuard("sanitizer-coverage-trace-pc-guard", cl::desc("pc tracing with a guard"), cl::Hidden, cl::init(false)); +static cl::opt<bool> ClInline8bitCounters("sanitizer-coverage-inline-8bit-counters", + cl::desc("increments 8-bit counter for every edge"), + cl::Hidden, cl::init(false)); + static cl::opt<bool> ClCMPTracing("sanitizer-coverage-trace-compares", cl::desc("Tracing of CMP and similar instructions"), @@ -123,9 +132,10 @@ SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) { Options.TraceCmp |= ClCMPTracing; Options.TraceDiv |= ClDIVTracing; Options.TraceGep |= ClGEPTracing; - Options.TracePC |= ClExperimentalTracePC; + Options.TracePC |= ClTracePC; Options.TracePCGuard |= ClTracePCGuard; - if (!Options.TracePCGuard && !Options.TracePC) + Options.Inline8bitCounters |= ClInline8bitCounters; + if (!Options.TracePCGuard && !Options.TracePC && !Options.Inline8bitCounters) Options.TracePCGuard = true; // TracePCGuard is default. Options.NoPrune |= !ClPruneBlocks; return Options; @@ -159,11 +169,22 @@ private: void InjectTraceForSwitch(Function &F, ArrayRef<Instruction *> SwitchTraceTargets); bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks); - void CreateFunctionGuardArray(size_t NumGuards, Function &F); + GlobalVariable *CreateFunctionLocalArrayInSection(size_t NumElements, + Function &F, Type *Ty, + const char *Section); + void CreateFunctionLocalArrays(size_t NumGuards, Function &F); void InjectCoverageAtBlock(Function &F, BasicBlock &BB, size_t Idx); - StringRef getSanCovTracePCGuardSection() const; - StringRef getSanCovTracePCGuardSectionStart() const; - StringRef getSanCovTracePCGuardSectionEnd() const; + void CreateInitCallForSection(Module &M, const char *InitFunctionName, + Type *Ty, const std::string &Section); + + void SetNoSanitizeMetadata(Instruction *I) { + I->setMetadata(I->getModule()->getMDKindID("nosanitize"), + MDNode::get(*C, None)); + } + + std::string getSectionName(const std::string &Section) const; + std::string getSectionStart(const std::string &Section) const; + std::string getSectionEnd(const std::string &Section) const; Function *SanCovTracePCIndir; Function *SanCovTracePC, *SanCovTracePCGuard; Function *SanCovTraceCmpFunction[4]; @@ -171,20 +192,48 @@ private: Function *SanCovTraceGepFunction; Function *SanCovTraceSwitchFunction; InlineAsm *EmptyAsm; - Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy; + Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy, + *Int8Ty, *Int8PtrTy; Module *CurModule; Triple TargetTriple; LLVMContext *C; const DataLayout *DL; GlobalVariable *FunctionGuardArray; // for trace-pc-guard. - bool HasSancovGuardsSection; + GlobalVariable *Function8bitCounterArray; // for inline-8bit-counters. SanitizerCoverageOptions Options; }; } // namespace +void SanitizerCoverageModule::CreateInitCallForSection( + Module &M, const char *InitFunctionName, Type *Ty, + const std::string &Section) { + IRBuilder<> IRB(M.getContext()); + Function *CtorFunc; + GlobalVariable *SecStart = + new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, nullptr, + getSectionStart(Section)); + SecStart->setVisibility(GlobalValue::HiddenVisibility); + GlobalVariable *SecEnd = + new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, + nullptr, getSectionEnd(Section)); + SecEnd->setVisibility(GlobalValue::HiddenVisibility); + + std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions( + M, SanCovModuleCtorName, InitFunctionName, {Ty, Ty}, + {IRB.CreatePointerCast(SecStart, Ty), IRB.CreatePointerCast(SecEnd, Ty)}); + + if (TargetTriple.supportsCOMDAT()) { + // Use comdat to dedup CtorFunc. + CtorFunc->setComdat(M.getOrInsertComdat(SanCovModuleCtorName)); + appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc); + } else { + appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority); + } +} + bool SanitizerCoverageModule::runOnModule(Module &M) { if (Options.CoverageType == SanitizerCoverageOptions::SCK_None) return false; @@ -192,15 +241,18 @@ bool SanitizerCoverageModule::runOnModule(Module &M) { DL = &M.getDataLayout(); CurModule = &M; TargetTriple = Triple(M.getTargetTriple()); - HasSancovGuardsSection = false; + FunctionGuardArray = nullptr; + Function8bitCounterArray = nullptr; IntptrTy = Type::getIntNTy(*C, DL->getPointerSizeInBits()); IntptrPtrTy = PointerType::getUnqual(IntptrTy); Type *VoidTy = Type::getVoidTy(*C); IRBuilder<> IRB(*C); Int64PtrTy = PointerType::getUnqual(IRB.getInt64Ty()); Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty()); + Int8PtrTy = PointerType::getUnqual(IRB.getInt8Ty()); Int64Ty = IRB.getInt64Ty(); Int32Ty = IRB.getInt32Ty(); + Int8Ty = IRB.getInt8Ty(); SanCovTracePCIndir = checkSanitizerInterfaceFunction( M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy)); @@ -243,34 +295,13 @@ bool SanitizerCoverageModule::runOnModule(Module &M) { for (auto &F : M) runOnFunction(F); - // Create variable for module (compilation unit) name - if (Options.TracePCGuard) { - if (HasSancovGuardsSection) { - Function *CtorFunc; - GlobalVariable *SecStart = new GlobalVariable( - M, Int32PtrTy, false, GlobalVariable::ExternalLinkage, nullptr, - getSanCovTracePCGuardSectionStart()); - SecStart->setVisibility(GlobalValue::HiddenVisibility); - GlobalVariable *SecEnd = new GlobalVariable( - M, Int32PtrTy, false, GlobalVariable::ExternalLinkage, nullptr, - getSanCovTracePCGuardSectionEnd()); - SecEnd->setVisibility(GlobalValue::HiddenVisibility); - - std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions( - M, SanCovModuleCtorName, SanCovTracePCGuardInitName, - {Int32PtrTy, Int32PtrTy}, - {IRB.CreatePointerCast(SecStart, Int32PtrTy), - IRB.CreatePointerCast(SecEnd, Int32PtrTy)}); - - if (TargetTriple.supportsCOMDAT()) { - // Use comdat to dedup CtorFunc. - CtorFunc->setComdat(M.getOrInsertComdat(SanCovModuleCtorName)); - appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority, CtorFunc); - } else { - appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority); - } - } - } + if (FunctionGuardArray) + CreateInitCallForSection(M, SanCovTracePCGuardInitName, Int32PtrTy, + SanCovGuardsSectionName); + if (Function8bitCounterArray) + CreateInitCallForSection(M, SanCov8bitCountersInitName, Int8PtrTy, + SanCovCountersSectionName); + return true; } @@ -393,17 +424,26 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) { InjectTraceForGep(F, GepTraceTargets); return true; } -void SanitizerCoverageModule::CreateFunctionGuardArray(size_t NumGuards, - Function &F) { - if (!Options.TracePCGuard) return; - HasSancovGuardsSection = true; - ArrayType *ArrayOfInt32Ty = ArrayType::get(Int32Ty, NumGuards); - FunctionGuardArray = new GlobalVariable( - *CurModule, ArrayOfInt32Ty, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayOfInt32Ty), "__sancov_gen_"); + +GlobalVariable *SanitizerCoverageModule::CreateFunctionLocalArrayInSection( + size_t NumElements, Function &F, Type *Ty, const char *Section) { + ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); + auto Array = new GlobalVariable( + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); if (auto Comdat = F.getComdat()) - FunctionGuardArray->setComdat(Comdat); - FunctionGuardArray->setSection(getSanCovTracePCGuardSection()); + Array->setComdat(Comdat); + Array->setSection(getSectionName(Section)); + return Array; +} +void SanitizerCoverageModule::CreateFunctionLocalArrays(size_t NumGuards, + Function &F) { + if (Options.TracePCGuard) + FunctionGuardArray = CreateFunctionLocalArrayInSection( + NumGuards, F, Int32Ty, SanCovGuardsSectionName); + if (Options.Inline8bitCounters) + Function8bitCounterArray = CreateFunctionLocalArrayInSection( + NumGuards, F, Int8Ty, SanCovCountersSectionName); } bool SanitizerCoverageModule::InjectCoverage(Function &F, @@ -413,11 +453,11 @@ bool SanitizerCoverageModule::InjectCoverage(Function &F, case SanitizerCoverageOptions::SCK_None: return false; case SanitizerCoverageOptions::SCK_Function: - CreateFunctionGuardArray(1, F); + CreateFunctionLocalArrays(1, F); InjectCoverageAtBlock(F, F.getEntryBlock(), 0); return true; default: { - CreateFunctionGuardArray(AllBlocks.size(), F); + CreateFunctionLocalArrays(AllBlocks.size(), F); for (size_t i = 0, N = AllBlocks.size(); i < N; i++) InjectCoverageAtBlock(F, *AllBlocks[i], i); return true; @@ -436,7 +476,7 @@ void SanitizerCoverageModule::InjectCoverageForIndirectCalls( Function &F, ArrayRef<Instruction *> IndirCalls) { if (IndirCalls.empty()) return; - assert(Options.TracePC || Options.TracePCGuard); + assert(Options.TracePC || Options.TracePCGuard || Options.Inline8bitCounters); for (auto I : IndirCalls) { IRBuilder<> IRB(I); CallSite CS(I); @@ -564,8 +604,8 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB, if (Options.TracePC) { IRB.CreateCall(SanCovTracePC); // gets the PC using GET_CALLER_PC. IRB.CreateCall(EmptyAsm, {}); // Avoids callback merge. - } else { - assert(Options.TracePCGuard); + } + if (Options.TracePCGuard) { auto GuardPtr = IRB.CreateIntToPtr( IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), ConstantInt::get(IntptrTy, Idx * 4)), @@ -573,26 +613,39 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB, IRB.CreateCall(SanCovTracePCGuard, GuardPtr); IRB.CreateCall(EmptyAsm, {}); // Avoids callback merge. } + if (Options.Inline8bitCounters) { + auto CounterPtr = IRB.CreateGEP( + Function8bitCounterArray, + {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); + auto Load = IRB.CreateLoad(CounterPtr); + auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1)); + auto Store = IRB.CreateStore(Inc, CounterPtr); + SetNoSanitizeMetadata(Load); + SetNoSanitizeMetadata(Store); + } } -StringRef SanitizerCoverageModule::getSanCovTracePCGuardSection() const { +std::string +SanitizerCoverageModule::getSectionName(const std::string &Section) const { if (TargetTriple.getObjectFormat() == Triple::COFF) return ".SCOV$M"; if (TargetTriple.isOSBinFormatMachO()) - return "__DATA,__sancov_guards"; - return "__sancov_guards"; + return "__DATA,__" + Section; + return "__" + Section; } -StringRef SanitizerCoverageModule::getSanCovTracePCGuardSectionStart() const { +std::string +SanitizerCoverageModule::getSectionStart(const std::string &Section) const { if (TargetTriple.isOSBinFormatMachO()) - return "\1section$start$__DATA$__sancov_guards"; - return "__start___sancov_guards"; + return "\1section$start$__DATA$__" + Section; + return "__start___" + Section; } -StringRef SanitizerCoverageModule::getSanCovTracePCGuardSectionEnd() const { +std::string +SanitizerCoverageModule::getSectionEnd(const std::string &Section) const { if (TargetTriple.isOSBinFormatMachO()) - return "\1section$end$__DATA$__sancov_guards"; - return "__stop___sancov_guards"; + return "\1section$end$__DATA$__" + Section; + return "__stop___" + Section; } diff --git a/lib/Transforms/Scalar/IndVarSimplify.cpp b/lib/Transforms/Scalar/IndVarSimplify.cpp index 3953198fe605..9a7882211bac 100644 --- a/lib/Transforms/Scalar/IndVarSimplify.cpp +++ b/lib/Transforms/Scalar/IndVarSimplify.cpp @@ -1823,6 +1823,7 @@ static PHINode *getLoopPhiForCounter(Value *IncV, Loop *L, DominatorTree *DT) { // An IV counter must preserve its type. if (IncI->getNumOperands() == 2) break; + LLVM_FALLTHROUGH; default: return nullptr; } diff --git a/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp index 930696b036c0..7d8da9b453f9 100644 --- a/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp +++ b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp @@ -14,6 +14,7 @@ #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" @@ -83,6 +84,149 @@ static bool handleSwitchExpect(SwitchInst &SI) { return true; } +/// Handler for PHINodes that define the value argument to an +/// @llvm.expect call. +/// +/// If the operand of the phi has a constant value and it 'contradicts' +/// with the expected value of phi def, then the corresponding incoming +/// edge of the phi is unlikely to be taken. Using that information, +/// the branch probability info for the originating branch can be inferred. +static void handlePhiDef(CallInst *Expect) { + Value &Arg = *Expect->getArgOperand(0); + ConstantInt *ExpectedValue = cast<ConstantInt>(Expect->getArgOperand(1)); + const APInt &ExpectedPhiValue = ExpectedValue->getValue(); + + // Walk up in backward a list of instructions that + // have 'copy' semantics by 'stripping' the copies + // until a PHI node or an instruction of unknown kind + // is reached. Negation via xor is also handled. + // + // C = PHI(...); + // B = C; + // A = B; + // D = __builtin_expect(A, 0); + // + Value *V = &Arg; + SmallVector<Instruction *, 4> Operations; + while (!isa<PHINode>(V)) { + if (ZExtInst *ZExt = dyn_cast<ZExtInst>(V)) { + V = ZExt->getOperand(0); + Operations.push_back(ZExt); + continue; + } + + if (SExtInst *SExt = dyn_cast<SExtInst>(V)) { + V = SExt->getOperand(0); + Operations.push_back(SExt); + continue; + } + + BinaryOperator *BinOp = dyn_cast<BinaryOperator>(V); + if (!BinOp || BinOp->getOpcode() != Instruction::Xor) + return; + + ConstantInt *CInt = dyn_cast<ConstantInt>(BinOp->getOperand(1)); + if (!CInt) + return; + + V = BinOp->getOperand(0); + Operations.push_back(BinOp); + } + + // Executes the recorded operations on input 'Value'. + auto ApplyOperations = [&](const APInt &Value) { + APInt Result = Value; + for (auto Op : llvm::reverse(Operations)) { + switch (Op->getOpcode()) { + case Instruction::Xor: + Result ^= cast<ConstantInt>(Op->getOperand(1))->getValue(); + break; + case Instruction::ZExt: + Result = Result.zext(Op->getType()->getIntegerBitWidth()); + break; + case Instruction::SExt: + Result = Result.sext(Op->getType()->getIntegerBitWidth()); + break; + default: + llvm_unreachable("Unexpected operation"); + } + } + return Result; + }; + + auto *PhiDef = dyn_cast<PHINode>(V); + + // Get the first dominating conditional branch of the operand + // i's incoming block. + auto GetDomConditional = [&](unsigned i) -> BranchInst * { + BasicBlock *BB = PhiDef->getIncomingBlock(i); + BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); + if (BI && BI->isConditional()) + return BI; + BB = BB->getSinglePredecessor(); + if (!BB) + return nullptr; + BI = dyn_cast<BranchInst>(BB->getTerminator()); + if (!BI || BI->isUnconditional()) + return nullptr; + return BI; + }; + + // Now walk through all Phi operands to find phi oprerands with values + // conflicting with the expected phi output value. Any such operand + // indicates the incoming edge to that operand is unlikely. + for (unsigned i = 0, e = PhiDef->getNumIncomingValues(); i != e; ++i) { + + Value *PhiOpnd = PhiDef->getIncomingValue(i); + ConstantInt *CI = dyn_cast<ConstantInt>(PhiOpnd); + if (!CI) + continue; + + // Not an interesting case when IsUnlikely is false -- we can not infer + // anything useful when the operand value matches the expected phi + // output. + if (ExpectedPhiValue == ApplyOperations(CI->getValue())) + continue; + + BranchInst *BI = GetDomConditional(i); + if (!BI) + continue; + + MDBuilder MDB(PhiDef->getContext()); + + // There are two situations in which an operand of the PhiDef comes + // from a given successor of a branch instruction BI. + // 1) When the incoming block of the operand is the successor block; + // 2) When the incoming block is BI's enclosing block and the + // successor is the PhiDef's enclosing block. + // + // Returns true if the operand which comes from OpndIncomingBB + // comes from outgoing edge of BI that leads to Succ block. + auto *OpndIncomingBB = PhiDef->getIncomingBlock(i); + auto IsOpndComingFromSuccessor = [&](BasicBlock *Succ) { + if (OpndIncomingBB == Succ) + // If this successor is the incoming block for this + // Phi operand, then this successor does lead to the Phi. + return true; + if (OpndIncomingBB == BI->getParent() && Succ == PhiDef->getParent()) + // Otherwise, if the edge is directly from the branch + // to the Phi, this successor is the one feeding this + // Phi operand. + return true; + return false; + }; + + if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) + BI->setMetadata( + LLVMContext::MD_prof, + MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight)); + else if (IsOpndComingFromSuccessor(BI->getSuccessor(0))) + BI->setMetadata( + LLVMContext::MD_prof, + MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight)); + } +} + // Handle both BranchInst and SelectInst. template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { @@ -99,25 +243,31 @@ template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition()); CmpInst::Predicate Predicate; - uint64_t ValueComparedTo = 0; + ConstantInt *CmpConstOperand = nullptr; if (!CmpI) { CI = dyn_cast<CallInst>(BSI.getCondition()); Predicate = CmpInst::ICMP_NE; - ValueComparedTo = 0; } else { Predicate = CmpI->getPredicate(); if (Predicate != CmpInst::ICMP_NE && Predicate != CmpInst::ICMP_EQ) return false; - ConstantInt *CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1)); + + CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1)); if (!CmpConstOperand) return false; - ValueComparedTo = CmpConstOperand->getZExtValue(); CI = dyn_cast<CallInst>(CmpI->getOperand(0)); } if (!CI) return false; + uint64_t ValueComparedTo = 0; + if (CmpConstOperand) { + if (CmpConstOperand->getBitWidth() > 64) + return false; + ValueComparedTo = CmpConstOperand->getZExtValue(); + } + Function *Fn = CI->getCalledFunction(); if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) return false; @@ -181,6 +331,10 @@ static bool lowerExpectIntrinsic(Function &F) { Function *Fn = CI->getCalledFunction(); if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { + // Before erasing the llvm.expect, walk backward to find + // phi that define llvm.expect's first arg, and + // infer branch probability: + handlePhiDef(CI); Value *Exp = CI->getArgOperand(0); CI->replaceAllUsesWith(Exp); CI->eraseFromParent(); diff --git a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 77b2bd84f9b6..350b50ffcdd4 100644 --- a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// // -// Rewrite an existing set of gc.statepoints such that they make potential -// relocations performed by the garbage collector explicit in the IR. +// Rewrite call/invoke instructions so as to make potential relocations +// performed by the garbage collector explicit in the IR. // //===----------------------------------------------------------------------===// @@ -2094,9 +2094,9 @@ static bool insertParsePoints(Function &F, DominatorTree &DT, // live in the IR. We'll remove all of these when done. SmallVector<CallInst *, 64> Holders; - // Insert a dummy call with all of the arguments to the vm_state we'll need - // for the actual safepoint insertion. This ensures reference arguments in - // the deopt argument list are considered live through the safepoint (and + // Insert a dummy call with all of the deopt operands we'll need for the + // actual safepoint insertion as arguments. This ensures reference operands + // in the deopt argument list are considered live through the safepoint (and // thus makes sure they get relocated.) for (CallSite CS : ToUpdate) { SmallVector<Value *, 64> DeoptValues; diff --git a/lib/Transforms/Scalar/SROA.cpp b/lib/Transforms/Scalar/SROA.cpp index 6e113bccff94..fb1b5813fd79 100644 --- a/lib/Transforms/Scalar/SROA.cpp +++ b/lib/Transforms/Scalar/SROA.cpp @@ -3698,7 +3698,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { int Idx = 0, Size = Offsets.Splits.size(); for (;;) { auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); - auto *PartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); + auto *LoadPartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace()); + auto *StorePartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); // Either lookup a split load or create one. LoadInst *PLoad; @@ -3709,7 +3710,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { PLoad = IRB.CreateAlignedLoad( getAdjustedPtr(IRB, DL, LoadBasePtr, APInt(DL.getPointerSizeInBits(), PartOffset), - PartPtrTy, LoadBasePtr->getName() + "."), + LoadPartPtrTy, LoadBasePtr->getName() + "."), getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false, LI->getName()); } @@ -3719,7 +3720,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { StoreInst *PStore = IRB.CreateAlignedStore( PLoad, getAdjustedPtr(IRB, DL, StoreBasePtr, APInt(DL.getPointerSizeInBits(), PartOffset), - PartPtrTy, StoreBasePtr->getName() + "."), + StorePartPtrTy, StoreBasePtr->getName() + "."), getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false); // Now build a new slice for the alloca. diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp index 1ec3d0d49637..1c1a75c111e9 100644 --- a/lib/Transforms/Utils/CloneFunction.cpp +++ b/lib/Transforms/Utils/CloneFunction.cpp @@ -37,10 +37,10 @@ using namespace llvm; /// See comments in Cloning.h. -BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, - ValueToValueMapTy &VMap, +BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix, Function *F, - ClonedCodeInfo *CodeInfo) { + ClonedCodeInfo *CodeInfo, + DebugInfoFinder *DIFinder) { DenseMap<const MDNode *, MDNode *> Cache; BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "", F); if (BB->hasName()) NewBB->setName(BB->getName()+NameSuffix); @@ -50,10 +50,11 @@ BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, // Loop over all instructions, and copy them over. for (BasicBlock::const_iterator II = BB->begin(), IE = BB->end(); II != IE; ++II) { + + if (DIFinder && F->getParent() && II->getDebugLoc()) + DIFinder->processLocation(*F->getParent(), II->getDebugLoc().get()); + Instruction *NewInst = II->clone(); - if (F && F->getSubprogram()) - DebugLoc::reparentDebugInfo(*NewInst, BB->getParent()->getSubprogram(), - F->getSubprogram(), Cache); if (II->hasName()) NewInst->setName(II->getName()+NameSuffix); NewBB->getInstList().push_back(NewInst); @@ -122,31 +123,38 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, AttributeList::get(NewFunc->getContext(), OldAttrs.getFnAttributes(), OldAttrs.getRetAttributes(), NewArgAttrs)); + bool MustCloneSP = + OldFunc->getParent() && OldFunc->getParent() == NewFunc->getParent(); + DISubprogram *SP = OldFunc->getSubprogram(); + if (SP) { + assert(!MustCloneSP || ModuleLevelChanges); + // Add mappings for some DebugInfo nodes that we don't want duplicated + // even if they're distinct. + auto &MD = VMap.MD(); + MD[SP->getUnit()].reset(SP->getUnit()); + MD[SP->getType()].reset(SP->getType()); + MD[SP->getFile()].reset(SP->getFile()); + // If we're not cloning into the same module, no need to clone the + // subprogram + if (!MustCloneSP) + MD[SP].reset(SP); + } + SmallVector<std::pair<unsigned, MDNode *>, 1> MDs; OldFunc->getAllMetadata(MDs); for (auto MD : MDs) { - MDNode *NewMD; - bool MustCloneSP = - (MD.first == LLVMContext::MD_dbg && OldFunc->getParent() && - OldFunc->getParent() == NewFunc->getParent()); - if (MustCloneSP) { - auto *SP = cast<DISubprogram>(MD.second); - NewMD = DISubprogram::getDistinct( - NewFunc->getContext(), SP->getScope(), SP->getName(), - SP->getLinkageName(), SP->getFile(), SP->getLine(), SP->getType(), - SP->isLocalToUnit(), SP->isDefinition(), SP->getScopeLine(), - SP->getContainingType(), SP->getVirtuality(), SP->getVirtualIndex(), - SP->getThisAdjustment(), SP->getFlags(), SP->isOptimized(), - SP->getUnit(), SP->getTemplateParams(), SP->getDeclaration(), - SP->getVariables(), SP->getThrownTypes()); - } else - NewMD = - MapMetadata(MD.second, VMap, - ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges, - TypeMapper, Materializer); - NewFunc->addMetadata(MD.first, *NewMD); + NewFunc->addMetadata( + MD.first, + *MapMetadata(MD.second, VMap, + ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges, + TypeMapper, Materializer)); } + // When we remap instructions, we want to avoid duplicating inlined + // DISubprograms, so record all subprograms we find as we duplicate + // instructions and then freeze them in the MD map. + DebugInfoFinder DIFinder; + // Loop over all of the basic blocks in the function, cloning them as // appropriate. Note that we save BE this way in order to handle cloning of // recursive functions into themselves. @@ -156,7 +164,8 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, const BasicBlock &BB = *BI; // Create a new basic block and copy instructions into it! - BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo); + BasicBlock *CBB = CloneBasicBlock(&BB, VMap, NameSuffix, NewFunc, CodeInfo, + SP ? &DIFinder : nullptr); // Add basic block mapping. VMap[&BB] = CBB; @@ -178,6 +187,12 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, Returns.push_back(RI); } + for (DISubprogram *ISP : DIFinder.subprograms()) { + if (ISP != SP) { + VMap.MD()[ISP].reset(ISP); + } + } + // Loop over all of the instructions in the function, fixing up operand // references as we go. This uses VMap to do all the hard work. for (Function::iterator BB = @@ -226,7 +241,7 @@ Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap, } SmallVector<ReturnInst*, 8> Returns; // Ignore returns cloned. - CloneFunctionInto(NewF, F, VMap, /*ModuleLevelChanges=*/false, Returns, "", + CloneFunctionInto(NewF, F, VMap, F->getSubprogram() != nullptr, Returns, "", CodeInfo); return NewF; diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index 8b9a64c220cc..799eef21dc4e 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4779,6 +4779,7 @@ void InnerLoopVectorizer::vectorizeInstruction(Instruction &I) { scalarizeInstruction(&I, true); break; } + LLVM_FALLTHROUGH; case Instruction::Add: case Instruction::FAdd: case Instruction::Sub: @@ -7396,6 +7397,7 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I, // likely. return Cost / getReciprocalPredBlockProb(); } + LLVM_FALLTHROUGH; case Instruction::Add: case Instruction::FAdd: case Instruction::Sub: diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp index e6f78e6b94a3..d1349535f298 100644 --- a/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -259,6 +259,7 @@ static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst, if (hasVectorInstrinsicScalarOpd(ID, 1)) { return (CI->getArgOperand(1) == Scalar); } + LLVM_FALLTHROUGH; } default: return false; @@ -4749,56 +4750,18 @@ static Value *getReductionValue(const DominatorTree *DT, PHINode *P, return nullptr; } -namespace { -/// Tracks instructons and its children. -class WeakTrackingVHWithLevel final : public CallbackVH { - /// Operand index of the instruction currently beeing analized. - unsigned Level = 0; - /// Is this the instruction that should be vectorized, or are we now - /// processing children (i.e. operands of this instruction) for potential - /// vectorization? - bool IsInitial = true; - -public: - explicit WeakTrackingVHWithLevel() = default; - WeakTrackingVHWithLevel(Value *V) : CallbackVH(V){}; - /// Restart children analysis each time it is repaced by the new instruction. - void allUsesReplacedWith(Value *New) override { - setValPtr(New); - Level = 0; - IsInitial = true; - } - /// Check if the instruction was not deleted during vectorization. - bool isValid() const { return !getValPtr(); } - /// Is the istruction itself must be vectorized? - bool isInitial() const { return IsInitial; } - /// Try to vectorize children. - void clearInitial() { IsInitial = false; } - /// Are all children processed already? - bool isFinal() const { - assert(getValPtr() && - (isa<Instruction>(getValPtr()) && - cast<Instruction>(getValPtr())->getNumOperands() >= Level)); - return getValPtr() && - cast<Instruction>(getValPtr())->getNumOperands() == Level; - } - /// Get next child operation. - Value *nextOperand() { - assert(getValPtr() && isa<Instruction>(getValPtr()) && - cast<Instruction>(getValPtr())->getNumOperands() > Level); - return cast<Instruction>(getValPtr())->getOperand(Level++); - } - virtual ~WeakTrackingVHWithLevel() = default; -}; -} // namespace - -/// \brief Attempt to reduce a horizontal reduction. -/// If it is legal to match a horizontal reduction feeding -/// the phi node P with reduction operators Root in a basic block BB, then check -/// if it can be done. -/// \returns true if a horizontal reduction was matched and reduced. -/// \returns false if a horizontal reduction was not matched. -static bool canBeVectorized( +/// Attempt to reduce a horizontal reduction. +/// If it is legal to match a horizontal reduction feeding the phi node \a P +/// with reduction operators \a Root (or one of its operands) in a basic block +/// \a BB, then check if it can be done. If horizontal reduction is not found +/// and root instruction is a binary operation, vectorization of the operands is +/// attempted. +/// \returns true if a horizontal reduction was matched and reduced or operands +/// of one of the binary instruction were vectorized. +/// \returns false if a horizontal reduction was not matched (or not possible) +/// or no vectorization of any binary operation feeding \a Root instruction was +/// performed. +static bool tryToVectorizeHorReductionOrInstOperands( PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R, TargetTransformInfo *TTI, const function_ref<bool(BinaryOperator *, BoUpSLP &)> Vectorize) { @@ -4810,56 +4773,62 @@ static bool canBeVectorized( if (Root->getParent() != BB) return false; - SmallVector<WeakTrackingVHWithLevel, 8> Stack(1, Root); + // Start analysis starting from Root instruction. If horizontal reduction is + // found, try to vectorize it. If it is not a horizontal reduction or + // vectorization is not possible or not effective, and currently analyzed + // instruction is a binary operation, try to vectorize the operands, using + // pre-order DFS traversal order. If the operands were not vectorized, repeat + // the same procedure considering each operand as a possible root of the + // horizontal reduction. + // Interrupt the process if the Root instruction itself was vectorized or all + // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized. + SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0}); SmallSet<Value *, 8> VisitedInstrs; bool Res = false; while (!Stack.empty()) { - Value *V = Stack.back(); - if (!V) { - Stack.pop_back(); + Value *V; + unsigned Level; + std::tie(V, Level) = Stack.pop_back_val(); + if (!V) continue; - } auto *Inst = dyn_cast<Instruction>(V); - if (!Inst || isa<PHINode>(Inst)) { - Stack.pop_back(); + if (!Inst || isa<PHINode>(Inst)) continue; - } - if (Stack.back().isInitial()) { - Stack.back().clearInitial(); - if (auto *BI = dyn_cast<BinaryOperator>(Inst)) { - HorizontalReduction HorRdx; - if (HorRdx.matchAssociativeReduction(P, BI)) { - if (HorRdx.tryToReduce(R, TTI)) { - Res = true; - P = nullptr; - continue; - } - } - if (P) { - Inst = dyn_cast<Instruction>(BI->getOperand(0)); - if (Inst == P) - Inst = dyn_cast<Instruction>(BI->getOperand(1)); - if (!Inst) { - P = nullptr; - continue; - } + if (auto *BI = dyn_cast<BinaryOperator>(Inst)) { + HorizontalReduction HorRdx; + if (HorRdx.matchAssociativeReduction(P, BI)) { + if (HorRdx.tryToReduce(R, TTI)) { + Res = true; + // Set P to nullptr to avoid re-analysis of phi node in + // matchAssociativeReduction function unless this is the root node. + P = nullptr; + continue; } } - P = nullptr; - if (Vectorize(dyn_cast<BinaryOperator>(Inst), R)) { - Res = true; - continue; + if (P) { + Inst = dyn_cast<Instruction>(BI->getOperand(0)); + if (Inst == P) + Inst = dyn_cast<Instruction>(BI->getOperand(1)); + if (!Inst) { + // Set P to nullptr to avoid re-analysis of phi node in + // matchAssociativeReduction function unless this is the root node. + P = nullptr; + continue; + } } } - if (Stack.back().isFinal()) { - Stack.pop_back(); + // Set P to nullptr to avoid re-analysis of phi node in + // matchAssociativeReduction function unless this is the root node. + P = nullptr; + if (Vectorize(dyn_cast<BinaryOperator>(Inst), R)) { + Res = true; continue; } - if (auto *NextV = dyn_cast<Instruction>(Stack.back().nextOperand())) - if (NextV->getParent() == BB && VisitedInstrs.insert(NextV).second && - Stack.size() < RecursionMaxDepth) - Stack.push_back(NextV); + // Try to vectorize operands. + if (++Level < RecursionMaxDepth) + for (auto *Op : Inst->operand_values()) + Stack.emplace_back(Op, Level); } return Res; } @@ -4876,10 +4845,10 @@ bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V, if (!isa<BinaryOperator>(I)) P = nullptr; // Try to match and vectorize a horizontal reduction. - return canBeVectorized(P, I, BB, R, TTI, - [this](BinaryOperator *BI, BoUpSLP &R) -> bool { - return tryToVectorize(BI, R); - }); + return tryToVectorizeHorReductionOrInstOperands( + P, I, BB, R, TTI, [this](BinaryOperator *BI, BoUpSLP &R) -> bool { + return tryToVectorize(BI, R); + }); } bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt index ef56fa1b9367..6793a49a2ddc 100644 --- a/runtimes/CMakeLists.txt +++ b/runtimes/CMakeLists.txt @@ -188,6 +188,7 @@ else() # if this is included from LLVM's CMake else() get_cmake_property(variableNames VARIABLES) add_custom_target(builtins) + add_custom_target(install-builtins) foreach(target ${LLVM_BUILTIN_TARGETS}) string(REPLACE "-" ";" builtin_target_list ${target}) foreach(item ${builtin_target_list}) @@ -218,6 +219,7 @@ else() # if this is included from LLVM's CMake USE_TOOLCHAIN ${EXTRA_ARGS}) add_dependencies(builtins builtins-${target}) + add_dependencies(install-builtins install-builtins-${target}) endforeach() endif() set(deps builtins) diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir index 0557008ceb4f..b3e41c7751c5 100644 --- a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir +++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir @@ -10,18 +10,27 @@ entry: ret void } + + declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1 + + attributes #1 = { nounwind } + ... --- name: test_constant registers: - { id: 0, class: _ } + - { id: 1, class: _ } body: | bb.0.entry: ; CHECK-LABEL: name: test_constant ; CHECK: %0(s32) = G_CONSTANT i32 5 + ; CHECK: %1(s1) = G_CONSTANT i1 false %0(s32) = G_CONSTANT i32 5 + %1(s1) = G_CONSTANT i1 0 + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), %0, %0, %0, %0, %0, %0, %1, %1; ... --- diff --git a/test/CodeGen/AMDGPU/basic-branch.ll b/test/CodeGen/AMDGPU/basic-branch.ll index e245e4296df2..d8f9e4f51ff4 100644 --- a/test/CodeGen/AMDGPU/basic-branch.ll +++ b/test/CodeGen/AMDGPU/basic-branch.ll @@ -34,8 +34,6 @@ end: ; GCN: s_cbranch_vccnz [[END:BB[0-9]+_[0-9]+]] ; GCN: buffer_store_dword -; GCNOPT-NEXT: s_waitcnt vmcnt(0) expcnt(0) -; TODO: This waitcnt can be eliminated ; GCN: {{^}}[[END]]: ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/branch-condition-and.ll b/test/CodeGen/AMDGPU/branch-condition-and.ll index 68b77ea3490e..662ea37a2b99 100644 --- a/test/CodeGen/AMDGPU/branch-condition-and.ll +++ b/test/CodeGen/AMDGPU/branch-condition-and.ll @@ -19,9 +19,8 @@ ; GCN-NEXT: BB{{[0-9]+_[0-9]+}}: ; %bb4 ; GCN: ds_write_b32 -; GCN: s_waitcnt -; GCN-NEXT: [[BB5]] +; GCN: [[BB5]] ; GCN: s_or_b64 exec, exec ; GCN-NEXT: s_endpgm ; GCN-NEXT: .Lfunc_end diff --git a/test/CodeGen/AMDGPU/branch-relaxation.ll b/test/CodeGen/AMDGPU/branch-relaxation.ll index 263059d4a6ed..d3f835bdf163 100644 --- a/test/CodeGen/AMDGPU/branch-relaxation.ll +++ b/test/CodeGen/AMDGPU/branch-relaxation.ll @@ -223,7 +223,6 @@ bb3: ; GCN-NEXT: [[BB2]]: ; %bb2 ; GCN: v_mov_b32_e32 [[BB2_K:v[0-9]+]], 17 ; GCN: buffer_store_dword [[BB2_K]] -; GCN: s_waitcnt vmcnt(0) ; GCN-NEXT: [[LONG_JUMP1:BB[0-9]+_[0-9]+]]: ; %bb2 ; GCN-NEXT: s_getpc_b64 vcc @@ -393,7 +392,6 @@ bb3: ; GCN-NEXT: ; BB#2: ; %if_uniform ; GCN: buffer_store_dword -; GCN: s_waitcnt vmcnt(0) ; GCN-NEXT: [[ENDIF]]: ; %endif ; GCN-NEXT: s_or_b64 exec, exec, [[MASK]] diff --git a/test/CodeGen/AMDGPU/commute-compares.ll b/test/CodeGen/AMDGPU/commute-compares.ll index 66148a43a271..caba83c50428 100644 --- a/test/CodeGen/AMDGPU/commute-compares.ll +++ b/test/CodeGen/AMDGPU/commute-compares.ll @@ -35,7 +35,7 @@ define amdgpu_kernel void @commute_ne_64_i32(i32 addrspace(1)* %out, i32 addrspa ; FIXME: Why isn't this being folded as a constant? ; GCN-LABEL: {{^}}commute_ne_litk_i32: ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3039 -; GCN: v_cmp_ne_u32_e32 vcc, [[K]], v{{[0-9]+}} +; GCN: v_cmp_ne_u32_e32 vcc, v{{[0-9]+}}, [[K]] define amdgpu_kernel void @commute_ne_litk_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i32, i32 addrspace(1)* %in, i32 %tid @@ -99,11 +99,9 @@ define amdgpu_kernel void @commute_ule_63_i32(i32 addrspace(1)* %out, i32 addrsp ret void } -; FIXME: Undo canonicalization to gt (x + 1) since it doesn't use the inline imm - ; GCN-LABEL: {{^}}commute_ule_64_i32: ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x41{{$}} -; GCN: v_cmp_gt_u32_e32 vcc, [[K]], v{{[0-9]+}} +; GCN: v_cmp_lt_u32_e32 vcc, v{{[0-9]+}}, [[K]] define amdgpu_kernel void @commute_ule_64_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i32, i32 addrspace(1)* %in, i32 %tid @@ -702,7 +700,7 @@ define amdgpu_kernel void @commute_uno_2.0_f64(i32 addrspace(1)* %out, double ad ; XGCN: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}} ; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 4{{$}} -; GCN: v_cmp_eq_u32_e32 vcc, [[FI]], v{{[0-9]+}} +; GCN: v_cmp_eq_u32_e32 vcc, v{{[0-9]+}}, [[FI]] define amdgpu_kernel void @commute_frameindex(i32 addrspace(1)* nocapture %out) #0 { entry: %stack0 = alloca i32 diff --git a/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll index d3e6c11ef908..79d9b1691878 100644 --- a/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll +++ b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll @@ -37,22 +37,21 @@ ; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; GCN: mask branch [[ENDIF:BB[0-9]+_[0-9]+]] ; GCN: {{^}}BB{{[0-9]+}}_1: ; %if ; GCN: s_mov_b32 m0, -1 ; GCN: ds_read_b32 [[LOAD1:v[0-9]+]] +; GCN: s_waitcnt lgkmcnt(0) ; GCN: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], s7 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload -; GCN: s_waitcnt vmcnt(0) ; Spill val register ; GCN: v_add_i32_e32 [[VAL:v[0-9]+]], vcc, [[LOAD1]], [[RELOAD_LOAD0]] ; GCN: buffer_store_dword [[VAL]], off, s[0:3], s7 offset:[[VAL_OFFSET:[0-9]+]] ; 4-byte Folded Spill -; GCN: s_waitcnt vmcnt(0) ; VMEM: [[ENDIF]]: ; Reload and restore exec mask +; VGPR: s_waitcnt lgkmcnt(0) ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] ; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] @@ -119,7 +118,6 @@ endif: ; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; GCN-NEXT: ; mask branch [[END:BB[0-9]+_[0-9]+]] ; GCN-NEXT: s_cbranch_execz [[END]] @@ -130,7 +128,6 @@ endif: ; GCN: v_cmp_ne_u32_e32 vcc, ; GCN: s_and_b64 vcc, exec, vcc ; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; GCN-NEXT: s_cbranch_vccnz [[LOOP]] @@ -197,7 +194,6 @@ end: ; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:[[SAVEEXEC_HI_OFFSET:[0-9]+]] ; 4-byte Folded Spill ; GCN: s_mov_b64 exec, [[CMP0]] -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; FIXME: It makes no sense to put this skip here ; GCN-NEXT: ; mask branch [[FLOW:BB[0-9]+_[0-9]+]] @@ -235,7 +231,6 @@ end: ; GCN: buffer_store_dword [[FLOW_VAL]], off, s[0:3], s7 offset:[[RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill ; GCN: s_xor_b64 exec, exec, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} -; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) ; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9]+_[0-9]+]] ; GCN-NEXT: s_cbranch_execz [[ENDIF]] @@ -245,14 +240,12 @@ end: ; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload ; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] ; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[RESULT_OFFSET]] ; 4-byte Folded Spill -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; GCN-NEXT: s_branch [[ENDIF:BB[0-9]+_[0-9]+]] ; GCN: [[ELSE]]: ; %else ; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload ; GCN: v_subrev_i32_e32 [[SUB:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] ; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[FLOW_RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill -; GCN: s_waitcnt vmcnt(0) expcnt(0) ; GCN-NEXT: s_branch [[FLOW]] ; GCN: [[ENDIF]]: diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/test/CodeGen/AMDGPU/indirect-addressing-si.ll index b18ae353ca4c..fab1f8d12253 100644 --- a/test/CodeGen/AMDGPU/indirect-addressing-si.ll +++ b/test/CodeGen/AMDGPU/indirect-addressing-si.ll @@ -120,8 +120,7 @@ entry: ; FIXME: The waitcnt for the argument load can go after the loop ; IDXMODE: s_set_gpr_idx_on 0, src0 ; GCN: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec -; GCN: s_waitcnt lgkmcnt(0) - +; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v{{[0-9]+}} ; MOVREL: s_add_i32 m0, [[READLANE]], 0xfffffe0 @@ -250,8 +249,6 @@ entry: ; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 4{{$}} ; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec -; GCN: s_waitcnt lgkmcnt(0) - ; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] @@ -290,7 +287,6 @@ entry: ; IDXMODE: s_set_gpr_idx_on 0, dst ; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec -; GCN: s_waitcnt lgkmcnt(0) ; The offset depends on the register that holds the first element of the vector. ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] @@ -330,9 +326,9 @@ entry: ; IDXMODE: s_set_gpr_idx_on 0, src0 ; GCN: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec -; GCN: s_waitcnt vmcnt(0) ; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] ; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] @@ -411,6 +407,7 @@ bb2: ; IDXMODE: s_set_gpr_idx_on 0, dst ; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] ; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] diff --git a/test/CodeGen/AMDGPU/infinite-loop.ll b/test/CodeGen/AMDGPU/infinite-loop.ll index 73482756b8c8..3caffc342c7e 100644 --- a/test/CodeGen/AMDGPU/infinite-loop.ll +++ b/test/CodeGen/AMDGPU/infinite-loop.ll @@ -4,8 +4,8 @@ ; SI-LABEL: {{^}}infinite_loop: ; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3e7 ; SI: BB0_1: +; SI: s_waitcnt lgkmcnt(0) ; SI: buffer_store_dword [[REG]] -; SI: s_waitcnt vmcnt(0) expcnt(0) ; SI: s_branch BB0_1 define amdgpu_kernel void @infinite_loop(i32 addrspace(1)* %out) { entry: diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll index 350dd38ef583..1edccff3bf15 100644 --- a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll +++ b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll @@ -421,11 +421,10 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspac } ; GCN-LABEL: {{^}}v_insertelement_v2i16_dynamic_vgpr: -; GFX89: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}} -; CI: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7 ; GCN: flat_load_dword [[IDX:v[0-9]+]] ; GCN: flat_load_dword [[VEC:v[0-9]+]] -; GFX89-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7 +; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}} +; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7 ; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 16, [[IDX]] ; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]] @@ -449,11 +448,10 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_vgpr(<2 x i16> addrspac } ; GCN-LABEL: {{^}}v_insertelement_v2f16_dynamic_vgpr: -; GFX89: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}} -; CI: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234 ; GCN: flat_load_dword [[IDX:v[0-9]+]] ; GCN: flat_load_dword [[VEC:v[0-9]+]] -; GFX89-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234 +; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}} +; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234 ; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 16, [[IDX]] ; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]] diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll index 555a1d23ebe9..e50455f6f9a1 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll @@ -58,7 +58,7 @@ main_body: ; ;CHECK-LABEL: {{^}}buffer_store_wait: ;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen -;CHECK: s_waitcnt vmcnt(0) expcnt(0) +;CHECK: s_waitcnt expcnt(0) ;CHECK: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen ;CHECK: s_waitcnt vmcnt(0) ;CHECK: buffer_store_format_xyzw v[0:3], v6, s[0:3], 0 idxen diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll index 5ae255c7a26c..81597516d5f2 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll @@ -58,7 +58,7 @@ main_body: ; ;CHECK-LABEL: {{^}}buffer_store_wait: ;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen -;CHECK: s_waitcnt vmcnt(0) expcnt(0) +;CHECK: s_waitcnt expcnt(0) ;CHECK: buffer_load_dwordx4 v[0:3], v5, s[0:3], 0 idxen ;CHECK: s_waitcnt vmcnt(0) ;CHECK: buffer_store_dwordx4 v[0:3], v6, s[0:3], 0 idxen diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll index 02642142ae2c..d97644262016 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll @@ -5,7 +5,6 @@ declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #0 ; FUNC-LABEL: {{^}}ds_swizzle: ; CHECK: ds_swizzle_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:swizzle(BITMASK_PERM,"00p11") -; CHECK: s_waitcnt lgkmcnt define amdgpu_kernel void @ds_swizzle(i32 addrspace(1)* %out, i32 %src) nounwind { %swizzle = call i32 @llvm.amdgcn.ds.swizzle(i32 %src, i32 100) #0 store i32 %swizzle, i32 addrspace(1)* %out, align 4 diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll index c74c0fa15855..a289f7b0cfb1 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll @@ -130,7 +130,7 @@ main_body: ; ; GCN-LABEL: {{^}}image_store_wait: ; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm -; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN: s_waitcnt expcnt(0) ; GCN: image_load v[0:3], v4, s[8:15] dmask:0xf unorm ; GCN: s_waitcnt vmcnt(0) ; GCN: image_store v[0:3], v4, s[16:23] dmask:0xf unorm diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll index 055dddbfa8af..9a27809f37bb 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll @@ -2,6 +2,8 @@ ; RUN: llc -mtriple=amdgcn--amdhsa-opencl -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL,HSA-OPENCL %s ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,OS-MESA3D,MESA,ALL %s ; RUN: llc -mtriple=amdgcn-mesa-unknown -verify-machineinstrs < %s | FileCheck -check-prefixes=OS-UNKNOWN,MESA,ALL %s +; RUN: llc -mtriple=amdgcn--amdhsa-amdgiz -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL,HSA-NOENV %s +; RUN: llc -mtriple=amdgcn--amdhsa-amdgizcl -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL,HSA-OPENCL %s ; ALL-LABEL: {{^}}test: ; CO-V2: enable_sgpr_kernarg_segment_ptr = 1 diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll index ef9cda142850..3d815cca5be2 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll @@ -1,10 +1,13 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 -check-prefix=NOAUTO %s +; RUN: llc -march=amdgcn -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 -check-prefix=AUTO %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=NOAUTO %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=AUTO %s ; GCN-LABEL: {{^}}test_barrier: ; GFX8: buffer_store_dword ; GFX9: flat_store_dword -; GCN: s_waitcnt +; NOAUTO: s_waitcnt +; AUTO-NOT: s_waitcnt ; GCN: s_barrier define amdgpu_kernel void @test_barrier(i32 addrspace(1)* %out, i32 %size) #0 { entry: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll index b488565c6b3a..224b2ed72e3b 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll @@ -20,7 +20,7 @@ define amdgpu_kernel void @test_s_dcache_inv() #0 { ; GCN: s_waitcnt lgkmcnt(0) ; encoding define amdgpu_kernel void @test_s_dcache_inv_insert_wait() #0 { call void @llvm.amdgcn.s.dcache.inv() - call void @llvm.amdgcn.s.waitcnt(i32 0) + call void @llvm.amdgcn.s.waitcnt(i32 127) br label %end end: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll index a3a5c329f411..f96d5db5794a 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll @@ -20,7 +20,7 @@ define amdgpu_kernel void @test_s_dcache_inv_vol() #0 { ; GCN: s_waitcnt lgkmcnt(0) ; encoding define amdgpu_kernel void @test_s_dcache_inv_vol_insert_wait() #0 { call void @llvm.amdgcn.s.dcache.inv.vol() - call void @llvm.amdgcn.s.waitcnt(i32 0) + call void @llvm.amdgcn.s.waitcnt(i32 127) br label %end end: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll index 909a85dda3e8..99b651350439 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll @@ -18,7 +18,7 @@ define amdgpu_kernel void @test_s_dcache_wb() #0 { ; VI: s_waitcnt lgkmcnt(0) ; encoding define amdgpu_kernel void @test_s_dcache_wb_insert_wait() #0 { call void @llvm.amdgcn.s.dcache.wb() - call void @llvm.amdgcn.s.waitcnt(i32 0) + call void @llvm.amdgcn.s.waitcnt(i32 127) br label %end end: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll index 217bf97c41a4..844fcecdb48b 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll @@ -18,7 +18,7 @@ define amdgpu_kernel void @test_s_dcache_wb_vol() #0 { ; VI: s_waitcnt lgkmcnt(0) ; encoding define amdgpu_kernel void @test_s_dcache_wb_vol_insert_wait() #0 { call void @llvm.amdgcn.s.dcache.wb.vol() - call void @llvm.amdgcn.s.waitcnt(i32 0) + call void @llvm.amdgcn.s.waitcnt(i32 127) br label %end end: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll index 6083ec885a86..ee58d359a935 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll @@ -18,8 +18,8 @@ define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <4 x float> %d0, <4 x float> ; ; CHECK-LABEL: {{^}}test2: ; CHECK: image_load -; CHECK-NOT: s_waitcnt vmcnt(0){{$}} -; CHECK: s_waitcnt +; CHECK-NEXT: s_waitcnt +; CHECK: s_waitcnt vmcnt(0){{$}} ; CHECK-NEXT: image_store define amdgpu_ps void @test2(<8 x i32> inreg %rsrc, i32 %c) { %t = call <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32 %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) diff --git a/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll b/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll index 9d0b6b395996..82c27f204a47 100644 --- a/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll +++ b/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll @@ -362,6 +362,7 @@ exit1: ; preds = %LeafBlock, %LeafBlock1 ; GCN: {{^BB[0-9]+_[0-9]+}}: ; %UnifiedReturnBlock ; GCN-NEXT: s_or_b64 exec, exec +; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) ; GCN-NEXT: ; return define amdgpu_ps float @uniform_branch_to_multi_divergent_region_exit_ret_ret_return_value(i32 inreg %sgpr, i32 %vgpr) #0 { diff --git a/test/CodeGen/AMDGPU/not-scalarize-volatile-load.ll b/test/CodeGen/AMDGPU/not-scalarize-volatile-load.ll new file mode 100644 index 000000000000..bced3c408c52 --- /dev/null +++ b/test/CodeGen/AMDGPU/not-scalarize-volatile-load.ll @@ -0,0 +1,15 @@ +; RUN: llc -mtriple amdgcn--amdhsa -mcpu=fiji -amdgpu-scalarize-global-loads < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: @volatile_load +; GCN: s_load_dwordx2 s{{\[}}[[LO_SREG:[0-9]+]]:[[HI_SREG:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0 +; GCN: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], s[[LO_SREG]] +; GCN: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], s[[HI_SREG]] +; GCN: flat_load_dword v{{[0-9]+}}, v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} + +define amdgpu_kernel void @volatile_load(i32 addrspace(1)* %arg, i32 addrspace(1)* nocapture %arg1) { +bb: + %tmp18 = load volatile i32, i32 addrspace(1)* %arg, align 4 + %tmp26 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 5 + store i32 %tmp18, i32 addrspace(1)* %tmp26, align 4 + ret void +} diff --git a/test/CodeGen/AMDGPU/ret_jump.ll b/test/CodeGen/AMDGPU/ret_jump.ll index f2fbacbab82e..e7a05d94cdc4 100644 --- a/test/CodeGen/AMDGPU/ret_jump.ll +++ b/test/CodeGen/AMDGPU/ret_jump.ll @@ -65,7 +65,6 @@ ret.bb: ; preds = %else, %main_body ; GCN-NEXT: ; %unreachable.bb ; GCN: ds_write_b32 -; GCN: s_waitcnt ; GCN: ; divergent unreachable ; GCN: ; %ret.bb @@ -73,6 +72,7 @@ ret.bb: ; preds = %else, %main_body ; GCN: ; %UnifiedReturnBlock ; GCN-NEXT: s_or_b64 exec, exec +; GCN-NEXT: s_waitcnt ; GCN-NEXT: ; return ; GCN-NEXT: .Lfunc_end define amdgpu_ps <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> @uniform_br_nontrivial_ret_divergent_br_nontrivial_unreachable([9 x <16 x i8>] addrspace(2)* byval %arg, [17 x <16 x i8>] addrspace(2)* byval %arg1, [17 x <8 x i32>] addrspace(2)* byval %arg2, i32 addrspace(2)* byval %arg3, float inreg %arg4, i32 inreg %arg5, <2 x i32> %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <3 x i32> %arg9, <2 x i32> %arg10, <2 x i32> %arg11, <2 x i32> %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, i32 inreg %arg18, i32 %arg19, float %arg20, i32 %arg21) #0 { diff --git a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll index cb010cf15300..5b0d5274d5bc 100644 --- a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll +++ b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll @@ -9,7 +9,6 @@ ; GCN-NEXT: BB{{[0-9]+_[0-9]+}}: ; %unreachable ; GCN: ds_write_b32 ; GCN: ; divergent unreachable -; GCN: s_waitcnt ; GCN-NEXT: [[RET]]: ; %UnifiedReturnBlock ; GCN-NEXT: s_or_b64 exec, exec @@ -38,7 +37,6 @@ ret: ; GCN-NEXT: {{^BB[0-9]+_[0-9]+}}: ; %unreachable ; GCN: ds_write_b32 ; GCN: ; divergent unreachable -; GCN: s_waitcnt ; GCN: [[RETURN]]: ; GCN-NEXT: s_or_b64 exec, exec @@ -66,7 +64,6 @@ unreachable: ; GCN: [[UNREACHABLE]]: ; GCN: ds_write_b32 -; GCN: s_waitcnt define amdgpu_kernel void @uniform_lower_control_flow_unreachable_terminator(i32 %arg0) #0 { bb: %tmp63 = icmp eq i32 %arg0, 32 diff --git a/test/CodeGen/AMDGPU/smrd-vccz-bug.ll b/test/CodeGen/AMDGPU/smrd-vccz-bug.ll index 343211b0219c..333113e8a9b6 100644 --- a/test/CodeGen/AMDGPU/smrd-vccz-bug.ll +++ b/test/CodeGen/AMDGPU/smrd-vccz-bug.ll @@ -5,7 +5,7 @@ ; GCN-FUNC: {{^}}vccz_workaround: ; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x0 ; GCN: v_cmp_neq_f32_e64 vcc, s{{[0-9]+}}, 0{{$}} -; GCN: s_waitcnt lgkmcnt(0) +; VCCZ-BUG: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; VCCZ-BUG: s_mov_b64 vcc, vcc ; NOVCCZ-BUG-NOT: s_mov_b64 vcc, vcc ; GCN: s_cbranch_vccnz [[EXIT:[0-9A-Za-z_]+]] diff --git a/test/CodeGen/AMDGPU/spill-m0.ll b/test/CodeGen/AMDGPU/spill-m0.ll index 8f1aebfe9ceb..7e8fa118c2c2 100644 --- a/test/CodeGen/AMDGPU/spill-m0.ll +++ b/test/CodeGen/AMDGPU/spill-m0.ll @@ -18,13 +18,11 @@ ; TOVMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 ; TOVMEM-DAG: v_mov_b32_e32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]] ; TOVMEM: buffer_store_dword [[SPILL_VREG]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4 ; 4-byte Folded Spill -; TOVMEM: s_waitcnt vmcnt(0) ; TOSMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 ; TOSMEM: s_add_u32 m0, s3, 0x100{{$}} ; TOSMEM-NOT: [[M0_COPY]] ; TOSMEM: s_buffer_store_dword [[M0_COPY]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Spill -; TOSMEM: s_waitcnt lgkmcnt(0) ; GCN: s_cbranch_scc1 [[ENDIF:BB[0-9]+_[0-9]+]] diff --git a/test/CodeGen/AMDGPU/sub.i16.ll b/test/CodeGen/AMDGPU/sub.i16.ll index cf9e714ea6d3..1d407ea9bcda 100644 --- a/test/CodeGen/AMDGPU/sub.i16.ll +++ b/test/CodeGen/AMDGPU/sub.i16.ll @@ -85,9 +85,9 @@ define amdgpu_kernel void @v_test_sub_i16_zext_to_i32(i32 addrspace(1)* %out, i1 ; FIXME: Need to handle non-uniform case for function below (load without gep). ; GCN-LABEL: {{^}}v_test_sub_i16_zext_to_i64: +; VI: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0 ; VI: flat_load_ushort [[A:v[0-9]+]] ; VI: flat_load_ushort [[B:v[0-9]+]] -; VI-DAG: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0 ; VI-DAG: v_subrev_u16_e32 v[[ADD:[0-9]+]], [[B]], [[A]] ; VI: buffer_store_dwordx2 v{{\[}}[[ADD]]:[[VZERO]]{{\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0{{$}} define amdgpu_kernel void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { diff --git a/test/CodeGen/AMDGPU/valu-i1.ll b/test/CodeGen/AMDGPU/valu-i1.ll index 85a8929ebe58..a67f36d0a7e8 100644 --- a/test/CodeGen/AMDGPU/valu-i1.ll +++ b/test/CodeGen/AMDGPU/valu-i1.ll @@ -11,7 +11,6 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone ; SI: v_cmp_lt_i32_e32 vcc, 0, ; SI-NEXT: s_and_saveexec_b64 [[SAVE1:s\[[0-9]+:[0-9]+\]]], vcc ; SI-NEXT: s_xor_b64 [[SAVE2:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE1]] -; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: ; mask branch [[FLOW_BB:BB[0-9]+_[0-9]+]] ; SI-NEXT: s_cbranch_execz [[FLOW_BB]] @@ -72,7 +71,6 @@ end: ; SI-NEXT: BB{{[0-9]+_[0-9]+}}: ; SI: buffer_store_dword -; SI-NEXT: s_waitcnt ; SI-NEXT: {{^}}[[EXIT]]: ; SI: s_or_b64 exec, exec, [[BR_SREG]] @@ -101,7 +99,6 @@ exit: ; SI-NEXT: BB{{[0-9]+_[0-9]+}}: ; SI: buffer_store_dword -; SI-NEXT: s_waitcnt ; SI-NEXT: {{^}}[[EXIT]]: ; SI: s_or_b64 exec, exec, [[BR_SREG]] @@ -132,7 +129,6 @@ exit: ; SI-NEXT: {{^BB[0-9]+_[0-9]+}}: ; %exit ; SI: ds_write_b32 -; SI: s_waitcnt ; SI-NEXT: {{^}}[[FLOW]]: ; SI-NEXT: s_or_saveexec_b64 @@ -140,8 +136,8 @@ exit: ; SI-NEXT: ; mask branch [[UNIFIED_RETURN:BB[0-9]+_[0-9]+]] ; SI-NEXT: {{^BB[0-9]+_[0-9]+}}: ; %then -; SI: buffer_store_dword -; SI-NEXT: s_waitcnt +; SI: s_waitcnt +; SI-NEXT: buffer_store_dword ; SI-NEXT: {{^}}[[UNIFIED_RETURN]]: ; %UnifiedReturnBlock ; SI: s_or_b64 exec, exec diff --git a/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll b/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll index f4aba880ff76..1c7769894a27 100644 --- a/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll +++ b/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll @@ -974,6 +974,68 @@ entry: ret [2 x i32*] %r } +declare arm_aapcscc {i32, i32} @structs_target({i32, i32}, {i32*, float, i32, double}) + +define arm_aapcscc {i32, i32} @test_structs({i32, i32} %x, {i32*, float, i32, double} %y) { +; CHECK-LABEL: test_structs +; CHECK: fixedStack: +; CHECK-DAG: id: [[Y2_ID:[0-9]+]], offset: 0, size: 4 +; CHECK-DAG: id: [[Y3_ID:[0-9]+]], offset: 8, size: 8 +; CHECK: liveins: %r0, %r1, %r2, %r3 +; CHECK-DAG: [[X0:%[0-9]+]](s32) = COPY %r0 +; CHECK-DAG: [[X1:%[0-9]+]](s32) = COPY %r1 +; CHECK-DAG: [[Y0:%[0-9]+]](s32) = COPY %r2 +; CHECK-DAG: [[Y1:%[0-9]+]](s32) = COPY %r3 +; CHECK: [[Y2_ADDR:%[0-9]+]](p0) = G_FRAME_INDEX %fixed-stack.[[Y2_ID]] +; CHECK: [[Y2:%[0-9]+]](s32) = G_LOAD [[Y2_ADDR]](p0){{.*}}load 4 +; CHECK: [[Y3_ADDR:%[0-9]+]](p0) = G_FRAME_INDEX %fixed-stack.[[Y3_ID]] +; CHECK: [[Y3:%[0-9]+]](s64) = G_LOAD [[Y3_ADDR]](p0){{.*}}load 8 +; CHECK: [[X_0:%[0-9]+]](s64) = IMPLICIT_DEF +; CHECK: [[X_1:%[0-9]+]](s64) = G_INSERT [[X_0]], [[X0]](s32), 0 +; CHECK: [[X_2:%[0-9]+]](s64) = G_INSERT [[X_1]], [[X1]](s32), 32 +; CHECK: [[X:%[0-9]+]](s64) = COPY [[X_2]] +; CHECK: [[Y_0:%[0-9]+]](s192) = IMPLICIT_DEF +; CHECK: [[Y_1:%[0-9]+]](s192) = G_INSERT [[Y_0]], [[Y0]](s32), 0 +; CHECK: [[Y_2:%[0-9]+]](s192) = G_INSERT [[Y_1]], [[Y1]](s32), 32 +; CHECK: [[Y_3:%[0-9]+]](s192) = G_INSERT [[Y_2]], [[Y2]](s32), 64 +; CHECK: [[Y_4:%[0-9]+]](s192) = G_INSERT [[Y_3]], [[Y3]](s64), 128 +; CHECK: [[Y:%[0-9]+]](s192) = COPY [[Y_4]] +; CHECK: ADJCALLSTACKDOWN 16, 0, 14, _, implicit-def %sp, implicit %sp +; CHECK: [[X0:%[0-9]+]](s32) = G_EXTRACT [[X]](s64), 0 +; CHECK: [[X1:%[0-9]+]](s32) = G_EXTRACT [[X]](s64), 32 +; CHECK: [[Y0:%[0-9]+]](s32) = G_EXTRACT [[Y]](s192), 0 +; CHECK: [[Y1:%[0-9]+]](s32) = G_EXTRACT [[Y]](s192), 32 +; CHECK: [[Y2:%[0-9]+]](s32) = G_EXTRACT [[Y]](s192), 64 +; CHECK: [[Y3:%[0-9]+]](s64) = G_EXTRACT [[Y]](s192), 128 +; CHECK-DAG: %r0 = COPY [[X0]](s32) +; CHECK-DAG: %r1 = COPY [[X1]](s32) +; CHECK-DAG: %r2 = COPY [[Y0]](s32) +; CHECK-DAG: %r3 = COPY [[Y1]](s32) +; CHECK: [[SP:%[0-9]+]](p0) = COPY %sp +; CHECK: [[Y2_OFF:%[0-9]+]](s32) = G_CONSTANT i32 0 +; CHECK: [[Y2_ADDR:%[0-9]+]](p0) = G_GEP [[SP]], [[Y2_OFF]](s32) +; CHECK: G_STORE [[Y2]](s32), [[Y2_ADDR]](p0){{.*}}store 4 +; CHECK: [[SP:%[0-9]+]](p0) = COPY %sp +; CHECK: [[Y3_OFF:%[0-9]+]](s32) = G_CONSTANT i32 8 +; CHECK: [[Y3_ADDR:%[0-9]+]](p0) = G_GEP [[SP]], [[Y3_OFF]](s32) +; CHECK: G_STORE [[Y3]](s64), [[Y3_ADDR]](p0){{.*}}store 8 +; CHECK: BLX @structs_target, csr_aapcs, implicit-def %lr, implicit %sp, implicit %r0, implicit %r1, implicit %r2, implicit %r3, implicit-def %r0, implicit-def %r1 +; CHECK: [[R0:%[0-9]+]](s32) = COPY %r0 +; CHECK: [[R1:%[0-9]+]](s32) = COPY %r1 +; CHECK: [[R_0:%[0-9]+]](s64) = IMPLICIT_DEF +; CHECK: [[R_1:%[0-9]+]](s64) = G_INSERT [[R_0]], [[R0]](s32), 0 +; CHECK: [[R_2:%[0-9]+]](s64) = G_INSERT [[R_1]], [[R1]](s32), 32 +; CHECK: [[R:%[0-9]+]](s64) = COPY [[R_2]] +; CHECK: ADJCALLSTACKUP 16, 0, 14, _, implicit-def %sp, implicit %sp +; CHECK: [[R0:%[0-9]+]](s32) = G_EXTRACT [[R]](s64), 0 +; CHECK: [[R1:%[0-9]+]](s32) = G_EXTRACT [[R]](s64), 32 +; CHECK: %r0 = COPY [[R0]](s32) +; CHECK: %r1 = COPY [[R1]](s32) +; CHECK: BX_RET 14, _, implicit %r0, implicit %r1 + %r = notail call arm_aapcscc {i32, i32} @structs_target({i32, i32} %x, {i32*, float, i32, double} %y) + ret {i32, i32} %r +} + define i32 @test_shufflevector_s32_v2s32(i32 %arg) { ; CHECK-LABEL: name: test_shufflevector_s32_v2s32 ; CHECK: [[ARG:%[0-9]+]](s32) = COPY %r0 diff --git a/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll b/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll index ef30cb1063f8..34f00aebe1be 100644 --- a/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll +++ b/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll @@ -54,10 +54,15 @@ define [16 x i32] @test_ret_demotion() { ret [16 x i32] %res } -define void @test_structs({i32, i32} %struct) { -; CHECK: remark: {{.*}} unable to lower arguments: void ({ i32, i32 })* -; CHECK-LABEL: warning: Instruction selection used fallback path for test_structs - ret void +%large.struct = type { i32, i32, i32, i32, i32} ; Doesn't fit in R0-R3 + +declare %large.struct @large_struct_return_target() + +define %large.struct @test_large_struct_return() { +; CHECK: remark: {{.*}} unable to translate instruction: call{{.*}} @large_struct_return_target +; CHECK-LABEL: warning: Instruction selection used fallback path for test_large_struct_return + %r = call %large.struct @large_struct_return_target() + ret %large.struct %r } define void @test_vararg_definition(i32 %a, ...) { diff --git a/test/CodeGen/ARM/cortex-a57-misched-alu.ll b/test/CodeGen/ARM/cortex-a57-misched-alu.ll new file mode 100644 index 000000000000..960ee87532b0 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-alu.ll @@ -0,0 +1,81 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s + +; Check the latency for ALU shifted operand variants. +; +; CHECK: ********** MI Scheduling ********** +; CHECK: foo:BB#0 entry + +; ALU, basic - 1 cyc I0/I1 +; CHECK: EORrr +; CHECK: rdefs left +; CHECK-NEXT: Latency : 1 + +; ALU, shift by immed - 2 cyc M +; CHECK: ADDrsi +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 + +; ALU, shift by register, unconditional - 2 cyc M +; CHECK: RSBrsr +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 + +; ALU, shift by register, conditional - 2 cyc I0/I1 +; CHECK: ANDrsr +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 + +; Checking scheduling units + +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; Skipping COPY +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; CHECK: Scheduling +; CHECK-SAME: ANDrsr +; CHECK: Ready +; CHECK-NEXT: A57UnitI + +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; CHECK: Scheduling +; CHECK-SAME: CMPri +; CHECK: Ready +; CHECK-NEXT: A57UnitI + +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; CHECK: Scheduling +; CHECK-SAME: RSBrsr +; CHECK: Ready +; CHECK-NEXT: A57UnitM + +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; CHECK: Scheduling +; CHECK-SAME: ADDrsi +; CHECK: Ready +; CHECK-NEXT: A57UnitM + +; CHECK: ** ScheduleDAGMILive::schedule picking next node +; CHECK: Scheduling +; CHECK-SAME: EORrr +; CHECK: Ready +; CHECK-NEXT: A57UnitI + + +target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" +target triple = "armv8r-arm-none-eabi" + +; Function Attrs: norecurse nounwind readnone +define hidden i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d) local_unnamed_addr #0 { +entry: + %xor = xor i32 %a, %b + %xor_shl = shl i32 %xor, 2 + %add = add i32 %xor_shl, %d + %add_ashr = ashr i32 %add, %a + %sub = sub i32 %add_ashr, %a + %sub_lshr_pred = lshr i32 %sub, %c + %pred = icmp sgt i32 %a, 4 + %and = and i32 %sub_lshr_pred, %b + %rv = select i1 %pred, i32 %and, i32 %d + ret i32 %rv +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-basic.ll b/test/CodeGen/ARM/cortex-a57-misched-basic.ll new file mode 100644 index 000000000000..2ec50b9d3343 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-basic.ll @@ -0,0 +1,53 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=A57_SCHED +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=generic -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC + +; Check the latency for instructions for both generic and cortex-a57. +; SDIV should be scheduled at the block's begin (20 cyc of independent M unit). +; +; CHECK: ********** MI Scheduling ********** +; CHECK: foo:BB#0 entry + +; GENERIC: SDIV +; GENERIC: Latency : 1 +; GENERIC: EORrr +; GENERIC: Latency : 1 +; GENERIC: LDRi12 +; GENERIC: Latency : 4 +; GENERIC: ADDrr +; GENERIC: Latency : 1 +; GENERIC: SUBrr +; GENERIC: Latency : 1 + +; A57_SCHED: SDIV +; A57_SCHED: Latency : 20 +; A57_SCHED: EORrr +; A57_SCHED: Latency : 1 +; A57_SCHED: LDRi12 +; A57_SCHED: Latency : 4 +; A57_SCHED: ADDrr +; A57_SCHED: Latency : 1 +; A57_SCHED: SUBrr +; A57_SCHED: Latency : 1 + +; CHECK: ** Final schedule for BB#0 *** +; GENERIC: LDRi12 +; GENERIC: SDIV +; A57_SCHED: SDIV +; A57_SCHED: LDRi12 +; CHECK: ********** INTERVALS ********** + +target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" +target triple = "armv8r-arm-none-eabi" + +; Function Attrs: norecurse nounwind readnone +define hidden i32 @foo(i32 %a, i32 %b, i32 %c, i32* %d) local_unnamed_addr #0 { +entry: + %xor = xor i32 %c, %b + %ld = load i32, i32* %d + %add = add nsw i32 %xor, %ld + %div = sdiv i32 %a, %b + %sub = sub i32 %div, %add + ret i32 %sub +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll b/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll new file mode 100644 index 000000000000..d54848a6bcf1 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-ldm-wrback.ll @@ -0,0 +1,37 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s +; + +@a = global i32 0, align 4 +@b = global i32 0, align 4 +@c = global i32 0, align 4 + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have LDM instruction combined from single-loads +; CHECK: ********** MI Scheduling ********** +; CHECK: LDMIA_UPD +; CHECK: rdefs left +; CHECK-NEXT: Latency : 4 +; CHECK: Successors: +; CHECK: data +; CHECK-SAME: Latency=1 +; CHECK-NEXT: data +; CHECK-SAME: Latency=3 +; CHECK-NEXT: data +; CHECK-SAME: Latency=3 +; CHECK-NEXT: data +; CHECK-SAME: Latency=4 +define i32 @bar(i32 %a1, i32 %b1, i32 %c1) minsize optsize { + %1 = load i32, i32* @a, align 4 + %2 = load i32, i32* @b, align 4 + %3 = load i32, i32* @c, align 4 + + %ptr_after = getelementptr i32, i32* @a, i32 3 + + %ptr_val = ptrtoint i32* %ptr_after to i32 + %mul1 = mul i32 %ptr_val, %1 + %mul2 = mul i32 %mul1, %2 + %mul3 = mul i32 %mul2, %3 + ret i32 %mul3 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-ldm.ll b/test/CodeGen/ARM/cortex-a57-misched-ldm.ll new file mode 100644 index 000000000000..9cb076651f5b --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-ldm.ll @@ -0,0 +1,28 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have LDM instruction combined from single-loads +; CHECK: ********** MI Scheduling ********** +; CHECK: LDMIA +; CHECK: rdefs left +; CHECK-NEXT: Latency : 3 +; CHECK: Successors: +; CHECK: data +; CHECK-SAME: Latency=3 +; CHECK-NEXT: data +; CHECK-SAME: Latency=3 + +define i32 @foo(i32* %a) nounwind optsize { +entry: + %b = getelementptr i32, i32* %a, i32 1 + %c = getelementptr i32, i32* %a, i32 2 + %0 = load i32, i32* %a, align 4 + %1 = load i32, i32* %b, align 4 + %2 = load i32, i32* %c, align 4 + + %mul1 = mul i32 %0, %1 + %mul2 = mul i32 %mul1, %2 + ret i32 %mul2 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll b/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll new file mode 100644 index 000000000000..774b0a907e39 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-stm-wrback.ll @@ -0,0 +1,36 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s +; N=3 STMIA_UPD should have latency 2cyc and writeback latency 1cyc + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have STM instruction combined from single-stores +; CHECK: ********** MI Scheduling ********** +; CHECK: schedule starting +; CHECK: STMIA_UPD +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 +; CHECK: Successors +; CHECK: data +; CHECK-SAME: Latency=1 + +define i32 @bar(i32 %v0, i32 %v1, i32 %v2, i32* %addr) { + + %addr.1 = getelementptr i32, i32* %addr, i32 0 + store i32 %v0, i32* %addr.1 + + %addr.2 = getelementptr i32, i32* %addr, i32 1 + store i32 %v1, i32* %addr.2 + + %addr.3 = getelementptr i32, i32* %addr, i32 2 + store i32 %v2, i32* %addr.3 + + %ptr_after = getelementptr i32, i32* %addr, i32 3 + %val = ptrtoint i32* %ptr_after to i32 + + %rv1 = mul i32 %val, %v0 + %rv2 = mul i32 %rv1, %v1 + %rv3 = mul i32 %rv2, %v2 + + ret i32 %rv3 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-stm.ll b/test/CodeGen/ARM/cortex-a57-misched-stm.ll new file mode 100644 index 000000000000..474f39d84bae --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-stm.ll @@ -0,0 +1,29 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s +; N=3 STMIB should have latency 2cyc + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have STM instruction combined from single-stores +; CHECK: ********** MI Scheduling ********** +; CHECK: schedule starting +; CHECK: STMIB +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 + +define i32 @test_stm(i32 %v0, i32 %v1, i32* %addr) { + + %addr.1 = getelementptr i32, i32* %addr, i32 1 + store i32 %v0, i32* %addr.1 + + %addr.2 = getelementptr i32, i32* %addr, i32 2 + store i32 %v1, i32* %addr.2 + + %addr.3 = getelementptr i32, i32* %addr, i32 3 + %val = ptrtoint i32* %addr to i32 + store i32 %val, i32* %addr.3 + + %rv = add i32 %v0, %v1 + + ret i32 %rv +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-vfma.ll b/test/CodeGen/ARM/cortex-a57-misched-vfma.ll new file mode 100644 index 000000000000..a9223e1e2a99 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-vfma.ll @@ -0,0 +1,77 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s +; Check latencies of vmul/vfma accumulate chains. + +define float @Test1(float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) { +; CHECK: ********** MI Scheduling ********** +; CHECK: Test1:BB#0 + +; CHECK: VMULS +; > VMULS common latency = 5 +; CHECK: Latency : 5 +; CHECK: Successors: +; CHECK: data +; > VMULS read-advanced latency to VMLAS = 0 +; CHECK-SAME: Latency=0 + +; CHECK: VMLAS +; > VMLAS common latency = 9 +; CHECK: Latency : 9 +; CHECK: Successors: +; CHECK: data +; > VMLAS read-advanced latency to the next VMLAS = 4 +; CHECK-SAME: Latency=4 + +; CHECK: VMLAS +; CHECK: Latency : 9 +; CHECK: Successors: +; CHECK: data +; > VMLAS not-optimized latency to VMOVRS = 9 +; CHECK-SAME: Latency=9 + +; f1 * f2 + f3 * f4 + f5 * f6 ==> VMULS, VMLAS, VMLAS + %mul1 = fmul float %f1, %f2 + %mul2 = fmul float %f3, %f4 + %mul3 = fmul float %f5, %f6 + %add1 = fadd float %mul1, %mul2 + %add2 = fadd float %add1, %mul3 + ret float %add2 +} + +; ASIMD form +define <2 x float> @Test2(<2 x float> %f1, <2 x float> %f2, <2 x float> %f3, <2 x float> %f4, <2 x float> %f5, <2 x float> %f6) { +; CHECK: ********** MI Scheduling ********** +; CHECK: Test2:BB#0 + +; CHECK: VMULfd +; > VMULfd common latency = 5 +; CHECK: Latency : 5 +; CHECK: Successors: +; CHECK: data +; VMULfd read-advanced latency to VMLAfd = 0 +; CHECK-SAME: Latency=0 + +; CHECK: VMLAfd +; > VMLAfd common latency = 9 +; CHECK: Latency : 9 +; CHECK: Successors: +; CHECK: data +; > VMLAfd read-advanced latency to the next VMLAfd = 4 +; CHECK-SAME: Latency=4 + +; CHECK: VMLAfd +; CHECK: Latency : 9 +; CHECK: Successors: +; CHECK: data +; > VMLAfd not-optimized latency to VMOVRRD = 9 +; CHECK-SAME: Latency=9 + +; f1 * f2 + f3 * f4 + f5 * f6 ==> VMULS, VMLAS, VMLAS + %mul1 = fmul <2 x float> %f1, %f2 + %mul2 = fmul <2 x float> %f3, %f4 + %mul3 = fmul <2 x float> %f5, %f6 + %add1 = fadd <2 x float> %mul1, %mul2 + %add2 = fadd <2 x float> %add1, %mul3 + ret <2 x float> %add2 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll b/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll new file mode 100644 index 000000000000..6cfa823fb969 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-vldm-wrback.ll @@ -0,0 +1,50 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s +; + +@a = global double 0.0, align 4 +@b = global double 0.0, align 4 +@c = global double 0.0, align 4 + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have VLDM instruction combined from single-loads +; CHECK: ********** MI Scheduling ********** +; CHECK: VLDMDIA_UPD +; CHECK: rdefs left +; CHECK-NEXT: Latency : 6 +; CHECK: Successors: +; CHECK: data +; CHECK-SAME: Latency=1 +; CHECK-NEXT: data +; CHECK-SAME: Latency=1 +; CHECK-NEXT: data +; CHECK-SAME: Latency=5 +; CHECK-NEXT: data +; CHECK-SAME: Latency=5 +; CHECK-NEXT: data +; CHECK-SAME: Latency=6 +define i32 @bar(i32* %iptr) minsize optsize { + %1 = load double, double* @a, align 8 + %2 = load double, double* @b, align 8 + %3 = load double, double* @c, align 8 + + %ptr_after = getelementptr double, double* @a, i32 3 + + %ptr_new_ival = ptrtoint double* %ptr_after to i32 + %ptr_new = inttoptr i32 %ptr_new_ival to i32* + + store i32 %ptr_new_ival, i32* %iptr, align 8 + + %v1 = fptoui double %1 to i32 + + %mul1 = mul i32 %ptr_new_ival, %v1 + + %v2 = fptoui double %2 to i32 + %v3 = fptoui double %3 to i32 + + %mul2 = mul i32 %mul1, %v2 + %mul3 = mul i32 %mul2, %v3 + + ret i32 %mul3 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-vldm.ll b/test/CodeGen/ARM/cortex-a57-misched-vldm.ll new file mode 100644 index 000000000000..218b5b41a7e4 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-vldm.ll @@ -0,0 +1,30 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have VLDM instruction combined from single-loads +; CHECK: ********** MI Scheduling ********** +; CHECK: VLDMDIA +; CHECK: rdefs left +; CHECK-NEXT: Latency : 6 +; CHECK: Successors: +; CHECK: data +; CHECK-SAME: Latency=5 +; CHECK-NEXT: data +; CHECK-SAME: Latency=5 +; CHECK-NEXT: data +; CHECK-SAME: Latency=6 + +define double @foo(double* %a) nounwind optsize { +entry: + %b = getelementptr double, double* %a, i32 1 + %c = getelementptr double, double* %a, i32 2 + %0 = load double, double* %a, align 4 + %1 = load double, double* %b, align 4 + %2 = load double, double* %c, align 4 + + %mul1 = fmul double %0, %1 + %mul2 = fmul double %mul1, %2 + ret double %mul2 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll b/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll new file mode 100644 index 000000000000..af1c469d4443 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-vstm-wrback.ll @@ -0,0 +1,43 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have VSTM instruction combined from single-stores +; CHECK: ********** MI Scheduling ********** +; CHECK: schedule starting +; CHECK: VSTMDIA_UPD +; CHECK: rdefs left +; CHECK-NEXT: Latency : 4 +; CHECK: Successors: +; CHECK: data +; CHECK-SAME: Latency=1 + +@a = global double 0.0, align 4 +@b = global double 0.0, align 4 +@c = global double 0.0, align 4 + +define i32 @bar(double* %vptr, i32 %iv1, i32* %iptr) minsize { + + %vp2 = getelementptr double, double* %vptr, i32 1 + %vp3 = getelementptr double, double* %vptr, i32 2 + + %v1 = load double, double* %vptr, align 8 + %v2 = load double, double* %vp2, align 8 + %v3 = load double, double* %vp3, align 8 + + store double %v1, double* @a, align 8 + store double %v2, double* @b, align 8 + store double %v3, double* @c, align 8 + + %ptr_after = getelementptr double, double* @a, i32 3 + + %ptr_new_ival = ptrtoint double* %ptr_after to i32 + %ptr_new = inttoptr i32 %ptr_new_ival to i32* + + store i32 %ptr_new_ival, i32* %iptr, align 8 + + %mul1 = mul i32 %ptr_new_ival, %iv1 + + ret i32 %mul1 +} + diff --git a/test/CodeGen/ARM/cortex-a57-misched-vstm.ll b/test/CodeGen/ARM/cortex-a57-misched-vstm.ll new file mode 100644 index 000000000000..f31474f66558 --- /dev/null +++ b/test/CodeGen/ARM/cortex-a57-misched-vstm.ll @@ -0,0 +1,23 @@ +; REQUIRES: asserts +; RUN: llc < %s -mtriple=armv8r-eabi -mcpu=cortex-a57 -misched-postra -enable-misched -verify-misched -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s + +; CHECK: ********** MI Scheduling ********** +; We need second, post-ra scheduling to have VSTM instruction combined from single-stores +; CHECK: ********** MI Scheduling ********** +; CHECK: schedule starting +; CHECK: VSTMDIA +; CHECK: rdefs left +; CHECK-NEXT: Latency : 2 + +%bigVec = type [2 x double] + +@var = global %bigVec zeroinitializer + +define void @bar(%bigVec* %ptr) { + + %tmp = load %bigVec, %bigVec* %ptr + store %bigVec %tmp, %bigVec* @var + + ret void +} + diff --git a/test/CodeGen/ARM/global-merge-external.ll b/test/CodeGen/ARM/global-merge-external.ll index a9e0d199705a..03c977614320 100644 --- a/test/CodeGen/ARM/global-merge-external.ll +++ b/test/CodeGen/ARM/global-merge-external.ll @@ -2,6 +2,7 @@ ; RUN: llc < %s -mtriple=arm-eabi -arm-global-merge -global-merge-on-external=true | FileCheck %s --check-prefix=CHECK-MERGE ; RUN: llc < %s -mtriple=arm-eabi -arm-global-merge -global-merge-on-external=false | FileCheck %s --check-prefix=CHECK-NO-MERGE ; RUN: llc < %s -mtriple=arm-macho -arm-global-merge | FileCheck %s --check-prefix=CHECK-NO-MERGE +; RUN: llc < %s -mtriple=arm-eabi -arm-global-merge -relocation-model=pic | FileCheck %s --check-prefix=CHECK-NO-MERGE @x = global i32 0, align 4 @y = global i32 0, align 4 diff --git a/test/CodeGen/Hexagon/newify-crash.ll b/test/CodeGen/Hexagon/newify-crash.ll new file mode 100644 index 000000000000..705170b13a59 --- /dev/null +++ b/test/CodeGen/Hexagon/newify-crash.ll @@ -0,0 +1,44 @@ +; RUN: llc -march=hexagon < %s | FileCheck %s +; +; Check that this testcase doesn't crash. +; CHECK: vadd + +target triple = "hexagon" + +define void @fred() #0 { +b0: + br label %b1 + +b1: ; preds = %b7, %b0 + %v2 = phi i32 [ 0, %b0 ], [ %v16, %b7 ] + %v3 = phi <32 x i32> [ undef, %b0 ], [ %v15, %b7 ] + %v4 = icmp slt i32 %v2, undef + br i1 %v4, label %b5, label %b7 + +b5: ; preds = %b1 + %v6 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v3, <32 x i32> undef) + br label %b7 + +b7: ; preds = %b5, %b1 + %v8 = phi <32 x i32> [ %v6, %b5 ], [ %v3, %b1 ] + %v9 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v8, <32 x i32> undef) + %v10 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v9, <32 x i32> undef) + %v11 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v10, <32 x i32> undef) + %v12 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v11, <32 x i32> undef) + %v13 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v12, <32 x i32> zeroinitializer) + %v14 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v13, <32 x i32> undef) + %v15 = tail call <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32> %v14, <32 x i32> undef) + %v16 = add nsw i32 %v2, 8 + %v17 = icmp eq i32 %v16, 64 + br i1 %v17, label %b18, label %b1 + +b18: ; preds = %b7 + tail call void @f0() #0 + ret void +} + +declare <32 x i32> @llvm.hexagon.V6.vaddhsat.128B(<32 x i32>, <32 x i32>) #1 +declare void @f0() #0 + +attributes #0 = { nounwind "target-cpu"="hexagonv60" "target-features"="+hvx,+hvx-double" } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/MIR/Generic/runPass.mir b/test/CodeGen/MIR/Generic/runPass.mir index eeef9d526510..33380d4c6bb4 100644 --- a/test/CodeGen/MIR/Generic/runPass.mir +++ b/test/CodeGen/MIR/Generic/runPass.mir @@ -1,4 +1,5 @@ # RUN: llc -run-pass=greedy -debug-pass=Arguments -o - %s | FileCheck %s +# RUN: llc -run-pass=regallocbasic -debug-pass=Arguments -o - %s | FileCheck %s # Check that passes are initialized correctly, so that it's possible to # use -run-pass. diff --git a/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll b/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll new file mode 100644 index 000000000000..804ea1e5c438 --- /dev/null +++ b/test/CodeGen/Mips/micromips-sizereduction/micromips-lbu16-lhu16-sb16-sh16.ll @@ -0,0 +1,40 @@ +; RUN: llc -march=mipsel -mcpu=mips32r2 -mattr=+micromips -verify-machineinstrs < %s | FileCheck %s + +define void @f1(i8* %p) { +entry: +; CHECK-LABEL: f1: +; CHECK: lbu16 +; CHECK: sb16 + %0 = load i8, i8* %p, align 4 + %a = zext i8 %0 to i32 + %and = and i32 %a, 1 + %cmp = icmp eq i32 %and, 0 + br i1 %cmp, label %if.then, label %if.end + +if.then: + store i8 0, i8* %p, align 1 + br label %if.end + +if.end: + ret void +} + +define void @f2(i16* %p) { +entry: +; CHECK-LABEL: f2: +; CHECK: lhu16 +; CHECK: sh16 + %0 = load i16, i16* %p, align 2 + %a = zext i16 %0 to i32 + %and = and i32 %a, 2 + %cmp = icmp eq i32 %and, 0 + br i1 %cmp, label %if.then, label %if.end + +if.then: + store i16 0, i16* %p, align 2 + br label %if.end + +if.end: + ret void +} + diff --git a/test/CodeGen/PowerPC/scavenging.mir b/test/CodeGen/PowerPC/scavenging.mir new file mode 100644 index 000000000000..8b5c26230bc6 --- /dev/null +++ b/test/CodeGen/PowerPC/scavenging.mir @@ -0,0 +1,149 @@ +# RUN: llc -mtriple=ppc64-- -run-pass scavenger-test -verify-machineinstrs -o - %s | FileCheck %s +--- +# CHECK-LABEL: name: noscav0 +name: noscav0 +tracksRegLiveness: true +body: | + bb.0: + ; CHECK: [[REG0:%r[0-9]+]] = LI 42 + ; CHECK-NEXT: NOP implicit [[REG0]] + %0 : gprc = LI 42 + NOP implicit %0 + + ; CHECK: [[REG1:%r[0-9]+]] = LI 42 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP implicit [[REG1]] + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP implicit [[REG1]] + %1 : gprc = LI 42 + NOP + NOP implicit %1 + NOP + NOP implicit %1 + + ; CHECK: [[REG2:%r[0-9]+]] = LI 42 + ; CHECK-NEXT: NOP implicit [[REG2]] + %2 : gprc = LI 42 + NOP implicit %2 + + %x0 = IMPLICIT_DEF + %x1 = IMPLICIT_DEF + %x2 = IMPLICIT_DEF + %x3 = IMPLICIT_DEF + %x4 = IMPLICIT_DEF + %x27 = IMPLICIT_DEF + %x28 = IMPLICIT_DEF + %x29 = IMPLICIT_DEF + %x30 = IMPLICIT_DEF + + ; CHECK-NOT: %x0 = LI 42 + ; CHECK-NOT: %x1 = LI 42 + ; CHECK-NOT: %x2 = LI 42 + ; CHECK-NOT: %x3 = LI 42 + ; CHECK-NOT: %x4 = LI 42 + ; CHECK-NOT: %x5 = LI 42 + ; CHECK-NOT: %x27 = LI 42 + ; CHECK-NOT: %x28 = LI 42 + ; CHECK-NOT: %x29 = LI 42 + ; CHECK-NOT: %x30 = LI 42 + ; CHECK: [[REG3:%r[0-9]+]] = LI 42 + ; CHECK-NEXT: %x5 = IMPLICIT_DEF + ; CHECK-NEXT: NOP implicit [[REG2]] + ; CHECK-NEXT: NOP implicit [[REG3]] + %3 : gprc = LI 42 + %x5 = IMPLICIT_DEF + NOP implicit %2 + NOP implicit %3 + + NOP implicit %x0 + NOP implicit %x1 + NOP implicit %x2 + NOP implicit %x3 + NOP implicit %x4 + NOP implicit %x5 + NOP implicit %x27 + NOP implicit %x28 + NOP implicit %x29 + NOP implicit %x30 +... +--- +# CHECK-LABEL: name: scav0 +name: scav0 +tracksRegLiveness: true +stack: + # variable-sized object should be a reason to reserve an emergency spillslot + # in the RegScavenger + - { id: 0, type: variable-sized, offset: -32, alignment: 1 } +body: | + bb.0: + %x0 = IMPLICIT_DEF + %x1 = IMPLICIT_DEF + %x2 = IMPLICIT_DEF + %x3 = IMPLICIT_DEF + %x4 = IMPLICIT_DEF + %x5 = IMPLICIT_DEF + %x6 = IMPLICIT_DEF + %x7 = IMPLICIT_DEF + %x8 = IMPLICIT_DEF + %x9 = IMPLICIT_DEF + %x10 = IMPLICIT_DEF + %x11 = IMPLICIT_DEF + %x12 = IMPLICIT_DEF + %x13 = IMPLICIT_DEF + %x14 = IMPLICIT_DEF + %x15 = IMPLICIT_DEF + %x16 = IMPLICIT_DEF + %x17 = IMPLICIT_DEF + %x18 = IMPLICIT_DEF + %x19 = IMPLICIT_DEF + %x20 = IMPLICIT_DEF + %x21 = IMPLICIT_DEF + %x22 = IMPLICIT_DEF + %x23 = IMPLICIT_DEF + %x24 = IMPLICIT_DEF + %x25 = IMPLICIT_DEF + %x26 = IMPLICIT_DEF + %x27 = IMPLICIT_DEF + %x28 = IMPLICIT_DEF + %x29 = IMPLICIT_DEF + %x30 = IMPLICIT_DEF + + ; CHECK: STD killed [[SPILLEDREG:%x[0-9]+]] + ; CHECK: [[SPILLEDREG]] = LI8 42 + ; CHECK: NOP implicit [[SPILLEDREG]] + ; CHECK: [[SPILLEDREG]] = LD + %0 : g8rc = LI8 42 + NOP implicit %0 + + NOP implicit %x0 + NOP implicit %x1 + NOP implicit %x2 + NOP implicit %x3 + NOP implicit %x4 + NOP implicit %x5 + NOP implicit %x6 + NOP implicit %x7 + NOP implicit %x8 + NOP implicit %x9 + NOP implicit %x10 + NOP implicit %x11 + NOP implicit %x12 + NOP implicit %x13 + NOP implicit %x14 + NOP implicit %x15 + NOP implicit %x16 + NOP implicit %x17 + NOP implicit %x18 + NOP implicit %x19 + NOP implicit %x20 + NOP implicit %x21 + NOP implicit %x22 + NOP implicit %x23 + NOP implicit %x24 + NOP implicit %x25 + NOP implicit %x26 + NOP implicit %x27 + NOP implicit %x28 + NOP implicit %x29 + NOP implicit %x30 +... diff --git a/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir new file mode 100644 index 000000000000..2f532f0a5efb --- /dev/null +++ b/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -0,0 +1,267 @@ +# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -run-pass=regallocbasic %s -o - | FileCheck %s +# This test used to assert in RABasic. The problem was when we split live-ranges, +# we were not updating the LiveRegMatrix properly and the interference calculation +# wouldn't match what the assignment thought it could do. +# In other words, this test case needs to trigger live-range splitting to exercise +# the problem. +# +# PR33057 +--- | + target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64" + target triple = "s390x--linux-gnu" + + define void @autogen_SD21418() #0 { + ret void + } + + attributes #0 = { "target-cpu"="z13" } + +... + +# CHECK: name: autogen_SD21418 +# Check that at least one live-range has been split +# CHECK: id: 114, class +--- +name: autogen_SD21418 +alignment: 2 +tracksRegLiveness: true +registers: + - { id: 0, class: vr128bit } + - { id: 1, class: vr128bit } + - { id: 2, class: vr128bit } + - { id: 3, class: vr64bit } + - { id: 4, class: gr64bit } + - { id: 5, class: vr128bit } + - { id: 6, class: grx32bit } + - { id: 7, class: vr128bit } + - { id: 8, class: vr128bit } + - { id: 9, class: gr32bit } + - { id: 10, class: gr64bit } + - { id: 11, class: vr128bit } + - { id: 12, class: fp64bit } + - { id: 13, class: vr64bit } + - { id: 14, class: vr64bit } + - { id: 15, class: gr64bit } + - { id: 16, class: gr128bit } + - { id: 17, class: gr64bit } + - { id: 18, class: gr32bit } + - { id: 19, class: gr32bit } + - { id: 20, class: gr128bit } + - { id: 21, class: gr32bit } + - { id: 22, class: gr64bit } + - { id: 23, class: gr32bit } + - { id: 24, class: gr32bit } + - { id: 25, class: gr128bit } + - { id: 26, class: grx32bit } + - { id: 27, class: gr64bit } + - { id: 28, class: gr64bit } + - { id: 29, class: vr128bit } + - { id: 30, class: vr128bit } + - { id: 31, class: gr64bit } + - { id: 32, class: gr32bit } + - { id: 33, class: gr32bit } + - { id: 34, class: gr128bit } + - { id: 35, class: gr32bit } + - { id: 36, class: vr128bit } + - { id: 37, class: gr64bit } + - { id: 38, class: gr32bit } + - { id: 39, class: gr32bit } + - { id: 40, class: gr128bit } + - { id: 41, class: gr32bit } + - { id: 42, class: addr64bit } + - { id: 43, class: grx32bit } + - { id: 44, class: addr64bit } + - { id: 45, class: vr64bit } + - { id: 46, class: vr64bit } + - { id: 47, class: gr32bit } + - { id: 48, class: gr32bit } + - { id: 49, class: grx32bit } + - { id: 50, class: vr64bit } + - { id: 51, class: gr64bit } + - { id: 52, class: grx32bit } + - { id: 53, class: gr32bit } + - { id: 54, class: gr64bit } + - { id: 55, class: grx32bit } + - { id: 56, class: gr32bit } + - { id: 57, class: gr128bit } + - { id: 58, class: gr128bit } + - { id: 59, class: gr32bit } + - { id: 60, class: gr64bit } + - { id: 61, class: grx32bit } + - { id: 62, class: gr32bit } + - { id: 63, class: gr64bit } + - { id: 64, class: grx32bit } + - { id: 65, class: gr32bit } + - { id: 66, class: gr128bit } + - { id: 67, class: gr128bit } + - { id: 68, class: grx32bit } + - { id: 69, class: gr64bit } + - { id: 70, class: gr64bit } + - { id: 71, class: vr128bit } + - { id: 72, class: vr128bit } + - { id: 73, class: gr64bit } + - { id: 74, class: grx32bit } + - { id: 75, class: gr32bit } + - { id: 76, class: gr64bit } + - { id: 77, class: grx32bit } + - { id: 78, class: gr32bit } + - { id: 79, class: gr128bit } + - { id: 80, class: gr128bit } + - { id: 81, class: gr32bit } + - { id: 82, class: vr128bit } + - { id: 83, class: gr64bit } + - { id: 84, class: grx32bit } + - { id: 85, class: gr32bit } + - { id: 86, class: gr64bit } + - { id: 87, class: grx32bit } + - { id: 88, class: gr32bit } + - { id: 89, class: gr128bit } + - { id: 90, class: gr128bit } + - { id: 91, class: gr32bit } + - { id: 92, class: grx32bit } + - { id: 93, class: gr64bit } + - { id: 94, class: gr32bit } + - { id: 95, class: gr32bit } + - { id: 96, class: gr32bit } + - { id: 97, class: gr64bit } + - { id: 98, class: gr64bit } + - { id: 99, class: grx32bit } + - { id: 100, class: grx32bit } + - { id: 101, class: gr128bit } + - { id: 102, class: gr128bit } + - { id: 103, class: gr128bit } + - { id: 104, class: gr64bit } + - { id: 105, class: gr128bit } + - { id: 106, class: gr128bit } + - { id: 107, class: gr64bit } + - { id: 108, class: gr128bit } + - { id: 109, class: gr128bit } + - { id: 110, class: gr64bit } + - { id: 111, class: gr128bit } + - { id: 112, class: gr128bit } + - { id: 113, class: gr64bit } +constants: + - id: 0 + value: double 0xD55960F86F577076 + alignment: 8 +body: | + bb.0: + %11 = VGBM 0 + %43 = LHIMux 0 + %44 = LARL %const.0 + %45 = VL64 %44, 0, _ :: (load 8 from constant-pool) + + bb.1: + ADJCALLSTACKDOWN 0, 0 + %12 = LZDR + %f0d = COPY %12 + CallBRASL $fmod, killed %f0d, undef %f2d, csr_systemz, implicit-def dead %r14d, implicit-def dead %cc, implicit-def %f0d + ADJCALLSTACKUP 0, 0 + KILL killed %f0d + + bb.2: + %17 = VLGVH %11, _, 0 + %19 = LHR %17.subreg_l32 + undef %20.subreg_l64 = LGHI 0 + %20 = DSGFR %20, %19 + %22 = VLGVH %11, _, 3 + %24 = LHR %22.subreg_l32 + undef %25.subreg_l64 = LGHI 0 + %25 = DSGFR %25, %24 + %31 = VLGVH %11, _, 1 + %33 = LHR %31.subreg_l32 + undef %34.subreg_l64 = LGHI 0 + %34 = DSGFR %34, %33 + %37 = VLGVH %11, _, 2 + %39 = LHR %37.subreg_l32 + undef %40.subreg_l64 = LGHI 0 + %40 = DSGFR %40, %39 + CHIMux %43, 0, implicit-def %cc + BRC 14, 6, %bb.2, implicit killed %cc + J %bb.3 + + bb.3: + WFCDB undef %46, %45, implicit-def %cc + %48 = IPM implicit killed %cc + %48 = AFIMux %48, 268435456, implicit-def dead %cc + %6 = RISBMux undef %6, %48, 31, 159, 35 + WFCDB undef %50, %45, implicit-def %cc + BRC 15, 6, %bb.1, implicit killed %cc + J %bb.4 + + bb.4: + %36 = VLVGP %25.subreg_l64, %25.subreg_l64 + %36 = VLVGH %36, %20.subreg_l32, _, 0 + %36 = VLVGH %36, %34.subreg_l32, _, 1 + dead %36 = VLVGH %36, %40.subreg_l32, _, 2 + %4 = LG undef %42, 0, _ :: (load 8 from `i64* undef`) + undef %57.subreg_h64 = LLILL 0 + undef %66.subreg_h64 = LLILL 0 + undef %79.subreg_h64 = LLILL 0 + undef %89.subreg_h64 = LLILL 0 + %92 = LHIMux 0 + + bb.5: + + bb.6: + %51 = VLGVH undef %7, _, 0 + %53 = LLHRMux %51.subreg_l32 + %54 = VLGVH undef %1, _, 0 + %57.subreg_l32 = LLHRMux %54.subreg_l32 + %58 = COPY %57 + %58 = DLR %58, %53 + %60 = VLGVH undef %7, _, 3 + %62 = LLHRMux %60.subreg_l32 + %63 = VLGVH undef %1, _, 3 + %66.subreg_l32 = LLHRMux %63.subreg_l32 + %67 = COPY %66 + %67 = DLR %67, %62 + %73 = VLGVH undef %7, _, 1 + %75 = LLHRMux %73.subreg_l32 + %76 = VLGVH undef %1, _, 1 + %79.subreg_l32 = LLHRMux %76.subreg_l32 + %80 = COPY %79 + %80 = DLR %80, %75 + %83 = VLGVH undef %7, _, 2 + %85 = LLHRMux %83.subreg_l32 + %86 = VLGVH undef %1, _, 2 + %89.subreg_l32 = LLHRMux %86.subreg_l32 + %90 = COPY %89 + %90 = DLR %90, %85 + CHIMux %92, 0, implicit-def %cc + BRC 14, 6, %bb.7, implicit killed %cc + J %bb.6 + + bb.7: + CGHI undef %93, 0, implicit-def %cc + %96 = IPM implicit killed %cc + CGHI undef %97, 0, implicit-def %cc + BRC 14, 6, %bb.6, implicit killed %cc + + bb.8: + CHIMux %6, 0, implicit-def %cc + %10 = LLILL 41639 + dead %10 = LOCGR %10, %4, 14, 6, implicit killed %cc + CHIMux %92, 0, implicit-def %cc + BRC 14, 6, %bb.5, implicit killed %cc + J %bb.9 + + bb.9: + %82 = VLVGP %67.subreg_h64, %67.subreg_h64 + %82 = VLVGH %82, %58.subreg_hl32, _, 0 + %82 = VLVGH %82, %80.subreg_hl32, _, 1 + dead %82 = VLVGH %82, %90.subreg_hl32, _, 2 + %96 = AFIMux %96, 1879048192, implicit-def dead %cc + %96 = SRL %96, _, 31 + dead %11 = VLVGF %11, %96, _, 1 + %100 = LHIMux 0 + + bb.10: + CHIMux %100, 0, implicit-def %cc + BRC 14, 6, %bb.10, implicit killed %cc + J %bb.11 + + bb.11: + Return + +... diff --git a/test/CodeGen/X86/and-sink.ll b/test/CodeGen/X86/and-sink.ll index 46e50f2a6a74..0f877e778c70 100644 --- a/test/CodeGen/X86/and-sink.ll +++ b/test/CodeGen/X86/and-sink.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=i686-unknown -verify-machineinstrs < %s | FileCheck %s ; RUN: opt < %s -codegenprepare -S -mtriple=x86_64-unknown-unknown | FileCheck --check-prefix=CHECK-CGP %s @@ -8,12 +9,20 @@ ; Test that 'and' is sunk into bb0. define i32 @and_sink1(i32 %a, i1 %c) { ; CHECK-LABEL: and_sink1: -; CHECK: testb $1, -; CHECK: je -; CHECK-NOT: andl $4, -; CHECK: movl $0, A -; CHECK: testb $4, -; CHECK: jne +; CHECK: # BB#0: +; CHECK-NEXT: testb $1, {{[0-9]+}}(%esp) +; CHECK-NEXT: je .LBB0_3 +; CHECK-NEXT: # BB#1: # %bb0 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: movl $0, A +; CHECK-NEXT: testb $4, %al +; CHECK-NEXT: jne .LBB0_3 +; CHECK-NEXT: # BB#2: # %bb1 +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .LBB0_3: # %bb2 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl ; CHECK-CGP-LABEL: @and_sink1( ; CHECK-CGP-NOT: and i32 @@ -37,16 +46,30 @@ bb2: ; Test that both 'and' and cmp get sunk to bb1. define i32 @and_sink2(i32 %a, i1 %c, i1 %c2) { ; CHECK-LABEL: and_sink2: -; CHECK: movl $0, A -; CHECK: testb $1, -; CHECK: je -; CHECK-NOT: andl $4, -; CHECK: movl $0, B -; CHECK: testb $1, -; CHECK: je -; CHECK: movl $0, C -; CHECK: testb $4, -; CHECK: jne +; CHECK: # BB#0: +; CHECK-NEXT: movl $0, A +; CHECK-NEXT: testb $1, {{[0-9]+}}(%esp) +; CHECK-NEXT: je .LBB1_5 +; CHECK-NEXT: # BB#1: # %bb0.preheader +; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: .p2align 4, 0x90 +; CHECK-NEXT: .LBB1_2: # %bb0 +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: movl $0, B +; CHECK-NEXT: testb $1, %al +; CHECK-NEXT: je .LBB1_5 +; CHECK-NEXT: # BB#3: # %bb1 +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: movl $0, C +; CHECK-NEXT: testb $4, %cl +; CHECK-NEXT: jne .LBB1_2 +; CHECK-NEXT: # BB#4: # %bb2 +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .LBB1_5: # %bb3 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl ; CHECK-CGP-LABEL: @and_sink2( ; CHECK-CGP-NOT: and i32 @@ -77,12 +100,21 @@ bb3: ; Test that CodeGenPrepare doesn't get stuck in a loop sinking and hoisting a masked load. define i32 @and_sink3(i1 %c, i32* %p) { ; CHECK-LABEL: and_sink3: -; CHECK: testb $1, -; CHECK: je -; CHECK: movzbl -; CHECK-DAG: movl $0, A -; CHECK-DAG: testl % -; CHECK: je +; CHECK: # BB#0: +; CHECK-NEXT: testb $1, {{[0-9]+}}(%esp) +; CHECK-NEXT: je .LBB2_3 +; CHECK-NEXT: # BB#1: # %bb0 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: movzbl (%eax), %eax +; CHECK-NEXT: testl %eax, %eax +; CHECK-NEXT: movl $0, A +; CHECK-NEXT: je .LBB2_2 +; CHECK-NEXT: .LBB2_3: # %bb2 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .LBB2_2: # %bb1 +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl ; CHECK-CGP-LABEL: @and_sink3( ; CHECK-CGP: load i32 @@ -106,15 +138,26 @@ bb2: ; Test that CodeGenPrepare sinks/duplicates non-immediate 'and'. define i32 @and_sink4(i32 %a, i32 %b, i1 %c) { ; CHECK-LABEL: and_sink4: -; CHECK: testb $1, -; CHECK: je -; CHECK-NOT: andl -; CHECK-DAG: movl $0, A -; CHECK-DAG: testl [[REG1:%[a-z0-9]+]], [[REG2:%[a-z0-9]+]] -; CHECK: jne -; CHECK-DAG: movl {{%[a-z0-9]+}}, B -; CHECK-DAG: testl [[REG1]], [[REG2]] -; CHECK: je +; CHECK: # BB#0: +; CHECK-NEXT: testb $1, {{[0-9]+}}(%esp) +; CHECK-NEXT: je .LBB3_4 +; CHECK-NEXT: # BB#1: # %bb0 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: testl %eax, %ecx +; CHECK-NEXT: movl $0, A +; CHECK-NEXT: jne .LBB3_4 +; CHECK-NEXT: # BB#2: # %bb1 +; CHECK-NEXT: leal (%ecx,%eax), %edx +; CHECK-NEXT: testl %eax, %ecx +; CHECK-NEXT: movl %edx, B +; CHECK-NEXT: je .LBB3_3 +; CHECK-NEXT: .LBB3_4: # %bb3 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .LBB3_3: # %bb2 +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl ; CHECK-CGP-LABEL: @and_sink4( ; CHECK-CGP-NOT: and i32 @@ -146,14 +189,26 @@ bb3: ; when it would increase register pressure. define i32 @and_sink5(i32 %a, i32 %b, i32 %a2, i32 %b2, i1 %c) { ; CHECK-LABEL: and_sink5: -; CHECK: testb $1, -; CHECK: je -; CHECK-DAG: andl {{[0-9]+\(%[a-z0-9]+\)}}, [[REG:%[a-z0-9]+]] -; CHECK-DAG: movl $0, A -; CHECK: jne -; CHECK-DAG: movl {{%[a-z0-9]+}}, B -; CHECK-DAG: testl [[REG]], [[REG]] -; CHECK: je +; CHECK: # BB#0: +; CHECK-NEXT: testb $1, {{[0-9]+}}(%esp) +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: # BB#1: # %bb0 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: andl {{[0-9]+}}(%esp), %eax +; CHECK-NEXT: movl $0, A +; CHECK-NEXT: jne .LBB4_4 +; CHECK-NEXT: # BB#2: # %bb1 +; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: addl {{[0-9]+}}(%esp), %ecx +; CHECK-NEXT: testl %eax, %eax +; CHECK-NEXT: movl %ecx, B +; CHECK-NEXT: je .LBB4_3 +; CHECK-NEXT: .LBB4_4: # %bb3 +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: retl +; CHECK-NEXT: .LBB4_3: # %bb2 +; CHECK-NEXT: movl $1, %eax +; CHECK-NEXT: retl ; CHECK-CGP-LABEL: @and_sink5( ; CHECK-CGP: and i32 diff --git a/test/CodeGen/X86/avx512-cvt.ll b/test/CodeGen/X86/avx512-cvt.ll index 33ac15de9de9..8f6afa8785d0 100644 --- a/test/CodeGen/X86/avx512-cvt.ll +++ b/test/CodeGen/X86/avx512-cvt.ll @@ -1,16 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl | FileCheck %s --check-prefix=ALL --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=KNL -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=skx | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLBW --check-prefix=SKX -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLNOBW --check-prefix=AVX512VL -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=DQ --check-prefix=AVX512DQ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=AVX512BW -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512vl,avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLNOBW --check-prefix=AVX512VLDQ -; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=avx512vl,avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLBW --check-prefix=AVX512VLBW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=ALL --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=KNL +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLBW --check-prefix=SKX +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLNOBW --check-prefix=AVX512VL +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=DQ --check-prefix=AVX512DQ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NOVL --check-prefix=NODQ --check-prefix=NOVLDQ --check-prefix=AVX512BW +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=DQ --check-prefix=VL --check-prefix=VLDQ --check-prefix=VLNOBW --check-prefix=AVX512VLDQ +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=NODQ --check-prefix=VL --check-prefix=VLNODQ --check-prefix=VLBW --check-prefix=AVX512VLBW define <16 x float> @sitof32(<16 x i32> %a) nounwind { ; ALL-LABEL: sitof32: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtdq2ps %zmm0, %zmm0 ; ALL-NEXT: retq %b = sitofp <16 x i32> %a to <16 x float> @@ -19,7 +19,7 @@ define <16 x float> @sitof32(<16 x i32> %a) nounwind { define <8 x double> @sltof864(<8 x i64> %a) { ; NODQ-LABEL: sltof864: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; NODQ-NEXT: vpextrq $1, %xmm1, %rax ; NODQ-NEXT: vcvtsi2sdq %rax, %xmm2, %xmm2 @@ -49,7 +49,7 @@ define <8 x double> @sltof864(<8 x i64> %a) { ; NODQ-NEXT: retq ; ; DQ-LABEL: sltof864: -; DQ: ## BB#0: +; DQ: # BB#0: ; DQ-NEXT: vcvtqq2pd %zmm0, %zmm0 ; DQ-NEXT: retq %b = sitofp <8 x i64> %a to <8 x double> @@ -58,7 +58,7 @@ define <8 x double> @sltof864(<8 x i64> %a) { define <4 x double> @sltof464(<4 x i64> %a) { ; NODQ-LABEL: sltof464: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vextracti128 $1, %ymm0, %xmm1 ; NODQ-NEXT: vpextrq $1, %xmm1, %rax ; NODQ-NEXT: vcvtsi2sdq %rax, %xmm2, %xmm2 @@ -74,15 +74,15 @@ define <4 x double> @sltof464(<4 x i64> %a) { ; NODQ-NEXT: retq ; ; VLDQ-LABEL: sltof464: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvtqq2pd %ymm0, %ymm0 ; VLDQ-NEXT: retq ; ; AVX512DQ-LABEL: sltof464: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0 -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; AVX512DQ-NEXT: retq %b = sitofp <4 x i64> %a to <4 x double> ret <4 x double> %b @@ -90,7 +90,7 @@ define <4 x double> @sltof464(<4 x i64> %a) { define <2 x float> @sltof2f32(<2 x i64> %a) { ; NODQ-LABEL: sltof2f32: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vpextrq $1, %xmm0, %rax ; NODQ-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 ; NODQ-NEXT: vmovq %xmm0, %rax @@ -101,15 +101,15 @@ define <2 x float> @sltof2f32(<2 x i64> %a) { ; NODQ-NEXT: retq ; ; VLDQ-LABEL: sltof2f32: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvtqq2ps %xmm0, %xmm0 ; VLDQ-NEXT: retq ; ; AVX512DQ-LABEL: sltof2f32: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvtqq2ps %zmm0, %ymm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq %b = sitofp <2 x i64> %a to <2 x float> @@ -118,7 +118,7 @@ define <2 x float> @sltof2f32(<2 x i64> %a) { define <4 x float> @sltof4f32_mem(<4 x i64>* %a) { ; KNL-LABEL: sltof4f32_mem: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vmovdqu (%rdi), %ymm0 ; KNL-NEXT: vpextrq $1, %xmm0, %rax ; KNL-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 @@ -135,12 +135,12 @@ define <4 x float> @sltof4f32_mem(<4 x i64>* %a) { ; KNL-NEXT: retq ; ; VLDQ-LABEL: sltof4f32_mem: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvtqq2psy (%rdi), %xmm0 ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sltof4f32_mem: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vmovdqu (%rdi), %ymm0 ; VLNODQ-NEXT: vpextrq $1, %xmm0, %rax ; VLNODQ-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 @@ -158,15 +158,15 @@ define <4 x float> @sltof4f32_mem(<4 x i64>* %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: sltof4f32_mem: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: vmovups (%rdi), %ymm0 ; AVX512DQ-NEXT: vcvtqq2ps %zmm0, %ymm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: sltof4f32_mem: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: vmovdqu (%rdi), %ymm0 ; AVX512BW-NEXT: vpextrq $1, %xmm0, %rax ; AVX512BW-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 @@ -189,7 +189,7 @@ define <4 x float> @sltof4f32_mem(<4 x i64>* %a) { define <4 x i64> @f64tosl(<4 x double> %a) { ; NODQ-LABEL: f64tosl: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vextractf128 $1, %ymm0, %xmm1 ; NODQ-NEXT: vcvttsd2si %xmm1, %rax ; NODQ-NEXT: vmovq %rax, %xmm2 @@ -207,15 +207,15 @@ define <4 x i64> @f64tosl(<4 x double> %a) { ; NODQ-NEXT: retq ; ; VLDQ-LABEL: f64tosl: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvttpd2qq %ymm0, %ymm0 ; VLDQ-NEXT: retq ; ; AVX512DQ-LABEL: f64tosl: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvttpd2qq %zmm0, %zmm0 -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; AVX512DQ-NEXT: retq %b = fptosi <4 x double> %a to <4 x i64> ret <4 x i64> %b @@ -223,7 +223,7 @@ define <4 x i64> @f64tosl(<4 x double> %a) { define <4 x i64> @f32tosl(<4 x float> %a) { ; NODQ-LABEL: f32tosl: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[3,1,2,3] ; NODQ-NEXT: vcvttss2si %xmm1, %rax ; NODQ-NEXT: vmovq %rax, %xmm1 @@ -241,15 +241,15 @@ define <4 x i64> @f32tosl(<4 x float> %a) { ; NODQ-NEXT: retq ; ; VLDQ-LABEL: f32tosl: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvttps2qq %xmm0, %ymm0 ; VLDQ-NEXT: retq ; ; AVX512DQ-LABEL: f32tosl: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<def> ; AVX512DQ-NEXT: vcvttps2qq %ymm0, %zmm0 -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; AVX512DQ-NEXT: retq %b = fptosi <4 x float> %a to <4 x i64> ret <4 x i64> %b @@ -257,7 +257,7 @@ define <4 x i64> @f32tosl(<4 x float> %a) { define <4 x float> @sltof432(<4 x i64> %a) { ; KNL-LABEL: sltof432: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vpextrq $1, %xmm0, %rax ; KNL-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 ; KNL-NEXT: vmovq %xmm0, %rax @@ -273,13 +273,13 @@ define <4 x float> @sltof432(<4 x i64> %a) { ; KNL-NEXT: retq ; ; VLDQ-LABEL: sltof432: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvtqq2ps %ymm0, %xmm0 ; VLDQ-NEXT: vzeroupper ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sltof432: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpextrq $1, %xmm0, %rax ; VLNODQ-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 ; VLNODQ-NEXT: vmovq %xmm0, %rax @@ -296,15 +296,15 @@ define <4 x float> @sltof432(<4 x i64> %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: sltof432: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvtqq2ps %zmm0, %ymm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: sltof432: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: vpextrq $1, %xmm0, %rax ; AVX512BW-NEXT: vcvtsi2ssq %rax, %xmm1, %xmm1 ; AVX512BW-NEXT: vmovq %xmm0, %rax @@ -325,7 +325,7 @@ define <4 x float> @sltof432(<4 x i64> %a) { define <4 x float> @ultof432(<4 x i64> %a) { ; KNL-LABEL: ultof432: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vpextrq $1, %xmm0, %rax ; KNL-NEXT: vcvtusi2ssq %rax, %xmm1, %xmm1 ; KNL-NEXT: vmovq %xmm0, %rax @@ -341,13 +341,13 @@ define <4 x float> @ultof432(<4 x i64> %a) { ; KNL-NEXT: retq ; ; VLDQ-LABEL: ultof432: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vcvtuqq2ps %ymm0, %xmm0 ; VLDQ-NEXT: vzeroupper ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: ultof432: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpextrq $1, %xmm0, %rax ; VLNODQ-NEXT: vcvtusi2ssq %rax, %xmm1, %xmm1 ; VLNODQ-NEXT: vmovq %xmm0, %rax @@ -364,15 +364,15 @@ define <4 x float> @ultof432(<4 x i64> %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: ultof432: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvtuqq2ps %zmm0, %ymm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: ultof432: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: vpextrq $1, %xmm0, %rax ; AVX512BW-NEXT: vcvtusi2ssq %rax, %xmm1, %xmm1 ; AVX512BW-NEXT: vmovq %xmm0, %rax @@ -393,7 +393,7 @@ define <4 x float> @ultof432(<4 x i64> %a) { define <8 x double> @ultof64(<8 x i64> %a) { ; NODQ-LABEL: ultof64: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; NODQ-NEXT: vpextrq $1, %xmm1, %rax ; NODQ-NEXT: vcvtusi2sdq %rax, %xmm2, %xmm2 @@ -423,7 +423,7 @@ define <8 x double> @ultof64(<8 x i64> %a) { ; NODQ-NEXT: retq ; ; DQ-LABEL: ultof64: -; DQ: ## BB#0: +; DQ: # BB#0: ; DQ-NEXT: vcvtuqq2pd %zmm0, %zmm0 ; DQ-NEXT: retq %b = uitofp <8 x i64> %a to <8 x double> @@ -432,7 +432,7 @@ define <8 x double> @ultof64(<8 x i64> %a) { define <16 x i32> @fptosi00(<16 x float> %a) nounwind { ; ALL-LABEL: fptosi00: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttps2dq %zmm0, %zmm0 ; ALL-NEXT: retq %b = fptosi <16 x float> %a to <16 x i32> @@ -441,7 +441,7 @@ define <16 x i32> @fptosi00(<16 x float> %a) nounwind { define <16 x i32> @fptoui00(<16 x float> %a) nounwind { ; ALL-LABEL: fptoui00: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttps2udq %zmm0, %zmm0 ; ALL-NEXT: retq %b = fptoui <16 x float> %a to <16 x i32> @@ -450,14 +450,14 @@ define <16 x i32> @fptoui00(<16 x float> %a) nounwind { define <8 x i32> @fptoui_256(<8 x float> %a) nounwind { ; NOVL-LABEL: fptoui_256: -; NOVL: ## BB#0: -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; NOVL: # BB#0: +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; NOVL-NEXT: vcvttps2udq %zmm0, %zmm0 -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; NOVL-NEXT: retq ; ; VL-LABEL: fptoui_256: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvttps2udq %ymm0, %ymm0 ; VL-NEXT: retq %b = fptoui <8 x float> %a to <8 x i32> @@ -466,30 +466,30 @@ define <8 x i32> @fptoui_256(<8 x float> %a) nounwind { define <4 x i32> @fptoui_128(<4 x float> %a) nounwind { ; KNL-LABEL: fptoui_128: -; KNL: ## BB#0: -; KNL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; KNL: # BB#0: +; KNL-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; KNL-NEXT: vcvttps2udq %zmm0, %zmm0 -; KNL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; KNL-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; KNL-NEXT: retq ; ; VL-LABEL: fptoui_128: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvttps2udq %xmm0, %xmm0 ; VL-NEXT: retq ; ; AVX512DQ-LABEL: fptoui_128: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvttps2udq %zmm0, %zmm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: fptoui_128: -; AVX512BW: ## BB#0: -; AVX512BW-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; AVX512BW: # BB#0: +; AVX512BW-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; AVX512BW-NEXT: vcvttps2udq %zmm0, %zmm0 -; AVX512BW-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; AVX512BW-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq %b = fptoui <4 x float> %a to <4 x i32> @@ -498,7 +498,7 @@ define <4 x i32> @fptoui_128(<4 x float> %a) nounwind { define <8 x i32> @fptoui01(<8 x double> %a) nounwind { ; ALL-LABEL: fptoui01: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttpd2udq %zmm0, %ymm0 ; ALL-NEXT: retq %b = fptoui <8 x double> %a to <8 x i32> @@ -507,31 +507,31 @@ define <8 x i32> @fptoui01(<8 x double> %a) nounwind { define <4 x i32> @fptoui_256d(<4 x double> %a) nounwind { ; KNL-LABEL: fptoui_256d: -; KNL: ## BB#0: -; KNL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; KNL: # BB#0: +; KNL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; KNL-NEXT: vcvttpd2udq %zmm0, %ymm0 -; KNL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; KNL-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; KNL-NEXT: retq ; ; VL-LABEL: fptoui_256d: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvttpd2udq %ymm0, %xmm0 ; VL-NEXT: vzeroupper ; VL-NEXT: retq ; ; AVX512DQ-LABEL: fptoui_256d: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvttpd2udq %zmm0, %ymm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: fptoui_256d: -; AVX512BW: ## BB#0: -; AVX512BW-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512BW: # BB#0: +; AVX512BW-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512BW-NEXT: vcvttpd2udq %zmm0, %ymm0 -; AVX512BW-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<kill> +; AVX512BW-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<kill> ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq %b = fptoui <4 x double> %a to <4 x i32> @@ -540,7 +540,7 @@ define <4 x i32> @fptoui_256d(<4 x double> %a) nounwind { define <8 x double> @sitof64(<8 x i32> %a) { ; ALL-LABEL: sitof64: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtdq2pd %ymm0, %zmm0 ; ALL-NEXT: retq %b = sitofp <8 x i32> %a to <8 x double> @@ -548,31 +548,31 @@ define <8 x double> @sitof64(<8 x i32> %a) { } define <8 x double> @sitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind { ; KNL-LABEL: sitof64_mask: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: kmovw %edi, %k1 ; KNL-NEXT: vcvtdq2pd %ymm1, %zmm0 {%k1} ; KNL-NEXT: retq ; ; VLBW-LABEL: sitof64_mask: -; VLBW: ## BB#0: +; VLBW: # BB#0: ; VLBW-NEXT: kmovd %edi, %k1 ; VLBW-NEXT: vcvtdq2pd %ymm1, %zmm0 {%k1} ; VLBW-NEXT: retq ; ; VLNOBW-LABEL: sitof64_mask: -; VLNOBW: ## BB#0: +; VLNOBW: # BB#0: ; VLNOBW-NEXT: kmovw %edi, %k1 ; VLNOBW-NEXT: vcvtdq2pd %ymm1, %zmm0 {%k1} ; VLNOBW-NEXT: retq ; ; AVX512DQ-LABEL: sitof64_mask: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: kmovw %edi, %k1 ; AVX512DQ-NEXT: vcvtdq2pd %ymm1, %zmm0 {%k1} ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: sitof64_mask: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: kmovd %edi, %k1 ; AVX512BW-NEXT: vcvtdq2pd %ymm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq @@ -583,31 +583,31 @@ define <8 x double> @sitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind } define <8 x double> @sitof64_maskz(<8 x i32> %a, i8 %b) nounwind { ; KNL-LABEL: sitof64_maskz: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: kmovw %edi, %k1 ; KNL-NEXT: vcvtdq2pd %ymm0, %zmm0 {%k1} {z} ; KNL-NEXT: retq ; ; VLBW-LABEL: sitof64_maskz: -; VLBW: ## BB#0: +; VLBW: # BB#0: ; VLBW-NEXT: kmovd %edi, %k1 ; VLBW-NEXT: vcvtdq2pd %ymm0, %zmm0 {%k1} {z} ; VLBW-NEXT: retq ; ; VLNOBW-LABEL: sitof64_maskz: -; VLNOBW: ## BB#0: +; VLNOBW: # BB#0: ; VLNOBW-NEXT: kmovw %edi, %k1 ; VLNOBW-NEXT: vcvtdq2pd %ymm0, %zmm0 {%k1} {z} ; VLNOBW-NEXT: retq ; ; AVX512DQ-LABEL: sitof64_maskz: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: kmovw %edi, %k1 ; AVX512DQ-NEXT: vcvtdq2pd %ymm0, %zmm0 {%k1} {z} ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: sitof64_maskz: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: kmovd %edi, %k1 ; AVX512BW-NEXT: vcvtdq2pd %ymm0, %zmm0 {%k1} {z} ; AVX512BW-NEXT: retq @@ -619,7 +619,7 @@ define <8 x double> @sitof64_maskz(<8 x i32> %a, i8 %b) nounwind { define <8 x i32> @fptosi01(<8 x double> %a) { ; ALL-LABEL: fptosi01: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttpd2dq %zmm0, %ymm0 ; ALL-NEXT: retq %b = fptosi <8 x double> %a to <8 x i32> @@ -628,12 +628,12 @@ define <8 x i32> @fptosi01(<8 x double> %a) { define <4 x i32> @fptosi03(<4 x double> %a) { ; KNL-LABEL: fptosi03: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vcvttpd2dq %ymm0, %xmm0 ; KNL-NEXT: retq ; ; AVX512-LABEL: fptosi03: -; AVX512: ## BB#0: +; AVX512: # BB#0: ; AVX512-NEXT: vcvttpd2dq %ymm0, %xmm0 ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -643,14 +643,14 @@ define <4 x i32> @fptosi03(<4 x double> %a) { define <16 x float> @fptrunc00(<16 x double> %b) nounwind { ; NODQ-LABEL: fptrunc00: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vcvtpd2ps %zmm0, %ymm0 ; NODQ-NEXT: vcvtpd2ps %zmm1, %ymm1 ; NODQ-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 ; NODQ-NEXT: retq ; ; DQ-LABEL: fptrunc00: -; DQ: ## BB#0: +; DQ: # BB#0: ; DQ-NEXT: vcvtpd2ps %zmm0, %ymm0 ; DQ-NEXT: vcvtpd2ps %zmm1, %ymm1 ; DQ-NEXT: vinsertf32x8 $1, %ymm1, %zmm0, %zmm0 @@ -661,12 +661,12 @@ define <16 x float> @fptrunc00(<16 x double> %b) nounwind { define <4 x float> @fptrunc01(<4 x double> %b) { ; KNL-LABEL: fptrunc01: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vcvtpd2ps %ymm0, %xmm0 ; KNL-NEXT: retq ; ; AVX512-LABEL: fptrunc01: -; AVX512: ## BB#0: +; AVX512: # BB#0: ; AVX512-NEXT: vcvtpd2ps %ymm0, %xmm0 ; AVX512-NEXT: vzeroupper ; AVX512-NEXT: retq @@ -676,7 +676,7 @@ define <4 x float> @fptrunc01(<4 x double> %b) { define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { ; KNL-LABEL: fptrunc02: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: vpslld $31, %xmm1, %xmm1 ; KNL-NEXT: vpsrad $31, %xmm1, %xmm1 ; KNL-NEXT: vcvtpd2ps %ymm0, %xmm0 @@ -684,7 +684,7 @@ define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { ; KNL-NEXT: retq ; ; VL-LABEL: fptrunc02: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpslld $31, %xmm1, %xmm1 ; VL-NEXT: vptestmd %xmm1, %xmm1, %k1 ; VL-NEXT: vcvtpd2ps %ymm0, %xmm0 {%k1} {z} @@ -692,7 +692,7 @@ define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { ; VL-NEXT: retq ; ; AVX512DQ-LABEL: fptrunc02: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: vpslld $31, %xmm1, %xmm1 ; AVX512DQ-NEXT: vpsrad $31, %xmm1, %xmm1 ; AVX512DQ-NEXT: vcvtpd2ps %ymm0, %xmm0 @@ -701,7 +701,7 @@ define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: fptrunc02: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: vpslld $31, %xmm1, %xmm1 ; AVX512BW-NEXT: vpsrad $31, %xmm1, %xmm1 ; AVX512BW-NEXT: vcvtpd2ps %ymm0, %xmm0 @@ -715,7 +715,7 @@ define <4 x float> @fptrunc02(<4 x double> %b, <4 x i1> %mask) { define <4 x float> @fptrunc03(<2 x double> %a0, <4 x float> %a1) nounwind { ; ALL-LABEL: fptrunc03: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtsd2ss %xmm0, %xmm1, %xmm0 ; ALL-NEXT: retq %ext = extractelement <2 x double> %a0, i32 0 @@ -726,7 +726,7 @@ define <4 x float> @fptrunc03(<2 x double> %a0, <4 x float> %a1) nounwind { define <8 x double> @fpext00(<8 x float> %b) nounwind { ; ALL-LABEL: fpext00: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtps2pd %ymm0, %zmm0 ; ALL-NEXT: retq %a = fpext <8 x float> %b to <8 x double> @@ -735,14 +735,14 @@ define <8 x double> @fpext00(<8 x float> %b) nounwind { define <4 x double> @fpext01(<4 x float> %b, <4 x double>%b1, <4 x double>%a1) { ; NOVL-LABEL: fpext01: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vcvtps2pd %xmm0, %ymm0 ; NOVL-NEXT: vcmpltpd %ymm2, %ymm1, %ymm1 ; NOVL-NEXT: vandpd %ymm0, %ymm1, %ymm0 ; NOVL-NEXT: retq ; ; VL-LABEL: fpext01: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcmpltpd %ymm2, %ymm1, %k1 ; VL-NEXT: vcvtps2pd %xmm0, %ymm0 {%k1} {z} ; VL-NEXT: retq @@ -754,7 +754,7 @@ define <4 x double> @fpext01(<4 x float> %b, <4 x double>%b1, <4 x double>%a1) { define <2 x double> @fpext02(<2 x double> %a0, <4 x float> %a1) nounwind { ; ALL-LABEL: fpext02: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtss2sd %xmm1, %xmm0, %xmm0 ; ALL-NEXT: retq %ext = extractelement <4 x float> %a1, i32 0 @@ -765,7 +765,7 @@ define <2 x double> @fpext02(<2 x double> %a0, <4 x float> %a1) nounwind { define double @funcA(i64* nocapture %e) { ; ALL-LABEL: funcA: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vcvtsi2sdq (%rdi), %xmm0, %xmm0 ; ALL-NEXT: retq entry: @@ -776,7 +776,7 @@ entry: define double @funcB(i32* %e) { ; ALL-LABEL: funcB: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vcvtsi2sdl (%rdi), %xmm0, %xmm0 ; ALL-NEXT: retq entry: @@ -787,7 +787,7 @@ entry: define float @funcC(i32* %e) { ; ALL-LABEL: funcC: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vcvtsi2ssl (%rdi), %xmm0, %xmm0 ; ALL-NEXT: retq entry: @@ -798,7 +798,7 @@ entry: define float @i64tof32(i64* %e) { ; ALL-LABEL: i64tof32: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vcvtsi2ssq (%rdi), %xmm0, %xmm0 ; ALL-NEXT: retq entry: @@ -809,7 +809,7 @@ entry: define void @fpext() { ; ALL-LABEL: fpext: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; ALL-NEXT: vcvtss2sd %xmm0, %xmm0, %xmm0 ; ALL-NEXT: vmovsd %xmm0, -{{[0-9]+}}(%rsp) @@ -825,7 +825,7 @@ entry: define void @fpround_scalar() nounwind uwtable { ; ALL-LABEL: fpround_scalar: -; ALL: ## BB#0: ## %entry +; ALL: # BB#0: # %entry ; ALL-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; ALL-NEXT: vcvtsd2ss %xmm0, %xmm0, %xmm0 ; ALL-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp) @@ -841,7 +841,7 @@ entry: define double @long_to_double(i64 %x) { ; ALL-LABEL: long_to_double: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vmovq %rdi, %xmm0 ; ALL-NEXT: retq %res = bitcast i64 %x to double @@ -850,7 +850,7 @@ define double @long_to_double(i64 %x) { define i64 @double_to_long(double %x) { ; ALL-LABEL: double_to_long: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vmovq %xmm0, %rax ; ALL-NEXT: retq %res = bitcast double %x to i64 @@ -859,7 +859,7 @@ define i64 @double_to_long(double %x) { define float @int_to_float(i32 %x) { ; ALL-LABEL: int_to_float: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vmovd %edi, %xmm0 ; ALL-NEXT: retq %res = bitcast i32 %x to float @@ -868,7 +868,7 @@ define float @int_to_float(i32 %x) { define i32 @float_to_int(float %x) { ; ALL-LABEL: float_to_int: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vmovd %xmm0, %eax ; ALL-NEXT: retq %res = bitcast float %x to i32 @@ -877,7 +877,7 @@ define i32 @float_to_int(float %x) { define <16 x double> @uitof64(<16 x i32> %a) nounwind { ; NODQ-LABEL: uitof64: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vcvtudq2pd %ymm0, %zmm2 ; NODQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0 ; NODQ-NEXT: vcvtudq2pd %ymm0, %zmm1 @@ -885,7 +885,7 @@ define <16 x double> @uitof64(<16 x i32> %a) nounwind { ; NODQ-NEXT: retq ; ; DQ-LABEL: uitof64: -; DQ: ## BB#0: +; DQ: # BB#0: ; DQ-NEXT: vcvtudq2pd %ymm0, %zmm2 ; DQ-NEXT: vextracti32x8 $1, %zmm0, %ymm0 ; DQ-NEXT: vcvtudq2pd %ymm0, %zmm1 @@ -896,31 +896,31 @@ define <16 x double> @uitof64(<16 x i32> %a) nounwind { } define <8 x double> @uitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind { ; KNL-LABEL: uitof64_mask: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: kmovw %edi, %k1 ; KNL-NEXT: vcvtudq2pd %ymm1, %zmm0 {%k1} ; KNL-NEXT: retq ; ; VLBW-LABEL: uitof64_mask: -; VLBW: ## BB#0: +; VLBW: # BB#0: ; VLBW-NEXT: kmovd %edi, %k1 ; VLBW-NEXT: vcvtudq2pd %ymm1, %zmm0 {%k1} ; VLBW-NEXT: retq ; ; VLNOBW-LABEL: uitof64_mask: -; VLNOBW: ## BB#0: +; VLNOBW: # BB#0: ; VLNOBW-NEXT: kmovw %edi, %k1 ; VLNOBW-NEXT: vcvtudq2pd %ymm1, %zmm0 {%k1} ; VLNOBW-NEXT: retq ; ; AVX512DQ-LABEL: uitof64_mask: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: kmovw %edi, %k1 ; AVX512DQ-NEXT: vcvtudq2pd %ymm1, %zmm0 {%k1} ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: uitof64_mask: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: kmovd %edi, %k1 ; AVX512BW-NEXT: vcvtudq2pd %ymm1, %zmm0 {%k1} ; AVX512BW-NEXT: retq @@ -931,31 +931,31 @@ define <8 x double> @uitof64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind } define <8 x double> @uitof64_maskz(<8 x i32> %a, i8 %b) nounwind { ; KNL-LABEL: uitof64_maskz: -; KNL: ## BB#0: +; KNL: # BB#0: ; KNL-NEXT: kmovw %edi, %k1 ; KNL-NEXT: vcvtudq2pd %ymm0, %zmm0 {%k1} {z} ; KNL-NEXT: retq ; ; VLBW-LABEL: uitof64_maskz: -; VLBW: ## BB#0: +; VLBW: # BB#0: ; VLBW-NEXT: kmovd %edi, %k1 ; VLBW-NEXT: vcvtudq2pd %ymm0, %zmm0 {%k1} {z} ; VLBW-NEXT: retq ; ; VLNOBW-LABEL: uitof64_maskz: -; VLNOBW: ## BB#0: +; VLNOBW: # BB#0: ; VLNOBW-NEXT: kmovw %edi, %k1 ; VLNOBW-NEXT: vcvtudq2pd %ymm0, %zmm0 {%k1} {z} ; VLNOBW-NEXT: retq ; ; AVX512DQ-LABEL: uitof64_maskz: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: kmovw %edi, %k1 ; AVX512DQ-NEXT: vcvtudq2pd %ymm0, %zmm0 {%k1} {z} ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: uitof64_maskz: -; AVX512BW: ## BB#0: +; AVX512BW: # BB#0: ; AVX512BW-NEXT: kmovd %edi, %k1 ; AVX512BW-NEXT: vcvtudq2pd %ymm0, %zmm0 {%k1} {z} ; AVX512BW-NEXT: retq @@ -967,14 +967,14 @@ define <8 x double> @uitof64_maskz(<8 x i32> %a, i8 %b) nounwind { define <4 x double> @uitof64_256(<4 x i32> %a) nounwind { ; NOVL-LABEL: uitof64_256: -; NOVL: ## BB#0: -; NOVL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %YMM0<def> +; NOVL: # BB#0: +; NOVL-NEXT: # kill: %XMM0<def> %XMM0<kill> %YMM0<def> ; NOVL-NEXT: vcvtudq2pd %ymm0, %zmm0 -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; NOVL-NEXT: retq ; ; VL-LABEL: uitof64_256: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvtudq2pd %xmm0, %ymm0 ; VL-NEXT: retq %b = uitofp <4 x i32> %a to <4 x double> @@ -983,7 +983,7 @@ define <4 x double> @uitof64_256(<4 x i32> %a) nounwind { define <16 x float> @uitof32(<16 x i32> %a) nounwind { ; ALL-LABEL: uitof32: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtudq2ps %zmm0, %zmm0 ; ALL-NEXT: retq %b = uitofp <16 x i32> %a to <16 x float> @@ -992,14 +992,14 @@ define <16 x float> @uitof32(<16 x i32> %a) nounwind { define <8 x float> @uitof32_256(<8 x i32> %a) nounwind { ; NOVL-LABEL: uitof32_256: -; NOVL: ## BB#0: -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; NOVL: # BB#0: +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; NOVL-NEXT: vcvtudq2ps %zmm0, %zmm0 -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; NOVL-NEXT: retq ; ; VL-LABEL: uitof32_256: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvtudq2ps %ymm0, %ymm0 ; VL-NEXT: retq %b = uitofp <8 x i32> %a to <8 x float> @@ -1008,30 +1008,30 @@ define <8 x float> @uitof32_256(<8 x i32> %a) nounwind { define <4 x float> @uitof32_128(<4 x i32> %a) nounwind { ; KNL-LABEL: uitof32_128: -; KNL: ## BB#0: -; KNL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; KNL: # BB#0: +; KNL-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; KNL-NEXT: vcvtudq2ps %zmm0, %zmm0 -; KNL-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; KNL-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; KNL-NEXT: retq ; ; VL-LABEL: uitof32_128: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vcvtudq2ps %xmm0, %xmm0 ; VL-NEXT: retq ; ; AVX512DQ-LABEL: uitof32_128: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vcvtudq2ps %zmm0, %zmm0 -; AVX512DQ-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; AVX512DQ-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; AVX512DQ-NEXT: vzeroupper ; AVX512DQ-NEXT: retq ; ; AVX512BW-LABEL: uitof32_128: -; AVX512BW: ## BB#0: -; AVX512BW-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<def> +; AVX512BW: # BB#0: +; AVX512BW-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<def> ; AVX512BW-NEXT: vcvtudq2ps %zmm0, %zmm0 -; AVX512BW-NEXT: ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> +; AVX512BW-NEXT: # kill: %XMM0<def> %XMM0<kill> %ZMM0<kill> ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq %b = uitofp <4 x i32> %a to <4 x float> @@ -1040,7 +1040,7 @@ define <4 x float> @uitof32_128(<4 x i32> %a) nounwind { define i32 @fptosi02(float %a) nounwind { ; ALL-LABEL: fptosi02: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttss2si %xmm0, %eax ; ALL-NEXT: retq %b = fptosi float %a to i32 @@ -1049,7 +1049,7 @@ define i32 @fptosi02(float %a) nounwind { define i32 @fptoui02(float %a) nounwind { ; ALL-LABEL: fptoui02: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvttss2usi %xmm0, %eax ; ALL-NEXT: retq %b = fptoui float %a to i32 @@ -1058,7 +1058,7 @@ define i32 @fptoui02(float %a) nounwind { define float @uitofp02(i32 %a) nounwind { ; ALL-LABEL: uitofp02: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtusi2ssl %edi, %xmm0, %xmm0 ; ALL-NEXT: retq %b = uitofp i32 %a to float @@ -1067,7 +1067,7 @@ define float @uitofp02(i32 %a) nounwind { define double @uitofp03(i32 %a) nounwind { ; ALL-LABEL: uitofp03: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vcvtusi2sdl %edi, %xmm0, %xmm0 ; ALL-NEXT: retq %b = uitofp i32 %a to double @@ -1076,7 +1076,7 @@ define double @uitofp03(i32 %a) nounwind { define <16 x float> @sitofp_16i1_float(<16 x i32> %a) { ; NODQ-LABEL: sitofp_16i1_float: -; NODQ: ## BB#0: +; NODQ: # BB#0: ; NODQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; NODQ-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; NODQ-NEXT: vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} @@ -1084,7 +1084,7 @@ define <16 x float> @sitofp_16i1_float(<16 x i32> %a) { ; NODQ-NEXT: retq ; ; DQ-LABEL: sitofp_16i1_float: -; DQ: ## BB#0: +; DQ: # BB#0: ; DQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; DQ-NEXT: vpcmpgtd %zmm0, %zmm1, %k0 ; DQ-NEXT: vpmovm2d %k0, %zmm0 @@ -1097,7 +1097,7 @@ define <16 x float> @sitofp_16i1_float(<16 x i32> %a) { define <16 x float> @sitofp_16i8_float(<16 x i8> %a) { ; ALL-LABEL: sitofp_16i8_float: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovsxbd %xmm0, %zmm0 ; ALL-NEXT: vcvtdq2ps %zmm0, %zmm0 ; ALL-NEXT: retq @@ -1107,7 +1107,7 @@ define <16 x float> @sitofp_16i8_float(<16 x i8> %a) { define <16 x float> @sitofp_16i16_float(<16 x i16> %a) { ; ALL-LABEL: sitofp_16i16_float: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovsxwd %ymm0, %zmm0 ; ALL-NEXT: vcvtdq2ps %zmm0, %zmm0 ; ALL-NEXT: retq @@ -1117,7 +1117,7 @@ define <16 x float> @sitofp_16i16_float(<16 x i16> %a) { define <8 x double> @sitofp_8i16_double(<8 x i16> %a) { ; ALL-LABEL: sitofp_8i16_double: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovsxwd %xmm0, %ymm0 ; ALL-NEXT: vcvtdq2pd %ymm0, %zmm0 ; ALL-NEXT: retq @@ -1127,7 +1127,7 @@ define <8 x double> @sitofp_8i16_double(<8 x i16> %a) { define <8 x double> @sitofp_8i8_double(<8 x i8> %a) { ; ALL-LABEL: sitofp_8i8_double: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero ; ALL-NEXT: vpslld $24, %ymm0, %ymm0 ; ALL-NEXT: vpsrad $24, %ymm0, %ymm0 @@ -1139,7 +1139,7 @@ define <8 x double> @sitofp_8i8_double(<8 x i8> %a) { define <16 x double> @sitofp_16i1_double(<16 x double> %a) { ; NOVLDQ-LABEL: sitofp_16i1_double: -; NOVLDQ: ## BB#0: +; NOVLDQ: # BB#0: ; NOVLDQ-NEXT: vpxord %zmm2, %zmm2, %zmm2 ; NOVLDQ-NEXT: vcmpltpd %zmm1, %zmm2, %k1 ; NOVLDQ-NEXT: vcmpltpd %zmm0, %zmm2, %k2 @@ -1152,7 +1152,7 @@ define <16 x double> @sitofp_16i1_double(<16 x double> %a) { ; NOVLDQ-NEXT: retq ; ; VLDQ-LABEL: sitofp_16i1_double: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorpd %zmm2, %zmm2, %zmm2 ; VLDQ-NEXT: vcmpltpd %zmm1, %zmm2, %k0 ; VLDQ-NEXT: vcmpltpd %zmm0, %zmm2, %k1 @@ -1163,7 +1163,7 @@ define <16 x double> @sitofp_16i1_double(<16 x double> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_16i1_double: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxord %zmm2, %zmm2, %zmm2 ; VLNODQ-NEXT: vcmpltpd %zmm1, %zmm2, %k1 ; VLNODQ-NEXT: vcmpltpd %zmm0, %zmm2, %k2 @@ -1175,7 +1175,7 @@ define <16 x double> @sitofp_16i1_double(<16 x double> %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: sitofp_16i1_double: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: vxorpd %zmm2, %zmm2, %zmm2 ; AVX512DQ-NEXT: vcmpltpd %zmm1, %zmm2, %k0 ; AVX512DQ-NEXT: vcmpltpd %zmm0, %zmm2, %k1 @@ -1191,7 +1191,7 @@ define <16 x double> @sitofp_16i1_double(<16 x double> %a) { define <8 x double> @sitofp_8i1_double(<8 x double> %a) { ; NOVLDQ-LABEL: sitofp_8i1_double: -; NOVLDQ: ## BB#0: +; NOVLDQ: # BB#0: ; NOVLDQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; NOVLDQ-NEXT: vcmpltpd %zmm0, %zmm1, %k1 ; NOVLDQ-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z} @@ -1200,7 +1200,7 @@ define <8 x double> @sitofp_8i1_double(<8 x double> %a) { ; NOVLDQ-NEXT: retq ; ; VLDQ-LABEL: sitofp_8i1_double: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorpd %zmm1, %zmm1, %zmm1 ; VLDQ-NEXT: vcmpltpd %zmm0, %zmm1, %k0 ; VLDQ-NEXT: vpmovm2d %k0, %ymm0 @@ -1208,7 +1208,7 @@ define <8 x double> @sitofp_8i1_double(<8 x double> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_8i1_double: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; VLNODQ-NEXT: vcmpltpd %zmm0, %zmm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0 @@ -1217,7 +1217,7 @@ define <8 x double> @sitofp_8i1_double(<8 x double> %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: sitofp_8i1_double: -; AVX512DQ: ## BB#0: +; AVX512DQ: # BB#0: ; AVX512DQ-NEXT: vxorpd %zmm1, %zmm1, %zmm1 ; AVX512DQ-NEXT: vcmpltpd %zmm0, %zmm1, %k0 ; AVX512DQ-NEXT: vpmovm2d %k0, %zmm0 @@ -1230,8 +1230,8 @@ define <8 x double> @sitofp_8i1_double(<8 x double> %a) { define <8 x float> @sitofp_8i1_float(<8 x float> %a) { ; NOVLDQ-LABEL: sitofp_8i1_float: -; NOVLDQ: ## BB#0: -; NOVLDQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; NOVLDQ: # BB#0: +; NOVLDQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; NOVLDQ-NEXT: vxorps %ymm1, %ymm1, %ymm1 ; NOVLDQ-NEXT: vcmpltps %zmm0, %zmm1, %k1 ; NOVLDQ-NEXT: vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z} @@ -1240,7 +1240,7 @@ define <8 x float> @sitofp_8i1_float(<8 x float> %a) { ; NOVLDQ-NEXT: retq ; ; VLDQ-LABEL: sitofp_8i1_float: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorps %ymm1, %ymm1, %ymm1 ; VLDQ-NEXT: vcmpltps %ymm0, %ymm1, %k0 ; VLDQ-NEXT: vpmovm2d %k0, %ymm0 @@ -1248,7 +1248,7 @@ define <8 x float> @sitofp_8i1_float(<8 x float> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_8i1_float: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; VLNODQ-NEXT: vcmpltps %ymm0, %ymm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0 @@ -1257,8 +1257,8 @@ define <8 x float> @sitofp_8i1_float(<8 x float> %a) { ; VLNODQ-NEXT: retq ; ; AVX512DQ-LABEL: sitofp_8i1_float: -; AVX512DQ: ## BB#0: -; AVX512DQ-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; AVX512DQ: # BB#0: +; AVX512DQ-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; AVX512DQ-NEXT: vxorps %ymm1, %ymm1, %ymm1 ; AVX512DQ-NEXT: vcmpltps %zmm0, %zmm1, %k0 ; AVX512DQ-NEXT: vpmovm2d %k0, %zmm0 @@ -1271,14 +1271,14 @@ define <8 x float> @sitofp_8i1_float(<8 x float> %a) { define <4 x float> @sitofp_4i1_float(<4 x float> %a) { ; NOVL-LABEL: sitofp_4i1_float: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vcmpltps %xmm0, %xmm1, %xmm0 ; NOVL-NEXT: vcvtdq2ps %xmm0, %xmm0 ; NOVL-NEXT: retq ; ; VLDQ-LABEL: sitofp_4i1_float: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; VLDQ-NEXT: vcmpltps %xmm0, %xmm1, %k0 ; VLDQ-NEXT: vpmovm2d %k0, %xmm0 @@ -1286,7 +1286,7 @@ define <4 x float> @sitofp_4i1_float(<4 x float> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_4i1_float: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VLNODQ-NEXT: vcmpltps %xmm0, %xmm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 @@ -1300,7 +1300,7 @@ define <4 x float> @sitofp_4i1_float(<4 x float> %a) { define <4 x double> @sitofp_4i1_double(<4 x double> %a) { ; NOVL-LABEL: sitofp_4i1_double: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; NOVL-NEXT: vcmpltpd %ymm0, %ymm1, %ymm0 ; NOVL-NEXT: vpmovqd %zmm0, %ymm0 @@ -1308,7 +1308,7 @@ define <4 x double> @sitofp_4i1_double(<4 x double> %a) { ; NOVL-NEXT: retq ; ; VLDQ-LABEL: sitofp_4i1_double: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorpd %ymm1, %ymm1, %ymm1 ; VLDQ-NEXT: vcmpltpd %ymm0, %ymm1, %k0 ; VLDQ-NEXT: vpmovm2d %k0, %xmm0 @@ -1316,7 +1316,7 @@ define <4 x double> @sitofp_4i1_double(<4 x double> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_4i1_double: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; VLNODQ-NEXT: vcmpltpd %ymm0, %ymm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 @@ -1330,14 +1330,14 @@ define <4 x double> @sitofp_4i1_double(<4 x double> %a) { define <2 x float> @sitofp_2i1_float(<2 x float> %a) { ; NOVL-LABEL: sitofp_2i1_float: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vcmpltps %xmm0, %xmm1, %xmm0 ; NOVL-NEXT: vcvtdq2ps %xmm0, %xmm0 ; NOVL-NEXT: retq ; ; VLDQ-LABEL: sitofp_2i1_float: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; VLDQ-NEXT: vcmpltps %xmm0, %xmm1, %k0 ; VLDQ-NEXT: vpmovm2d %k0, %xmm0 @@ -1345,7 +1345,7 @@ define <2 x float> @sitofp_2i1_float(<2 x float> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_2i1_float: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VLNODQ-NEXT: vcmpltps %xmm0, %xmm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 @@ -1359,7 +1359,7 @@ define <2 x float> @sitofp_2i1_float(<2 x float> %a) { define <2 x double> @sitofp_2i1_double(<2 x double> %a) { ; NOVL-LABEL: sitofp_2i1_double: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vcmpltpd %xmm0, %xmm1, %xmm0 ; NOVL-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3] @@ -1367,7 +1367,7 @@ define <2 x double> @sitofp_2i1_double(<2 x double> %a) { ; NOVL-NEXT: retq ; ; VLDQ-LABEL: sitofp_2i1_double: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; VLDQ-NEXT: vcmpltpd %xmm0, %xmm1, %k0 ; VLDQ-NEXT: vpmovm2q %k0, %xmm0 @@ -1375,7 +1375,7 @@ define <2 x double> @sitofp_2i1_double(<2 x double> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: sitofp_2i1_double: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VLNODQ-NEXT: vcmpltpd %xmm0, %xmm1, %k1 ; VLNODQ-NEXT: vpcmpeqd %xmm0, %xmm0, %xmm0 @@ -1393,7 +1393,7 @@ define <2 x double> @sitofp_2i1_double(<2 x double> %a) { define <16 x float> @uitofp_16i8(<16 x i8>%a) { ; ALL-LABEL: uitofp_16i8: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero ; ALL-NEXT: vcvtdq2ps %zmm0, %zmm0 ; ALL-NEXT: retq @@ -1403,7 +1403,7 @@ define <16 x float> @uitofp_16i8(<16 x i8>%a) { define <16 x float> @uitofp_16i16(<16 x i16>%a) { ; ALL-LABEL: uitofp_16i16: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero ; ALL-NEXT: vcvtdq2ps %zmm0, %zmm0 ; ALL-NEXT: retq @@ -1413,7 +1413,7 @@ define <16 x float> @uitofp_16i16(<16 x i16>%a) { define <16 x float> @uitofp_16i1_float(<16 x i32> %a) { ; ALL-LABEL: uitofp_16i1_float: -; ALL: ## BB#0: +; ALL: # BB#0: ; ALL-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; ALL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; ALL-NEXT: vpbroadcastd {{.*}}(%rip), %zmm0 {%k1} {z} @@ -1426,7 +1426,7 @@ define <16 x float> @uitofp_16i1_float(<16 x i32> %a) { define <16 x double> @uitofp_16i1_double(<16 x i32> %a) { ; NOVL-LABEL: uitofp_16i1_double: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; NOVL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; NOVL-NEXT: movq {{.*}}(%rip), %rax @@ -1440,7 +1440,7 @@ define <16 x double> @uitofp_16i1_double(<16 x i32> %a) { ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_16i1_double: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxord %zmm1, %zmm1, %zmm1 ; VL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; VL-NEXT: movl {{.*}}(%rip), %eax @@ -1457,18 +1457,18 @@ define <16 x double> @uitofp_16i1_double(<16 x i32> %a) { define <8 x float> @uitofp_8i1_float(<8 x i32> %a) { ; NOVL-LABEL: uitofp_8i1_float: -; NOVL: ## BB#0: -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; NOVL: # BB#0: +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; NOVL-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; NOVL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; NOVL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} ; NOVL-NEXT: vpmovqd %zmm0, %ymm0 ; NOVL-NEXT: vcvtudq2ps %zmm0, %zmm0 -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<kill> ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_8i1_float: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; VL-NEXT: vpcmpgtd %ymm0, %ymm1, %k1 ; VL-NEXT: vpbroadcastd {{.*}}(%rip), %ymm0 {%k1} {z} @@ -1481,8 +1481,8 @@ define <8 x float> @uitofp_8i1_float(<8 x i32> %a) { define <8 x double> @uitofp_8i1_double(<8 x i32> %a) { ; NOVL-LABEL: uitofp_8i1_double: -; NOVL: ## BB#0: -; NOVL-NEXT: ## kill: %YMM0<def> %YMM0<kill> %ZMM0<def> +; NOVL: # BB#0: +; NOVL-NEXT: # kill: %YMM0<def> %YMM0<kill> %ZMM0<def> ; NOVL-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; NOVL-NEXT: vpcmpgtd %zmm0, %zmm1, %k1 ; NOVL-NEXT: vpbroadcastq {{.*}}(%rip), %zmm0 {%k1} {z} @@ -1491,7 +1491,7 @@ define <8 x double> @uitofp_8i1_double(<8 x i32> %a) { ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_8i1_double: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxor %ymm1, %ymm1, %ymm1 ; VL-NEXT: vpcmpgtd %ymm0, %ymm1, %k1 ; VL-NEXT: vpbroadcastd {{.*}}(%rip), %ymm0 {%k1} {z} @@ -1504,7 +1504,7 @@ define <8 x double> @uitofp_8i1_double(<8 x i32> %a) { define <4 x float> @uitofp_4i1_float(<4 x i32> %a) { ; NOVL-LABEL: uitofp_4i1_float: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 ; NOVL-NEXT: vpbroadcastd {{.*}}(%rip), %xmm1 @@ -1512,7 +1512,7 @@ define <4 x float> @uitofp_4i1_float(<4 x i32> %a) { ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_4i1_float: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VL-NEXT: vpcmpgtd %xmm0, %xmm1, %k1 ; VL-NEXT: vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z} @@ -1525,7 +1525,7 @@ define <4 x float> @uitofp_4i1_float(<4 x i32> %a) { define <4 x double> @uitofp_4i1_double(<4 x i32> %a) { ; NOVL-LABEL: uitofp_4i1_double: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 ; NOVL-NEXT: vpsrld $31, %xmm0, %xmm0 @@ -1533,7 +1533,7 @@ define <4 x double> @uitofp_4i1_double(<4 x i32> %a) { ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_4i1_double: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VL-NEXT: vpcmpgtd %xmm0, %xmm1, %k1 ; VL-NEXT: vpbroadcastd {{.*}}(%rip), %xmm0 {%k1} {z} @@ -1546,7 +1546,7 @@ define <4 x double> @uitofp_4i1_double(<4 x i32> %a) { define <2 x float> @uitofp_2i1_float(<2 x i32> %a) { ; NOVL-LABEL: uitofp_2i1_float: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; NOVL-NEXT: vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808] @@ -1562,7 +1562,7 @@ define <2 x float> @uitofp_2i1_float(<2 x i32> %a) { ; NOVL-NEXT: retq ; ; VL-LABEL: uitofp_2i1_float: -; VL: ## BB#0: +; VL: # BB#0: ; VL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; VL-NEXT: vpcmpltuq %xmm1, %xmm0, %k1 @@ -1576,7 +1576,7 @@ define <2 x float> @uitofp_2i1_float(<2 x i32> %a) { define <2 x double> @uitofp_2i1_double(<2 x i32> %a) { ; NOVL-LABEL: uitofp_2i1_double: -; NOVL: ## BB#0: +; NOVL: # BB#0: ; NOVL-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; NOVL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; NOVL-NEXT: vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808] @@ -1586,7 +1586,7 @@ define <2 x double> @uitofp_2i1_double(<2 x i32> %a) { ; NOVL-NEXT: retq ; ; VLDQ-LABEL: uitofp_2i1_double: -; VLDQ: ## BB#0: +; VLDQ: # BB#0: ; VLDQ-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VLDQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; VLDQ-NEXT: vpcmpltuq %xmm1, %xmm0, %k1 @@ -1595,7 +1595,7 @@ define <2 x double> @uitofp_2i1_double(<2 x i32> %a) { ; VLDQ-NEXT: retq ; ; VLNODQ-LABEL: uitofp_2i1_double: -; VLNODQ: ## BB#0: +; VLNODQ: # BB#0: ; VLNODQ-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; VLNODQ-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] ; VLNODQ-NEXT: vpcmpltuq %xmm1, %xmm0, %k1 diff --git a/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll b/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll index b13965a30ed8..bbe31c5c2ac5 100644 --- a/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll +++ b/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll @@ -1203,3 +1203,35 @@ define <8 x double> @f8xf64_f256(<8 x double> %a) { ret <8 x double> %res2 } + + +; ALL: .LCPI38 +; ALL-NEXT: .long 4290379776 # 0xffba0000 + +; AVX: .LCPI38 +; AVX-NEXT: .long 4290379776 # float NaN + +define <8 x i16> @f8xi16_i32_NaN(<8 x i16> %a) { +; ALL32-LABEL: f8xi16_i32_NaN: +; ALL32: # BB#0: +; ALL32-NEXT: vpbroadcastd {{\.LCPI.*}}, %xmm1 +; ALL32-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; ALL32-NEXT: vpand %xmm1, %xmm0, %xmm0 +; ALL32-NEXT: retl +; +; ALL64-LABEL: f8xi16_i32_NaN: +; ALL64: # BB#0: +; ALL64-NEXT: vpbroadcastd {{.*}}(%rip), %xmm1 +; ALL64-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; ALL64-NEXT: vpand %xmm1, %xmm0, %xmm0 +; ALL64-NEXT: retq +; +; AVX-LABEL: f8xi16_i32_NaN: +; AVX: # BB#0: +; AVX-NEXT: vbroadcastss {{\.LCPI.*}}, %xmm1 +; AVX-NEXT: vpaddw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpand %xmm1, %xmm0, %xmm0 + %res1 = add <8 x i16> <i16 0, i16 -70, i16 0, i16 -70, i16 0, i16 -70, i16 0, i16 -70>, %a + %res2 = and <8 x i16> <i16 0, i16 -70, i16 0, i16 -70, i16 0, i16 -70, i16 0, i16 -70>, %res1 + ret <8 x i16> %res2 +} diff --git a/test/CodeGen/X86/clear_upper_vector_element_bits.ll b/test/CodeGen/X86/clear_upper_vector_element_bits.ll index ae0f4406ba0d..1218b68b1be4 100644 --- a/test/CodeGen/X86/clear_upper_vector_element_bits.ll +++ b/test/CodeGen/X86/clear_upper_vector_element_bits.ll @@ -405,12 +405,7 @@ define <16 x i8> @_clearupper16xi8a(<16 x i8>) nounwind { ; ; AVX-LABEL: _clearupper16xi8a: ; AVX: # BB#0: -; AVX-NEXT: vpextrb $0, %xmm0, %eax -; AVX-NEXT: vpextrb $1, %xmm0, %ecx -; AVX-NEXT: vmovd %eax, %xmm1 -; AVX-NEXT: vpinsrb $1, %ecx, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7] -; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vandps {{.*}}(%rip), %xmm0, %xmm0 ; AVX-NEXT: retq %x0 = extractelement <16 x i8> %0, i32 0 %x1 = extractelement <16 x i8> %0, i32 1 @@ -575,39 +570,10 @@ define <32 x i8> @_clearupper32xi8a(<32 x i8>) nounwind { ; SSE-NEXT: pand %xmm2, %xmm1 ; SSE-NEXT: retq ; -; AVX1-LABEL: _clearupper32xi8a: -; AVX1: # BB#0: -; AVX1-NEXT: vpextrb $0, %xmm0, %eax -; AVX1-NEXT: vpextrb $1, %xmm0, %ecx -; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX1-NEXT: vpextrb $0, %xmm1, %edx -; AVX1-NEXT: vpextrb $1, %xmm1, %esi -; AVX1-NEXT: vmovd %edx, %xmm2 -; AVX1-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5,6,7] -; AVX1-NEXT: vmovd %eax, %xmm2 -; AVX1-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2 -; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3,4,5,6,7] -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX1-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 -; AVX1-NEXT: retq -; -; AVX2-LABEL: _clearupper32xi8a: -; AVX2: # BB#0: -; AVX2-NEXT: vpextrb $0, %xmm0, %eax -; AVX2-NEXT: vpextrb $1, %xmm0, %ecx -; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 -; AVX2-NEXT: vpextrb $0, %xmm1, %edx -; AVX2-NEXT: vpextrb $1, %xmm1, %esi -; AVX2-NEXT: vmovd %edx, %xmm2 -; AVX2-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2 -; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5,6,7] -; AVX2-NEXT: vmovd %eax, %xmm2 -; AVX2-NEXT: vpinsrb $1, %ecx, %xmm2, %xmm2 -; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3,4,5,6,7] -; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 -; AVX2-NEXT: vpand {{.*}}(%rip), %ymm0, %ymm0 -; AVX2-NEXT: retq +; AVX-LABEL: _clearupper32xi8a: +; AVX: # BB#0: +; AVX-NEXT: vandps {{.*}}(%rip), %ymm0, %ymm0 +; AVX-NEXT: retq %x0 = extractelement <32 x i8> %0, i32 0 %x1 = extractelement <32 x i8> %0, i32 1 %x2 = extractelement <32 x i8> %0, i32 2 diff --git a/test/CodeGen/X86/scavenger.mir b/test/CodeGen/X86/scavenger.mir new file mode 100644 index 000000000000..8d97aeb22cb9 --- /dev/null +++ b/test/CodeGen/X86/scavenger.mir @@ -0,0 +1,54 @@ +# RUN: llc -mtriple=i386-- -run-pass scavenger-test -verify-machineinstrs -o - %s | FileCheck %s +--- +# CHECK-LABEL: name: func0 +name: func0 +tracksRegLiveness: true +body: | + bb.0: + %0 : gr32 = MOV32ri 42 + %ebp = COPY %0 +... +--- +# CHECK-LABEL: name: func2 +name: func2 +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-NOT: %eax = MOV32ri 42 + ; CHECK: [[REG0:%e[a-z]+]] = MOV32ri 42 + ; CHECK: %ebp = COPY [[REG0]] + %eax = MOV32ri 13 + %0 : gr32 = MOV32ri 42 + %ebp = COPY %0 + + ; CHECK: [[REG1:%e[a-z]+]] = MOV32ri 23 + ; CHECK: [[REG2:%e[a-z]+]] = MOV32ri 7 + ; CHECK: [[REG1]] = ADD32ri8 [[REG1]], 5, implicit-def dead %eflags + %1 : gr32 = MOV32ri 23 + %2 : gr32 = MOV32ri 7 + %1 = ADD32ri8 %1, 5, implicit-def dead %eflags + + NOOP implicit %ebp + + ; CHECK: NOOP implicit [[REG2]] + ; CHECK: NOOP implicit [[REG1]] + NOOP implicit %2 + NOOP implicit %1 + RETQ %eax +... +--- +# Defs without uses are currently broken +#name: func3 +#tracksRegLiveness: true +#body: | +# bb.0: +# dead %0 : gr32 = MOV32ri 42 +... +--- +# Uses without defs are currently broken (and honestly not that useful). +#name: func3 +#tracksRegLiveness: true +#body: | +# bb.0: +# NOOP undef implicit %0 : gr32 +... diff --git a/test/CodeGen/X86/select.ll b/test/CodeGen/X86/select.ll index 1afef86a5f11..7c2937936313 100644 --- a/test/CodeGen/X86/select.ll +++ b/test/CodeGen/X86/select.ll @@ -15,6 +15,7 @@ define i32 @test1(%0* %p, %0* %q, i1 %r) nounwind { ; CHECK-NEXT: cmovneq %rdi, %rsi ; CHECK-NEXT: movl (%rsi), %eax ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test1: ; MCU: # BB#0: @@ -55,6 +56,7 @@ define i32 @test2() nounwind { ; CHECK-NEXT: popq %rcx ; CHECK-NEXT: retq ; CHECK-NEXT: LBB1_1: ## %bb90 +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test2: ; MCU: # BB#0: # %entry @@ -100,6 +102,7 @@ define float @test3(i32 %x) nounwind readnone { ; CHECK-NEXT: leaq {{.*}}(%rip), %rcx ; CHECK-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test3: ; MCU: # BB#0: # %entry @@ -123,6 +126,7 @@ define signext i8 @test4(i8* nocapture %P, double %F) nounwind readonly { ; CHECK-NEXT: seta %al ; CHECK-NEXT: movsbl (%rdi,%rax,4), %eax ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test4: ; MCU: # BB#0: # %entry @@ -157,6 +161,7 @@ define void @test5(i1 %c, <2 x i16> %a, <2 x i16> %b, <2 x i16>* %p) nounwind { ; CHECK-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; CHECK-NEXT: movd %xmm0, (%rsi) ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test5: ; MCU: # BB#0: @@ -196,6 +201,7 @@ define void @test6(i32 %C, <4 x float>* %A, <4 x float>* %B) nounwind { ; CHECK-NEXT: mulps %xmm0, %xmm0 ; CHECK-NEXT: movaps %xmm0, (%rsi) ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test6: ; MCU: # BB#0: @@ -267,6 +273,7 @@ define x86_fp80 @test7(i32 %tmp8) nounwind { ; CHECK-NEXT: leaq {{.*}}(%rip), %rcx ; CHECK-NEXT: fldt (%rax,%rcx) ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test7: ; MCU: # BB#0: @@ -319,6 +326,7 @@ define void @test8(i1 %c, <6 x i32>* %dst.addr, <6 x i32> %src1,<6 x i32> %src2) ; GENERIC-NEXT: movq %xmm0, 16(%rsi) ; GENERIC-NEXT: movdqa %xmm1, (%rsi) ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test8: ; ATOM: ## BB#0: @@ -358,6 +366,7 @@ define void @test8(i1 %c, <6 x i32>* %dst.addr, <6 x i32> %src1,<6 x i32> %src2) ; ATOM-NEXT: movq %xmm0, 16(%rsi) ; ATOM-NEXT: movdqa %xmm1, (%rsi) ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test8: ; MCU: # BB#0: @@ -448,6 +457,7 @@ define i64 @test9(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; GENERIC-NEXT: sbbq %rax, %rax ; GENERIC-NEXT: orq %rsi, %rax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test9: ; ATOM: ## BB#0: @@ -457,6 +467,7 @@ define i64 @test9(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test9: ; MCU: # BB#0: @@ -483,6 +494,7 @@ define i64 @test9a(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; GENERIC-NEXT: sbbq %rax, %rax ; GENERIC-NEXT: orq %rsi, %rax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test9a: ; ATOM: ## BB#0: @@ -492,6 +504,7 @@ define i64 @test9a(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test9a: ; MCU: # BB#0: @@ -516,6 +529,7 @@ define i64 @test9b(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; GENERIC-NEXT: sbbq %rax, %rax ; GENERIC-NEXT: orq %rsi, %rax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test9b: ; ATOM: ## BB#0: @@ -525,6 +539,7 @@ define i64 @test9b(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test9b: ; MCU: # BB#0: @@ -552,6 +567,7 @@ define i64 @test10(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; GENERIC-NEXT: sbbq %rax, %rax ; GENERIC-NEXT: orq $1, %rax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test10: ; ATOM: ## BB#0: @@ -561,6 +577,7 @@ define i64 @test10(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test10: ; MCU: # BB#0: @@ -586,6 +603,7 @@ define i64 @test11(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; CHECK-NEXT: notq %rax ; CHECK-NEXT: orq %rsi, %rax ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test11: ; MCU: # BB#0: @@ -612,6 +630,7 @@ define i64 @test11a(i64 %x, i64 %y) nounwind readnone ssp noredzone { ; CHECK-NEXT: notq %rax ; CHECK-NEXT: orq %rsi, %rax ; CHECK-NEXT: retq +; CHECK-NEXT: ## -- End function ; ; MCU-LABEL: test11a: ; MCU: # BB#0: @@ -641,6 +660,7 @@ define noalias i8* @test12(i64 %count) nounwind ssp noredzone { ; GENERIC-NEXT: movq $-1, %rdi ; GENERIC-NEXT: cmovnoq %rax, %rdi ; GENERIC-NEXT: jmp __Znam ## TAILCALL +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test12: ; ATOM: ## BB#0: ## %entry @@ -650,6 +670,7 @@ define noalias i8* @test12(i64 %count) nounwind ssp noredzone { ; ATOM-NEXT: movq $-1, %rdi ; ATOM-NEXT: cmovnoq %rax, %rdi ; ATOM-NEXT: jmp __Znam ## TAILCALL +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test12: ; MCU: # BB#0: # %entry @@ -700,6 +721,7 @@ define i32 @test13(i32 %a, i32 %b) nounwind { ; GENERIC-NEXT: cmpl %esi, %edi ; GENERIC-NEXT: sbbl %eax, %eax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test13: ; ATOM: ## BB#0: @@ -710,6 +732,7 @@ define i32 @test13(i32 %a, i32 %b) nounwind { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test13: ; MCU: # BB#0: @@ -728,6 +751,7 @@ define i32 @test14(i32 %a, i32 %b) nounwind { ; GENERIC-NEXT: sbbl %eax, %eax ; GENERIC-NEXT: notl %eax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test14: ; ATOM: ## BB#0: @@ -737,6 +761,7 @@ define i32 @test14(i32 %a, i32 %b) nounwind { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test14: ; MCU: # BB#0: @@ -756,6 +781,7 @@ define i32 @test15(i32 %x) nounwind { ; GENERIC-NEXT: negl %edi ; GENERIC-NEXT: sbbl %eax, %eax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test15: ; ATOM: ## BB#0: ## %entry @@ -766,6 +792,7 @@ define i32 @test15(i32 %x) nounwind { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test15: ; MCU: # BB#0: # %entry @@ -817,6 +844,7 @@ define i16 @test17(i16 %x) nounwind { ; GENERIC-NEXT: negw %di ; GENERIC-NEXT: sbbw %ax, %ax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test17: ; ATOM: ## BB#0: ## %entry @@ -827,6 +855,7 @@ define i16 @test17(i16 %x) nounwind { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test17: ; MCU: # BB#0: # %entry @@ -846,6 +875,7 @@ define i8 @test18(i32 %x, i8 zeroext %a, i8 zeroext %b) nounwind { ; GENERIC-NEXT: cmovgel %edx, %esi ; GENERIC-NEXT: movl %esi, %eax ; GENERIC-NEXT: retq +; GENERIC-NEXT: ## -- End function ; ; ATOM-LABEL: test18: ; ATOM: ## BB#0: @@ -855,6 +885,7 @@ define i8 @test18(i32 %x, i8 zeroext %a, i8 zeroext %b) nounwind { ; ATOM-NEXT: nop ; ATOM-NEXT: nop ; ATOM-NEXT: retq +; ATOM-NEXT: ## -- End function ; ; MCU-LABEL: test18: ; MCU: # BB#0: diff --git a/test/CodeGen/X86/shrink-compare.ll b/test/CodeGen/X86/shrink-compare.ll index 41f5d2d5be23..7f35258377ec 100644 --- a/test/CodeGen/X86/shrink-compare.ll +++ b/test/CodeGen/X86/shrink-compare.ll @@ -1,8 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s declare void @bar() define void @test1(i32* nocapture %X) nounwind minsize { +; CHECK-LABEL: test1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $47, (%rdi) +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %tmp1 = load i32, i32* %X, align 4 %and = and i32 %tmp1, 255 @@ -15,11 +22,15 @@ if.then: if.end: ret void -; CHECK-LABEL: test1: -; CHECK: cmpb $47, (%{{rdi|rcx}}) } define void @test2(i32 %X) nounwind minsize { +; CHECK-LABEL: test2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $47, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %and = and i32 %X, 255 %cmp = icmp eq i32 %and, 47 @@ -31,11 +42,15 @@ if.then: if.end: ret void -; CHECK-LABEL: test2: -; CHECK: cmpb $47, %{{dil|cl}} } define void @test3(i32 %X) nounwind minsize { +; CHECK-LABEL: test3: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $-1, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %and = and i32 %X, 255 %cmp = icmp eq i32 %and, 255 @@ -47,12 +62,22 @@ if.then: if.end: ret void -; CHECK-LABEL: test3: -; CHECK: cmpb $-1, %{{dil|cl}} } ; PR16083 define i1 @test4(i64 %a, i32 %b) { +; CHECK-LABEL: test4: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: testl %esi, %esi +; CHECK-NEXT: je .LBB3_1 +; CHECK-NEXT: # BB#2: # %lor.end +; CHECK-NEXT: # kill: %AL<def> %AL<kill> %EAX<kill> +; CHECK-NEXT: retq +; CHECK-NEXT: .LBB3_1: # %lor.rhs +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: # kill: %AL<def> %AL<kill> %EAX<kill> +; CHECK-NEXT: retq entry: %tobool = icmp ne i32 %b, 0 br i1 %tobool, label %lor.end, label %lor.rhs @@ -71,6 +96,16 @@ lor.end: ; preds = %lor.rhs, %entry ; PR16551 define void @test5(i32 %X) nounwind minsize { +; CHECK-LABEL: test5: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: movzbl x+{{.*}}(%rip), %eax +; CHECK-NEXT: shll $16, %eax +; CHECK-NEXT: movzwl x+{{.*}}(%rip), %ecx +; CHECK-NEXT: orl %eax, %ecx +; CHECK-NEXT: cmpl $1, %ecx +; CHECK-NEXT: jne bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %bf.load = load i56, i56* bitcast ({ i8, i8, i8, i8, i8, i8, i8, i8 }* @x to i56*), align 4 %bf.lshr = lshr i56 %bf.load, 32 @@ -84,17 +119,16 @@ if.then: if.end: ret void - -; CHECK-LABEL: test5: -; CHECK-NOT: cmpl $1,{{.*}}x+4 -; CHECK: ret } -; CHECK-LABEL: test2_1: -; CHECK: movzbl -; CHECK: cmpl $256 -; CHECK: je bar define void @test2_1(i32 %X) nounwind minsize { +; CHECK-LABEL: test2_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: movzbl %dil, %eax +; CHECK-NEXT: cmpl $256, %eax # imm = 0x100 +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %and = and i32 %X, 255 %cmp = icmp eq i32 %and, 256 @@ -108,9 +142,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_1: -; CHECK: cmpb $1, %{{dil|cl}} define void @test_sext_i8_icmp_1(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $1, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, 1 @@ -124,9 +162,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_47: -; CHECK: cmpb $47, %{{dil|cl}} define void @test_sext_i8_icmp_47(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_47: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $47, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, 47 @@ -140,9 +182,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_127: -; CHECK: cmpb $127, %{{dil|cl}} define void @test_sext_i8_icmp_127(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_127: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $127, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, 127 @@ -156,9 +202,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_neg1: -; CHECK: cmpb $-1, %{{dil|cl}} define void @test_sext_i8_icmp_neg1(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_neg1: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $-1, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, -1 @@ -172,9 +222,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_neg2: -; CHECK: cmpb $-2, %{{dil|cl}} define void @test_sext_i8_icmp_neg2(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_neg2: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $-2, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, -2 @@ -188,9 +242,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_neg127: -; CHECK: cmpb $-127, %{{dil|cl}} define void @test_sext_i8_icmp_neg127(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_neg127: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $-127, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, -127 @@ -204,9 +262,13 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_neg128: -; CHECK: cmpb $-128, %{{dil|cl}} define void @test_sext_i8_icmp_neg128(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_neg128: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: cmpb $-128, %dil +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, -128 @@ -220,11 +282,14 @@ if.end: ret void } -; CHECK-LABEL: test_sext_i8_icmp_255: -; CHECK: movb $1, -; CHECK: testb -; CHECK: je bar define void @test_sext_i8_icmp_255(i8 %x) nounwind minsize { +; CHECK-LABEL: test_sext_i8_icmp_255: +; CHECK: # BB#0: # %entry +; CHECK-NEXT: movb $1, %al +; CHECK-NEXT: testb %al, %al +; CHECK-NEXT: je bar # TAILCALL +; CHECK-NEXT: # BB#1: # %if.end +; CHECK-NEXT: retq entry: %sext = sext i8 %x to i32 %cmp = icmp eq i32 %sext, 255 diff --git a/test/CodeGen/X86/sse3.ll b/test/CodeGen/X86/sse3.ll index 6d51fb54f8b8..79b949a6ccb1 100644 --- a/test/CodeGen/X86/sse3.ll +++ b/test/CodeGen/X86/sse3.ll @@ -14,6 +14,7 @@ define void @t0(<8 x i16>* %dest, <8 x i16>* %old) nounwind { ; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] ; X64-NEXT: movdqa %xmm0, (%rdi) ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp3 = load <8 x i16>, <8 x i16>* %old %tmp6 = shufflevector <8 x i16> %tmp3, @@ -32,6 +33,7 @@ define <8 x i16> @t1(<8 x i16>* %A, <8 x i16>* %B) nounwind { ; X64-NEXT: andps (%rdi), %xmm0 ; X64-NEXT: orps %xmm1, %xmm0 ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp1 = load <8 x i16>, <8 x i16>* %A %tmp2 = load <8 x i16>, <8 x i16>* %B %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> < i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7 > @@ -48,6 +50,7 @@ define <8 x i16> @t2(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64-NEXT: pandn %xmm1, %xmm2 ; X64-NEXT: por %xmm2, %xmm0 ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> < i32 9, i32 1, i32 2, i32 9, i32 4, i32 5, i32 6, i32 7 > ret <8 x i16> %tmp } @@ -61,6 +64,7 @@ define <8 x i16> @t3(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] ; X64-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %A, <8 x i32> < i32 8, i32 3, i32 2, i32 13, i32 7, i32 6, i32 5, i32 4 > ret <8 x i16> %tmp } @@ -73,6 +77,7 @@ define <8 x i16> @t4(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] ; X64-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,4,7] ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> < i32 0, i32 7, i32 2, i32 3, i32 1, i32 5, i32 6, i32 5 > ret <8 x i16> %tmp } @@ -83,6 +88,7 @@ define <8 x i16> @t5(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] ; X64-NEXT: movdqa %xmm1, %xmm0 ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> < i32 8, i32 9, i32 0, i32 1, i32 10, i32 11, i32 2, i32 3 > ret <8 x i16> %tmp } @@ -92,6 +98,7 @@ define <8 x i16> @t6(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64: ## BB#0: ; X64-NEXT: movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> < i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7 > ret <8 x i16> %tmp } @@ -102,6 +109,7 @@ define <8 x i16> @t7(<8 x i16> %A, <8 x i16> %B) nounwind { ; X64-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,3,2,4,5,6,7] ; X64-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,4,7] ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = shufflevector <8 x i16> %A, <8 x i16> %B, <8 x i32> < i32 0, i32 0, i32 3, i32 2, i32 4, i32 6, i32 4, i32 7 > ret <8 x i16> %tmp } @@ -113,6 +121,7 @@ define void @t8(<2 x i64>* %res, <2 x i64>* %A) nounwind { ; X64-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7] ; X64-NEXT: movdqa %xmm0, (%rdi) ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = load <2 x i64>, <2 x i64>* %A %tmp.upgrd.1 = bitcast <2 x i64> %tmp to <8 x i16> %tmp0 = extractelement <8 x i16> %tmp.upgrd.1, i32 0 @@ -143,6 +152,7 @@ define void @t9(<4 x float>* %r, <2 x i32>* %A) nounwind { ; X64-NEXT: movhpd {{.*#+}} xmm0 = xmm0[0],mem[0] ; X64-NEXT: movapd %xmm0, (%rdi) ; X64-NEXT: retq +; X64-NEXT: ## -- End function %tmp = load <4 x float>, <4 x float>* %r %tmp.upgrd.3 = bitcast <2 x i32>* %A to double* %tmp.upgrd.4 = load double, double* %tmp.upgrd.3 @@ -179,6 +189,7 @@ define void @t10() nounwind { ; X64-NEXT: movq _g2@{{.*}}(%rip), %rax ; X64-NEXT: movq %xmm0, (%rax) ; X64-NEXT: retq +; X64-NEXT: ## -- End function load <4 x i32>, <4 x i32>* @g1, align 16 bitcast <4 x i32> %1 to <8 x i16> shufflevector <8 x i16> %2, <8 x i16> undef, <8 x i32> < i32 0, i32 2, i32 4, i32 6, i32 undef, i32 undef, i32 undef, i32 undef > @@ -196,6 +207,7 @@ define <8 x i16> @t11(<8 x i16> %T0, <8 x i16> %T1) nounwind readnone { ; X64-NEXT: psrld $16, %xmm0 ; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp7 = shufflevector <8 x i16> %T0, <8 x i16> %T1, <8 x i32> < i32 1, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef , i32 undef > ret <8 x i16> %tmp7 @@ -209,6 +221,7 @@ define <8 x i16> @t12(<8 x i16> %T0, <8 x i16> %T1) nounwind readnone { ; X64-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7] ; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,3,3] ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp9 = shufflevector <8 x i16> %T0, <8 x i16> %T1, <8 x i32> < i32 0, i32 1, i32 undef, i32 undef, i32 3, i32 11, i32 undef , i32 undef > ret <8 x i16> %tmp9 @@ -222,6 +235,7 @@ define <8 x i16> @t13(<8 x i16> %T0, <8 x i16> %T1) nounwind readnone { ; X64-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,2,2,3,4,5,6,7] ; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,3,3] ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp9 = shufflevector <8 x i16> %T0, <8 x i16> %T1, <8 x i32> < i32 8, i32 9, i32 undef, i32 undef, i32 11, i32 3, i32 undef , i32 undef > ret <8 x i16> %tmp9 @@ -234,6 +248,7 @@ define <8 x i16> @t14(<8 x i16> %T0, <8 x i16> %T1) nounwind readnone { ; X64-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; X64-NEXT: movdqa %xmm1, %xmm0 ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp9 = shufflevector <8 x i16> %T0, <8 x i16> %T1, <8 x i32> < i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 2, i32 undef , i32 undef > ret <8 x i16> %tmp9 @@ -247,6 +262,7 @@ define <8 x i16> @t15(<8 x i16> %T0, <8 x i16> %T1) nounwind readnone { ; X64-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,2,4,5,6,7] ; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp8 = shufflevector <8 x i16> %T0, <8 x i16> %T1, <8 x i32> < i32 undef, i32 undef, i32 7, i32 2, i32 8, i32 undef, i32 undef , i32 undef > ret <8 x i16> %tmp8 @@ -260,6 +276,7 @@ define <16 x i8> @t16(<16 x i8> %T0) nounwind readnone { ; X64-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] ; X64-NEXT: movdqa %xmm1, %xmm0 ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp8 = shufflevector <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> %T0, <16 x i32> < i32 0, i32 1, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef , i32 undef > %tmp9 = shufflevector <16 x i8> %tmp8, <16 x i8> %T0, <16 x i32> < i32 0, i32 1, i32 2, i32 17, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef , i32 undef > @@ -275,6 +292,7 @@ define <4 x i32> @t17() nounwind { ; X64-NEXT: pxor %xmm1, %xmm1 ; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] ; X64-NEXT: retq +; X64-NEXT: ## -- End function entry: %tmp1 = load <4 x float>, <4 x float>* undef, align 16 %tmp2 = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> <i32 4, i32 1, i32 2, i32 3> diff --git a/test/CodeGen/X86/stack-folding-fp-avx1.ll b/test/CodeGen/X86/stack-folding-fp-avx1.ll index a00d47bb13e9..f937d484ce0d 100644 --- a/test/CodeGen/X86/stack-folding-fp-avx1.ll +++ b/test/CodeGen/X86/stack-folding-fp-avx1.ll @@ -1926,5 +1926,19 @@ define <8 x float> @stack_fold_xorps_ymm(<8 x float> %a0, <8 x float> %a1) { ret <8 x float> %6 } +define <4 x float> @stack_nofold_insertps(<8 x float> %a0, <8 x float> %a1) { +; Cannot fold this without changing the immediate. +; CHECK-LABEL: stack_nofold_insertps +; CHECK: 32-byte Spill +; CHECK: nop +; CHECK: 32-byte Reload +; CHECK: vinsertps $179, {{%xmm., %xmm., %xmm.}} + %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"() + %v0 = shufflevector <8 x float> %a0, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3> + %v1 = shufflevector <8 x float> %a1, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3> + %res = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %v0, <4 x float> %v1, i8 179) + ret <4 x float> %res +} + attributes #0 = { "unsafe-fp-math"="false" } attributes #1 = { "unsafe-fp-math"="true" } diff --git a/test/CodeGen/X86/statepoint-allocas.ll b/test/CodeGen/X86/statepoint-allocas.ll index 9f5418432abc..b8e5c82913a5 100644 --- a/test/CodeGen/X86/statepoint-allocas.ll +++ b/test/CodeGen/X86/statepoint-allocas.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s | FileCheck %s +; RUN: llc -verify-machineinstrs < %s | FileCheck %s ; Check that we can lower a use of an alloca both as a deopt value (where the ; exact meaning is up to the consumer of the stackmap) and as an explicit spill ; slot used for GC. diff --git a/test/CodeGen/X86/statepoint-call-lowering.ll b/test/CodeGen/X86/statepoint-call-lowering.ll index 6e5cdd605122..bd2dd53b654a 100644 --- a/test/CodeGen/X86/statepoint-call-lowering.ll +++ b/test/CodeGen/X86/statepoint-call-lowering.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s | FileCheck %s +; RUN: llc -verify-machineinstrs < %s | FileCheck %s ; This file contains a collection of basic tests to ensure we didn't ; screw up normal call lowering when there are no deopt or gc arguments. diff --git a/test/CodeGen/X86/statepoint-far-call.ll b/test/CodeGen/X86/statepoint-far-call.ll index dc49061f6461..9f9b684efae8 100644 --- a/test/CodeGen/X86/statepoint-far-call.ll +++ b/test/CodeGen/X86/statepoint-far-call.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s | FileCheck %s +; RUN: llc -verify-machineinstrs < %s | FileCheck %s ; Test to check that Statepoints with X64 far-immediate targets ; are lowered correctly to an indirect call via a scratch register. diff --git a/test/CodeGen/X86/statepoint-forward.ll b/test/CodeGen/X86/statepoint-forward.ll index d97bc0c75602..bee4b5ac884e 100644 --- a/test/CodeGen/X86/statepoint-forward.ll +++ b/test/CodeGen/X86/statepoint-forward.ll @@ -1,5 +1,5 @@ ; RUN: opt -O3 -S < %s | FileCheck --check-prefix=CHECK-OPT %s -; RUN: llc < %s | FileCheck --check-prefix=CHECK-LLC %s +; RUN: llc -verify-machineinstrs < %s | FileCheck --check-prefix=CHECK-LLC %s ; These tests are targetted at making sure we don't retain information ; about memory which contains potential gc references across a statepoint. ; They're carefully written to only outlaw forwarding of references. diff --git a/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll index 11dbe9e2e6c1..b88ca03805f2 100644 --- a/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll +++ b/test/CodeGen/X86/statepoint-gctransition-call-lowering.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s | FileCheck %s +; RUN: llc -verify-machineinstrs < %s | FileCheck %s ; This file contains a collection of basic tests to ensure we didn't ; screw up normal call lowering when a statepoint is a GC transition. diff --git a/test/CodeGen/X86/statepoint-invoke.ll b/test/CodeGen/X86/statepoint-invoke.ll index 3e8b8ca49f1d..29f8e3ed4f78 100644 --- a/test/CodeGen/X86/statepoint-invoke.ll +++ b/test/CodeGen/X86/statepoint-invoke.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s 2>&1 | FileCheck %s +; RUN: llc -verify-machineinstrs < %s 2>&1 | FileCheck %s target triple = "x86_64-pc-linux-gnu" diff --git a/test/CodeGen/X86/statepoint-live-in.ll b/test/CodeGen/X86/statepoint-live-in.ll index abe2b0a7acc8..aaa4d7c8422a 100644 --- a/test/CodeGen/X86/statepoint-live-in.ll +++ b/test/CodeGen/X86/statepoint-live-in.ll @@ -1,4 +1,5 @@ -; RUN: llc -O3 < %s | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: llc -verify-machineinstrs -O3 < %s | FileCheck %s target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.11.0" @@ -6,38 +7,70 @@ declare void @bar() #0 declare void @baz() define void @test1(i32 %a) gc "statepoint-example" { +; CHECK-LABEL: test1: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: Lcfi0: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp0: +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq +; entry: ; We expect the argument to be passed in an extra register to bar -; CHECK-LABEL: test1 -; CHECK: pushq %rax -; CHECK-NEXT: Lcfi0: -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: callq _bar %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 1, i32 %a) ret void } define void @test2(i32 %a, i32 %b) gc "statepoint-example" { +; CHECK-LABEL: test2: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: Lcfi1: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: Lcfi2: +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: Lcfi3: +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: Lcfi4: +; CHECK-NEXT: .cfi_offset %rbx, -24 +; CHECK-NEXT: Lcfi5: +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movl %esi, %ebx +; CHECK-NEXT: movl %edi, %ebp +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp1: +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp2: +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq +; entry: ; Because the first call clobbers esi, we have to move the values into ; new registers. Note that they stay in the registers for both calls. -; CHECK-LABEL: @test2 -; CHECK: movl %esi, %ebx -; CHECK-NEXT: movl %edi, %ebp -; CHECK-NEXT: callq _bar call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 2, i32 %a, i32 %b) call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 2, i32 %b, i32 %a) ret void } define void @test3(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i) gc "statepoint-example" { +; CHECK-LABEL: test3: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: Lcfi6: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp3: +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq +; entry: -; TODO: We should have folded the reload into the statepoint. -; CHECK-LABEL: @test3 -; CHECK: pushq %rax -; CHECK-NEXT: Lcfi -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: callq _bar +; We directly reference the argument slot %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 9, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i) ret void } @@ -47,25 +80,39 @@ entry: ; also ends up being a good test of whether we can fold loads from immutable ; stack slots into the statepoint. define void @test4(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" { +; CHECK-LABEL: test4: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: Lcfi7: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp4: +; CHECK-NEXT: popq %rax +; CHECK-NEXT: retq +; entry: -; CHECK-LABEL: test4 -; CHECK: pushq %rax -; CHECK-NEXT: Lcfi -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: callq _bar %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) ret void } ; A live-through gc-value must be spilled even if it is also a live-in deopt ; value. For live-in, we could technically report the register copy, but from -; a code quality perspective it's better to reuse the required stack slot so +; a code quality perspective it's better to reuse the required stack slot so ; as to put less stress on the register allocator for no benefit. define i32 addrspace(1)* @test5(i32 %a, i32 addrspace(1)* %p) gc "statepoint-example" { +; CHECK-LABEL: test5: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: Lcfi8: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: movq %rsi, (%rsp) +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp5: +; CHECK-NEXT: movq (%rsp), %rax +; CHECK-NEXT: popq %rcx +; CHECK-NEXT: retq +; entry: -; CHECK-LABEL: test5 -; CHECK: movq %rsi, (%rsp) -; CHECK-NEXT: callq _bar %token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 1, i32 %a, i32 addrspace(1)* %p, i32 addrspace(1)* %p) %p2 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %token, i32 9, i32 9) ret i32 addrspace(1)* %p2 @@ -73,14 +120,27 @@ entry: ; Show the interaction of live-through spilling followed by live-in. define void @test6(i32 %a) gc "statepoint-example" { +; CHECK-LABEL: test6: +; CHECK: ## BB#0: ## %entry +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: Lcfi9: +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: subq $16, %rsp +; CHECK-NEXT: Lcfi10: +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: Lcfi11: +; CHECK-NEXT: .cfi_offset %rbx, -16 +; CHECK-NEXT: movl %edi, %ebx +; CHECK-NEXT: movl %ebx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: callq _baz +; CHECK-NEXT: Ltmp6: +; CHECK-NEXT: callq _bar +; CHECK-NEXT: Ltmp7: +; CHECK-NEXT: addq $16, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: retq +; entry: -; TODO: We could have reused the previous spill slot at zero additional cost. -; CHECK-LABEL: test6 -; CHECK: movl %edi, %ebx -; CHECK: movl %ebx, 12(%rsp) -; CHECK-NEXT: callq _baz -; CHECK-NEXT: Ltmp -; CHECK-NEXT: callq _bar call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @baz, i32 0, i32 0, i32 0, i32 1, i32 %a) call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 1, i32 %a) ret void diff --git a/test/CodeGen/X86/statepoint-stack-usage.ll b/test/CodeGen/X86/statepoint-stack-usage.ll index 5c27898f284a..b16426eae3d5 100644 --- a/test/CodeGen/X86/statepoint-stack-usage.ll +++ b/test/CodeGen/X86/statepoint-stack-usage.ll @@ -1,4 +1,4 @@ -; RUN: llc -stack-symbol-ordering=0 < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -stack-symbol-ordering=0 < %s | FileCheck %s target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-linux-gnu" diff --git a/test/CodeGen/X86/statepoint-stackmap-format.ll b/test/CodeGen/X86/statepoint-stackmap-format.ll index 0506381b9ec2..966f66815f92 100644 --- a/test/CodeGen/X86/statepoint-stackmap-format.ll +++ b/test/CodeGen/X86/statepoint-stackmap-format.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -stack-symbol-ordering=0 -mtriple="x86_64-pc-linux-gnu" | FileCheck %s -; RUN: llc < %s -stack-symbol-ordering=0 -mtriple="x86_64-pc-unknown-elf" | FileCheck %s +; RUN: llc < %s -verify-machineinstrs -stack-symbol-ordering=0 -mtriple="x86_64-pc-linux-gnu" | FileCheck %s +; RUN: llc < %s -verify-machineinstrs -stack-symbol-ordering=0 -mtriple="x86_64-pc-unknown-elf" | FileCheck %s ; This test is a sanity check to ensure statepoints are generating StackMap ; sections correctly. This is not intended to be a rigorous test of the diff --git a/test/CodeGen/X86/statepoint-uniqueing.ll b/test/CodeGen/X86/statepoint-uniqueing.ll index e791bc6b2333..a5fa1f2d99c9 100644 --- a/test/CodeGen/X86/statepoint-uniqueing.ll +++ b/test/CodeGen/X86/statepoint-uniqueing.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s | FileCheck %s +; RUN: llc -verify-machineinstrs < %s | FileCheck %s ; Checks for a crash we had when two gc.relocate calls would ; relocating identical values diff --git a/test/CodeGen/X86/statepoint-vector-bad-spill.ll b/test/CodeGen/X86/statepoint-vector-bad-spill.ll index 848988589cb0..7c55491bb1be 100644 --- a/test/CodeGen/X86/statepoint-vector-bad-spill.ll +++ b/test/CodeGen/X86/statepoint-vector-bad-spill.ll @@ -1,4 +1,4 @@ -; RUN: llc -O3 < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -O3 < %s | FileCheck %s ; This is checking for a crash. diff --git a/test/CodeGen/X86/statepoint-vector.ll b/test/CodeGen/X86/statepoint-vector.ll index 000e88742880..5bc8f983ff06 100644 --- a/test/CodeGen/X86/statepoint-vector.ll +++ b/test/CodeGen/X86/statepoint-vector.ll @@ -1,4 +1,4 @@ -; RUN: llc -stack-symbol-ordering=0 -mcpu=nehalem -debug-only=stackmaps < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -stack-symbol-ordering=0 -mcpu=nehalem -debug-only=stackmaps < %s | FileCheck %s ; REQUIRES: asserts target triple = "x86_64-pc-linux-gnu" diff --git a/test/CodeGen/X86/vector-unsigned-cmp.ll b/test/CodeGen/X86/vector-unsigned-cmp.ll new file mode 100644 index 000000000000..fc246669992c --- /dev/null +++ b/test/CodeGen/X86/vector-unsigned-cmp.ll @@ -0,0 +1,519 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=SSE --check-prefix=SSE2 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE --check-prefix=SSE41 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2 + +; PR33276 - https://bugs.llvm.org/show_bug.cgi?id=33276 +; If both operands of an unsigned icmp are known non-negative, then +; we don't need to flip the sign bits in order to map to signed pcmpgt*. + +define <2 x i1> @ugt_v2i64(<2 x i64> %x, <2 x i64> %y) { +; SSE-LABEL: ugt_v2i64: +; SSE: # BB#0: +; SSE-NEXT: psrlq $1, %xmm0 +; SSE-NEXT: psrlq $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ugt_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <2 x i64> %x, <i64 1, i64 1> + %sh2 = lshr <2 x i64> %y, <i64 1, i64 1> + %cmp = icmp ugt <2 x i64> %sh1, %sh2 + ret <2 x i1> %cmp +} + +define <2 x i1> @ult_v2i64(<2 x i64> %x, <2 x i64> %y) { +; SSE-LABEL: ult_v2i64: +; SSE: # BB#0: +; SSE-NEXT: psrlq $1, %xmm0 +; SSE-NEXT: psrlq $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3] +; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ult_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <2 x i64> %x, <i64 1, i64 1> + %sh2 = lshr <2 x i64> %y, <i64 1, i64 1> + %cmp = icmp ult <2 x i64> %sh1, %sh2 + ret <2 x i1> %cmp +} + +define <2 x i1> @uge_v2i64(<2 x i64> %x, <2 x i64> %y) { +; SSE-LABEL: uge_v2i64: +; SSE: # BB#0: +; SSE-NEXT: psrlq $1, %xmm0 +; SSE-NEXT: psrlq $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE-NEXT: pcmpeqd %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uge_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtq %xmm0, %xmm1, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <2 x i64> %x, <i64 1, i64 1> + %sh2 = lshr <2 x i64> %y, <i64 1, i64 1> + %cmp = icmp uge <2 x i64> %sh1, %sh2 + ret <2 x i1> %cmp +} + +define <2 x i1> @ule_v2i64(<2 x i64> %x, <2 x i64> %y) { +; SSE-LABEL: ule_v2i64: +; SSE: # BB#0: +; SSE-NEXT: psrlq $1, %xmm0 +; SSE-NEXT: psrlq $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pcmpgtd %xmm1, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2] +; SSE-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ule_v2i64: +; AVX: # BB#0: +; AVX-NEXT: vpsrlq $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlq $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808] +; AVX-NEXT: vpor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <2 x i64> %x, <i64 1, i64 1> + %sh2 = lshr <2 x i64> %y, <i64 1, i64 1> + %cmp = icmp ule <2 x i64> %sh1, %sh2 + ret <2 x i1> %cmp +} + +define <4 x i1> @ugt_v4i32(<4 x i32> %x, <4 x i32> %y) { +; SSE-LABEL: ugt_v4i32: +; SSE: # BB#0: +; SSE-NEXT: psrld $1, %xmm0 +; SSE-NEXT: psrld $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: ugt_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: ugt_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpcmpgtd %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: retq + %sh1 = lshr <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1> + %sh2 = lshr <4 x i32> %y, <i32 1, i32 1, i32 1, i32 1> + %cmp = icmp ugt <4 x i32> %sh1, %sh2 + ret <4 x i1> %cmp +} + +define <4 x i1> @ult_v4i32(<4 x i32> %x, <4 x i32> %y) { +; SSE-LABEL: ult_v4i32: +; SSE: # BB#0: +; SSE-NEXT: psrld $1, %xmm0 +; SSE-NEXT: psrld $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX1-LABEL: ult_v4i32: +; AVX1: # BB#0: +; AVX1-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX1-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: ult_v4i32: +; AVX2: # BB#0: +; AVX2-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX2-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX2-NEXT: vpcmpgtd %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: retq + %sh1 = lshr <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1> + %sh2 = lshr <4 x i32> %y, <i32 1, i32 1, i32 1, i32 1> + %cmp = icmp ult <4 x i32> %sh1, %sh2 + ret <4 x i1> %cmp +} + +define <4 x i1> @uge_v4i32(<4 x i32> %x, <4 x i32> %y) { +; SSE2-LABEL: uge_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: psrld $1, %xmm0 +; SSE2-NEXT: psrld $1, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm2 +; SSE2-NEXT: pcmpgtd %xmm0, %xmm2 +; SSE2-NEXT: pcmpeqd %xmm0, %xmm0 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: uge_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: psrld $1, %xmm0 +; SSE41-NEXT: psrld $1, %xmm1 +; SSE41-NEXT: pmaxud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: uge_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1> + %sh2 = lshr <4 x i32> %y, <i32 1, i32 1, i32 1, i32 1> + %cmp = icmp uge <4 x i32> %sh1, %sh2 + ret <4 x i1> %cmp +} + +define <4 x i1> @ule_v4i32(<4 x i32> %x, <4 x i32> %y) { +; SSE2-LABEL: ule_v4i32: +; SSE2: # BB#0: +; SSE2-NEXT: psrld $1, %xmm0 +; SSE2-NEXT: psrld $1, %xmm1 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648] +; SSE2-NEXT: pxor %xmm2, %xmm1 +; SSE2-NEXT: pxor %xmm2, %xmm0 +; SSE2-NEXT: pcmpgtd %xmm1, %xmm0 +; SSE2-NEXT: pcmpeqd %xmm1, %xmm1 +; SSE2-NEXT: pxor %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ule_v4i32: +; SSE41: # BB#0: +; SSE41-NEXT: psrld $1, %xmm0 +; SSE41-NEXT: psrld $1, %xmm1 +; SSE41-NEXT: pminud %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqd %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: ule_v4i32: +; AVX: # BB#0: +; AVX-NEXT: vpsrld $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrld $1, %xmm1, %xmm1 +; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1> + %sh2 = lshr <4 x i32> %y, <i32 1, i32 1, i32 1, i32 1> + %cmp = icmp ule <4 x i32> %sh1, %sh2 + ret <4 x i1> %cmp +} + +define <8 x i1> @ugt_v8i16(<8 x i16> %x, <8 x i16> %y) { +; SSE-LABEL: ugt_v8i16: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE-NEXT: pxor %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtw %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ugt_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %sh2 = lshr <8 x i16> %y, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %cmp = icmp ugt <8 x i16> %sh1, %sh2 + ret <8 x i1> %cmp +} + +define <8 x i1> @ult_v8i16(<8 x i16> %x, <8 x i16> %y) { +; SSE-LABEL: ult_v8i16: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtw %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ult_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768] +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtw %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %sh2 = lshr <8 x i16> %y, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %cmp = icmp ult <8 x i16> %sh1, %sh2 + ret <8 x i1> %cmp +} + +define <8 x i1> @uge_v8i16(<8 x i16> %x, <8 x i16> %y) { +; SSE2-LABEL: uge_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm1 +; SSE2-NEXT: psubusw %xmm0, %xmm1 +; SSE2-NEXT: pxor %xmm0, %xmm0 +; SSE2-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: uge_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: psrlw $1, %xmm0 +; SSE41-NEXT: psrlw $1, %xmm1 +; SSE41-NEXT: pmaxuw %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: uge_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %sh2 = lshr <8 x i16> %y, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %cmp = icmp uge <8 x i16> %sh1, %sh2 + ret <8 x i1> %cmp +} + +define <8 x i1> @ule_v8i16(<8 x i16> %x, <8 x i16> %y) { +; SSE2-LABEL: ule_v8i16: +; SSE2: # BB#0: +; SSE2-NEXT: psrlw $1, %xmm0 +; SSE2-NEXT: psrlw $1, %xmm1 +; SSE2-NEXT: psubusw %xmm1, %xmm0 +; SSE2-NEXT: pxor %xmm1, %xmm1 +; SSE2-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; SSE41-LABEL: ule_v8i16: +; SSE41: # BB#0: +; SSE41-NEXT: psrlw $1, %xmm0 +; SSE41-NEXT: psrlw $1, %xmm1 +; SSE41-NEXT: pminuw %xmm0, %xmm1 +; SSE41-NEXT: pcmpeqw %xmm1, %xmm0 +; SSE41-NEXT: retq +; +; AVX-LABEL: ule_v8i16: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %sh2 = lshr <8 x i16> %y, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1> + %cmp = icmp ule <8 x i16> %sh1, %sh2 + ret <8 x i1> %cmp +} + +define <16 x i1> @ugt_v16i8(<16 x i8> %x, <16 x i8> %y) { +; SSE-LABEL: ugt_v16i8: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: pand {{.*}}(%rip), %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; SSE-NEXT: por %xmm2, %xmm1 +; SSE-NEXT: pxor %xmm2, %xmm0 +; SSE-NEXT: pcmpgtb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ugt_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; AVX-NEXT: vpor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpxor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpcmpgtb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %sh2 = lshr <16 x i8> %y, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %cmp = icmp ugt <16 x i8> %sh1, %sh2 + ret <16 x i1> %cmp +} + +define <16 x i1> @ult_v16i8(<16 x i8> %x, <16 x i8> %y) { +; SSE-LABEL: ult_v16i8: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: pand {{.*}}(%rip), %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm2 +; SSE-NEXT: pcmpgtb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ult_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vpand {{.*}}(%rip), %xmm1, %xmm1 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128] +; AVX-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpxor %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpcmpgtb %xmm0, %xmm1, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %sh2 = lshr <16 x i8> %y, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %cmp = icmp ult <16 x i8> %sh1, %sh2 + ret <16 x i1> %cmp +} + +define <16 x i1> @uge_v16i8(<16 x i8> %x, <16 x i8> %y) { +; SSE-LABEL: uge_v16i8: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: pmaxub %xmm0, %xmm1 +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: uge_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %sh2 = lshr <16 x i8> %y, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %cmp = icmp uge <16 x i8> %sh1, %sh2 + ret <16 x i1> %cmp +} + +define <16 x i1> @ule_v16i8(<16 x i8> %x, <16 x i8> %y) { +; SSE-LABEL: ule_v16i8: +; SSE: # BB#0: +; SSE-NEXT: psrlw $1, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: psrlw $1, %xmm1 +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: pminub %xmm0, %xmm1 +; SSE-NEXT: pcmpeqb %xmm1, %xmm0 +; SSE-NEXT: retq +; +; AVX-LABEL: ule_v16i8: +; AVX: # BB#0: +; AVX-NEXT: vpsrlw $1, %xmm0, %xmm0 +; AVX-NEXT: vmovdqa {{.*#+}} xmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127] +; AVX-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX-NEXT: vpsrlw $1, %xmm1, %xmm1 +; AVX-NEXT: vpand %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm1 +; AVX-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0 +; AVX-NEXT: retq + %sh1 = lshr <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %sh2 = lshr <16 x i8> %y, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1> + %cmp = icmp ule <16 x i8> %sh1, %sh2 + ret <16 x i1> %cmp +} + diff --git a/test/CodeGen/X86/wide-fma-contraction.ll b/test/CodeGen/X86/wide-fma-contraction.ll index f51f917fbac9..99e03c891c00 100644 --- a/test/CodeGen/X86/wide-fma-contraction.ll +++ b/test/CodeGen/X86/wide-fma-contraction.ll @@ -1,26 +1,48 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -march=x86 -mcpu=bdver2 -mattr=-fma -mtriple=x86_64-apple-darwin < %s | FileCheck %s ; RUN: llc -march=x86 -mcpu=bdver2 -mattr=-fma,-fma4 -mtriple=x86_64-apple-darwin < %s | FileCheck %s --check-prefix=CHECK-NOFMA ; CHECK-LABEL: fmafunc ; CHECK-NOFMA-LABEL: fmafunc define <16 x float> @fmafunc(<16 x float> %a, <16 x float> %b, <16 x float> %c) { +; CHECK-LABEL: fmafunc: +; CHECK: ## BB#0: +; CHECK-NEXT: pushl %ebp +; CHECK-NEXT: Lcfi0: +; CHECK-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NEXT: Lcfi1: +; CHECK-NEXT: .cfi_offset %ebp, -8 +; CHECK-NEXT: movl %esp, %ebp +; CHECK-NEXT: Lcfi2: +; CHECK-NEXT: .cfi_def_cfa_register %ebp +; CHECK-NEXT: andl $-32, %esp +; CHECK-NEXT: subl $32, %esp +; CHECK-NEXT: vfmaddps 8(%ebp), %ymm2, %ymm0, %ymm0 +; CHECK-NEXT: vfmaddps 40(%ebp), %ymm3, %ymm1, %ymm1 +; CHECK-NEXT: movl %ebp, %esp +; CHECK-NEXT: popl %ebp +; CHECK-NEXT: retl +; +; CHECK-NOFMA-LABEL: fmafunc: +; CHECK-NOFMA: ## BB#0: +; CHECK-NOFMA-NEXT: pushl %ebp +; CHECK-NOFMA-NEXT: Lcfi0: +; CHECK-NOFMA-NEXT: .cfi_def_cfa_offset 8 +; CHECK-NOFMA-NEXT: Lcfi1: +; CHECK-NOFMA-NEXT: .cfi_offset %ebp, -8 +; CHECK-NOFMA-NEXT: movl %esp, %ebp +; CHECK-NOFMA-NEXT: Lcfi2: +; CHECK-NOFMA-NEXT: .cfi_def_cfa_register %ebp +; CHECK-NOFMA-NEXT: andl $-32, %esp +; CHECK-NOFMA-NEXT: subl $32, %esp +; CHECK-NOFMA-NEXT: vmulps %ymm2, %ymm0, %ymm0 +; CHECK-NOFMA-NEXT: vaddps 8(%ebp), %ymm0, %ymm0 +; CHECK-NOFMA-NEXT: vmulps %ymm3, %ymm1, %ymm1 +; CHECK-NOFMA-NEXT: vaddps 40(%ebp), %ymm1, %ymm1 +; CHECK-NOFMA-NEXT: movl %ebp, %esp +; CHECK-NOFMA-NEXT: popl %ebp +; CHECK-NOFMA-NEXT: retl -; CHECK-NOT: vmulps -; CHECK-NOT: vaddps -; CHECK: vfmaddps -; CHECK-NOT: vmulps -; CHECK-NOT: vaddps -; CHECK: vfmaddps -; CHECK-NOT: vmulps -; CHECK-NOT: vaddps - -; CHECK-NOFMA-NOT: calll -; CHECK-NOFMA: vmulps -; CHECK-NOFMA: vaddps -; CHECK-NOFMA-NOT: calll -; CHECK-NOFMA: vmulps -; CHECK-NOFMA: vaddps -; CHECK-NOFMA-NOT: calll %ret = tail call <16 x float> @llvm.fmuladd.v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %c) ret <16 x float> %ret diff --git a/test/CodeGen/X86/xor-icmp.ll b/test/CodeGen/X86/xor-icmp.ll index 397e5bc10f5b..cd58dd1e7604 100644 --- a/test/CodeGen/X86/xor-icmp.ll +++ b/test/CodeGen/X86/xor-icmp.ll @@ -1,21 +1,33 @@ -; RUN: llc < %s -march=x86 | FileCheck %s -check-prefix=X32 -; RUN: llc < %s -march=x86-64 | FileCheck %s -check-prefix=X64 +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s -check-prefix=X32 +; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s -check-prefix=X64 ; rdar://7367229 define i32 @t(i32 %a, i32 %b) nounwind ssp { +; X32-LABEL: t: +; X32: # BB#0: # %entry +; X32-NEXT: movb {{[0-9]+}}(%esp), %al +; X32-NEXT: xorb {{[0-9]+}}(%esp), %al +; X32-NEXT: testb $64, %al +; X32-NEXT: je .LBB0_1 +; X32-NEXT: # BB#2: # %bb1 +; X32-NEXT: jmp bar # TAILCALL +; X32-NEXT: .LBB0_1: # %bb +; X32-NEXT: jmp foo # TAILCALL +; +; X64-LABEL: t: +; X64: # BB#0: # %entry +; X64-NEXT: movl %edi, %eax +; X64-NEXT: xorl %esi, %eax +; X64-NEXT: testb $64, %ah +; X64-NEXT: je .LBB0_1 +; X64-NEXT: # BB#2: # %bb1 +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: jmp bar # TAILCALL +; X64-NEXT: .LBB0_1: # %bb +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: jmp foo # TAILCALL entry: -; X32-LABEL: t: -; X32: xorb -; X32-NOT: andb -; X32-NOT: shrb -; X32: testb $64 -; X32: je - -; X64-LABEL: t: -; X64-NOT: setne -; X64: xorl -; X64: testb $64 -; X64: je %0 = and i32 %a, 16384 %1 = icmp ne i32 %0, 0 %2 = and i32 %b, 16384 @@ -38,20 +50,32 @@ declare i32 @bar(...) define i32 @t2(i32 %x, i32 %y) nounwind ssp { ; X32-LABEL: t2: -; X32: cmpl -; X32: sete -; X32: cmpl -; X32: sete -; X32-NOT: xor -; X32: je - +; X32: # BB#0: # %entry +; X32-NEXT: cmpl $0, {{[0-9]+}}(%esp) +; X32-NEXT: sete %al +; X32-NEXT: cmpl $0, {{[0-9]+}}(%esp) +; X32-NEXT: sete %cl +; X32-NEXT: cmpb %al, %cl +; X32-NEXT: je .LBB1_1 +; X32-NEXT: # BB#2: # %bb +; X32-NEXT: jmp foo # TAILCALL +; X32-NEXT: .LBB1_1: # %return +; X32-NEXT: retl +; ; X64-LABEL: t2: -; X64: testl -; X64: sete -; X64: testl -; X64: sete -; X64-NOT: xor -; X64: je +; X64: # BB#0: # %entry +; X64-NEXT: testl %edi, %edi +; X64-NEXT: sete %al +; X64-NEXT: testl %esi, %esi +; X64-NEXT: sete %cl +; X64-NEXT: cmpb %al, %cl +; X64-NEXT: je .LBB1_1 +; X64-NEXT: # BB#2: # %bb +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: jmp foo # TAILCALL +; X64-NEXT: .LBB1_1: # %return +; X64-NEXT: retq + entry: %0 = icmp eq i32 %x, 0 ; <i1> [#uses=1] %1 = icmp eq i32 %y, 0 ; <i1> [#uses=1] diff --git a/test/DebugInfo/MIR/AArch64/clobber-sp.mir b/test/DebugInfo/MIR/AArch64/clobber-sp.mir new file mode 100644 index 000000000000..444faee81cb3 --- /dev/null +++ b/test/DebugInfo/MIR/AArch64/clobber-sp.mir @@ -0,0 +1,181 @@ +# RUN: llc -start-after=livedebugvalues -filetype=obj -o - %s \ +# RUN: | llvm-dwarfdump - | FileCheck %s +# CHECK: .debug_info contents: +# CHECK: DW_TAG_formal_parameter +# CHECK: DW_TAG_formal_parameter +# CHECK-NEXT: DW_AT_location [DW_FORM_data4] ([[LOC:.*]]) +# CHECK-NEXT: DW_AT_name {{.*}}"y" +# CHECK: .debug_loc contents: +# CHECK: [[LOC]]: +# CHECK-SAME: Beginning address offset: 0x0000000000000000 +# CHECK-NEXT: Ending address offset: 0x0000000000000014 +# CHECK-NEXT: Location description: 51 +# reg1 +# +# The range of y's [SP+8] location must not be interrupted by the call to h. +# CHECK: Beginning address offset: 0x0000000000000014 +# CHECK-NEXT: Ending address offset: 0x0000000000000038 +# CHECK-NEXT: Location description: 8f 08 +# breg31 +8 +--- | + ; Generated at -Os from: + ; struct Rect { + ; double x, y, w, h; + ; }; + ; void g(struct Rect); + ; void h(int *); + ; int f(int x, int y, struct Rect s) { + ; g(s); + ; if (y) + ; h(&x); + ; return 0; + ; } + source_filename = "/tmp/clobber.c" + target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" + target triple = "arm64-apple-ios" + + %struct.Rect = type { double, double, double, double } + + ; Function Attrs: nounwind optsize ssp + define i32 @f(i32 %x, i32 %y, [4 x double] %s.coerce) local_unnamed_addr #0 !dbg !7 { + entry: + %x.addr = alloca i32, align 4 + tail call void @llvm.dbg.value(metadata i32 %x, i64 0, metadata !19, metadata !22), !dbg !23 + store i32 %x, i32* %x.addr, align 4, !tbaa !24 + tail call void @llvm.dbg.value(metadata i32 %y, i64 0, metadata !20, metadata !22), !dbg !28 + tail call void @llvm.dbg.declare(metadata %struct.Rect* undef, metadata !21, metadata !22), !dbg !29 + tail call void @g([4 x double] %s.coerce) #4, !dbg !30 + %tobool = icmp eq i32 %y, 0, !dbg !31 + br i1 %tobool, label %if.end, label %if.then, !dbg !33 + + if.then: ; preds = %entry + tail call void @llvm.dbg.value(metadata i32* %x.addr, i64 0, metadata !19, metadata !22), !dbg !23 + call void @h(i32* nonnull %x.addr) #4, !dbg !34 + br label %if.end, !dbg !34 + + if.end: ; preds = %if.then, %entry + ret i32 0, !dbg !35 + } + + declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + declare void @g([4 x double]) local_unnamed_addr #2 + declare void @h(i32*) local_unnamed_addr #2 + declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1 + declare void @llvm.stackprotector(i8*, i8**) #3 + + attributes #0 = { nounwind optsize ssp } + attributes #1 = { nounwind readnone speculatable } + attributes #2 = { optsize } + attributes #3 = { nounwind } + attributes #4 = { nounwind optsize } + + !llvm.dbg.cu = !{!0} + !llvm.module.flags = !{!3, !4, !5} + !llvm.ident = !{!6} + + !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (trunk 302682) (llvm/trunk 302683)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2) + !1 = !DIFile(filename: "/tmp/clobber.c", directory: "/Volumes/Data/apple-internal/swift") + !2 = !{} + !3 = !{i32 2, !"Dwarf Version", i32 2} + !4 = !{i32 2, !"Debug Info Version", i32 3} + !5 = !{i32 1, !"PIC Level", i32 2} + !6 = !{!"clang version 5.0.0 (trunk 302682) (llvm/trunk 302683)"} + !7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 7, type: !8, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: true, unit: !0, variables: !18) + !8 = !DISubroutineType(types: !9) + !9 = !{!10, !10, !10, !11} + !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) + !11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Rect", file: !1, line: 1, size: 256, elements: !12) + !12 = !{!13, !15, !16, !17} + !13 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !11, file: !1, line: 2, baseType: !14, size: 64) + !14 = !DIBasicType(name: "double", size: 64, encoding: DW_ATE_float) + !15 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !11, file: !1, line: 2, baseType: !14, size: 64, offset: 64) + !16 = !DIDerivedType(tag: DW_TAG_member, name: "w", scope: !11, file: !1, line: 2, baseType: !14, size: 64, offset: 128) + !17 = !DIDerivedType(tag: DW_TAG_member, name: "h", scope: !11, file: !1, line: 2, baseType: !14, size: 64, offset: 192) + !18 = !{!19, !20, !21} + !19 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 7, type: !10) + !20 = !DILocalVariable(name: "y", arg: 2, scope: !7, file: !1, line: 7, type: !10) + !21 = !DILocalVariable(name: "s", arg: 3, scope: !7, file: !1, line: 7, type: !11) + !22 = !DIExpression() + !23 = !DILocation(line: 7, column: 11, scope: !7) + !24 = !{!25, !25, i64 0} + !25 = !{!"int", !26, i64 0} + !26 = !{!"omnipotent char", !27, i64 0} + !27 = !{!"Simple C/C++ TBAA"} + !28 = !DILocation(line: 7, column: 18, scope: !7) + !29 = !DILocation(line: 7, column: 33, scope: !7) + !30 = !DILocation(line: 8, column: 3, scope: !7) + !31 = !DILocation(line: 9, column: 7, scope: !32) + !32 = distinct !DILexicalBlock(scope: !7, file: !1, line: 9, column: 7) + !33 = !DILocation(line: 9, column: 7, scope: !7) + !34 = !DILocation(line: 10, column: 5, scope: !32) + !35 = !DILocation(line: 12, column: 3, scope: !7) + +... +--- +name: f +alignment: 2 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%w0' } + - { reg: '%w1' } + - { reg: '%d0' } + - { reg: '%d1' } + - { reg: '%d2' } + - { reg: '%d3' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 32 + offsetAdjustment: 0 + maxAlignment: 8 + adjustsStack: true + hasCalls: true + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +stack: + - { id: 0, name: x.addr, offset: -20, size: 4, alignment: 4, local-offset: -4 } + - { id: 1, type: spill-slot, offset: -24, size: 4, alignment: 4 } + - { id: 2, type: spill-slot, offset: -8, size: 8, alignment: 8, callee-saved-register: '%lr' } + - { id: 3, type: spill-slot, offset: -16, size: 8, alignment: 8, callee-saved-register: '%fp' } +body: | + bb.0.entry: + successors: %bb.2.if.end(0x40000000), %bb.1.if.then(0x40000000) + liveins: %w0, %w1, %d0, %d1, %d2, %d3, %lr + + %sp = frame-setup SUBXri %sp, 32, 0 + frame-setup STPXi killed %fp, killed %lr, %sp, 2 :: (store 8 into %stack.3), (store 8 into %stack.2) + %fp = frame-setup ADDXri %sp, 16, 0 + DBG_VALUE debug-use %w0, debug-use _, !19, !22, debug-location !23 + STURWi killed %w0, %fp, -4 :: (store 4 into %stack.0.x.addr) + DBG_VALUE debug-use %w1, debug-use _, !20, !22, debug-location !28 + STRWui killed %w1, %sp, 2, debug-location !30 :: (store 4 into %stack.1) + DBG_VALUE %sp, 8, !20, !22, debug-location !28 + BL @g, csr_aarch64_aapcs, implicit-def dead %lr, implicit %sp, implicit killed %d0, implicit killed %d1, implicit killed %d2, implicit killed %d3, implicit-def %sp, debug-location !30 + %w0 = LDRWui %sp, 2, debug-location !33 :: (load 4 from %stack.1) + CBZW killed %w0, %bb.2.if.end, debug-location !33 + + bb.1.if.then: + successors: %bb.2.if.end(0x80000000) + + DBG_VALUE debug-use %sp, 8, !20, !22, debug-location !28 + %x0 = SUBXri %fp, 4, 0 + DBG_VALUE debug-use %x0, debug-use _, !19, !22, debug-location !23 + BL @h, csr_aarch64_aapcs, implicit-def dead %lr, implicit %sp, implicit killed %x0, debug-location !34 + + bb.2.if.end: + DBG_VALUE debug-use %sp, 8, !20, !22, debug-location !28 + %w8 = MOVZWi 0, 0 + %x0 = ORRXrs %xzr, undef %x8, 0, implicit killed %w8, debug-location !35 + %fp, %lr = LDPXi %sp, 2, debug-location !35 :: (load 8 from %stack.3), (load 8 from %stack.2) + %sp = ADDXri %sp, 32, 0, debug-location !35 + RET undef %lr, implicit killed %w0, debug-location !35 + +... diff --git a/test/DebugInfo/MIR/AArch64/lit.local.cfg b/test/DebugInfo/MIR/AArch64/lit.local.cfg new file mode 100644 index 000000000000..cec29af5bbe4 --- /dev/null +++ b/test/DebugInfo/MIR/AArch64/lit.local.cfg @@ -0,0 +1,3 @@ +if not 'AArch64' in config.root.targets: + config.unsupported = True + diff --git a/test/DebugInfo/PDB/Inputs/simple-line-info.yaml b/test/DebugInfo/PDB/Inputs/simple-line-info.yaml index 66030020f8f4..d1324d26d8bb 100644 --- a/test/DebugInfo/PDB/Inputs/simple-line-info.yaml +++ b/test/DebugInfo/PDB/Inputs/simple-line-info.yaml @@ -5,39 +5,40 @@ DbiStream: ObjFile: 'd:\src\llvm\test\DebugInfo\PDB\Inputs\empty.obj' SourceFiles: - 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' - LineInfo: - Checksums: - - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' - Kind: MD5 - Checksum: A0A5BD0D3ECD93FC29D19DE826FBF4BC - - FileName: 'f:\dd\externalapis\windows\10\sdk\inc\winerror.h' - Kind: MD5 - Checksum: 1154D69F5B2650196E1FC34F4134E56B - Lines: - - CodeSize: 10 - Flags: [ ] - RelocOffset: 16 - RelocSegment: 1 - Blocks: - - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' - Lines: - - Offset: 0 - LineStart: 5 - IsStatement: true - EndDelta: 0 - - Offset: 3 - LineStart: 6 - IsStatement: true - EndDelta: 0 - - Offset: 8 - LineStart: 7 - IsStatement: true - EndDelta: 0 - Columns: - InlineeLines: - - HasExtraFiles: false - Sites: - - FileName: 'f:\dd\externalapis\windows\10\sdk\inc\winerror.h' - LineNum: 26950 - Inlinee: 22767 + Subsections: + - !FileChecksums + Checksums: + - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' + Kind: MD5 + Checksum: A0A5BD0D3ECD93FC29D19DE826FBF4BC + - FileName: 'f:\dd\externalapis\windows\10\sdk\inc\winerror.h' + Kind: MD5 + Checksum: 1154D69F5B2650196E1FC34F4134E56B + - !Lines + CodeSize: 10 + Flags: [ ] + RelocOffset: 16 + RelocSegment: 1 + Blocks: + - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' + Lines: + - Offset: 0 + LineStart: 5 + IsStatement: true + EndDelta: 0 + - Offset: 3 + LineStart: 6 + IsStatement: true + EndDelta: 0 + - Offset: 8 + LineStart: 7 + IsStatement: true + EndDelta: 0 + Columns: + - !InlineeLines + HasExtraFiles: false + Sites: + - FileName: 'f:\dd\externalapis\windows\10\sdk\inc\winerror.h' + LineNum: 26950 + Inlinee: 22767 ... diff --git a/test/DebugInfo/PDB/pdbdump-write.test b/test/DebugInfo/PDB/pdbdump-write.test index f56b4fbe3624..393473a53af1 100644 --- a/test/DebugInfo/PDB/pdbdump-write.test +++ b/test/DebugInfo/PDB/pdbdump-write.test @@ -11,10 +11,10 @@ ; (for example if we don't write the entire stream) ; ; RUN: llvm-pdbdump pdb2yaml -stream-metadata -stream-directory \ -; RUN: -pdb-stream -tpi-stream %p/Inputs/empty.pdb > %t.1 +; RUN: -pdb-stream -tpi-stream -dbi-module-syms %p/Inputs/empty.pdb > %t.1 ; RUN: llvm-pdbdump yaml2pdb -pdb=%t.2 %t.1 ; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream \ -; RUN: -no-file-headers %p/Inputs/empty.pdb > %t.3 +; RUN: -dbi-module-syms -no-file-headers %p/Inputs/empty.pdb > %t.3 ; RUN: llvm-pdbdump pdb2yaml -pdb-stream -tpi-stream \ -; RUN: -no-file-headers %t.2 > %t.4 +; RUN: -dbi-module-syms -no-file-headers %t.2 > %t.4 ; RUN: diff %t.3 %t.4 diff --git a/test/DebugInfo/PDB/pdbdump-yaml-lineinfo.test b/test/DebugInfo/PDB/pdbdump-yaml-lineinfo.test index ca7427c0099b..f959805c7474 100644 --- a/test/DebugInfo/PDB/pdbdump-yaml-lineinfo.test +++ b/test/DebugInfo/PDB/pdbdump-yaml-lineinfo.test @@ -28,12 +28,8 @@ YAML: - Module: 'd:\src\llvm\test\DebugInfo\PDB\Inputs\empty.obj' YAML: ObjFile: 'd:\src\llvm\test\DebugInfo\PDB\Inputs\empty.obj' YAML: SourceFiles: YAML: - 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' -YAML: LineInfo: -YAML: Checksums: -YAML: - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' -YAML: Kind: MD5 -YAML: Checksum: A0A5BD0D3ECD93FC29D19DE826FBF4BC -YAML: Lines: +YAML: Subsections: +YAML: - !Lines YAML: CodeSize: 10 YAML: Flags: [ ] YAML: RelocOffset: 16 @@ -54,6 +50,11 @@ YAML: LineStart: 7 YAML: IsStatement: true YAML: EndDelta: 0 YAML: Columns: +YAML: - !FileChecksums +YAML: Checksums: +YAML: - FileName: 'd:\src\llvm\test\debuginfo\pdb\inputs\empty.cpp' +YAML: Kind: MD5 +YAML: Checksum: A0A5BD0D3ECD93FC29D19DE826FBF4BC YAML: - Module: '* Linker *' YAML: ObjFile: '' YAML: ...
\ No newline at end of file diff --git a/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll b/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll new file mode 100644 index 000000000000..4df6ffeb5a8c --- /dev/null +++ b/test/Instrumentation/SanitizerCoverage/inline-8bit-counters.ll @@ -0,0 +1,13 @@ +; Test -sanitizer-coverage-inline-8bit-counters=1 +; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64" +target triple = "x86_64-unknown-linux-gnu" +define void @foo() { +entry: +; CHECK: %0 = load i8, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__sancov_gen_, i64 0, i64 0), !nosanitize +; CHECK: %1 = add i8 %0, 1 +; CHECK: store i8 %1, i8* getelementptr inbounds ([1 x i8], [1 x i8]* @__sancov_gen_, i64 0, i64 0), !nosanitize + ret void +} +; CHECK: call void @__sanitizer_cov_8bit_counters_init(i8* bitcast (i8** @__start___sancov_counters to i8*), i8* bitcast (i8** @__stop___sancov_counters to i8*)) diff --git a/test/MC/WebAssembly/external-data.ll b/test/MC/WebAssembly/external-data.ll new file mode 100644 index 000000000000..91e05b3f13a6 --- /dev/null +++ b/test/MC/WebAssembly/external-data.ll @@ -0,0 +1,21 @@ +; RUN: llc -mtriple wasm32-unknown-unknown-wasm -filetype=obj %s -o - | obj2yaml | FileCheck %s +; Verify relocations are correctly generated for addresses of externals +; in the data section. + +declare i32 @f1(...) + +@foo = global i64 7, align 4 +@far = local_unnamed_addr global i32 (...)* @f1, align 4 + +; CHECK: - Type: DATA +; CHECK: Relocations: +; CHECK: - Type: R_WEBASSEMBLY_GLOBAL_ADDR_I32 +; CHECK: Index: 0 +; CHECK: Offset: 0x0000000E +; CHECK: Segments: +; CHECK: - Index: 0 +; CHECK: Offset: +; CHECK: Opcode: I32_CONST +; CHECK: Value: 0 +; CHECK: Content: 0700000000000000FFFFFFFF + diff --git a/test/ThinLTO/X86/deadstrip.ll b/test/ThinLTO/X86/deadstrip.ll index 0c85322eb565..c19ccb01be3c 100644 --- a/test/ThinLTO/X86/deadstrip.ll +++ b/test/ThinLTO/X86/deadstrip.ll @@ -22,6 +22,20 @@ ; RUN: llvm-dis < %t.out.1.3.import.bc | FileCheck %s --check-prefix=CHECK2 ; RUN: llvm-nm %t.out.1 | FileCheck %s --check-prefix=CHECK2-NM +; RUN: llvm-bcanalyzer -dump %t.out.index.bc | FileCheck %s --check-prefix=COMBINED +; Live, NotEligibleForImport, Internal +; COMBINED-DAG: <COMBINED {{.*}} op2=55 +; Live, Internal +; COMBINED-DAG: <COMBINED {{.*}} op2=39 +; Live, External +; COMBINED-DAG: <COMBINED {{.*}} op2=32 +; COMBINED-DAG: <COMBINED {{.*}} op2=32 +; COMBINED-DAG: <COMBINED {{.*}} op2=32 +; (Dead) +; COMBINED-DAG: <COMBINED {{.*}} op2=0 +; COMBINED-DAG: <COMBINED {{.*}} op2=0 +; COMBINED-DAG: <COMBINED {{.*}} op2=0 + ; Dead-stripping on the index allows to internalize these, ; and limit the import of @baz thanks to early pruning. ; CHECK-NOT: available_externally {{.*}} @baz() @@ -35,7 +49,7 @@ ; Make sure we didn't internalize @boo, which is reachable via ; llvm.global_ctors ; CHECK2: define void @boo() -; We should have eventually revoved @baz since it was internalized and unused +; We should have eventually removed @baz since it was internalized and unused ; CHECK2-NM-NOT: _baz ; The final binary should not contain any of the dead functions, diff --git a/test/ThinLTO/X86/newpm-basic.ll b/test/ThinLTO/X86/newpm-basic.ll index d357cbc85d00..bfcc60c6807b 100644 --- a/test/ThinLTO/X86/newpm-basic.ll +++ b/test/ThinLTO/X86/newpm-basic.ll @@ -1,7 +1,7 @@ ; RUN: opt -module-summary %s -o %t1.bc ; RUN: llvm-lto2 run %t1.bc -o %t.o \ ; RUN: -r=%t1.bc,_tinkywinky,pxl \ -; RUN: -lto-use-new-pm +; RUN: -use-new-pm target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.11.0" diff --git a/test/Transforms/CodeExtractor/cost.ll b/test/Transforms/CodeExtractor/cost.ll new file mode 100644 index 000000000000..4ac5acee019a --- /dev/null +++ b/test/Transforms/CodeExtractor/cost.ll @@ -0,0 +1,64 @@ +; RUN: opt -S < %s -partial-inliner -partial-inlining-extra-penalty=10 | FileCheck %s +; RUN: opt -S < %s -passes=partial-inliner -partial-inlining-extra-penalty=10 | FileCheck %s +define i32 @outline_region_notlikely(i32* %arg) local_unnamed_addr { +bb: +; ptr != null is predicted to be true + %tmp = icmp ne i32* %arg, null + br i1 %tmp, label %bb8, label %bb1 + +; bb1 is not likely +bb1: ; preds = %bb + %tmp2 = tail call i32 @foo(i32* nonnull %arg) + %tmp3 = tail call i32 @foo(i32* nonnull %arg) + %tmp4 = tail call i32 @foo(i32* nonnull %arg) + %tmp5 = tail call i32 @foo(i32* nonnull %arg) + %tmp6 = tail call i32 @foo(i32* nonnull %arg) + %tmp7 = tail call i32 @foo(i32* nonnull %arg) + br label %bb8 + +bb8: ; preds = %bb1, %bb + %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ] + ret i32 %tmp9 +} + +define i32 @outline_region_likely(i32* %arg) local_unnamed_addr { +bb: +; ptr == null is predicted to be false + %tmp = icmp eq i32* %arg, null + br i1 %tmp, label %bb8, label %bb1 + +; bb1 is likely +bb1: ; preds = %bb + %tmp2 = tail call i32 @foo(i32* nonnull %arg) + %tmp3 = tail call i32 @foo(i32* nonnull %arg) + %tmp4 = tail call i32 @foo(i32* nonnull %arg) + %tmp5 = tail call i32 @foo(i32* nonnull %arg) + %tmp6 = tail call i32 @foo(i32* nonnull %arg) + %tmp7 = tail call i32 @foo(i32* nonnull %arg) + br label %bb8 + +bb8: ; preds = %bb1, %bb + %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ] + ret i32 %tmp9 +} + +declare i32 @foo(i32* %arg) + +define i32 @dummy_caller(i32* %arg) local_unnamed_addr { +; CHECK-LABEL: @dummy_caller + %tmp = call i32 @outline_region_notlikely(i32* %arg) +; CHECK: call void @outline_region_notlikely.2_bb1 + %tmp2 = tail call i32 @outline_region_likely(i32* %arg) +; CHECK: %tmp2 = tail call i32 @outline_region_likely(i32* %arg) + ret i32 %tmp + +} + +; CHECK-LABEL: define internal void @outline_region_notlikely.2_bb1(i32* %arg) { +; CHECK-NEXT: newFuncRoot: + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 5.0.0 (trunk 304489)"} diff --git a/test/Transforms/CodeExtractor/cost_meta.ll b/test/Transforms/CodeExtractor/cost_meta.ll new file mode 100644 index 000000000000..2e4467a8d0c9 --- /dev/null +++ b/test/Transforms/CodeExtractor/cost_meta.ll @@ -0,0 +1,41 @@ +; RUN: opt -S < %s -partial-inliner -partial-inlining-extra-penalty=2000 | FileCheck %s +; RUN: opt -S < %s -passes=partial-inliner -partial-inlining-extra-penalty=2000 | FileCheck %s +define i32 @outline_region_notlikely(i32* %arg) local_unnamed_addr { +bb: +; ptr != null is predicted to be true + %tmp = icmp ne i32* %arg, null + br i1 %tmp, label %bb8, label %bb1, !prof !2 + +; bb1 is not likely +bb1: ; preds = %bb + %tmp2 = tail call i32 @foo(i32* nonnull %arg) + %tmp3 = tail call i32 @foo(i32* nonnull %arg) + %tmp4 = tail call i32 @foo(i32* nonnull %arg) + %tmp5 = tail call i32 @foo(i32* nonnull %arg) + %tmp6 = tail call i32 @foo(i32* nonnull %arg) + %tmp7 = tail call i32 @foo(i32* nonnull %arg) + br label %bb8 + +bb8: ; preds = %bb1, %bb + %tmp9 = phi i32 [ 0, %bb1 ], [ 1, %bb ] + ret i32 %tmp9 +} + +define i32 @dummy_caller(i32* %arg) local_unnamed_addr { +; CHECK-LABEL: @dummy_caller + %tmp = call i32 @outline_region_notlikely(i32* %arg) + ret i32 %tmp + } + + +; CHECK-LABEL: define internal void @outline_region_notlikely.1_bb1(i32* %arg) { +; CHECK-NEXT: newFuncRoot: + +declare i32 @foo(i32 * %arg) + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 5.0.0 (trunk 304489)"} +!2 = !{!"branch_weights", i32 2000, i32 1} diff --git a/test/Transforms/Coroutines/coro-split-02.ll b/test/Transforms/Coroutines/coro-split-02.ll index 953c25088652..4dc8921cd69a 100644 --- a/test/Transforms/Coroutines/coro-split-02.ll +++ b/test/Transforms/Coroutines/coro-split-02.ll @@ -1,5 +1,6 @@ ; Tests that coro-split can handle the case when a code after coro.suspend uses ; a value produces between coro.save and coro.suspend (%Result.i19) +; and checks whether stray coro.saves are properly removed ; RUN: opt < %s -coro-split -S | FileCheck %s %"struct.std::coroutine_handle" = type { i8* } @@ -24,9 +25,10 @@ entry: i8 1, label %exit ] await.ready: + %StrayCoroSave = call token @llvm.coro.save(i8* null) %val = load i32, i32* %Result.i19 call void @print(i32 %val) - br label %exit + br label %exit exit: call i1 @llvm.coro.end(i8* null, i1 false) ret void @@ -35,6 +37,7 @@ exit: ; CHECK-LABEL: @a.resume( ; CHECK: getelementptr inbounds %a.Frame ; CHECK-NEXT: getelementptr inbounds %"struct.lean_future<int>::Awaiter" +; CHECK-NOT: call token @llvm.coro.save(i8* null) ; CHECK-NEXT: %val = load i32, i32* %Result ; CHECK-NEXT: call void @print(i32 %val) ; CHECK-NEXT: ret void diff --git a/test/Transforms/Inline/AArch64/switch.ll b/test/Transforms/Inline/AArch64/switch.ll index 96d6bf2db682..a530ba734705 100644 --- a/test/Transforms/Inline/AArch64/switch.ll +++ b/test/Transforms/Inline/AArch64/switch.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -inline -inline-threshold=20 -S -mtriple=aarch64-none-linux -inline-generic-switch-cost=true | FileCheck %s -; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=20 -S -mtriple=aarch64-none-linux -inline-generic-switch-cost=true | FileCheck %s +; RUN: opt < %s -inline -inline-threshold=20 -S -mtriple=aarch64-none-linux | FileCheck %s +; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=20 -S -mtriple=aarch64-none-linux | FileCheck %s define i32 @callee_range(i32 %a, i32* %P) { switch i32 %a, label %sw.default [ diff --git a/test/Transforms/InstCombine/not.ll b/test/Transforms/InstCombine/not.ll index 6ff0a50318d2..8352c07a816b 100644 --- a/test/Transforms/InstCombine/not.ll +++ b/test/Transforms/InstCombine/not.ll @@ -33,17 +33,46 @@ define i1 @invert_fcmp(float %X, float %Y) { ; PR2298 -define zeroext i8 @test6(i32 %a, i32 %b) { -; CHECK-LABEL: @test6( -; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i32 %b, %a -; CHECK-NEXT: [[RETVAL67:%.*]] = zext i1 [[TMP3]] to i8 -; CHECK-NEXT: ret i8 [[RETVAL67]] +define i1 @not_not_cmp(i32 %a, i32 %b) { +; CHECK-LABEL: @not_not_cmp( +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 %b, %a +; CHECK-NEXT: ret i1 [[CMP]] +; + %nota = xor i32 %a, -1 + %notb = xor i32 %b, -1 + %cmp = icmp slt i32 %nota, %notb + ret i1 %cmp +} + +define <2 x i1> @not_not_cmp_vector(<2 x i32> %a, <2 x i32> %b) { +; CHECK-LABEL: @not_not_cmp_vector( +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <2 x i32> %b, %a +; CHECK-NEXT: ret <2 x i1> [[CMP]] +; + %nota = xor <2 x i32> %a, <i32 -1, i32 -1> + %notb = xor <2 x i32> %b, <i32 -1, i32 -1> + %cmp = icmp ugt <2 x i32> %nota, %notb + ret <2 x i1> %cmp +} + +define i1 @not_cmp_constant(i32 %a) { +; CHECK-LABEL: @not_cmp_constant( +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 %a, -43 +; CHECK-NEXT: ret i1 [[CMP]] +; + %nota = xor i32 %a, -1 + %cmp = icmp ugt i32 %nota, 42 + ret i1 %cmp +} + +define <2 x i1> @not_cmp_constant_vector(<2 x i32> %a) { +; CHECK-LABEL: @not_cmp_constant_vector( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt <2 x i32> %a, <i32 -43, i32 -43> +; CHECK-NEXT: ret <2 x i1> [[CMP]] ; - %tmp1not = xor i32 %a, -1 - %tmp2not = xor i32 %b, -1 - %tmp3 = icmp slt i32 %tmp1not, %tmp2not - %retval67 = zext i1 %tmp3 to i8 - ret i8 %retval67 + %nota = xor <2 x i32> %a, <i32 -1, i32 -1> + %cmp = icmp slt <2 x i32> %nota, <i32 42, i32 42> + ret <2 x i1> %cmp } define <2 x i1> @test7(<2 x i32> %A, <2 x i32> %B) { diff --git a/test/Transforms/InstSimplify/compare.ll b/test/Transforms/InstSimplify/compare.ll index d6f1b634102f..20ebd36991a5 100644 --- a/test/Transforms/InstSimplify/compare.ll +++ b/test/Transforms/InstSimplify/compare.ll @@ -1278,3 +1278,19 @@ define void @icmp_slt_sge_or(i32 %Ax, i32 %Bx) { ; CHECK: call void @helper_i1(i1 true) ret void } + +define i1 @constant_fold_inttoptr_null() { +; CHECK-LABEL: @constant_fold_inttoptr_null( +; CHECK-NEXT: ret i1 false +; + %x = icmp eq i32* inttoptr (i64 32 to i32*), null + ret i1 %x +} + +define i1 @constant_fold_null_inttoptr() { +; CHECK-LABEL: @constant_fold_null_inttoptr( +; CHECK-NEXT: ret i1 false +; + %x = icmp eq i32* null, inttoptr (i64 32 to i32*) + ret i1 %x +} diff --git a/test/Transforms/LowerExpectIntrinsic/phi_merge.ll b/test/Transforms/LowerExpectIntrinsic/phi_merge.ll new file mode 100644 index 000000000000..3b407c0f3a5a --- /dev/null +++ b/test/Transforms/LowerExpectIntrinsic/phi_merge.ll @@ -0,0 +1,356 @@ +; RUN: opt -lower-expect -S -o - < %s | FileCheck %s +; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s + +; The C case +; if (__builtin_expect((x > goo() && y > hoo() && z > too()), 1)) +; For the above case, all 3 branches should be annotated. +; +; if (__builtin_expect((x > goo() && y > hoo() && z > too()), 0)) +; For the above case, we don't have enough information, so +; only the last branch is annotated. + +define void @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: !prof [[WEIGHT:![0-9]+]] + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13, {{.*}}!prof [[WEIGHT]] + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i1 [ false, %bb10 ], [ false, %bb ], [ %tmp17, %bb14 ] + %tmp20 = xor i1 %tmp19, true + %tmp21 = xor i1 %tmp20, true + %tmp22 = zext i1 %tmp21 to i32 + %tmp23 = sext i32 %tmp22 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 1) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 + +bb30: ; preds = %bb28, %bb26 + ret void +} + +define void @foo2(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo2 +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9 +; CHECK-NOT: !prof + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13 +; CHECK-NOT: !prof + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i1 [ false, %bb10 ], [ false, %bb ], [ %tmp17, %bb14 ] + %tmp20 = xor i1 %tmp19, true + %tmp21 = xor i1 %tmp20, true + %tmp22 = zext i1 %tmp21 to i32 + %tmp23 = sext i32 %tmp22 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 0) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT2:![0-9]+]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 + +bb30: ; preds = %bb28, %bb26 + ret void +} + +define void @foo_i32(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i32 +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: !prof [[WEIGHT]] + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13, {{.*}}!prof [[WEIGHT]] + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i32 [ 5, %bb10 ], [ 5, %bb ], [ %tmp16, %bb14 ] + %tmp23 = sext i32 %tmp19 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 + +bb30: ; preds = %bb28, %bb26 + ret void +} + + +define void @foo_i32_not_unlikely(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i32_not_unlikely +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9 +; CHECK-NOT: !prof + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13 +; CHECK-NOT: !prof + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i32 [ 4, %bb10 ], [ 4, %bb ], [ %tmp16, %bb14 ] + %tmp23 = sext i32 %tmp19 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 + +bb30: ; preds = %bb28, %bb26 + ret void +} + +define void @foo_i32_xor(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i32_xor +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9,{{.*}}!prof [[WEIGHT]] + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13,{{.*}}!prof [[WEIGHT]] + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i32 [ 6, %bb10 ], [ 6, %bb ], [ %tmp16, %bb14 ] + %tmp20 = xor i32 %tmp19, 3 + %tmp23 = sext i32 %tmp20 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 +bb30: ; preds = %bb28, %bb26 + ret void +} + +define void @foo_i8_sext(i32 %arg, i32 %arg1, i8 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i8_sext +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9,{{.*}}!prof [[WEIGHT]] + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13,{{.*}}!prof [[WEIGHT]] + +bb14: ; preds = %bb10 + %tmp16 = call i8 @too8() + %tmp17 = icmp sgt i8 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i8 [ 255, %bb10 ], [ 255, %bb ], [ %tmp16, %bb14 ] + %tmp23 = sext i8 %tmp19 to i64 +; after sign extension, the operand value becomes -1 which does not match 255 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 255) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 +bb30: ; preds = %bb28, %bb26 + ret void +} + +define void @foo_i8_sext_not_unlikely(i32 %arg, i32 %arg1, i8 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i8_sext_not_unlikely +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9 +; CHECK-NOT: !prof + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13 +; CHECK-NOT: !prof + +bb14: ; preds = %bb10 + %tmp16 = call i8 @too8() + %tmp17 = icmp sgt i8 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i8 [ 255, %bb10 ], [ 255, %bb ], [ %tmp16, %bb14 ] + %tmp23 = sext i8 %tmp19 to i64 +; after sign extension, the operand value becomes -1 which matches -1 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 -1) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 +bb30: ; preds = %bb28, %bb26 + ret void +} + + +define void @foo_i32_xor_not_unlikely(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo_i32_xor_not_unlikely +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp sgt i32 %tmp8, %arg + br i1 %tmp9, label %bb10, label %bb18 +; CHECK: br i1 %tmp9 +; CHECK-NOT: !prof + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br i1 %tmp13, label %bb14, label %bb18 +; CHECK: br i1 %tmp13 +; CHECK-NOT: !prof + +bb14: ; preds = %bb10 + %tmp16 = call i32 @too() + %tmp17 = icmp sgt i32 %arg2, %tmp16 + br label %bb18 + +bb18: ; preds = %bb14, %bb10, %bb + %tmp19 = phi i32 [ 6, %bb10 ], [ 6, %bb ], [ %tmp16, %bb14 ] + %tmp20 = xor i32 %tmp19, 2 + %tmp23 = sext i32 %tmp20 to i64 + %tmp24 = call i64 @llvm.expect.i64(i64 %tmp23, i64 4) + %tmp25 = icmp ne i64 %tmp24, 0 + br i1 %tmp25, label %bb26, label %bb28 +; CHECK: br i1 %tmp25,{{.*}}!prof [[WEIGHT]] + +bb26: ; preds = %bb18 + %tmp27 = call i32 @goo() + br label %bb30 + +bb28: ; preds = %bb18 + %tmp29 = call i32 @hoo() + br label %bb30 + +bb30: ; preds = %bb28, %bb26 + ret void +} + +declare i32 @goo() + +declare i32 @hoo() + +declare i32 @too() + +declare i8 @too8() + +; Function Attrs: nounwind readnone +declare i64 @llvm.expect.i64(i64, i64) + +!llvm.ident = !{!0} + +!0 = !{!"clang version 5.0.0 (trunk 302965)"} +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/test/Transforms/LowerExpectIntrinsic/phi_or.ll b/test/Transforms/LowerExpectIntrinsic/phi_or.ll new file mode 100644 index 000000000000..849baef3dca8 --- /dev/null +++ b/test/Transforms/LowerExpectIntrinsic/phi_or.ll @@ -0,0 +1,103 @@ +; RUN: opt -lower-expect -S -o - < %s | FileCheck %s +; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s +; +; if (__builtin_expect((x > goo() || y > hoo()), 1)) { +; .. +; } +; For the above case, only the second branch should be +; annotated. +; if (__builtin_expect((x > goo() || y > hoo()), 0)) { +; .. +; } +; For the above case, two branches should be annotated. +; Function Attrs: noinline nounwind uwtable +define void @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo +bb: + %tmp8 = call i32 @goo() + %tmp9 = icmp slt i32 %arg, %tmp8 + br i1 %tmp9, label %bb14, label %bb10 +; CHECK: br i1 %tmp9 +; CHECK-NOT: br i1 %tmp9{{.*}}!prof + +bb10: ; preds = %bb + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %arg1, %tmp12 + br label %bb14 + +bb14: ; preds = %bb10, %bb + %tmp15 = phi i1 [ true, %bb ], [ %tmp13, %bb10 ] + %tmp16 = zext i1 %tmp15 to i32 + %tmp17 = sext i32 %tmp16 to i64 + %expect = call i64 @llvm.expect.i64(i64 %tmp17, i64 1) + %tmp18 = icmp ne i64 %expect, 0 + br i1 %tmp18, label %bb19, label %bb21 +; CHECK: br i1 %tmp18{{.*}}!prof [[WEIGHT:![0-9]+]] + +bb19: ; preds = %bb14 + %tmp20 = call i32 @goo() + br label %bb23 + +bb21: ; preds = %bb14 + %tmp22 = call i32 @hoo() + br label %bb23 + +bb23: ; preds = %bb21, %bb19 + ret void +} + +define void @foo2(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3) { +; CHECK-LABEL: void @foo2 +bb: + %tmp = alloca i32, align 4 + %tmp4 = alloca i32, align 4 + %tmp5 = alloca i32, align 4 + %tmp6 = alloca i32, align 4 + store i32 %arg, i32* %tmp, align 4 + store i32 %arg1, i32* %tmp4, align 4 + store i32 %arg2, i32* %tmp5, align 4 + store i32 %arg3, i32* %tmp6, align 4 + %tmp7 = load i32, i32* %tmp, align 4 + %tmp8 = call i32 @goo() + %tmp9 = icmp slt i32 %tmp7, %tmp8 + br i1 %tmp9, label %bb14, label %bb10 +; CHECK: br i1 %tmp9{{.*}}!prof [[WEIGHT2:![0-9]+]] + +bb10: ; preds = %bb + %tmp11 = load i32, i32* %tmp5, align 4 + %tmp12 = call i32 @hoo() + %tmp13 = icmp sgt i32 %tmp11, %tmp12 + br label %bb14 + +bb14: ; preds = %bb10, %bb + %tmp15 = phi i1 [ true, %bb ], [ %tmp13, %bb10 ] + %tmp16 = zext i1 %tmp15 to i32 + %tmp17 = sext i32 %tmp16 to i64 + %expect = call i64 @llvm.expect.i64(i64 %tmp17, i64 0) + %tmp18 = icmp ne i64 %expect, 0 + br i1 %tmp18, label %bb19, label %bb21 +; CHECK: br i1 %tmp18{{.*}}!prof [[WEIGHT2]] + +bb19: ; preds = %bb14 + %tmp20 = call i32 @goo() + br label %bb23 + +bb21: ; preds = %bb14 + %tmp22 = call i32 @hoo() + br label %bb23 + +bb23: ; preds = %bb21, %bb19 + ret void +} + +declare i32 @goo() +declare i32 @hoo() +declare i64 @llvm.expect.i64(i64, i64) + + +!llvm.ident = !{!0} + + +!0 = !{!"clang version 5.0.0 (trunk 302965)"} +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 2000, i32 1} +; CHECK: [[WEIGHT2]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/test/Transforms/LowerExpectIntrinsic/phi_tern.ll b/test/Transforms/LowerExpectIntrinsic/phi_tern.ll new file mode 100644 index 000000000000..3c603d51b438 --- /dev/null +++ b/test/Transforms/LowerExpectIntrinsic/phi_tern.ll @@ -0,0 +1,56 @@ +; RUN: opt -lower-expect -S -o - < %s | FileCheck %s +; RUN: opt -S -passes='function(lower-expect)' < %s | FileCheck %s + +; return __builtin_expect((a > b ? 1, goo(), 0); +; +; Function Attrs: noinline nounwind uwtable +define i32 @foo(i32 %arg, i32 %arg1) { +; CHECK-LABEL: i32 @foo +bb: + %tmp5 = icmp sgt i32 %arg, %arg1 + br i1 %tmp5, label %bb9, label %bb7 +; CHECK: br i1 %tmp5{{.*}}!prof [[WEIGHT:![0-9]+]] + +bb7: ; preds = %bb + %tmp8 = call i32 @goo() + br label %bb9 + +bb9: ; preds = %bb7, %bb9 + %tmp10 = phi i32 [ 1, %bb ], [ %tmp8, %bb7 ] + %tmp11 = sext i32 %tmp10 to i64 + %expect = call i64 @llvm.expect.i64(i64 %tmp11, i64 0) + %tmp12 = trunc i64 %expect to i32 + ret i32 %tmp12 +} + +define i32 @foo2(i32 %arg, i32 %arg1) { +bb: + %tmp5 = icmp sgt i32 %arg, %arg1 + br i1 %tmp5, label %bb6, label %bb7 +; CHECK: br i1 %tmp5{{.*}}!prof [[WEIGHT:![0-9]+]] + +bb6: ; preds = %bb + br label %bb9 + +bb7: ; preds = %bb + %tmp8 = call i32 @goo() + br label %bb9 + +bb9: ; preds = %bb7, %bb6 + %tmp10 = phi i32 [ 1, %bb6 ], [ %tmp8, %bb7 ] + %tmp11 = sext i32 %tmp10 to i64 + %expect = call i64 @llvm.expect.i64(i64 %tmp11, i64 0) + %tmp12 = trunc i64 %expect to i32 + ret i32 %tmp12 +} + +declare i32 @goo() +declare i64 @llvm.expect.i64(i64, i64) + + + +!llvm.ident = !{!0} + +!0 = !{!"clang version 5.0.0 (trunk 302965)"} + +; CHECK: [[WEIGHT]] = !{!"branch_weights", i32 1, i32 2000} diff --git a/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml b/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml index b7a1d208fc6f..cfac37986bda 100644 --- a/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml +++ b/test/Transforms/LowerTypeTests/Inputs/import-unsat.yaml @@ -1,7 +1,8 @@ --- GlobalValueMap: 42: - - TypeTests: [123] + - Live: true + TypeTests: [123] TypeIdMap: typeid1: TTRes: diff --git a/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml b/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml new file mode 100644 index 000000000000..7baa02ada86c --- /dev/null +++ b/test/Transforms/LowerTypeTests/Inputs/use-typeid1-dead.yaml @@ -0,0 +1,7 @@ +--- +GlobalValueMap: + 42: + - Live: false + TypeTests: [14276520915468743435] # guid("typeid1") +WithGlobalValueDeadStripping: true +... diff --git a/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml b/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml index 031b2e8de04e..f30257cfc0d4 100644 --- a/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml +++ b/test/Transforms/LowerTypeTests/Inputs/use-typeid1-typeid2.yaml @@ -1,5 +1,6 @@ --- GlobalValueMap: 42: - - TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2") + - Live: true + TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2") ... diff --git a/test/Transforms/LowerTypeTests/export-dead.ll b/test/Transforms/LowerTypeTests/export-dead.ll new file mode 100644 index 000000000000..265402b34a6e --- /dev/null +++ b/test/Transforms/LowerTypeTests/export-dead.ll @@ -0,0 +1,14 @@ +; The only use of "typeid1" is in a dead function. Export nothing. + +; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-dead.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s +; RUN: FileCheck --check-prefix=SUMMARY %s < %t + +@foo = constant i32 42, !type !0 + +!0 = !{i32 0, !"typeid1"} + +; CHECK-NOT: @__typeid_typeid1_global_addr = + +; SUMMARY: TypeIdMap: +; SUMMARY-NEXT: WithGlobalValueDeadStripping: true +; SUMMARY-NEXT: ... diff --git a/test/Transforms/LowerTypeTests/export-nothing.ll b/test/Transforms/LowerTypeTests/export-nothing.ll index 9ab41b5f6cb6..8ad331539942 100644 --- a/test/Transforms/LowerTypeTests/export-nothing.ll +++ b/test/Transforms/LowerTypeTests/export-nothing.ll @@ -4,4 +4,5 @@ ; CHECK: --- ; CHECK-NEXT: GlobalValueMap: ; CHECK-NEXT: TypeIdMap: +; CHECK-NEXT: WithGlobalValueDeadStripping: false ; CHECK-NEXT: ... diff --git a/test/Transforms/LowerTypeTests/import-unsat.ll b/test/Transforms/LowerTypeTests/import-unsat.ll index 76b244001986..6cb9b26fb574 100644 --- a/test/Transforms/LowerTypeTests/import-unsat.ll +++ b/test/Transforms/LowerTypeTests/import-unsat.ll @@ -4,7 +4,10 @@ ; SUMMARY: GlobalValueMap: ; SUMMARY-NEXT: 42: -; SUMMARY-NEXT: - TypeTests: [ 123 ] +; SUMMARY-NEXT: - Linkage: 0 +; SUMMARY-NEXT: NotEligibleToImport: false +; SUMMARY-NEXT: Live: true +; SUMMARY-NEXT: TypeTests: [ 123 ] ; SUMMARY-NEXT: TypeIdMap: ; SUMMARY-NEXT: typeid1: ; SUMMARY-NEXT: TTRes: diff --git a/test/Transforms/SROA/address-spaces.ll b/test/Transforms/SROA/address-spaces.ll index 119f2252d95e..8fba30c2720f 100644 --- a/test/Transforms/SROA/address-spaces.ll +++ b/test/Transforms/SROA/address-spaces.ll @@ -83,3 +83,21 @@ define void @pr27557() { store i32 addrspace(3)* @l, i32 addrspace(3)** %3, align 8 ret void } + +; Make sure pre-splitting doesn't try to introduce an illegal bitcast +define float @presplit(i64 addrspace(1)* %p) { +entry: +; CHECK-LABEL: @presplit( +; CHECK: %[[CAST:.*]] = bitcast i64 addrspace(1)* {{.*}} to i32 addrspace(1)* +; CHECK: load i32, i32 addrspace(1)* %[[CAST]] + %b = alloca i64 + %b.cast = bitcast i64* %b to [2 x float]* + %b.gep1 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 0 + %b.gep2 = getelementptr [2 x float], [2 x float]* %b.cast, i32 0, i32 1 + %l = load i64, i64 addrspace(1)* %p + store i64 %l, i64* %b + %f1 = load float, float* %b.gep1 + %f2 = load float, float* %b.gep2 + %ret = fadd float %f1, %f2 + ret float %ret +} diff --git a/test/Transforms/Util/PredicateInfo/condprop.ll b/test/Transforms/Util/PredicateInfo/condprop.ll index 61f59f03e1bc..496bb8385217 100644 --- a/test/Transforms/Util/PredicateInfo/condprop.ll +++ b/test/Transforms/Util/PredicateInfo/condprop.ll @@ -1,6 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -print-predicateinfo -analyze < %s 2>&1 | FileCheck %s -; RUN: opt -print-predicateinfo -analyze -reverse-iterate < %s 2>&1 | FileCheck %s @a = external global i32 ; <i32*> [#uses=7] diff --git a/test/Transforms/Util/PredicateInfo/condprop2.ll b/test/Transforms/Util/PredicateInfo/condprop2.ll new file mode 100644 index 000000000000..415fa7c879e3 --- /dev/null +++ b/test/Transforms/Util/PredicateInfo/condprop2.ll @@ -0,0 +1,474 @@ +; REQUIRES: asserts +; NOTE: The flag -reverse-iterate is present only in a +Asserts build. +; Hence, this test has been split from condprop.ll to test with -reverse-iterate. +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -print-predicateinfo -analyze -reverse-iterate < %s 2>&1 | FileCheck %s + +@a = external global i32 ; <i32*> [#uses=7] + +define i32 @test1() nounwind { +; CHECK-LABEL: @test1( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 4 +; CHECK-NEXT: br i1 [[TMP1]], label [[BB:%.*]], label [[BB1:%.*]] +; CHECK: bb: +; CHECK-NEXT: br label [[BB8:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 5 +; CHECK-NEXT: br i1 [[TMP3]], label [[BB2:%.*]], label [[BB3:%.*]] +; CHECK: bb2: +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb3: +; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 4 +; CHECK-NEXT: br i1 [[TMP5]], label [[BB4:%.*]], label [[BB5:%.*]] +; CHECK: bb4: +; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP6]], 5 +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb5: +; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 5 +; CHECK-NEXT: br i1 [[TMP9]], label [[BB6:%.*]], label [[BB7:%.*]] +; CHECK: bb6: +; CHECK-NEXT: [[TMP10:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], 4 +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb7: +; CHECK-NEXT: [[TMP12:%.*]] = load i32, i32* @a, align 4 +; CHECK-NEXT: br label [[BB8]] +; CHECK: bb8: +; CHECK-NEXT: [[DOT0:%.*]] = phi i32 [ [[TMP12]], [[BB7]] ], [ [[TMP11]], [[BB6]] ], [ [[TMP7]], [[BB4]] ], [ 4, [[BB2]] ], [ 5, [[BB]] ] +; CHECK-NEXT: br label [[RETURN:%.*]] +; CHECK: return: +; CHECK-NEXT: ret i32 [[DOT0]] +; +entry: + %0 = load i32, i32* @a, align 4 + %1 = icmp eq i32 %0, 4 + br i1 %1, label %bb, label %bb1 + +bb: ; preds = %entry + br label %bb8 + +bb1: ; preds = %entry + %2 = load i32, i32* @a, align 4 + %3 = icmp eq i32 %2, 5 + br i1 %3, label %bb2, label %bb3 + +bb2: ; preds = %bb1 + br label %bb8 + +bb3: ; preds = %bb1 + %4 = load i32, i32* @a, align 4 + %5 = icmp eq i32 %4, 4 + br i1 %5, label %bb4, label %bb5 + +bb4: ; preds = %bb3 + %6 = load i32, i32* @a, align 4 + %7 = add i32 %6, 5 + br label %bb8 + +bb5: ; preds = %bb3 + %8 = load i32, i32* @a, align 4 + %9 = icmp eq i32 %8, 5 + br i1 %9, label %bb6, label %bb7 + +bb6: ; preds = %bb5 + %10 = load i32, i32* @a, align 4 + %11 = add i32 %10, 4 + br label %bb8 + +bb7: ; preds = %bb5 + %12 = load i32, i32* @a, align 4 + br label %bb8 + +bb8: ; preds = %bb7, %bb6, %bb4, %bb2, %bb + %.0 = phi i32 [ %12, %bb7 ], [ %11, %bb6 ], [ %7, %bb4 ], [ 4, %bb2 ], [ 5, %bb ] + br label %return + +return: ; preds = %bb8 + ret i32 %.0 +} + +declare void @foo(i1) +declare void @bar(i32) + +define void @test3(i32 %x, i32 %y) { +; CHECK-LABEL: @test3( +; CHECK-NEXT: [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0 +; CHECK-NEXT: [[Z:%.*]] = and i1 [[XZ]], [[YZ]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]]) +; CHECK: [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]]) +; CHECK: [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: br i1 [[Z]], label [[BOTH_ZERO:%.*]], label [[NOPE:%.*]] +; CHECK: both_zero: +; CHECK-NEXT: call void @foo(i1 [[XZ_0]]) +; CHECK-NEXT: call void @foo(i1 [[YZ_0]]) +; CHECK-NEXT: call void @bar(i32 [[X_0]]) +; CHECK-NEXT: call void @bar(i32 [[Y_0]]) +; CHECK-NEXT: ret void +; CHECK: nope: +; CHECK-NEXT: call void @foo(i1 [[Z_0]]) +; CHECK-NEXT: ret void +; + %xz = icmp eq i32 %x, 0 + %yz = icmp eq i32 %y, 0 + %z = and i1 %xz, %yz + br i1 %z, label %both_zero, label %nope +both_zero: + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + ret void +nope: + call void @foo(i1 %z) + ret void +} + +define void @test4(i1 %b, i32 %x) { +; CHECK-LABEL: @test4( +; CHECK-NEXT: br i1 [[B:%.*]], label [[SW:%.*]], label [[CASE3:%.*]] +; CHECK: sw: +; CHECK: i32 0, label [[CASE0:%.*]] +; CHECK-NEXT: i32 1, label [[CASE1:%.*]] +; CHECK-NEXT: i32 2, label [[CASE0]] +; CHECK-NEXT: i32 3, label [[CASE3]] +; CHECK-NEXT: i32 4, label [[DEFAULT:%.*]] +; CHECK-NEXT: ] Edge: [label [[SW]],label %case1] } +; CHECK-NEXT: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X:%.*]]) +; CHECK-NEXT: switch i32 [[X]], label [[DEFAULT]] [ +; CHECK-NEXT: i32 0, label [[CASE0]] +; CHECK-NEXT: i32 1, label [[CASE1]] +; CHECK-NEXT: i32 2, label [[CASE0]] +; CHECK-NEXT: i32 3, label [[CASE3]] +; CHECK-NEXT: i32 4, label [[DEFAULT]] +; CHECK-NEXT: ] +; CHECK: default: +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: ret void +; CHECK: case0: +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: ret void +; CHECK: case1: +; CHECK-NEXT: call void @bar(i32 [[X_0]]) +; CHECK-NEXT: ret void +; CHECK: case3: +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: ret void +; + br i1 %b, label %sw, label %case3 +sw: + switch i32 %x, label %default [ + i32 0, label %case0 + i32 1, label %case1 + i32 2, label %case0 + i32 3, label %case3 + i32 4, label %default + ] +default: + call void @bar(i32 %x) + ret void +case0: + call void @bar(i32 %x) + ret void +case1: + call void @bar(i32 %x) + ret void +case3: + call void @bar(i32 %x) + ret void +} + +define i1 @test5(i32 %x, i32 %y) { +; CHECK-LABEL: @test5( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[X_0]], [[Y_0]] +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i32 [[X_1]], [[Y_1]] +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp = icmp eq i32 %x, %y + br i1 %cmp, label %same, label %different + +same: + %cmp2 = icmp ne i32 %x, %y + ret i1 %cmp2 + +different: + %cmp3 = icmp eq i32 %x, %y + ret i1 %cmp3 +} + +define i1 @test6(i32 %x, i32 %y) { +; CHECK-LABEL: @test6( +; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[X]], [[Y]] +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i32 [[X]], [[Y]] +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp2 = icmp ne i32 %x, %y + %cmp = icmp eq i32 %x, %y + %cmp3 = icmp eq i32 %x, %y + br i1 %cmp, label %same, label %different + +same: + ret i1 %cmp2 + +different: + ret i1 %cmp3 +} + +define i1 @test6_fp(float %x, float %y) { +; CHECK-LABEL: @test6_fp( +; CHECK-NEXT: [[CMP2:%.*]] = fcmp une float [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq float [[X]], [[Y]] +; CHECK-NEXT: [[CMP3:%.*]] = fcmp oeq float [[X]], [[Y]] +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp2 = fcmp une float %x, %y + %cmp = fcmp oeq float %x, %y + %cmp3 = fcmp oeq float %x, %y + br i1 %cmp, label %same, label %different + +same: + ret i1 %cmp2 + +different: + ret i1 %cmp3 +} + +define i1 @test7(i32 %x, i32 %y) { +; CHECK-LABEL: @test7( +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: [[CMP2:%.*]] = icmp sle i32 [[X_0]], [[Y_0]] +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[X_1]], [[Y_1]] +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp = icmp sgt i32 %x, %y + br i1 %cmp, label %same, label %different + +same: + %cmp2 = icmp sle i32 %x, %y + ret i1 %cmp2 + +different: + %cmp3 = icmp sgt i32 %x, %y + ret i1 %cmp3 +} + +define i1 @test7_fp(float %x, float %y) { +; CHECK-LABEL: @test7_fp( +; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]] +; CHECK: [[X_0:%.*]] = call float @llvm.ssa.copy.f32(float [[X]]) +; CHECK: [[X_1:%.*]] = call float @llvm.ssa.copy.f32(float [[X]]) +; CHECK: [[Y_0:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]]) +; CHECK: [[Y_1:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]]) +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: [[CMP2:%.*]] = fcmp ule float [[X_0]], [[Y_0]] +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: [[CMP3:%.*]] = fcmp ogt float [[X_1]], [[Y_1]] +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp = fcmp ogt float %x, %y + br i1 %cmp, label %same, label %different + +same: + %cmp2 = fcmp ule float %x, %y + ret i1 %cmp2 + +different: + %cmp3 = fcmp ogt float %x, %y + ret i1 %cmp3 +} + +define i1 @test8(i32 %x, i32 %y) { +; CHECK-LABEL: @test8( +; CHECK-NEXT: [[CMP2:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], [[Y]] +; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[X]], [[Y]] +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp2 = icmp sle i32 %x, %y + %cmp = icmp sgt i32 %x, %y + %cmp3 = icmp sgt i32 %x, %y + br i1 %cmp, label %same, label %different + +same: + ret i1 %cmp2 + +different: + ret i1 %cmp3 +} + +define i1 @test8_fp(float %x, float %y) { +; CHECK-LABEL: @test8_fp( +; CHECK-NEXT: [[CMP2:%.*]] = fcmp ule float [[X:%.*]], [[Y:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[X]], [[Y]] +; CHECK-NEXT: [[CMP3:%.*]] = fcmp ogt float [[X]], [[Y]] +; CHECK-NEXT: br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]] +; CHECK: same: +; CHECK-NEXT: ret i1 [[CMP2]] +; CHECK: different: +; CHECK-NEXT: ret i1 [[CMP3]] +; + %cmp2 = fcmp ule float %x, %y + %cmp = fcmp ogt float %x, %y + %cmp3 = fcmp ogt float %x, %y + br i1 %cmp, label %same, label %different + +same: + ret i1 %cmp2 + +different: + ret i1 %cmp3 +} + +define i32 @test9(i32 %i, i32 %j) { +; CHECK-LABEL: @test9( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]] +; CHECK: [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]]) +; CHECK: [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]]) +; CHECK-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]] +; CHECK: cond_true: +; CHECK-NEXT: [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]] +; CHECK-NEXT: ret i32 [[DIFF]] +; CHECK: ret: +; CHECK-NEXT: ret i32 5 +; + %cmp = icmp eq i32 %i, %j + br i1 %cmp, label %cond_true, label %ret + +cond_true: + %diff = sub i32 %i, %j + ret i32 %diff + +ret: + ret i32 5 +} + +define i32 @test10(i32 %j, i32 %i) { +; CHECK-LABEL: @test10( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]] +; CHECK: [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]]) +; CHECK: [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]]) +; CHECK-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]] +; CHECK: cond_true: +; CHECK-NEXT: [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]] +; CHECK-NEXT: ret i32 [[DIFF]] +; CHECK: ret: +; CHECK-NEXT: ret i32 5 +; + %cmp = icmp eq i32 %i, %j + br i1 %cmp, label %cond_true, label %ret + +cond_true: + %diff = sub i32 %i, %j + ret i32 %diff + +ret: + ret i32 5 +} + +declare i32 @yogibar() + +define i32 @test11(i32 %x) { +; CHECK-LABEL: @test11( +; CHECK-NEXT: [[V0:%.*]] = call i32 @yogibar() +; CHECK-NEXT: [[V1:%.*]] = call i32 @yogibar() +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[V0]], [[V1]] +; CHECK: [[V0_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0]]) +; CHECK: [[V1_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V1]]) +; CHECK-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[NEXT:%.*]] +; CHECK: cond_true: +; CHECK-NEXT: ret i32 [[V1_0]] +; CHECK: next: +; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 [[X:%.*]], [[V0_0]] +; CHECK: [[V0_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0_0]]) +; CHECK-NEXT: br i1 [[CMP2]], label [[COND_TRUE2:%.*]], label [[NEXT2:%.*]] +; CHECK: cond_true2: +; CHECK-NEXT: ret i32 [[V0_0_1]] +; CHECK: next2: +; CHECK-NEXT: ret i32 0 +; + %v0 = call i32 @yogibar() + %v1 = call i32 @yogibar() + %cmp = icmp eq i32 %v0, %v1 + br i1 %cmp, label %cond_true, label %next + +cond_true: + ret i32 %v1 + +next: + %cmp2 = icmp eq i32 %x, %v0 + br i1 %cmp2, label %cond_true2, label %next2 + +cond_true2: + ret i32 %v0 + +next2: + ret i32 0 +} + +define i32 @test12(i32 %x) { +; CHECK-LABEL: @test12( +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK-NEXT: br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]] +; CHECK: cond_true: +; CHECK-NEXT: br label [[RET:%.*]] +; CHECK: cond_false: +; CHECK-NEXT: br label [[RET]] +; CHECK: ret: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[X_0]], [[COND_TRUE]] ], [ [[X_1]], [[COND_FALSE]] ] +; CHECK-NEXT: ret i32 [[RES]] +; + %cmp = icmp eq i32 %x, 0 + br i1 %cmp, label %cond_true, label %cond_false + +cond_true: + br label %ret + +cond_false: + br label %ret + +ret: + %res = phi i32 [ %x, %cond_true ], [ %x, %cond_false ] + ret i32 %res +} diff --git a/test/Transforms/Util/PredicateInfo/testandor.ll b/test/Transforms/Util/PredicateInfo/testandor.ll index 43c508670908..c1048cf6d0f6 100644 --- a/test/Transforms/Util/PredicateInfo/testandor.ll +++ b/test/Transforms/Util/PredicateInfo/testandor.ll @@ -1,6 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -print-predicateinfo < %s 2>&1 | FileCheck %s -; RUN: opt -print-predicateinfo -reverse-iterate < %s 2>&1 | FileCheck %s declare void @foo(i1) declare void @bar(i32) diff --git a/test/Transforms/Util/PredicateInfo/testandor2.ll b/test/Transforms/Util/PredicateInfo/testandor2.ll new file mode 100644 index 000000000000..a03250c2f7a0 --- /dev/null +++ b/test/Transforms/Util/PredicateInfo/testandor2.ll @@ -0,0 +1,214 @@ +; REQUIRES: asserts +; NOTE: The flag -reverse-iterate is present only in a +Asserts build. +; Hence, this test has been split from testandor.ll to test with -reverse-iterate. +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -print-predicateinfo -reverse-iterate < %s 2>&1 | FileCheck %s + +declare void @foo(i1) +declare void @bar(i32) +declare void @llvm.assume(i1) + +define void @testor(i32 %x, i32 %y) { +; CHECK-LABEL: @testor( +; CHECK-NEXT: [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0 +; CHECK-NEXT: [[Z:%.*]] = or i1 [[XZ]], [[YZ]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]]) +; CHECK: [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]]) +; CHECK: [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]] +; CHECK: oneof: +; CHECK-NEXT: call void @foo(i1 [[XZ]]) +; CHECK-NEXT: call void @foo(i1 [[YZ]]) +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: call void @bar(i32 [[Y]]) +; CHECK-NEXT: ret void +; CHECK: neither: +; CHECK-NEXT: call void @foo(i1 [[XZ_0]]) +; CHECK-NEXT: call void @foo(i1 [[YZ_0]]) +; CHECK-NEXT: call void @bar(i32 [[X_0]]) +; CHECK-NEXT: call void @bar(i32 [[Y_0]]) +; CHECK-NEXT: call void @foo(i1 [[Z_0]]) +; CHECK-NEXT: ret void +; + %xz = icmp eq i32 %x, 0 + %yz = icmp eq i32 %y, 0 + %z = or i1 %xz, %yz + br i1 %z, label %oneof, label %neither +oneof: +;; Should not insert on the true edge for or + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + ret void +neither: + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + call void @foo(i1 %z) + ret void +} +define void @testand(i32 %x, i32 %y) { +; CHECK-LABEL: @testand( +; CHECK-NEXT: [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0 +; CHECK-NEXT: [[Z:%.*]] = and i1 [[XZ]], [[YZ]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]]) +; CHECK: [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]]) +; CHECK: [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]] +; CHECK: both: +; CHECK-NEXT: call void @foo(i1 [[XZ_0]]) +; CHECK-NEXT: call void @foo(i1 [[YZ_0]]) +; CHECK-NEXT: call void @bar(i32 [[X_0]]) +; CHECK-NEXT: call void @bar(i32 [[Y_0]]) +; CHECK-NEXT: ret void +; CHECK: nope: +; CHECK-NEXT: call void @foo(i1 [[XZ]]) +; CHECK-NEXT: call void @foo(i1 [[YZ]]) +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: call void @bar(i32 [[Y]]) +; CHECK-NEXT: call void @foo(i1 [[Z_0]]) +; CHECK-NEXT: ret void +; + %xz = icmp eq i32 %x, 0 + %yz = icmp eq i32 %y, 0 + %z = and i1 %xz, %yz + br i1 %z, label %both, label %nope +both: + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + ret void +nope: +;; Should not insert on the false edge for and + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + call void @foo(i1 %z) + ret void +} +define void @testandsame(i32 %x, i32 %y) { +; CHECK-LABEL: @testandsame( +; CHECK-NEXT: [[XGT:%.*]] = icmp sgt i32 [[X:%.*]], 0 +; CHECK-NEXT: [[XLT:%.*]] = icmp slt i32 [[X]], 100 +; CHECK-NEXT: [[Z:%.*]] = and i1 [[XGT]], [[XLT]] +; CHECK: [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]]) +; CHECK: [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XGT]]) +; CHECK: [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XLT]]) +; CHECK: [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]] +; CHECK: both: +; CHECK-NEXT: call void @foo(i1 [[XGT_0]]) +; CHECK-NEXT: call void @foo(i1 [[XLT_0]]) +; CHECK-NEXT: call void @bar(i32 [[X_0_1]]) +; CHECK-NEXT: ret void +; CHECK: nope: +; CHECK-NEXT: call void @foo(i1 [[XGT]]) +; CHECK-NEXT: call void @foo(i1 [[XLT]]) +; CHECK-NEXT: call void @foo(i1 [[Z_0]]) +; CHECK-NEXT: ret void +; + %xgt = icmp sgt i32 %x, 0 + %xlt = icmp slt i32 %x, 100 + %z = and i1 %xgt, %xlt + br i1 %z, label %both, label %nope +both: + call void @foo(i1 %xgt) + call void @foo(i1 %xlt) + call void @bar(i32 %x) + ret void +nope: + call void @foo(i1 %xgt) + call void @foo(i1 %xlt) + call void @foo(i1 %z) + ret void +} + +define void @testandassume(i32 %x, i32 %y) { +; CHECK-LABEL: @testandassume( +; CHECK-NEXT: [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0 +; CHECK-NEXT: [[Z:%.*]] = and i1 [[XZ]], [[YZ]] +; CHECK: [[TMP1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]]) +; CHECK: [[TMP2:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]]) +; CHECK: [[TMP3:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]]) +; CHECK: [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]]) +; CHECK: [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: call void @llvm.assume(i1 [[TMP5]]) +; CHECK: [[DOT0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP1]]) +; CHECK: [[DOT01:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP2]]) +; CHECK: [[DOT02:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP3]]) +; CHECK: [[DOT03:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP4]]) +; CHECK: [[DOT04:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP5]]) +; CHECK-NEXT: br i1 [[TMP5]], label [[BOTH:%.*]], label [[NOPE:%.*]] +; CHECK: both: +; CHECK-NEXT: call void @foo(i1 [[DOT02]]) +; CHECK-NEXT: call void @foo(i1 [[DOT03]]) +; CHECK-NEXT: call void @bar(i32 [[DOT0]]) +; CHECK-NEXT: call void @bar(i32 [[DOT01]]) +; CHECK-NEXT: ret void +; CHECK: nope: +; CHECK-NEXT: call void @foo(i1 [[DOT04]]) +; CHECK-NEXT: ret void +; + %xz = icmp eq i32 %x, 0 + %yz = icmp eq i32 %y, 0 + %z = and i1 %xz, %yz + call void @llvm.assume(i1 %z) + br i1 %z, label %both, label %nope +both: + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + ret void +nope: + call void @foo(i1 %z) + ret void +} + +;; Unlike and/or for branches, assume is *always* true, so we only match and for it +define void @testorassume(i32 %x, i32 %y) { +; +; CHECK-LABEL: @testorassume( +; CHECK-NEXT: [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0 +; CHECK-NEXT: [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0 +; CHECK-NEXT: [[Z:%.*]] = or i1 [[XZ]], [[YZ]] +; CHECK-NEXT: call void @llvm.assume(i1 [[Z]]) +; CHECK: [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]]) +; CHECK-NEXT: br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]] +; CHECK: both: +; CHECK-NEXT: call void @foo(i1 [[XZ]]) +; CHECK-NEXT: call void @foo(i1 [[YZ]]) +; CHECK-NEXT: call void @bar(i32 [[X]]) +; CHECK-NEXT: call void @bar(i32 [[Y]]) +; CHECK-NEXT: ret void +; CHECK: nope: +; CHECK-NEXT: call void @foo(i1 [[Z_0]]) +; CHECK-NEXT: ret void +; + %xz = icmp eq i32 %x, 0 + %yz = icmp eq i32 %y, 0 + %z = or i1 %xz, %yz + call void @llvm.assume(i1 %z) + br i1 %z, label %both, label %nope +both: + call void @foo(i1 %xz) + call void @foo(i1 %yz) + call void @bar(i32 %x) + call void @bar(i32 %y) + ret void +nope: + call void @foo(i1 %z) + ret void +} diff --git a/test/Transforms/WholeProgramDevirt/Inputs/export.yaml b/test/Transforms/WholeProgramDevirt/Inputs/export.yaml index 0f6f59de7522..71cf38b216c7 100644 --- a/test/Transforms/WholeProgramDevirt/Inputs/export.yaml +++ b/test/Transforms/WholeProgramDevirt/Inputs/export.yaml @@ -1,7 +1,8 @@ --- GlobalValueMap: 42: - - TypeTestAssumeVCalls: + - Live: true + TypeTestAssumeVCalls: - GUID: 14276520915468743435 # typeid1 Offset: 0 TypeCheckedLoadVCalls: diff --git a/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml b/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml index 1cb3ad3f134c..30159c5012b0 100644 --- a/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml +++ b/test/Transforms/WholeProgramDevirt/Inputs/import-indir.yaml @@ -1,7 +1,8 @@ --- GlobalValueMap: 42: - - TypeTestAssumeVCalls: + - Live: true + TypeTestAssumeVCalls: - GUID: 123 Offset: 0 - GUID: 456 diff --git a/test/Transforms/WholeProgramDevirt/export-nothing.ll b/test/Transforms/WholeProgramDevirt/export-nothing.ll index e0814efbf9c0..4707eaa17ead 100644 --- a/test/Transforms/WholeProgramDevirt/export-nothing.ll +++ b/test/Transforms/WholeProgramDevirt/export-nothing.ll @@ -4,4 +4,5 @@ ; CHECK: --- ; CHECK-NEXT: GlobalValueMap: ; CHECK-NEXT: TypeIdMap: +; CHECK-NEXT: WithGlobalValueDeadStripping: false ; CHECK-NEXT: ... diff --git a/test/Transforms/WholeProgramDevirt/export-single-impl.ll b/test/Transforms/WholeProgramDevirt/export-single-impl.ll index f4f3fd054c46..15de77381ed1 100644 --- a/test/Transforms/WholeProgramDevirt/export-single-impl.ll +++ b/test/Transforms/WholeProgramDevirt/export-single-impl.ll @@ -38,6 +38,7 @@ ; SUMMARY-NEXT: Kind: SingleImpl ; SUMMARY-NEXT: SingleImplName: 'vf4$merged' ; SUMMARY-NEXT: ResByArg: +; SUMMARY-NEXT: WithGlobalValueDeadStripping: false ; SUMMARY-NEXT: ... ; CHECK: @vt1 = constant void (i8*)* @vf1 diff --git a/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll b/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll index 1d7030c41fd0..11b1c5de4d83 100644 --- a/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll +++ b/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll @@ -1,8 +1,7 @@ ; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s ; RUN: FileCheck --check-prefix=SUMMARY %s < %t -; SUMMARY: - TypeTests: -; SUMMARY-NEXT: TypeTestAssumeVCalls: +; SUMMARY-NOT: TypeTests: ; SUMMARY: TypeIdMap: ; SUMMARY-NEXT: typeid4: diff --git a/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll b/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll index 174a573b5b0d..0878d01cce03 100644 --- a/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll +++ b/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll @@ -1,8 +1,7 @@ ; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s ; RUN: FileCheck --check-prefix=SUMMARY %s < %t -; SUMMARY: - TypeTests: -; SUMMARY-NEXT: TypeTestAssumeVCalls: +; SUMMARY-NOT: TypeTests: ; SUMMARY: TypeIdMap: ; SUMMARY-NEXT: typeid3: diff --git a/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll b/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll index 0785ade28570..3132444a9f36 100644 --- a/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll +++ b/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll @@ -1,7 +1,7 @@ ; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -o /dev/null %s ; RUN: FileCheck %s < %t -; CHECK: - TypeTests: [ 15427464259790519041, 17525413373118030901 ] +; CHECK: TypeTests: [ 15427464259790519041, 17525413373118030901 ] ; CHECK-NEXT: TypeTestAssumeVCalls: @vt1a = constant void (i8*)* @vf1a, !type !0 diff --git a/test/Transforms/WholeProgramDevirt/import-indir.ll b/test/Transforms/WholeProgramDevirt/import-indir.ll index 1de9352eeb22..73c982b17893 100644 --- a/test/Transforms/WholeProgramDevirt/import-indir.ll +++ b/test/Transforms/WholeProgramDevirt/import-indir.ll @@ -4,7 +4,9 @@ ; SUMMARY: GlobalValueMap: ; SUMMARY-NEXT: 42: -; SUMMARY-NEXT: - TypeTests: +; SUMMARY-NEXT: - Linkage: 0 +; SUMMARY-NEXT: NotEligibleToImport: false +; SUMMARY-NEXT: Live: true ; SUMMARY-NEXT: TypeTestAssumeVCalls: ; SUMMARY-NEXT: - GUID: 123 ; SUMMARY-NEXT: Offset: 0 diff --git a/test/tools/llvm-lto2/X86/pipeline.ll b/test/tools/llvm-lto2/X86/pipeline.ll index dbec9ab22527..7effb0c801b9 100644 --- a/test/tools/llvm-lto2/X86/pipeline.ll +++ b/test/tools/llvm-lto2/X86/pipeline.ll @@ -8,7 +8,7 @@ ; Try the new pass manager LTO default pipeline (make sure the option ; is accepted). -; RUN: llvm-lto2 run %t1.bc -o %t.o -lto-use-new-pm -r %t1.bc,patatino,px +; RUN: llvm-lto2 run %t1.bc -o %t.o -use-new-pm -r %t1.bc,patatino,px target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/tools/llc/llc.cpp b/tools/llc/llc.cpp index 589005943045..e10d112dcf90 100644 --- a/tools/llc/llc.cpp +++ b/tools/llc/llc.cpp @@ -304,6 +304,9 @@ int main(int argc, char **argv) { initializeScalarizeMaskedMemIntrinPass(*Registry); initializeExpandReductionsPass(*Registry); + // Initialize debugging passes. + initializeScavengerTestPass(*Registry); + // Register the target printer for --version. cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion); diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp index 888da7143c9f..08b096afb052 100644 --- a/tools/llvm-config/llvm-config.cpp +++ b/tools/llvm-config/llvm-config.cpp @@ -333,7 +333,7 @@ int main(int argc, char **argv) { } else { ActivePrefix = CurrentExecPrefix; ActiveIncludeDir = ActivePrefix + "/include"; - SmallString<PATH_MAX> path(StringRef(LLVM_TOOLS_INSTALL_DIR)); + SmallString<256> path(StringRef(LLVM_TOOLS_INSTALL_DIR)); sys::fs::make_absolute(ActivePrefix, path); ActiveBinDir = path.str(); ActiveLibDir = ActivePrefix + "/lib" + LLVM_LIBDIR_SUFFIX; diff --git a/tools/llvm-lto2/llvm-lto2.cpp b/tools/llvm-lto2/llvm-lto2.cpp index 3d2643db85bd..89f85157e1df 100644 --- a/tools/llvm-lto2/llvm-lto2.cpp +++ b/tools/llvm-lto2/llvm-lto2.cpp @@ -99,6 +99,11 @@ static cl::opt<bool> OptRemarksWithHotness( cl::desc("Whether to include hotness informations in the remarks.\n" "Has effect only if -pass-remarks-output is specified.")); +static cl::opt<bool> + UseNewPM("use-new-pm", + cl::desc("Run LTO passes using the new pass manager"), + cl::init(false), cl::Hidden); + static void check(Error E, std::string Msg) { if (!E) return; @@ -196,6 +201,7 @@ static int run(int argc, char **argv) { Conf.AAPipeline = AAPipeline; Conf.OptLevel = OptLevel - '0'; + Conf.UseNewPM = UseNewPM; switch (CGOptLevel) { case '0': Conf.CGOptLevel = CodeGenOpt::None; @@ -351,7 +357,7 @@ int main(int argc, char **argv) { // FIXME: This should use llvm::cl subcommands, but it isn't currently // possible to pass an argument not associated with a subcommand to a - // subcommand (e.g. -lto-use-new-pm). + // subcommand (e.g. -use-new-pm). if (argc < 2) return usage(); diff --git a/tools/llvm-pdbdump/LLVMOutputStyle.cpp b/tools/llvm-pdbdump/LLVMOutputStyle.cpp index d95eca1aeddb..31c342cd0f5a 100644 --- a/tools/llvm-pdbdump/LLVMOutputStyle.cpp +++ b/tools/llvm-pdbdump/LLVMOutputStyle.cpp @@ -483,8 +483,8 @@ Error LLVMOutputStyle::dumpStreamBytes() { if (SI >= File.getNumStreams()) return make_error<RawError>(raw_error_code::no_stream); - auto S = MappedBlockStream::createIndexedStream(File.getMsfLayout(), - File.getMsfBuffer(), SI); + auto S = MappedBlockStream::createIndexedStream( + File.getMsfLayout(), File.getMsfBuffer(), SI, File.getAllocator()); if (!S) continue; DictScope DD(P, "Stream"); @@ -791,7 +791,7 @@ Error LLVMOutputStyle::dumpDbiStream() { if (HasModuleDI && (ShouldDumpSymbols || opts::raw::DumpLineInfo)) { auto ModStreamData = MappedBlockStream::createIndexedStream( File.getMsfLayout(), File.getMsfBuffer(), - Modi.getModuleStreamIndex()); + Modi.getModuleStreamIndex(), File.getAllocator()); ModuleDebugStreamRef ModS(Modi, std::move(ModStreamData)); if (auto EC = ModS.reload()) @@ -804,7 +804,8 @@ Error LLVMOutputStyle::dumpDbiStream() { auto &Types = *ExpectedTypes; ListScope SS(P, "Symbols"); - codeview::CVSymbolDumper SD(P, Types, nullptr, false); + codeview::CVSymbolDumper SD(P, Types, CodeViewContainer::Pdb, nullptr, + false); bool HadError = false; for (auto S : ModS.symbols(&HadError)) { DictScope LL(P, ""); @@ -830,8 +831,7 @@ Error LLVMOutputStyle::dumpDbiStream() { return ExpectedTypes.takeError(); auto &IpiItems = *ExpectedTypes; C13RawVisitor V(P, File, IpiItems); - if (auto EC = - codeview::visitDebugSubsections(ModS.linesAndChecksums(), V)) + if (auto EC = codeview::visitDebugSubsections(ModS.subsections(), V)) return EC; } } @@ -952,7 +952,7 @@ Error LLVMOutputStyle::dumpPublicsStream() { return ExpectedTypes.takeError(); auto &Tpi = *ExpectedTypes; - codeview::CVSymbolDumper SD(P, Tpi, nullptr, false); + codeview::CVSymbolDumper SD(P, Tpi, CodeViewContainer::Pdb, nullptr, false); bool HadError = false; for (auto S : Publics->getSymbols(&HadError)) { DictScope DD(P, ""); diff --git a/tools/llvm-pdbdump/PdbYaml.cpp b/tools/llvm-pdbdump/PdbYaml.cpp index e288063e2afa..b4a41fbfdb8f 100644 --- a/tools/llvm-pdbdump/PdbYaml.cpp +++ b/tools/llvm-pdbdump/PdbYaml.cpp @@ -12,6 +12,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/DebugInfo/CodeView/CVSymbolVisitor.h" #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" +#include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h" #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" #include "llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h" #include "llvm/DebugInfo/CodeView/TypeSerializer.h" @@ -21,6 +22,7 @@ #include "llvm/DebugInfo/PDB/Native/TpiHashing.h" #include "llvm/DebugInfo/PDB/PDBExtras.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" +#include "llvm/ObjectYAML/CodeViewYAMLDebugSections.h" #include "llvm/ObjectYAML/CodeViewYAMLTypes.h" using namespace llvm; @@ -220,6 +222,6 @@ void MappingTraits<PdbDbiModuleInfo>::mapping(IO &IO, PdbDbiModuleInfo &Obj) { IO.mapRequired("Module", Obj.Mod); IO.mapOptional("ObjFile", Obj.Obj, Obj.Mod); IO.mapOptional("SourceFiles", Obj.SourceFiles); - IO.mapOptional("LineInfo", Obj.FileLineInfo); + IO.mapOptional("Subsections", Obj.Subsections); IO.mapOptional("Modi", Obj.Modi); } diff --git a/tools/llvm-pdbdump/PdbYaml.h b/tools/llvm-pdbdump/PdbYaml.h index deb500ec2074..62ed608916fc 100644 --- a/tools/llvm-pdbdump/PdbYaml.h +++ b/tools/llvm-pdbdump/PdbYaml.h @@ -28,6 +28,9 @@ #include <vector> namespace llvm { +namespace codeview { +class DebugStringTableSubsection; +} namespace pdb { namespace yaml { @@ -68,7 +71,7 @@ struct PdbDbiModuleInfo { StringRef Obj; StringRef Mod; std::vector<StringRef> SourceFiles; - Optional<CodeViewYAML::SourceFileInfo> FileLineInfo; + std::vector<CodeViewYAML::YAMLDebugSubsection> Subsections; Optional<PdbModiStream> Modi; }; diff --git a/tools/llvm-pdbdump/YAMLOutputStyle.cpp b/tools/llvm-pdbdump/YAMLOutputStyle.cpp index 18839a7679d3..ee72b90b12d1 100644 --- a/tools/llvm-pdbdump/YAMLOutputStyle.cpp +++ b/tools/llvm-pdbdump/YAMLOutputStyle.cpp @@ -101,117 +101,6 @@ Error YAMLOutputStyle::dump() { return Error::success(); } -namespace { -class C13YamlVisitor : public C13DebugFragmentVisitor { -public: - C13YamlVisitor(CodeViewYAML::SourceFileInfo &Info, PDBFile &F) - : C13DebugFragmentVisitor(F), Info(Info) {} - - Error handleFileChecksums() override { - for (const auto &C : *Checksums) { - CodeViewYAML::SourceFileChecksumEntry Entry; - if (auto Result = getNameFromStringTable(C.FileNameOffset)) - Entry.FileName = *Result; - else - return Result.takeError(); - - Entry.Kind = C.Kind; - Entry.ChecksumBytes.Bytes = C.Checksum; - Info.FileChecksums.push_back(Entry); - } - return Error::success(); - } - - Error handleLines() override { - for (const auto &LF : Lines) { - Info.LineFragments.emplace_back(); - auto &Fragment = Info.LineFragments.back(); - - Fragment.CodeSize = LF.header()->CodeSize; - Fragment.Flags = - static_cast<codeview::LineFlags>(uint16_t(LF.header()->Flags)); - Fragment.RelocOffset = LF.header()->RelocOffset; - Fragment.RelocSegment = LF.header()->RelocSegment; - - for (const auto &L : LF) { - Fragment.Blocks.emplace_back(); - auto &Block = Fragment.Blocks.back(); - - if (auto Result = getNameFromChecksumsBuffer(L.NameIndex)) - Block.FileName = *Result; - else - return Result.takeError(); - - for (const auto &N : L.LineNumbers) { - CodeViewYAML::SourceLineEntry Line; - Line.Offset = N.Offset; - codeview::LineInfo LI(N.Flags); - Line.LineStart = LI.getStartLine(); - Line.EndDelta = LI.getLineDelta(); - Line.IsStatement = LI.isStatement(); - Block.Lines.push_back(Line); - } - - if (LF.hasColumnInfo()) { - for (const auto &C : L.Columns) { - CodeViewYAML::SourceColumnEntry Column; - Column.StartColumn = C.StartColumn; - Column.EndColumn = C.EndColumn; - Block.Columns.push_back(Column); - } - } - } - } - return Error::success(); - } - - Error handleInlineeLines() override { - for (const auto &ILF : InlineeLines) { - Info.Inlinees.emplace_back(); - auto &Inlinee = Info.Inlinees.back(); - - Inlinee.HasExtraFiles = ILF.hasExtraFiles(); - for (const auto &IL : ILF) { - Inlinee.Sites.emplace_back(); - auto &Site = Inlinee.Sites.back(); - if (auto Result = getNameFromChecksumsBuffer(IL.Header->FileID)) - Site.FileName = *Result; - else - return Result.takeError(); - - Site.Inlinee = IL.Header->Inlinee.getIndex(); - Site.SourceLineNum = IL.Header->SourceLineNum; - if (ILF.hasExtraFiles()) { - for (const auto &EF : IL.ExtraFiles) { - if (auto Result = getNameFromChecksumsBuffer(EF)) - Site.ExtraFiles.push_back(*Result); - else - return Result.takeError(); - } - } - } - } - return Error::success(); - } - -private: - CodeViewYAML::SourceFileInfo &Info; -}; -} - -Expected<Optional<CodeViewYAML::SourceFileInfo>> -YAMLOutputStyle::getFileLineInfo(const pdb::ModuleDebugStreamRef &ModS) { - if (!ModS.hasLineInfo()) - return None; - - CodeViewYAML::SourceFileInfo Info; - C13YamlVisitor Visitor(Info, File); - if (auto EC = - codeview::visitDebugSubsections(ModS.linesAndChecksums(), Visitor)) - return std::move(EC); - - return Info; -} Error YAMLOutputStyle::dumpFileHeaders() { if (opts::pdb2yaml::NoFileHeaders) @@ -236,14 +125,17 @@ Error YAMLOutputStyle::dumpFileHeaders() { } Error YAMLOutputStyle::dumpStringTable() { - if (!opts::pdb2yaml::StringTable) + bool RequiresStringTable = opts::pdb2yaml::DbiModuleSourceFileInfo || + opts::pdb2yaml::DbiModuleSourceLineInfo; + bool RequestedStringTable = opts::pdb2yaml::StringTable; + if (!RequiresStringTable && !RequestedStringTable) return Error::success(); - Obj.StringTable.emplace(); auto ExpectedST = File.getStringTable(); if (!ExpectedST) return ExpectedST.takeError(); + Obj.StringTable.emplace(); const auto &ST = ExpectedST.get(); for (auto ID : ST.name_ids()) { auto S = ST.getStringForID(ID); @@ -337,17 +229,30 @@ Error YAMLOutputStyle::dumpDbiStream() { continue; auto ModStreamData = msf::MappedBlockStream::createIndexedStream( - File.getMsfLayout(), File.getMsfBuffer(), ModiStream); + File.getMsfLayout(), File.getMsfBuffer(), ModiStream, + File.getAllocator()); pdb::ModuleDebugStreamRef ModS(MI, std::move(ModStreamData)); if (auto EC = ModS.reload()) return EC; - if (opts::pdb2yaml::DbiModuleSourceLineInfo) { - auto ExpectedInfo = getFileLineInfo(ModS); - if (!ExpectedInfo) - return ExpectedInfo.takeError(); - DMI.FileLineInfo = *ExpectedInfo; + auto ExpectedST = File.getStringTable(); + if (!ExpectedST) + return ExpectedST.takeError(); + if (opts::pdb2yaml::DbiModuleSourceLineInfo && + ModS.hasDebugSubsections()) { + auto ExpectedChecksums = ModS.findChecksumsSubsection(); + if (!ExpectedChecksums) + return ExpectedChecksums.takeError(); + + for (const auto &SS : ModS.subsections()) { + auto Converted = + CodeViewYAML::YAMLDebugSubsection::fromCodeViewSubection( + ExpectedST->getStringTable(), *ExpectedChecksums, SS); + if (!Converted) + return Converted.takeError(); + DMI.Subsections.push_back(*Converted); + } } if (opts::pdb2yaml::DbiModuleSyms) { diff --git a/tools/llvm-pdbdump/YAMLOutputStyle.h b/tools/llvm-pdbdump/YAMLOutputStyle.h index 6e4067c48f88..3690e3529d4a 100644 --- a/tools/llvm-pdbdump/YAMLOutputStyle.h +++ b/tools/llvm-pdbdump/YAMLOutputStyle.h @@ -27,9 +27,6 @@ public: Error dump() override; private: - Expected<Optional<CodeViewYAML::SourceFileInfo>> - getFileLineInfo(const pdb::ModuleDebugStreamRef &ModS); - Error dumpStringTable(); Error dumpFileHeaders(); Error dumpStreamMetadata(); diff --git a/tools/llvm-pdbdump/fuzzer/llvm-pdbdump-fuzzer.cpp b/tools/llvm-pdbdump/fuzzer/llvm-pdbdump-fuzzer.cpp index 14cd222d138a..5f09416a9ff6 100644 --- a/tools/llvm-pdbdump/fuzzer/llvm-pdbdump-fuzzer.cpp +++ b/tools/llvm-pdbdump/fuzzer/llvm-pdbdump-fuzzer.cpp @@ -85,7 +85,7 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) { for (auto &Modi : DS.modules()) { auto ModStreamData = pdb::MappedBlockStream::createIndexedStream( - Modi.Info.getModuleStreamIndex(), *File); + Modi.Info.getModuleStreamIndex(), *File, File->getAllocator()); if (!ModStreamData) { consumeError(ModStreamData.takeError()); return 0; diff --git a/tools/llvm-pdbdump/llvm-pdbdump.cpp b/tools/llvm-pdbdump/llvm-pdbdump.cpp index 0b2b766a3c52..4626de9c4440 100644 --- a/tools/llvm-pdbdump/llvm-pdbdump.cpp +++ b/tools/llvm-pdbdump/llvm-pdbdump.cpp @@ -476,7 +476,6 @@ static void yamlToPdb(StringRef Path) { std::unique_ptr<MemoryBuffer> &Buffer = ErrorOrBuffer.get(); llvm::yaml::Input In(Buffer->getBuffer()); - In.setContext(&Allocator); pdb::yaml::PdbObject YamlObj(Allocator); In >> YamlObj; @@ -535,67 +534,16 @@ static void yamlToPdb(StringRef Path) { ExitOnErr(DbiBuilder.addModuleSourceFile(MI.Mod, S)); if (MI.Modi.hasValue()) { const auto &ModiStream = *MI.Modi; - for (auto Symbol : ModiStream.Symbols) - ModiBuilder.addSymbol(Symbol.toCodeViewSymbol(Allocator)); - } - if (MI.FileLineInfo.hasValue()) { - const auto &FLI = *MI.FileLineInfo; - - // File Checksums must be emitted before line information, because line - // info records use offsets into the checksum buffer to reference a file's - // source file name. - auto Checksums = llvm::make_unique<DebugChecksumsSubsection>(Strings); - auto &ChecksumRef = *Checksums; - if (!FLI.FileChecksums.empty()) { - for (auto &FC : FLI.FileChecksums) - Checksums->addChecksum(FC.FileName, FC.Kind, FC.ChecksumBytes.Bytes); - } - ModiBuilder.setC13FileChecksums(std::move(Checksums)); - - for (const auto &Fragment : FLI.LineFragments) { - auto Lines = - llvm::make_unique<DebugLinesSubsection>(ChecksumRef, Strings); - Lines->setCodeSize(Fragment.CodeSize); - Lines->setRelocationAddress(Fragment.RelocSegment, - Fragment.RelocOffset); - Lines->setFlags(Fragment.Flags); - for (const auto &LC : Fragment.Blocks) { - Lines->createBlock(LC.FileName); - if (Lines->hasColumnInfo()) { - for (const auto &Item : zip(LC.Lines, LC.Columns)) { - auto &L = std::get<0>(Item); - auto &C = std::get<1>(Item); - uint32_t LE = L.LineStart + L.EndDelta; - Lines->addLineAndColumnInfo( - L.Offset, LineInfo(L.LineStart, LE, L.IsStatement), - C.StartColumn, C.EndColumn); - } - } else { - for (const auto &L : LC.Lines) { - uint32_t LE = L.LineStart + L.EndDelta; - Lines->addLineInfo(L.Offset, - LineInfo(L.LineStart, LE, L.IsStatement)); - } - } - } - ModiBuilder.addC13Fragment(std::move(Lines)); + for (auto Symbol : ModiStream.Symbols) { + ModiBuilder.addSymbol( + Symbol.toCodeViewSymbol(Allocator, CodeViewContainer::Pdb)); } + } - for (const auto &Inlinee : FLI.Inlinees) { - auto Inlinees = llvm::make_unique<DebugInlineeLinesSubsection>( - ChecksumRef, Inlinee.HasExtraFiles); - for (const auto &Site : Inlinee.Sites) { - Inlinees->addInlineSite(TypeIndex(Site.Inlinee), Site.FileName, - Site.SourceLineNum); - if (!Inlinee.HasExtraFiles) - continue; - - for (auto EF : Site.ExtraFiles) { - Inlinees->addExtraFile(EF); - } - } - ModiBuilder.addC13Fragment(std::move(Inlinees)); - } + auto CodeViewSubsections = + ExitOnErr(CodeViewYAML::convertSubsectionList(MI.Subsections, Strings)); + for (auto &SS : CodeViewSubsections) { + ModiBuilder.addDebugSubsection(std::move(SS)); } } diff --git a/tools/llvm-readobj/COFFDumper.cpp b/tools/llvm-readobj/COFFDumper.cpp index 663f7b4c8a82..bc07bd296ad2 100644 --- a/tools/llvm-readobj/COFFDumper.cpp +++ b/tools/llvm-readobj/COFFDumper.cpp @@ -978,7 +978,8 @@ void COFFDumper::printCodeViewSymbolsSubsection(StringRef Subsection, Subsection.bytes_end()); auto CODD = llvm::make_unique<COFFObjectDumpDelegate>(*this, Section, Obj, SectionContents); - CVSymbolDumper CVSD(W, Types, std::move(CODD), opts::CodeViewSubsectionBytes); + CVSymbolDumper CVSD(W, Types, CodeViewContainer::ObjectFile, std::move(CODD), + opts::CodeViewSubsectionBytes); CVSymbolArray Symbols; BinaryStreamReader Reader(BinaryData, llvm::support::little); if (auto EC = Reader.readArray(Symbols, Reader.getLength())) { diff --git a/unittests/ADT/SmallVectorTest.cpp b/unittests/ADT/SmallVectorTest.cpp index 7367ad470e3a..ca6391024f27 100644 --- a/unittests/ADT/SmallVectorTest.cpp +++ b/unittests/ADT/SmallVectorTest.cpp @@ -424,6 +424,16 @@ TYPED_TEST(SmallVectorTest, AssignTest) { this->assertValuesInOrder(this->theVector, 2u, 77, 77); } +// Assign test +TYPED_TEST(SmallVectorTest, AssignRangeTest) { + SCOPED_TRACE("AssignTest"); + + this->theVector.push_back(Constructable(1)); + int arr[] = {1, 2, 3}; + this->theVector.assign(std::begin(arr), std::end(arr)); + this->assertValuesInOrder(this->theVector, 3u, 1, 2, 3); +} + // Move-assign test TYPED_TEST(SmallVectorTest, MoveAssignTest) { SCOPED_TRACE("MoveAssignTest"); diff --git a/unittests/Analysis/CMakeLists.txt b/unittests/Analysis/CMakeLists.txt index 40d5ea5f5ad7..8082c54b9c66 100644 --- a/unittests/Analysis/CMakeLists.txt +++ b/unittests/Analysis/CMakeLists.txt @@ -9,17 +9,18 @@ add_llvm_unittest(AnalysisTests AliasAnalysisTest.cpp BlockFrequencyInfoTest.cpp BranchProbabilityInfoTest.cpp + CallGraphTest.cpp CFGTest.cpp CGSCCPassManagerTest.cpp - CallGraphTest.cpp LazyCallGraphTest.cpp LoopInfoTest.cpp MemoryBuiltinsTest.cpp MemorySSA.cpp + OrderedBasicBlockTest.cpp ProfileSummaryInfoTest.cpp ScalarEvolutionTest.cpp - TBAATest.cpp TargetLibraryInfoTest.cpp + TBAATest.cpp UnrollAnalyzer.cpp ValueTrackingTest.cpp ) diff --git a/unittests/Analysis/OrderedBasicBlockTest.cpp b/unittests/Analysis/OrderedBasicBlockTest.cpp new file mode 100644 index 000000000000..b8b9ff04ce7c --- /dev/null +++ b/unittests/Analysis/OrderedBasicBlockTest.cpp @@ -0,0 +1,58 @@ +//===- OrderedBasicBlockTest.cpp - OrderedBasicBlock unit tests -----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/OrderedBasicBlock.h" +#include "llvm/AsmParser/Parser.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/SourceMgr.h" +#include "gtest/gtest.h" + +namespace llvm { +namespace { + +class OrderedBasicBlockTest : public testing::Test { +protected: + LLVMContext C; + + std::unique_ptr<Module> makeLLVMModule() { + const char *ModuleString = R"(define i32 @f(i32 %x) { + %add = add i32 %x, 42 + ret i32 %add + })"; + SMDiagnostic Err; + auto foo = parseAssemblyString(ModuleString, Err, C); + return foo; + } +}; + +TEST_F(OrderedBasicBlockTest, Basic) { + auto M = makeLLVMModule(); + Function *F = M->getFunction("f"); + BasicBlock::iterator I = F->front().begin(); + Instruction *Add = &*I++; + Instruction *Ret = &*I++; + + OrderedBasicBlock OBB(&F->front()); + // Intentionally duplicated to verify cached and uncached are the same. + EXPECT_FALSE(OBB.dominates(Add, Add)); + EXPECT_FALSE(OBB.dominates(Add, Add)); + EXPECT_TRUE(OBB.dominates(Add, Ret)); + EXPECT_TRUE(OBB.dominates(Add, Ret)); + EXPECT_FALSE(OBB.dominates(Ret, Add)); + EXPECT_FALSE(OBB.dominates(Ret, Add)); + EXPECT_FALSE(OBB.dominates(Ret, Ret)); + EXPECT_FALSE(OBB.dominates(Ret, Ret)); +} + +} // end anonymous namespace +} // end namespace llvm diff --git a/unittests/DebugInfo/PDB/MappedBlockStreamTest.cpp b/unittests/DebugInfo/PDB/MappedBlockStreamTest.cpp index 9d90e265df33..789fe515b018 100644 --- a/unittests/DebugInfo/PDB/MappedBlockStreamTest.cpp +++ b/unittests/DebugInfo/PDB/MappedBlockStreamTest.cpp @@ -70,6 +70,8 @@ public: return MSFStreamLayout{static_cast<uint32_t>(Data.size()), Blocks}; } + BumpPtrAllocator Allocator; + private: std::vector<support::ulittle32_t> Blocks; MutableArrayRef<uint8_t> Data; @@ -77,7 +79,8 @@ private: TEST(MappedBlockStreamTest, NumBlocks) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); EXPECT_EQ(F.block_size(), S->getBlockSize()); EXPECT_EQ(F.layout().Blocks.size(), S->getNumBlocks()); @@ -87,7 +90,8 @@ TEST(MappedBlockStreamTest, NumBlocks) { // and does not allocate. TEST(MappedBlockStreamTest, ReadBeyondEndOfStreamRef) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); BinaryStreamRef SR; @@ -102,13 +106,14 @@ TEST(MappedBlockStreamTest, ReadBeyondEndOfStreamRef) { // does not fail due to the length of the output buffer. TEST(MappedBlockStreamTest, ReadOntoNonEmptyBuffer) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str = "ZYXWVUTSRQPONMLKJIHGFEDCBA"; EXPECT_NO_ERROR(R.readFixedString(Str, 1)); EXPECT_EQ(Str, StringRef("A")); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Tests that a read which crosses a block boundary, but where the subsequent @@ -116,18 +121,18 @@ TEST(MappedBlockStreamTest, ReadOntoNonEmptyBuffer) { // not allocate memory. TEST(MappedBlockStreamTest, ZeroCopyReadContiguousBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), - F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; EXPECT_NO_ERROR(R.readFixedString(Str, 2)); EXPECT_EQ(Str, StringRef("AB")); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); R.setOffset(6); EXPECT_NO_ERROR(R.readFixedString(Str, 4)); EXPECT_EQ(Str, StringRef("GHIJ")); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Tests that a read which crosses a block boundary and cannot be referenced @@ -135,62 +140,67 @@ TEST(MappedBlockStreamTest, ZeroCopyReadContiguousBreak) { // requested. TEST(MappedBlockStreamTest, CopyReadNonContiguousBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; EXPECT_NO_ERROR(R.readFixedString(Str, 10)); EXPECT_EQ(Str, StringRef("ABCDEFGHIJ")); - EXPECT_EQ(10U, S->getNumBytesCopied()); + EXPECT_EQ(10U, F.Allocator.getBytesAllocated()); } // Test that an out of bounds read which doesn't cross a block boundary // fails and allocates no memory. TEST(MappedBlockStreamTest, InvalidReadSizeNoBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; R.setOffset(10); EXPECT_ERROR(R.readFixedString(Str, 1)); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Test that an out of bounds read which crosses a contiguous block boundary // fails and allocates no memory. TEST(MappedBlockStreamTest, InvalidReadSizeContiguousBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; R.setOffset(6); EXPECT_ERROR(R.readFixedString(Str, 5)); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Test that an out of bounds read which crosses a discontiguous block // boundary fails and allocates no memory. TEST(MappedBlockStreamTest, InvalidReadSizeNonContiguousBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; EXPECT_ERROR(R.readFixedString(Str, 11)); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Tests that a read which is entirely contained within a single block but // beyond the end of a StreamRef fails. TEST(MappedBlockStreamTest, ZeroCopyReadNoBreak) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str; EXPECT_NO_ERROR(R.readFixedString(Str, 1)); EXPECT_EQ(Str, StringRef("A")); - EXPECT_EQ(0U, S->getNumBytesCopied()); + EXPECT_EQ(0U, F.Allocator.getBytesAllocated()); } // Tests that a read which is not aligned on the same boundary as a previous @@ -198,19 +208,20 @@ TEST(MappedBlockStreamTest, ZeroCopyReadNoBreak) { // previous allocation. TEST(MappedBlockStreamTest, UnalignedOverlappingRead) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str1; StringRef Str2; EXPECT_NO_ERROR(R.readFixedString(Str1, 7)); EXPECT_EQ(Str1, StringRef("ABCDEFG")); - EXPECT_EQ(7U, S->getNumBytesCopied()); + EXPECT_EQ(7U, F.Allocator.getBytesAllocated()); R.setOffset(2); EXPECT_NO_ERROR(R.readFixedString(Str2, 3)); EXPECT_EQ(Str2, StringRef("CDE")); EXPECT_EQ(Str1.data() + 2, Str2.data()); - EXPECT_EQ(7U, S->getNumBytesCopied()); + EXPECT_EQ(7U, F.Allocator.getBytesAllocated()); } // Tests that a read which is not aligned on the same boundary as a previous @@ -218,18 +229,19 @@ TEST(MappedBlockStreamTest, UnalignedOverlappingRead) { // still works correctly and allocates again from the shared pool. TEST(MappedBlockStreamTest, UnalignedOverlappingReadFail) { DiscontiguousStream F(BlocksAry, DataAry); - auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F); + auto S = MappedBlockStream::createStream(F.block_size(), F.layout(), F, + F.Allocator); BinaryStreamReader R(*S); StringRef Str1; StringRef Str2; EXPECT_NO_ERROR(R.readFixedString(Str1, 6)); EXPECT_EQ(Str1, StringRef("ABCDEF")); - EXPECT_EQ(6U, S->getNumBytesCopied()); + EXPECT_EQ(6U, F.Allocator.getBytesAllocated()); R.setOffset(4); EXPECT_NO_ERROR(R.readFixedString(Str2, 4)); EXPECT_EQ(Str2, StringRef("EFGH")); - EXPECT_EQ(10U, S->getNumBytesCopied()); + EXPECT_EQ(10U, F.Allocator.getBytesAllocated()); } TEST(MappedBlockStreamTest, WriteBeyondEndOfStream) { @@ -241,8 +253,8 @@ TEST(MappedBlockStreamTest, WriteBeyondEndOfStream) { "LargeBuffer is not big enough"); DiscontiguousStream F(BlocksAry, Data); - auto S = WritableMappedBlockStream::createStream( - F.block_size(), F.layout(), F); + auto S = WritableMappedBlockStream::createStream(F.block_size(), F.layout(), + F, F.Allocator); ArrayRef<uint8_t> Buffer; EXPECT_ERROR(S->writeBytes(0, ArrayRef<uint8_t>(LargeBuffer))); @@ -254,8 +266,8 @@ TEST(MappedBlockStreamTest, WriteBeyondEndOfStream) { TEST(MappedBlockStreamTest, TestWriteBytesNoBreakBoundary) { static uint8_t Data[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'}; DiscontiguousStream F(BlocksAry, Data); - auto S = WritableMappedBlockStream::createStream( - F.block_size(), F.layout(), F); + auto S = WritableMappedBlockStream::createStream(F.block_size(), F.layout(), + F, F.Allocator); ArrayRef<uint8_t> Buffer; EXPECT_NO_ERROR(S->readBytes(0, 1, Buffer)); @@ -287,8 +299,8 @@ TEST(MappedBlockStreamTest, TestWriteBytesBreakBoundary) { 'T', 'G', '.', '0', '0'}; DiscontiguousStream F(BlocksAry, Data); - auto S = WritableMappedBlockStream::createStream( - F.block_size(), F.layout(), F); + auto S = WritableMappedBlockStream::createStream(F.block_size(), F.layout(), + F, F.Allocator); ArrayRef<uint8_t> Buffer; EXPECT_NO_ERROR(S->writeBytes(0, TestData)); @@ -306,8 +318,8 @@ TEST(MappedBlockStreamTest, TestWriteThenRead) { const uint32_t Blocks[] = {2, 1, 0, 6, 3, 4, 5, 7, 9, 8}; DiscontiguousStream F(Blocks, Data); - auto S = WritableMappedBlockStream::createStream( - F.block_size(), F.layout(), F); + auto S = WritableMappedBlockStream::createStream(F.block_size(), F.layout(), + F, F.Allocator); enum class MyEnum : uint32_t { Val1 = 2908234, Val2 = 120891234 }; using support::ulittle32_t; @@ -399,7 +411,7 @@ TEST(MappedBlockStreamTest, TestWriteContiguousStreamRef) { DiscontiguousStream F(DestBlocks, DestData); auto DestStream = WritableMappedBlockStream::createStream( - F.block_size(), F.layout(), F); + F.block_size(), F.layout(), F, F.Allocator); // First write "Test Str" into the source stream. MutableBinaryByteStream SourceStream(SrcData, little); @@ -434,9 +446,9 @@ TEST(MappedBlockStreamTest, TestWriteDiscontiguousStreamRef) { DiscontiguousStream SrcF(SrcBlocks, SrcData); auto Dest = WritableMappedBlockStream::createStream( - DestF.block_size(), DestF.layout(), DestF); + DestF.block_size(), DestF.layout(), DestF, DestF.Allocator); auto Src = WritableMappedBlockStream::createStream( - SrcF.block_size(), SrcF.layout(), SrcF); + SrcF.block_size(), SrcF.layout(), SrcF, SrcF.Allocator); // First write "Test Str" into the source stream. BinaryStreamWriter SourceWriter(*Src); @@ -457,4 +469,27 @@ TEST(MappedBlockStreamTest, TestWriteDiscontiguousStreamRef) { EXPECT_EQ(Result, "Test Str"); } +TEST(MappedBlockStreamTest, DataLivesAfterStreamDestruction) { + std::vector<uint8_t> DataBytes(10); + MutableArrayRef<uint8_t> Data(DataBytes); + const uint32_t Blocks[] = {2, 1, 0, 6, 3, 4, 5, 7, 9, 8}; + + StringRef Str[] = {"Zero Str", ""}; + + DiscontiguousStream F(Blocks, Data); + { + auto S = WritableMappedBlockStream::createStream(F.block_size(), F.layout(), + F, F.Allocator); + + BinaryStreamReader Reader(*S); + BinaryStreamWriter Writer(*S); + ::memset(DataBytes.data(), 0, 10); + EXPECT_NO_ERROR(Writer.writeCString(Str[0])); + EXPECT_NO_ERROR(Reader.readCString(Str[1])); + EXPECT_EQ(Str[0], Str[1]); + } + + EXPECT_EQ(Str[0], Str[1]); +} + } // end anonymous namespace diff --git a/unittests/Transforms/Utils/Cloning.cpp b/unittests/Transforms/Utils/Cloning.cpp index d13547a842e4..db3d10847cd8 100644 --- a/unittests/Transforms/Utils/Cloning.cpp +++ b/unittests/Transforms/Utils/Cloning.cpp @@ -361,7 +361,7 @@ TEST_F(CloneFunc, NewFunctionCreated) { // Test that a new subprogram entry was added and is pointing to the new // function, while the original subprogram still points to the old one. TEST_F(CloneFunc, Subprogram) { - EXPECT_FALSE(verifyModule(*M)); + EXPECT_FALSE(verifyModule(*M, &errs())); EXPECT_EQ(3U, Finder->subprogram_count()); EXPECT_NE(NewFunc->getSubprogram(), OldFunc->getSubprogram()); } diff --git a/utils/TableGen/X86FoldTablesEmitter.cpp b/utils/TableGen/X86FoldTablesEmitter.cpp index b89cee2ce4bb..34f5fbc6ea31 100644 --- a/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/utils/TableGen/X86FoldTablesEmitter.cpp @@ -101,6 +101,11 @@ const char *const NoFoldSet[] = { "BTS16rr", "BTS32rr", "BTS64rr", "BTS16mr", "BTS32mr", "BTS64mr", + // insertps cannot be folded without adjusting the immediate. There's custom + // code to handle it in X86InstrInfo.cpp, ignore it here. + "INSERTPSrr", "INSERTPSrm", + "VINSERTPSrr", "VINSERTPSrm", "VINSERTPSZrr", "VINSERTPSZrm", + // Memory folding is enabled only when optimizing for size by DAG // patterns only. (issue detailed in D28744 review) "VCVTSS2SDrm", "VCVTSS2SDrr", diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py index 104e9dac464d..8991588a868d 100644 --- a/utils/lit/lit/util.py +++ b/utils/lit/lit/util.py @@ -267,6 +267,20 @@ def usePlatformSdkOnDarwin(config, lit_config): lit_config.note('using SDKROOT: %r' % sdk_path) config.environment['SDKROOT'] = sdk_path +def findPlatformSdkVersionOnMacOS(config, lit_config): + if 'darwin' in config.target_triple: + try: + cmd = subprocess.Popen(['xcrun', '--show-sdk-version', '--sdk', 'macosx'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + out = out.strip() + res = cmd.wait() + except OSError: + res = -1 + if res == 0 and out: + return out + return None + def killProcessAndChildren(pid): """ This function kills a process with ``pid`` and all its |