diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2023-02-11 12:38:04 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2023-02-11 12:38:11 +0000 |
commit | e3b557809604d036af6e00c60f012c2025b59a5e (patch) | |
tree | 8a11ba2269a3b669601e2fd41145b174008f4da8 /llvm/lib/Target/SystemZ | |
parent | 08e8dd7b9db7bb4a9de26d44c1cbfd24e869c014 (diff) | |
download | src-e3b557809604d036af6e00c60f012c2025b59a5e.tar.gz src-e3b557809604d036af6e00c60f012c2025b59a5e.zip |
Vendor import of llvm-project main llvmorg-16-init-18548-gb0daacf58f41,vendor/llvm-project/llvmorg-16-init-18548-gb0daacf58f41
the last commit before the upstream release/17.x branch was created.
Diffstat (limited to 'llvm/lib/Target/SystemZ')
29 files changed, 421 insertions, 520 deletions
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp index 60e1b05a6d1a..4e7985bd4edc 100644 --- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp +++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp @@ -432,6 +432,7 @@ private: bool ParseDirectiveInsn(SMLoc L); bool ParseDirectiveMachine(SMLoc L); + bool ParseGNUAttribute(SMLoc L); OperandMatchResultTy parseAddress(OperandVector &Operands, MemoryKind MemKind, @@ -494,10 +495,11 @@ public: // Override MCTargetAsmParser. bool ParseDirective(AsmToken DirectiveID) override; - bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override; - bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc, + bool parseRegister(MCRegister &RegNo, SMLoc &StartLoc, + SMLoc &EndLoc) override; + bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc, bool RestoreOnFailure); - OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc, + OperandMatchResultTy tryParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override; bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc, OperandVector &Operands) override; @@ -1224,6 +1226,8 @@ bool SystemZAsmParser::ParseDirective(AsmToken DirectiveID) { return ParseDirectiveInsn(DirectiveID.getLoc()); if (IDVal == ".machine") return ParseDirectiveMachine(DirectiveID.getLoc()); + if (IDVal.startswith(".gnu_attribute")) + return ParseGNUAttribute(DirectiveID.getLoc()); return true; } @@ -1358,7 +1362,25 @@ bool SystemZAsmParser::ParseDirectiveMachine(SMLoc L) { return false; } -bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, +bool SystemZAsmParser::ParseGNUAttribute(SMLoc L) { + int64_t Tag; + int64_t IntegerValue; + if (!Parser.parseGNUAttribute(L, Tag, IntegerValue)) + return false; + + // Tag_GNU_S390_ABI_Vector tag is '8' and can be 0, 1, or 2. + if (Tag != 8 || (IntegerValue < 0 || IntegerValue > 2)) { + Error(Parser.getTok().getLoc(), + "Unrecognized .gnu_attribute tag/value pair."); + return false; + } + + Parser.getStreamer().emitGNUAttribute(Tag, IntegerValue); + + return true; +} + +bool SystemZAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc, bool RestoreOnFailure) { Register Reg; if (parseRegister(Reg, RestoreOnFailure)) @@ -1378,12 +1400,12 @@ bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, return false; } -bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, +bool SystemZAsmParser::parseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) { return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false); } -OperandMatchResultTy SystemZAsmParser::tryParseRegister(unsigned &RegNo, +OperandMatchResultTy SystemZAsmParser::tryParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) { bool Result = diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp index 0cb6bfaaebfb..3e0e385b25c4 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp @@ -9,6 +9,7 @@ #include "SystemZInstPrinter.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCRegister.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -23,8 +24,8 @@ using namespace llvm; #include "SystemZGenAsmWriter.inc" -void SystemZInstPrinter::printAddress(const MCAsmInfo *MAI, unsigned Base, - const MCOperand &DispMO, unsigned Index, +void SystemZInstPrinter::printAddress(const MCAsmInfo *MAI, MCRegister Base, + const MCOperand &DispMO, MCRegister Index, raw_ostream &O) { printOperand(DispMO, MAI, O); if (Base || Index) { @@ -49,7 +50,7 @@ void SystemZInstPrinter::printOperand(const MCOperand &MO, const MCAsmInfo *MAI, printFormattedRegName(MAI, MO.getReg(), O); } else if (MO.isImm()) - O << MO.getImm(); + O << markup("<imm:") << MO.getImm() << markup(">"); else if (MO.isExpr()) MO.getExpr()->print(O, MAI); else @@ -57,14 +58,19 @@ void SystemZInstPrinter::printOperand(const MCOperand &MO, const MCAsmInfo *MAI, } void SystemZInstPrinter::printFormattedRegName(const MCAsmInfo *MAI, - unsigned RegNo, raw_ostream &O) { - const char *RegName = getRegisterName(RegNo); + MCRegister Reg, + raw_ostream &O) const { + const char *RegName = getRegisterName(Reg); if (MAI->getAssemblerDialect() == AD_HLASM) { // Skip register prefix so that only register number is left assert(isalpha(RegName[0]) && isdigit(RegName[1])); - O << (RegName + 1); + O << markup("<reg:") << (RegName + 1) << markup(">"); } else - O << '%' << RegName; + O << markup("<reg:") << '%' << RegName << markup(">"); +} + +void SystemZInstPrinter::printRegName(raw_ostream &O, MCRegister Reg) const { + printFormattedRegName(&MAI, Reg, O); } void SystemZInstPrinter::printInst(const MCInst *MI, uint64_t Address, @@ -75,17 +81,19 @@ void SystemZInstPrinter::printInst(const MCInst *MI, uint64_t Address, } template <unsigned N> -static void printUImmOperand(const MCInst *MI, int OpNum, raw_ostream &O) { +void SystemZInstPrinter::printUImmOperand(const MCInst *MI, int OpNum, + raw_ostream &O) { int64_t Value = MI->getOperand(OpNum).getImm(); assert(isUInt<N>(Value) && "Invalid uimm argument"); - O << Value; + O << markup("<imm:") << Value << markup(">"); } template <unsigned N> -static void printSImmOperand(const MCInst *MI, int OpNum, raw_ostream &O) { +void SystemZInstPrinter::printSImmOperand(const MCInst *MI, int OpNum, + raw_ostream &O) { int64_t Value = MI->getOperand(OpNum).getImm(); assert(isInt<N>(Value) && "Invalid simm argument"); - O << Value; + O << markup("<imm:") << Value << markup(">"); } void SystemZInstPrinter::printU1ImmOperand(const MCInst *MI, int OpNum, @@ -157,8 +165,9 @@ void SystemZInstPrinter::printPCRelOperand(const MCInst *MI, int OpNum, raw_ostream &O) { const MCOperand &MO = MI->getOperand(OpNum); if (MO.isImm()) { - O << "0x"; + O << markup("<imm:") << "0x"; O.write_hex(MO.getImm()); + O << markup(">"); } else MO.getExpr()->print(O, &MAI); } diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h index 008bf747e5a1..6a188ff15039 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h @@ -30,29 +30,30 @@ public: // Automatically generated by tblgen. std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override; void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O); - static const char *getRegisterName(unsigned RegNo); + static const char *getRegisterName(MCRegister Reg); // Print an address with the given base, displacement and index. - static void printAddress(const MCAsmInfo *MAI, unsigned Base, - const MCOperand &DispMO, unsigned Index, - raw_ostream &O); + void printAddress(const MCAsmInfo *MAI, MCRegister Base, + const MCOperand &DispMO, MCRegister Index, raw_ostream &O); // Print the given operand. - static void printOperand(const MCOperand &MO, const MCAsmInfo *MAI, - raw_ostream &O); + void printOperand(const MCOperand &MO, const MCAsmInfo *MAI, raw_ostream &O); - static void printFormattedRegName(const MCAsmInfo *MAI, unsigned RegNo, - raw_ostream &O); + void printFormattedRegName(const MCAsmInfo *MAI, MCRegister Reg, + raw_ostream &O) const; // Override MCInstPrinter. - inline void printRegName(raw_ostream &O, unsigned RegNo) const override { - printFormattedRegName(&MAI, RegNo, O); - } + void printRegName(raw_ostream &O, MCRegister Reg) const override; void printInst(const MCInst *MI, uint64_t Address, StringRef Annot, const MCSubtargetInfo &STI, raw_ostream &O) override; private: + template <unsigned N> + void printUImmOperand(const MCInst *MI, int OpNum, raw_ostream &O); + template <unsigned N> + void printSImmOperand(const MCInst *MI, int OpNum, raw_ostream &O); + // Print various types of operand. void printOperand(const MCInst *MI, int OpNum, raw_ostream &O); void printOperand(const MCInst *MI, uint64_t /*Address*/, unsigned OpNum, diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp index 538380263c3c..d9f770a399f6 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp @@ -89,7 +89,7 @@ public: unsigned getNumFixupKinds() const override { return SystemZ::NumTargetFixupKinds; } - Optional<MCFixupKind> getFixupKind(StringRef Name) const override; + std::optional<MCFixupKind> getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target) override; @@ -111,7 +111,8 @@ public: }; } // end anonymous namespace -Optional<MCFixupKind> SystemZMCAsmBackend::getFixupKind(StringRef Name) const { +std::optional<MCFixupKind> +SystemZMCAsmBackend::getFixupKind(StringRef Name) const { unsigned Type = llvm::StringSwitch<unsigned>(Name) #define ELF_RELOC(X, Y) .Case(#X, Y) #include "llvm/BinaryFormat/ELFRelocs/SystemZ.def" @@ -124,7 +125,7 @@ Optional<MCFixupKind> SystemZMCAsmBackend::getFixupKind(StringRef Name) const { .Default(-1u); if (Type != -1u) return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type); - return None; + return std::nullopt; } const MCFixupKindInfo & diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp index 08886507fdb7..8e86c59bf919 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp @@ -236,6 +236,11 @@ createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) { return new SystemZTargetELFStreamer(S); } +static MCTargetStreamer * +createNullTargetStreamer(MCStreamer &S) { + return new SystemZTargetStreamer(S); +} + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSystemZTargetMC() { // Register the MCAsmInfo. TargetRegistry::RegisterMCAsmInfo(getTheSystemZTarget(), @@ -272,4 +277,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSystemZTargetMC() { // Register the obj streamer TargetRegistry::RegisterObjectTargetStreamer(getTheSystemZTarget(), createObjectTargetStreamer); + + // Register the null streamer + TargetRegistry::RegisterNullTargetStreamer(getTheSystemZTarget(), + createNullTargetStreamer); } diff --git a/llvm/lib/Target/SystemZ/SystemZ.h b/llvm/lib/Target/SystemZ/SystemZ.h index 5be19f0e3b46..cdd2850ad8e1 100644 --- a/llvm/lib/Target/SystemZ/SystemZ.h +++ b/llvm/lib/Target/SystemZ/SystemZ.h @@ -18,9 +18,9 @@ #include "llvm/Support/CodeGen.h" namespace llvm { -class SystemZTargetMachine; class FunctionPass; class PassRegistry; +class SystemZTargetMachine; namespace SystemZ { // Condition-code mask values. @@ -198,12 +198,13 @@ FunctionPass *createSystemZCopyPhysRegsPass(SystemZTargetMachine &TM); FunctionPass *createSystemZPostRewritePass(SystemZTargetMachine &TM); FunctionPass *createSystemZTDCPass(); +void initializeSystemZCopyPhysRegsPass(PassRegistry &); +void initializeSystemZDAGToDAGISelPass(PassRegistry &); void initializeSystemZElimComparePass(PassRegistry &); -void initializeSystemZShortenInstPass(PassRegistry &); -void initializeSystemZLongBranchPass(PassRegistry &); void initializeSystemZLDCleanupPass(PassRegistry &); -void initializeSystemZCopyPhysRegsPass(PassRegistry &); +void initializeSystemZLongBranchPass(PassRegistry &); void initializeSystemZPostRewritePass(PassRegistry &); +void initializeSystemZShortenInstPass(PassRegistry &); void initializeSystemZTDCPassPass(PassRegistry &); } // end namespace llvm diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp index 1d55bf9a5804..3e63f17c6518 100644 --- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp +++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp @@ -530,11 +530,6 @@ void SystemZAsmPrinter::emitInstruction(const MachineInstr *MI) { .addImm(15).addReg(SystemZ::R0D); break; - // Emit nothing here but a comment if we can. - case SystemZ::MemBarrier: - OutStreamer->emitRawComment("MEMBARRIER"); - return; - // We want to emit "j .+2" for traps, jumping to the relative immediate field // of the jump instruction, which is an illegal instruction. We cannot emit a // "." symbol, so create and emit a temp label before the instruction and use @@ -759,6 +754,17 @@ void SystemZAsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, getSubtargetInfo()); } +// The *alignment* of 128-bit vector types is different between the software +// and hardware vector ABIs. If the there is an externally visible use of a +// vector type in the module it should be annotated with an attribute. +void SystemZAsmPrinter::emitAttributes(Module &M) { + if (M.getModuleFlag("s390x-visible-vector-ABI")) { + bool HasVectorFeature = + TM.getMCSubtargetInfo()->getFeatureBits()[SystemZ::FeatureVector]; + OutStreamer->emitGNUAttribute(8, HasVectorFeature ? 2 : 1); + } +} + // Convert a SystemZ-specific constant pool modifier into the associated // MCSymbolRefExpr variant kind. static MCSymbolRefExpr::VariantKind @@ -785,6 +791,49 @@ void SystemZAsmPrinter::emitMachineConstantPoolValue( OutStreamer->emitValue(Expr, Size); } +static void printFormattedRegName(const MCAsmInfo *MAI, unsigned RegNo, + raw_ostream &OS) { + const char *RegName = SystemZInstPrinter::getRegisterName(RegNo); + if (MAI->getAssemblerDialect() == AD_HLASM) { + // Skip register prefix so that only register number is left + assert(isalpha(RegName[0]) && isdigit(RegName[1])); + OS << (RegName + 1); + } else + OS << '%' << RegName; +} + +static void printOperand(const MCOperand &MCOp, const MCAsmInfo *MAI, + raw_ostream &OS) { + if (MCOp.isReg()) { + if (!MCOp.getReg()) + OS << '0'; + else + printFormattedRegName(MAI, MCOp.getReg(), OS); + } else if (MCOp.isImm()) + OS << MCOp.getImm(); + else if (MCOp.isExpr()) + MCOp.getExpr()->print(OS, MAI); + else + llvm_unreachable("Invalid operand"); +} + +static void printAddress(const MCAsmInfo *MAI, unsigned Base, + const MCOperand &DispMO, unsigned Index, + raw_ostream &OS) { + printOperand(DispMO, MAI, OS); + if (Base || Index) { + OS << '('; + if (Index) { + printFormattedRegName(MAI, Index, OS); + if (Base) + OS << ','; + } + if (Base) + printFormattedRegName(MAI, Base, OS); + OS << ')'; + } +} + bool SystemZAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS) { @@ -802,7 +851,7 @@ bool SystemZAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, SystemZMCInstLower Lower(MF->getContext(), *this); MCOp = Lower.lowerOperand(MO); } - SystemZInstPrinter::printOperand(MCOp, MAI, OS); + printOperand(MCOp, MAI, OS); return false; } @@ -810,15 +859,14 @@ bool SystemZAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS) { - SystemZInstPrinter:: - printAddress(MAI, MI->getOperand(OpNo).getReg(), - MCOperand::createImm(MI->getOperand(OpNo + 1).getImm()), - MI->getOperand(OpNo + 2).getReg(), OS); + printAddress(MAI, MI->getOperand(OpNo).getReg(), + MCOperand::createImm(MI->getOperand(OpNo + 1).getImm()), + MI->getOperand(OpNo + 2).getReg(), OS); return false; } void SystemZAsmPrinter::emitEndOfAsmFile(Module &M) { - emitStackMaps(SM); + emitAttributes(M); } void SystemZAsmPrinter::emitFunctionBodyEnd() { diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.h b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.h index f14b4a184f62..c99fcda6dcc5 100644 --- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.h +++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.h @@ -25,7 +25,6 @@ class raw_ostream; class LLVM_LIBRARY_VISIBILITY SystemZAsmPrinter : public AsmPrinter { private: - StackMaps SM; MCSymbol *CurrentFnPPA1Sym; // PPA1 Symbol. MCSymbol *CurrentFnEPMarkerSym; // Entry Point Marker. @@ -51,8 +50,8 @@ private: public: SystemZAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) - : AsmPrinter(TM, std::move(Streamer)), SM(*this), - CurrentFnPPA1Sym(nullptr), CurrentFnEPMarkerSym(nullptr) {} + : AsmPrinter(TM, std::move(Streamer)), CurrentFnPPA1Sym(nullptr), + CurrentFnEPMarkerSym(nullptr) {} // Override AsmPrinter. StringRef getPassName() const override { return "SystemZ Assembly Printer"; } @@ -76,6 +75,7 @@ private: void LowerFENTRY_CALL(const MachineInstr &MI, SystemZMCInstLower &MCIL); void LowerSTACKMAP(const MachineInstr &MI); void LowerPATCHPOINT(const MachineInstr &MI, SystemZMCInstLower &Lower); + void emitAttributes(Module &M); }; } // end namespace llvm diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp index d943507b4112..d7a2a51d4652 100644 --- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp @@ -301,8 +301,8 @@ SystemZELFFrameLowering::SystemZELFFrameLowering() // Create a mapping from register number to save slot offset. // These offsets are relative to the start of the register save area. RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS); - for (unsigned I = 0, E = array_lengthof(ELFSpillOffsetTable); I != E; ++I) - RegSpillOffsets[ELFSpillOffsetTable[I].Reg] = ELFSpillOffsetTable[I].Offset; + for (const auto &Entry : ELFSpillOffsetTable) + RegSpillOffsets[Entry.Reg] = Entry.Offset; } // Add GPR64 to the save instruction being built by MIB, which is in basic @@ -370,12 +370,12 @@ bool SystemZELFFrameLowering::spillCalleeSavedRegisters( if (SystemZ::FP64BitRegClass.contains(Reg)) { MBB.addLiveIn(Reg); TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(), - &SystemZ::FP64BitRegClass, TRI); + &SystemZ::FP64BitRegClass, TRI, Register()); } if (SystemZ::VR128BitRegClass.contains(Reg)) { MBB.addLiveIn(Reg); TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(), - &SystemZ::VR128BitRegClass, TRI); + &SystemZ::VR128BitRegClass, TRI, Register()); } } @@ -399,10 +399,10 @@ bool SystemZELFFrameLowering::restoreCalleeSavedRegisters( Register Reg = I.getReg(); if (SystemZ::FP64BitRegClass.contains(Reg)) TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(), - &SystemZ::FP64BitRegClass, TRI); + &SystemZ::FP64BitRegClass, TRI, Register()); if (SystemZ::VR128BitRegClass.contains(Reg)) TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(), - &SystemZ::VR128BitRegClass, TRI); + &SystemZ::VR128BitRegClass, TRI, Register()); } // Restore call-saved GPRs (but not call-clobbered varargs, which at @@ -906,9 +906,8 @@ SystemZXPLINKFrameLowering::SystemZXPLINKFrameLowering() // Create a mapping from register number to save slot offset. // These offsets are relative to the start of the local are area. RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS); - for (unsigned I = 0, E = array_lengthof(XPLINKSpillOffsetTable); I != E; ++I) - RegSpillOffsets[XPLINKSpillOffsetTable[I].Reg] = - XPLINKSpillOffsetTable[I].Offset; + for (const auto &Entry : XPLINKSpillOffsetTable) + RegSpillOffsets[Entry.Reg] = Entry.Offset; } // Checks if the function is a potential candidate for being a XPLeaf routine. @@ -1114,12 +1113,12 @@ bool SystemZXPLINKFrameLowering::spillCalleeSavedRegisters( if (SystemZ::FP64BitRegClass.contains(Reg)) { MBB.addLiveIn(Reg); TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(), - &SystemZ::FP64BitRegClass, TRI); + &SystemZ::FP64BitRegClass, TRI, Register()); } if (SystemZ::VR128BitRegClass.contains(Reg)) { MBB.addLiveIn(Reg); TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(), - &SystemZ::VR128BitRegClass, TRI); + &SystemZ::VR128BitRegClass, TRI, Register()); } } @@ -1142,14 +1141,14 @@ bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters( DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); // Restore FPRs in the normal TargetInstrInfo way. - for (unsigned I = 0, E = CSI.size(); I != E; ++I) { - Register Reg = CSI[I].getReg(); + for (const CalleeSavedInfo &I : CSI) { + Register Reg = I.getReg(); if (SystemZ::FP64BitRegClass.contains(Reg)) - TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), - &SystemZ::FP64BitRegClass, TRI); + TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(), + &SystemZ::FP64BitRegClass, TRI, Register()); if (SystemZ::VR128BitRegClass.contains(Reg)) - TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), - &SystemZ::VR128BitRegClass, TRI); + TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(), + &SystemZ::VR128BitRegClass, TRI, Register()); } // Restore call-saved GPRs (but not call-clobbered varargs, which at @@ -1176,8 +1175,8 @@ bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters( MIB.addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset); // Do a second scan adding regs as being defined by instruction - for (unsigned I = 0, E = CSI.size(); I != E; ++I) { - Register Reg = CSI[I].getReg(); + for (const CalleeSavedInfo &I : CSI) { + Register Reg = I.getReg(); if (Reg > RestoreGPRs.LowGPR && Reg < RestoreGPRs.HighGPR) MIB.addReg(Reg, RegState::ImplicitDefine); } diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 9ac7eafd5f34..250edf64cb6c 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -21,6 +21,7 @@ using namespace llvm; #define DEBUG_TYPE "systemz-isel" +#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection" namespace { // Used to build addressing modes. @@ -345,8 +346,12 @@ class SystemZDAGToDAGISel : public SelectionDAGISel { SDValue expandSelectBoolean(SDNode *Node); public: + static char ID; + + SystemZDAGToDAGISel() = delete; + SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel) - : SelectionDAGISel(TM, OptLevel) {} + : SelectionDAGISel(ID, TM, OptLevel) {} bool runOnMachineFunction(MachineFunction &MF) override { const Function &F = MF.getFunction(); @@ -361,11 +366,6 @@ public: return SelectionDAGISel::runOnMachineFunction(MF); } - // Override MachineFunctionPass. - StringRef getPassName() const override { - return "SystemZ DAG->DAG Pattern Instruction Selection"; - } - // Override SelectionDAGISel. void Select(SDNode *Node) override; bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, @@ -378,6 +378,10 @@ public: }; } // end anonymous namespace +char SystemZDAGToDAGISel::ID = 0; + +INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) + FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel) { return new SystemZDAGToDAGISel(TM, OptLevel); @@ -860,7 +864,7 @@ bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const { RxSBG.Input = N.getOperand(0); return true; } - LLVM_FALLTHROUGH; + [[fallthrough]]; case ISD::SIGN_EXTEND: { // Check that the extension bits are don't-care (i.e. are masked out @@ -1067,10 +1071,13 @@ bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) { }; unsigned Count[] = { 0, 0 }; for (unsigned I = 0; I < 2; ++I) - while (expandRxSBG(RxSBG[I])) - // The widening or narrowing is expected to be free. - // Counting widening or narrowing as a saved operation will result in - // preferring an R*SBG over a simple shift/logical instruction. + while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I])) + // In cases of multiple users it seems better to keep the simple + // instruction as they are one cycle faster, and it also helps in cases + // where both inputs share a common node. + // The widening or narrowing is expected to be free. Counting widening + // or narrowing as a saved operation will result in preferring an R*SBG + // over a simple shift/logical instruction. if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND && RxSBG[I].Input.getOpcode() != ISD::TRUNCATE) Count[I] += 1; @@ -1349,7 +1356,7 @@ bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) { return false; case SystemZISD::SSUBO: NegateOperand = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case SystemZISD::SADDO: if (MemVT == MVT::i32) NewOpc = SystemZ::ASI; @@ -1360,7 +1367,7 @@ bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) { break; case SystemZISD::USUBO: NegateOperand = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case SystemZISD::UADDO: if (MemVT == MVT::i32) NewOpc = SystemZ::ALSI; @@ -1562,7 +1569,7 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) { if (Node->getOperand(1).getOpcode() != ISD::Constant) if (tryRxSBG(Node, SystemZ::RNSBG)) return; - LLVM_FALLTHROUGH; + [[fallthrough]]; case ISD::ROTL: case ISD::SHL: case ISD::SRL: diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index ac4531262187..5dca792dc89a 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/KnownBits.h" #include <cctype> +#include <optional> using namespace llvm; @@ -642,6 +643,8 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, setOperationAction(ISD::VACOPY, MVT::Other, Custom); setOperationAction(ISD::VAEND, MVT::Other, Expand); + setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom); + // Codes for which we want to perform some z-specific combinations. setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::SIGN_EXTEND, @@ -842,7 +845,7 @@ bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, } /// Returns true if stack probing through inline assembly is requested. -bool SystemZTargetLowering::hasInlineStackProbe(MachineFunction &MF) const { +bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const { // If the function specifically requests inline stack probes, emit them. if (MF.getFunction().hasFnAttribute("probe-stack")) return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() == @@ -861,12 +864,12 @@ bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const { } bool SystemZTargetLowering::allowsMisalignedMemoryAccesses( - EVT VT, unsigned, Align, MachineMemOperand::Flags, bool *Fast) const { + EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const { // Unaligned accesses should never be slower than the expanded version. // We check specifically for aligned accesses in the few cases where // they are required. if (Fast) - *Fast = true; + *Fast = 1; return true; } @@ -1021,8 +1024,8 @@ EVT SystemZTargetLowering::getOptimalMemOpType(const MemOp &Op, bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) return false; - unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedSize(); - unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedSize(); + unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue(); + unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue(); return FromBits > ToBits; } @@ -1450,7 +1453,7 @@ static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) { bool SystemZTargetLowering::splitValueIntoRegisterParts( SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, - unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const { + unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const { EVT ValueVT = Val.getValueType(); assert((ValueVT != MVT::i128 || ((NumParts == 1 && PartVT == MVT::Untyped) || @@ -1466,7 +1469,7 @@ bool SystemZTargetLowering::splitValueIntoRegisterParts( SDValue SystemZTargetLowering::joinRegisterPartsIntoValue( SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, - MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const { + MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const { assert((ValueVT != MVT::i128 || ((NumParts == 1 && PartVT == MVT::Untyped) || (NumParts == 2 && PartVT == MVT::i64))) && @@ -1627,8 +1630,8 @@ SDValue SystemZTargetLowering::LowerFormalArguments( } // Join the stores, which are independent of one another. Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, - makeArrayRef(&MemOps[NumFixedFPRs], - SystemZ::ELFNumArgFPRs-NumFixedFPRs)); + ArrayRef(&MemOps[NumFixedFPRs], + SystemZ::ELFNumArgFPRs - NumFixedFPRs)); } } @@ -1854,10 +1857,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, Glue = Chain.getValue(1); // Mark the end of the call, which is glued to the call itself. - Chain = DAG.getCALLSEQ_END(Chain, - DAG.getConstant(NumBytes, DL, PtrVT, true), - DAG.getConstant(0, DL, PtrVT, true), - Glue, DL); + Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL); Glue = Chain.getValue(1); // Assign locations to each value returned by this call. @@ -2491,8 +2491,8 @@ static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, C.Op1.getOpcode() == ISD::Constant && cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { auto *L = cast<LoadSDNode>(C.Op0.getOperand(0)); - if (L->getMemoryVT().getStoreSizeInBits().getFixedSize() <= - C.Op0.getValueSizeInBits().getFixedSize()) { + if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <= + C.Op0.getValueSizeInBits().getFixedValue()) { unsigned Type = L->getExtensionType(); if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) || (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) { @@ -3037,7 +3037,7 @@ SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG, // Handle tests for order using (or (ogt y x) (oge x y)). case ISD::SETUO: Invert = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case ISD::SETO: { assert(IsFP && "Unexpected integer comparison"); SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode), @@ -3054,7 +3054,7 @@ SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG, // Handle <> tests using (or (ogt y x) (ogt x y)). case ISD::SETUEQ: Invert = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case ISD::SETONE: { assert(IsFP && "Unexpected integer comparison"); SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode), @@ -3674,7 +3674,7 @@ SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op, bool IsReturnValueUsed = false; EVT VT = Op.getValueType(); SDValue AllocaCall = - makeExternalCall(Chain, DAG, "@@ALCAXP", VT, makeArrayRef(NeededSpace), + makeExternalCall(Chain, DAG, "@@ALCAXP", VT, ArrayRef(NeededSpace), CallingConv::C, IsSigned, DL, DoesNotReturn, IsReturnValueUsed) .first; @@ -4104,7 +4104,7 @@ SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op, // Skip known-zero high parts of the operand. int64_t OrigBitSize = VT.getSizeInBits(); - int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits); + int64_t BitSize = llvm::bit_ceil(NumSignificantBits); BitSize = std::min(BitSize, OrigBitSize); // The POPCNT instruction counts the number of bits in each byte. @@ -4148,7 +4148,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op, } // MEMBARRIER is a compiler barrier; it codegens to a no-op. - return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0)); + return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0)); } // Op is an atomic load. Lower it into a normal volatile load. @@ -5808,6 +5808,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op, return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR); case ISD::IS_FPCLASS: return lowerIS_FPCLASS(Op, DAG); + case ISD::GET_ROUNDING: + return lowerGET_ROUNDING(Op, DAG); default: llvm_unreachable("Unexpected node to lower"); } @@ -5946,7 +5948,6 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(STRCMP); OPCODE(SEARCH_STRING); OPCODE(IPM); - OPCODE(MEMBARRIER); OPCODE(TBEGIN); OPCODE(TBEGIN_NOFLOAT); OPCODE(TEND); @@ -7041,6 +7042,8 @@ SDValue SystemZTargetLowering::combineGET_CCMASK( int CCMaskVal = CCMask->getZExtValue(); SDValue Select = N->getOperand(0); + if (Select->getOpcode() == ISD::TRUNCATE) + Select = Select->getOperand(0); if (Select->getOpcode() != SystemZISD::SELECT_CCMASK) return SDValue(); @@ -7055,9 +7058,9 @@ SDValue SystemZTargetLowering::combineGET_CCMASK( auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1)); if (!TrueVal || !FalseVal) return SDValue(); - if (TrueVal->getZExtValue() != 0 && FalseVal->getZExtValue() == 0) + if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0) ; - else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() != 0) + else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1) SelectCCMaskVal ^= SelectCCValidVal; else return SDValue(); @@ -7320,7 +7323,7 @@ SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, case Intrinsic::s390_vupllh: case Intrinsic::s390_vupllf: IsLogical = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH case Intrinsic::s390_vuphh: case Intrinsic::s390_vuphf: @@ -7442,19 +7445,15 @@ SystemZTargetLowering::ComputeNumSignBitsForTargetNode( } unsigned -SystemZTargetLowering::getStackProbeSize(MachineFunction &MF) const { +SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const { const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); unsigned StackAlign = TFI->getStackAlignment(); assert(StackAlign >=1 && isPowerOf2_32(StackAlign) && "Unexpected stack alignment"); // The default stack probe size is 4096 if the function has no // stack-probe-size attribute. - unsigned StackProbeSize = 4096; - const Function &Fn = MF.getFunction(); - if (Fn.hasFnAttribute("stack-probe-size")) - Fn.getFnAttribute("stack-probe-size") - .getValueAsString() - .getAsInteger(0, StackProbeSize); + unsigned StackProbeSize = + MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size", 4096); // Round down to the stack alignment. StackProbeSize &= ~(StackAlign - 1); return StackProbeSize ? StackProbeSize : StackAlign; @@ -7557,7 +7556,7 @@ static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects, // destination registers, and the registers that went into the PHI. DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable; - for (auto MI : Selects) { + for (auto *MI : Selects) { Register DestReg = MI->getOperand(0).getReg(); Register TrueReg = MI->getOperand(1).getReg(); Register FalseReg = MI->getOperand(2).getReg(); @@ -7603,35 +7602,32 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI, SmallVector<MachineInstr*, 8> DbgValues; Selects.push_back(&MI); unsigned Count = 0; - for (MachineBasicBlock::iterator NextMIIt = - std::next(MachineBasicBlock::iterator(MI)); - NextMIIt != MBB->end(); ++NextMIIt) { - if (isSelectPseudo(*NextMIIt)) { - assert(NextMIIt->getOperand(3).getImm() == CCValid && + for (MachineInstr &NextMI : llvm::make_range( + std::next(MachineBasicBlock::iterator(MI)), MBB->end())) { + if (isSelectPseudo(NextMI)) { + assert(NextMI.getOperand(3).getImm() == CCValid && "Bad CCValid operands since CC was not redefined."); - if (NextMIIt->getOperand(4).getImm() == CCMask || - NextMIIt->getOperand(4).getImm() == (CCValid ^ CCMask)) { - Selects.push_back(&*NextMIIt); + if (NextMI.getOperand(4).getImm() == CCMask || + NextMI.getOperand(4).getImm() == (CCValid ^ CCMask)) { + Selects.push_back(&NextMI); continue; } break; } - if (NextMIIt->definesRegister(SystemZ::CC) || - NextMIIt->usesCustomInsertionHook()) + if (NextMI.definesRegister(SystemZ::CC) || NextMI.usesCustomInsertionHook()) break; bool User = false; - for (auto SelMI : Selects) - if (NextMIIt->readsVirtualRegister(SelMI->getOperand(0).getReg())) { + for (auto *SelMI : Selects) + if (NextMI.readsVirtualRegister(SelMI->getOperand(0).getReg())) { User = true; break; } - if (NextMIIt->isDebugInstr()) { + if (NextMI.isDebugInstr()) { if (User) { - assert(NextMIIt->isDebugValue() && "Unhandled debug opcode."); - DbgValues.push_back(&*NextMIIt); + assert(NextMI.isDebugValue() && "Unhandled debug opcode."); + DbgValues.push_back(&NextMI); } - } - else if (User || ++Count > 20) + } else if (User || ++Count > 20) break; } @@ -7668,11 +7664,11 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI, // ... MBB = JoinMBB; createPHIsForSelects(Selects, StartMBB, FalseMBB, MBB); - for (auto SelMI : Selects) + for (auto *SelMI : Selects) SelMI->eraseFromParent(); MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI(); - for (auto DbgMI : DbgValues) + for (auto *DbgMI : DbgValues) MBB->splice(InsertPos, StartMBB, DbgMI); return JoinMBB; @@ -9043,3 +9039,43 @@ SystemZTargetLowering::getRepRegClassFor(MVT VT) const { return &SystemZ::ADDR128BitRegClass; return TargetLowering::getRepRegClassFor(VT); } + +SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op, + SelectionDAG &DAG) const { + SDLoc dl(Op); + /* + The rounding method is in FPC Byte 3 bits 6-7, and has the following + settings: + 00 Round to nearest + 01 Round to 0 + 10 Round to +inf + 11 Round to -inf + + FLT_ROUNDS, on the other hand, expects the following: + -1 Undefined + 0 Round to 0 + 1 Round to nearest + 2 Round to +inf + 3 Round to -inf + */ + + // Save FPC to register. + SDValue Chain = Op.getOperand(0); + SDValue EFPC( + DAG.getMachineNode(SystemZ::EFPC, dl, {MVT::i32, MVT::Other}, Chain), 0); + Chain = EFPC.getValue(1); + + // Transform as necessary + SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, EFPC, + DAG.getConstant(3, dl, MVT::i32)); + // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1 + SDValue CWD2 = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, + DAG.getNode(ISD::SRL, dl, MVT::i32, CWD1, + DAG.getConstant(1, dl, MVT::i32))); + + SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD2, + DAG.getConstant(1, dl, MVT::i32)); + RetVal = DAG.getZExtOrTrunc(RetVal, dl, Op.getValueType()); + + return DAG.getMergeValues({RetVal, Chain}, dl); +} diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index b9c95274f62b..c5cc2cc3ae3a 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/TargetLowering.h" +#include <optional> namespace llvm { namespace SystemZISD { @@ -145,9 +146,6 @@ enum NodeType : unsigned { // Store the CC value in bits 29 and 28 of an integer. IPM, - // Compiler barrier only; generate a no-op. - MEMBARRIER, - // Transaction begin. The first operand is the chain, the second // the TDB pointer, and the third the immediate control field. // Returns CC value and chain. @@ -417,13 +415,13 @@ public: } unsigned getNumRegisters(LLVMContext &Context, EVT VT, - Optional<MVT> RegisterVT) const override { + std::optional<MVT> RegisterVT) const override { // i128 inline assembly operand. if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped) return 1; return TargetLowering::getNumRegisters(Context, VT); } - bool isCheapToSpeculateCtlz() const override { return true; } + bool isCheapToSpeculateCtlz(Type *) const override { return true; } bool preferZeroCompareBranch() const override { return true; } bool hasBitPreservingFPLogic(EVT VT) const override { EVT ScVT = VT.getScalarType(); @@ -447,7 +445,7 @@ public: // LD, and having the full constant in memory enables reg/mem opcodes. return VT != MVT::f64; } - bool hasInlineStackProbe(MachineFunction &MF) const override; + bool hasInlineStackProbe(const MachineFunction &MF) const override; bool isLegalICmpImmediate(int64_t Imm) const override; bool isLegalAddImmediate(int64_t Imm) const override; bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, @@ -455,7 +453,7 @@ public: Instruction *I = nullptr) const override; bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, - bool *Fast) const override; + unsigned *Fast) const override; bool findOptimalMemOpLowering(std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, @@ -556,15 +554,14 @@ public: const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override; bool allowTruncateForTailCall(Type *, Type *) const override; bool mayBeEmittedAsTailCall(const CallInst *CI) const override; - bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, - SDValue Val, SDValue *Parts, - unsigned NumParts, MVT PartVT, - Optional<CallingConv::ID> CC) const override; - SDValue - joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, - const SDValue *Parts, unsigned NumParts, - MVT PartVT, EVT ValueVT, - Optional<CallingConv::ID> CC) const override; + bool splitValueIntoRegisterParts( + SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, + unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) + const override; + SDValue joinRegisterPartsIntoValue( + SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts, + unsigned NumParts, MVT PartVT, EVT ValueVT, + std::optional<CallingConv::ID> CC) const override; SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl<ISD::InputArg> &Ins, @@ -614,7 +611,7 @@ public: return true; } - unsigned getStackProbeSize(MachineFunction &MF) const; + unsigned getStackProbeSize(const MachineFunction &MF) const; private: const SystemZSubtarget &Subtarget; @@ -688,6 +685,7 @@ private: SDValue lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const; SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const; SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const; + SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; bool canTreatAsByteVector(EVT VT) const; SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp index 1436be1e4052..2b9210f102de 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp @@ -629,7 +629,7 @@ bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, switch (UseOpc) { case SystemZ::SELRMux: TieOps = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case SystemZ::LOCRMux: if (!STI.hasLoadStoreOnCond2()) return false; @@ -643,7 +643,7 @@ bool SystemZInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, break; case SystemZ::SELGR: TieOps = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case SystemZ::LOCGR: if (!STI.hasLoadStoreOnCond2()) return false; @@ -869,7 +869,7 @@ void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB, void SystemZInstrInfo::storeRegToStackSlot( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIdx, const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const { + const TargetRegisterInfo *TRI, Register VReg) const { DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); // Callers may expect a single instruction, so keep 128-bit moves @@ -881,10 +881,12 @@ void SystemZInstrInfo::storeRegToStackSlot( FrameIdx); } -void SystemZInstrInfo::loadRegFromStackSlot( - MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, - int FrameIdx, const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const { +void SystemZInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + Register DestReg, int FrameIdx, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI, + Register VReg) const { DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); // Callers may expect a single instruction, so keep 128-bit moves @@ -1202,13 +1204,13 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( // to FP conversion. const MCInstrDesc &MCID = MI.getDesc(); for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) { - const MCOperandInfo &MCOI = MCID.OpInfo[I]; + const MCOperandInfo &MCOI = MCID.operands()[I]; if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum) continue; const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass); if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) { Register Reg = MI.getOperand(I).getReg(); - Register PhysReg = Register::isVirtualRegister(Reg) + Register PhysReg = Reg.isVirtual() ? (VRM ? Register(VRM->getPhys(Reg)) : Register()) : Reg; if (!PhysReg || @@ -1255,15 +1257,13 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( else { Register DstReg = MI.getOperand(0).getReg(); Register DstPhys = - (Register::isVirtualRegister(DstReg) ? Register(VRM->getPhys(DstReg)) - : DstReg); + (DstReg.isVirtual() ? Register(VRM->getPhys(DstReg)) : DstReg); Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg() : ((OpNum == 1 && MI.isCommutable()) ? MI.getOperand(2).getReg() : Register())); if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg && - Register::isVirtualRegister(SrcReg) && - DstPhys == VRM->getPhys(SrcReg)) + SrcReg.isVirtual() && DstPhys == VRM->getPhys(SrcReg)) NeedsCommute = (OpNum == 1); else return nullptr; @@ -1314,7 +1314,7 @@ MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl( // Constrain the register classes if converted from a vector opcode. The // allocated regs are in an FP reg-class per previous check above. for (const MachineOperand &MO : MIB->operands()) - if (MO.isReg() && Register::isVirtualRegister(MO.getReg())) { + if (MO.isReg() && MO.getReg().isVirtual()) { Register Reg = MO.getReg(); if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass) MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass); @@ -1711,20 +1711,6 @@ unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const { } } -// Return true if Mask matches the regexp 0*1+0*, given that zero masks -// have already been filtered out. Store the first set bit in LSB and -// the number of set bits in Length if so. -static bool isStringOfOnes(uint64_t Mask, unsigned &LSB, unsigned &Length) { - unsigned First = findFirstSet(Mask); - uint64_t Top = (Mask >> First) + 1; - if ((Top & -Top) == Top) { - LSB = First; - Length = findFirstSet(Top); - return true; - } - return false; -} - bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, unsigned &Start, unsigned &End) const { // Reject trivial all-zero masks. @@ -1735,7 +1721,7 @@ bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, // Handle the 1+0+ or 0+1+0* cases. Start then specifies the index of // the msb and End specifies the index of the lsb. unsigned LSB, Length; - if (isStringOfOnes(Mask, LSB, Length)) { + if (isShiftedMask_64(Mask, LSB, Length)) { Start = 63 - (LSB + Length - 1); End = 63 - LSB; return true; @@ -1743,7 +1729,7 @@ bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize, // Handle the wrap-around 1+0+1+ cases. Start then specifies the msb // of the low 1s and End specifies the lsb of the high 1s. - if (isStringOfOnes(Mask ^ allOnes(BitSize), LSB, Length)) { + if (isShiftedMask_64(Mask ^ allOnes(BitSize), LSB, Length)) { assert(LSB > 0 && "Bottom bit must be set"); assert(LSB + Length < BitSize && "Top bit must be set"); Start = 63 - (LSB - 1); @@ -1878,16 +1864,15 @@ prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const { MachineBasicBlock *MBB = MBBI->getParent(); bool CCLive = true; SmallVector<MachineInstr *, 4> CCUsers; - for (MachineBasicBlock::iterator Itr = std::next(MBBI); - Itr != MBB->end(); ++Itr) { - if (Itr->readsRegister(SystemZ::CC)) { - unsigned Flags = Itr->getDesc().TSFlags; + for (MachineInstr &MI : llvm::make_range(std::next(MBBI), MBB->end())) { + if (MI.readsRegister(SystemZ::CC)) { + unsigned Flags = MI.getDesc().TSFlags; if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast)) - CCUsers.push_back(&*Itr); + CCUsers.push_back(&MI); else return false; } - if (Itr->definesRegister(SystemZ::CC)) { + if (MI.definesRegister(SystemZ::CC)) { CCLive = false; break; } @@ -2000,7 +1985,7 @@ bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI, if (I >= MCID.getNumOperands()) break; const MachineOperand &Op = MI.getOperand(I); - const MCOperandInfo &MCOI = MCID.OpInfo[I]; + const MCOperandInfo &MCOI = MCID.operands()[I]; // Addressing modes have register and immediate operands. Op should be a // register (or frame index) operand if MCOI.RegClass contains a valid // register class, or an immediate otherwise. diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h index 0525f5827736..9ce75db6c177 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h @@ -262,15 +262,16 @@ public: const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override; void storeRegToStackSlot(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - Register SrcReg, bool isKill, int FrameIndex, + MachineBasicBlock::iterator MBBI, Register SrcReg, + bool isKill, int FrameIndex, const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const override; + const TargetRegisterInfo *TRI, + Register VReg) const override; void loadRegFromStackSlot(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MBBI, - Register DestReg, int FrameIdx, - const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) const override; + MachineBasicBlock::iterator MBBI, Register DestReg, + int FrameIdx, const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI, + Register VReg) const override; MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override; MachineInstr * diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td index ed7e3c02a10d..c53cb7cadadb 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td @@ -1716,10 +1716,6 @@ let Predicates = [FeatureExecutionHint], hasSideEffects = 1 in { let hasSideEffects = 1 in def Serialize : Alias<2, (outs), (ins), []>; -// A pseudo instruction that serves as a compiler barrier. -let hasSideEffects = 1, hasNoSchedulingInfo = 1 in -def MemBarrier : Pseudo<(outs), (ins), [(z_membarrier)]>; - let Predicates = [FeatureInterlockedAccess1], Defs = [CC] in { def LAA : LoadAndOpRSY<"laa", 0xEBF8, atomic_load_add_32, GR32>; def LAAG : LoadAndOpRSY<"laag", 0xEBE8, atomic_load_add_64, GR64>; diff --git a/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp b/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp index 1e6f971906e9..8073ed0e2a3c 100644 --- a/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp +++ b/llvm/lib/Target/SystemZ/SystemZLDCleanup.cpp @@ -105,8 +105,8 @@ bool SystemZLDCleanup::VisitNode(MachineDomTreeNode *Node, } // Visit the children of this block in the dominator tree. - for (auto I = Node->begin(), E = Node->end(); I != E; ++I) - Changed |= VisitNode(*I, TLSBaseAddrReg); + for (auto &N : *Node) + Changed |= VisitNode(N, TLSBaseAddrReg); return Changed; } diff --git a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp index d53693154d40..632218cc61ee 100644 --- a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp +++ b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp @@ -217,7 +217,7 @@ static unsigned getInstSizeInBytes(const MachineInstr &MI, assert((Size || // These do not have a size: MI.isDebugOrPseudoInstr() || MI.isPosition() || MI.isKill() || - MI.isImplicitDef() || MI.getOpcode() == SystemZ::MemBarrier || + MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::MEMBARRIER || // These have a size that may be zero: MI.isInlineAsm() || MI.getOpcode() == SystemZ::STACKMAP || MI.getOpcode() == SystemZ::PATCHPOINT) && diff --git a/llvm/lib/Target/SystemZ/SystemZMachineFunctionInfo.h b/llvm/lib/Target/SystemZ/SystemZMachineFunctionInfo.h index de73a5d86422..333195989a11 100644 --- a/llvm/lib/Target/SystemZ/SystemZMachineFunctionInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZMachineFunctionInfo.h @@ -37,9 +37,9 @@ class SystemZMachineFunctionInfo : public MachineFunctionInfo { unsigned NumLocalDynamics; public: - explicit SystemZMachineFunctionInfo(MachineFunction &MF) - : VarArgsFirstGPR(0), VarArgsFirstFPR(0), VarArgsFrameIndex(0), - RegSaveFrameIndex(0), FramePointerSaveIndex(0), NumLocalDynamics(0) {} + SystemZMachineFunctionInfo(const Function &F, const TargetSubtargetInfo *STI) + : VarArgsFirstGPR(0), VarArgsFirstFPR(0), VarArgsFrameIndex(0), + RegSaveFrameIndex(0), FramePointerSaveIndex(0), NumLocalDynamics(0) {} MachineFunctionInfo * clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF, diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td index 9935416559bc..4091c49cec28 100644 --- a/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -279,9 +279,6 @@ def z_usubo : SDNode<"SystemZISD::USUBO", SDT_ZBinaryWithFlags>; def z_addcarry_1 : SDNode<"SystemZISD::ADDCARRY", SDT_ZBinaryWithCarry>; def z_subcarry_1 : SDNode<"SystemZISD::SUBCARRY", SDT_ZBinaryWithCarry>; -def z_membarrier : SDNode<"SystemZISD::MEMBARRIER", SDTNone, - [SDNPHasChain, SDNPSideEffect]>; - def z_loadbswap : SDNode<"SystemZISD::LRV", SDTLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def z_storebswap : SDNode<"SystemZISD::STRV", SDTStore, diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp index be65fe55c634..7f3d8e8d311e 100644 --- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp @@ -107,9 +107,8 @@ bool SystemZRegisterInfo::getRegAllocationHints( auto tryAddHint = [&](const MachineOperand *MO) -> void { Register Reg = MO->getReg(); - Register PhysReg = Register::isPhysicalRegister(Reg) - ? Reg - : Register(VRM->getPhys(Reg)); + Register PhysReg = + Reg.isPhysical() ? Reg : Register(VRM->getPhys(Reg)); if (PhysReg) { if (MO->getSubReg()) PhysReg = getSubReg(PhysReg, MO->getSubReg()); @@ -282,7 +281,7 @@ SystemZRegisterInfo::getReservedRegs(const MachineFunction &MF) const { return Reserved; } -void +bool SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const { @@ -314,7 +313,7 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, MI->getDebugExpressionOp().setMetadata( DIExpression::appendOpsToArg(MI->getDebugExpression(), Ops, OpIdx)); } - return; + return false; } // See if the offset is in range, or if an equivalent instruction that @@ -374,6 +373,7 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, } MI->setDesc(TII->get(OpcodeForOffset)); MI->getOperand(FIOperandNum + 1).ChangeToImmediate(Offset); + return false; } bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI, @@ -429,7 +429,7 @@ bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI, MEE++; for (; MII != MEE; ++MII) { for (const MachineOperand &MO : MII->operands()) - if (MO.isReg() && Register::isPhysicalRegister(MO.getReg())) { + if (MO.isReg() && MO.getReg().isPhysical()) { for (MCSuperRegIterator SI(MO.getReg(), this, true/*IncludeSelf*/); SI.isValid(); ++SI) if (NewRC->contains(*SI)) { diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h index db0936f3f56b..19305d4e8957 100644 --- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h @@ -160,7 +160,7 @@ public: const uint32_t *getCallPreservedMask(const MachineFunction &MF, CallingConv::ID CC) const override; BitVector getReservedRegs(const MachineFunction &MF) const override; - void eliminateFrameIndex(MachineBasicBlock::iterator MI, + bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const override; diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp index ce30d8ef2cba..4eb58e27f7ad 100644 --- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp @@ -76,13 +76,13 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy( // MVI, MVHHI, MVHI and MVGHI respectively. static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, uint64_t ByteVal, uint64_t Size, - unsigned Align, MachinePointerInfo DstPtrInfo) { + Align Alignment, MachinePointerInfo DstPtrInfo) { uint64_t StoreVal = ByteVal; for (unsigned I = 1; I < Size; ++I) StoreVal |= ByteVal << (I * 8); return DAG.getStore( Chain, DL, DAG.getConstant(StoreVal, DL, MVT::getIntegerVT(Size * 8)), - Dst, DstPtrInfo, Align); + Dst, DstPtrInfo, Alignment); } SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset( @@ -102,24 +102,24 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset( if (CByte) { // Handle cases that can be done using at most two of // MVI, MVHI, MVHHI and MVGHI. The latter two can only be - // used if ByteVal is all zeros or all ones; in other casees, + // used if ByteVal is all zeros or all ones; in other cases, // we can move at most 2 halfwords. uint64_t ByteVal = CByte->getZExtValue(); - if (ByteVal == 0 || ByteVal == 255 ? - Bytes <= 16 && countPopulation(Bytes) <= 2 : - Bytes <= 4) { - unsigned Size1 = Bytes == 16 ? 8 : 1 << findLastSet(Bytes); + if (ByteVal == 0 || ByteVal == 255 + ? Bytes <= 16 && llvm::popcount(Bytes) <= 2 + : Bytes <= 4) { + unsigned Size1 = Bytes == 16 ? 8 : llvm::bit_floor(Bytes); unsigned Size2 = Bytes - Size1; SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1, - Alignment.value(), DstPtrInfo); + Alignment, DstPtrInfo); if (Size2 == 0) return Chain1; Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst, DAG.getConstant(Size1, DL, PtrVT)); DstPtrInfo = DstPtrInfo.getWithOffset(Size1); - SDValue Chain2 = memsetStore( - DAG, DL, Chain, Dst, ByteVal, Size2, - std::min((unsigned)Alignment.value(), Size1), DstPtrInfo); + SDValue Chain2 = + memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2, + std::min(Alignment, Align(Size1)), DstPtrInfo); return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2); } } else { diff --git a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp index f6889035b654..25b013ba1876 100644 --- a/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp +++ b/llvm/lib/Target/SystemZ/SystemZSubtarget.cpp @@ -68,27 +68,7 @@ SystemZSubtarget::SystemZSubtarget(const Triple &TT, const std::string &CPU, const std::string &TuneCPU, const std::string &FS, const TargetMachine &TM) - : SystemZGenSubtargetInfo(TT, CPU, TuneCPU, FS), - HasDistinctOps(false), HasLoadStoreOnCond(false), HasHighWord(false), - HasFPExtension(false), HasPopulationCount(false), - HasMessageSecurityAssist3(false), HasMessageSecurityAssist4(false), - HasResetReferenceBitsMultiple(false), HasFastSerialization(false), - HasInterlockedAccess1(false), HasMiscellaneousExtensions(false), - HasExecutionHint(false), HasLoadAndTrap(false), - HasTransactionalExecution(false), HasProcessorAssist(false), - HasDFPZonedConversion(false), HasEnhancedDAT2(false), HasVector(false), - HasLoadStoreOnCond2(false), HasLoadAndZeroRightmostByte(false), - HasMessageSecurityAssist5(false), HasDFPPackedConversion(false), - HasMiscellaneousExtensions2(false), HasGuardedStorage(false), - HasMessageSecurityAssist7(false), HasMessageSecurityAssist8(false), - HasVectorEnhancements1(false), HasVectorPackedDecimal(false), - HasInsertReferenceBitsMultiple(false), HasMiscellaneousExtensions3(false), - HasMessageSecurityAssist9(false), HasVectorEnhancements2(false), - HasVectorPackedDecimalEnhancement(false), HasEnhancedSort(false), - HasDeflateConversion(false), HasVectorPackedDecimalEnhancement2(false), - HasNNPAssist(false), HasBEAREnhancement(false), - HasResetDATProtection(false), HasProcessorActivityInstrumentation(false), - HasSoftFloat(false), TargetTriple(TT), + : SystemZGenSubtargetInfo(TT, CPU, TuneCPU, FS), TargetTriple(TT), SpecialRegisters(initializeSpecialRegisters()), InstrInfo(initializeSubtargetDependencies(CPU, TuneCPU, FS)), TLInfo(TM, *this), FrameLowering(SystemZFrameLowering::create(*this)) {} diff --git a/llvm/lib/Target/SystemZ/SystemZSubtarget.h b/llvm/lib/Target/SystemZ/SystemZSubtarget.h index cd16c19f9bfa..9d4c1f0fe710 100644 --- a/llvm/lib/Target/SystemZ/SystemZSubtarget.h +++ b/llvm/lib/Target/SystemZ/SystemZSubtarget.h @@ -33,47 +33,10 @@ class StringRef; class SystemZSubtarget : public SystemZGenSubtargetInfo { virtual void anchor(); protected: - bool HasDistinctOps; - bool HasLoadStoreOnCond; - bool HasHighWord; - bool HasFPExtension; - bool HasPopulationCount; - bool HasMessageSecurityAssist3; - bool HasMessageSecurityAssist4; - bool HasResetReferenceBitsMultiple; - bool HasFastSerialization; - bool HasInterlockedAccess1; - bool HasMiscellaneousExtensions; - bool HasExecutionHint; - bool HasLoadAndTrap; - bool HasTransactionalExecution; - bool HasProcessorAssist; - bool HasDFPZonedConversion; - bool HasEnhancedDAT2; - bool HasVector; - bool HasLoadStoreOnCond2; - bool HasLoadAndZeroRightmostByte; - bool HasMessageSecurityAssist5; - bool HasDFPPackedConversion; - bool HasMiscellaneousExtensions2; - bool HasGuardedStorage; - bool HasMessageSecurityAssist7; - bool HasMessageSecurityAssist8; - bool HasVectorEnhancements1; - bool HasVectorPackedDecimal; - bool HasInsertReferenceBitsMultiple; - bool HasMiscellaneousExtensions3; - bool HasMessageSecurityAssist9; - bool HasVectorEnhancements2; - bool HasVectorPackedDecimalEnhancement; - bool HasEnhancedSort; - bool HasDeflateConversion; - bool HasVectorPackedDecimalEnhancement2; - bool HasNNPAssist; - bool HasBEAREnhancement; - bool HasResetDATProtection; - bool HasProcessorActivityInstrumentation; - bool HasSoftFloat; +// Bool members corresponding to the SubtargetFeatures defined in tablegen. +#define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \ + bool ATTRIBUTE = DEFAULT; +#include "SystemZGenSubtargetInfo.inc" private: Triple TargetTriple; @@ -138,155 +101,10 @@ public: // Automatically generated by tblgen. void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); - // Return true if the target has the distinct-operands facility. - bool hasDistinctOps() const { return HasDistinctOps; } - - // Return true if the target has the load/store-on-condition facility. - bool hasLoadStoreOnCond() const { return HasLoadStoreOnCond; } - - // Return true if the target has the load/store-on-condition facility 2. - bool hasLoadStoreOnCond2() const { return HasLoadStoreOnCond2; } - - // Return true if the target has the high-word facility. - bool hasHighWord() const { return HasHighWord; } - - // Return true if the target has the floating-point extension facility. - bool hasFPExtension() const { return HasFPExtension; } - - // Return true if the target has the population-count facility. - bool hasPopulationCount() const { return HasPopulationCount; } - - // Return true if the target has the message-security-assist - // extension facility 3. - bool hasMessageSecurityAssist3() const { return HasMessageSecurityAssist3; } - - // Return true if the target has the message-security-assist - // extension facility 4. - bool hasMessageSecurityAssist4() const { return HasMessageSecurityAssist4; } - - // Return true if the target has the reset-reference-bits-multiple facility. - bool hasResetReferenceBitsMultiple() const { - return HasResetReferenceBitsMultiple; - } - - // Return true if the target has the fast-serialization facility. - bool hasFastSerialization() const { return HasFastSerialization; } - - // Return true if the target has interlocked-access facility 1. - bool hasInterlockedAccess1() const { return HasInterlockedAccess1; } - - // Return true if the target has the miscellaneous-extensions facility. - bool hasMiscellaneousExtensions() const { - return HasMiscellaneousExtensions; - } - - // Return true if the target has the execution-hint facility. - bool hasExecutionHint() const { return HasExecutionHint; } - - // Return true if the target has the load-and-trap facility. - bool hasLoadAndTrap() const { return HasLoadAndTrap; } - - // Return true if the target has the transactional-execution facility. - bool hasTransactionalExecution() const { return HasTransactionalExecution; } - - // Return true if the target has the processor-assist facility. - bool hasProcessorAssist() const { return HasProcessorAssist; } - - // Return true if the target has the DFP zoned-conversion facility. - bool hasDFPZonedConversion() const { return HasDFPZonedConversion; } - - // Return true if the target has the enhanced-DAT facility 2. - bool hasEnhancedDAT2() const { return HasEnhancedDAT2; } - - // Return true if the target has the load-and-zero-rightmost-byte facility. - bool hasLoadAndZeroRightmostByte() const { - return HasLoadAndZeroRightmostByte; - } - - // Return true if the target has the message-security-assist - // extension facility 5. - bool hasMessageSecurityAssist5() const { return HasMessageSecurityAssist5; } - - // Return true if the target has the DFP packed-conversion facility. - bool hasDFPPackedConversion() const { return HasDFPPackedConversion; } - - // Return true if the target has the vector facility. - bool hasVector() const { return HasVector; } - - // Return true if the target has the miscellaneous-extensions facility 2. - bool hasMiscellaneousExtensions2() const { - return HasMiscellaneousExtensions2; - } - - // Return true if the target has the guarded-storage facility. - bool hasGuardedStorage() const { return HasGuardedStorage; } - - // Return true if the target has the message-security-assist - // extension facility 7. - bool hasMessageSecurityAssist7() const { return HasMessageSecurityAssist7; } - - // Return true if the target has the message-security-assist - // extension facility 8. - bool hasMessageSecurityAssist8() const { return HasMessageSecurityAssist8; } - - // Return true if the target has the vector-enhancements facility 1. - bool hasVectorEnhancements1() const { return HasVectorEnhancements1; } - - // Return true if the target has the vector-packed-decimal facility. - bool hasVectorPackedDecimal() const { return HasVectorPackedDecimal; } - - // Return true if the target has the insert-reference-bits-multiple facility. - bool hasInsertReferenceBitsMultiple() const { - return HasInsertReferenceBitsMultiple; - } - - // Return true if the target has the miscellaneous-extensions facility 3. - bool hasMiscellaneousExtensions3() const { - return HasMiscellaneousExtensions3; - } - - // Return true if the target has the message-security-assist - // extension facility 9. - bool hasMessageSecurityAssist9() const { return HasMessageSecurityAssist9; } - - // Return true if the target has the vector-enhancements facility 2. - bool hasVectorEnhancements2() const { return HasVectorEnhancements2; } - - // Return true if the target has the vector-packed-decimal - // enhancement facility. - bool hasVectorPackedDecimalEnhancement() const { - return HasVectorPackedDecimalEnhancement; - } - - // Return true if the target has the enhanced-sort facility. - bool hasEnhancedSort() const { return HasEnhancedSort; } - - // Return true if the target has the deflate-conversion facility. - bool hasDeflateConversion() const { return HasDeflateConversion; } - - // Return true if the target has the vector-packed-decimal - // enhancement facility 2. - bool hasVectorPackedDecimalEnhancement2() const { - return HasVectorPackedDecimalEnhancement2; - } - - // Return true if the target has the NNP-assist facility. - bool hasNNPAssist() const { return HasNNPAssist; } - - // Return true if the target has the BEAR-enhancement facility. - bool hasBEAREnhancement() const { return HasBEAREnhancement; } - - // Return true if the target has the reset-DAT-protection facility. - bool hasResetDATProtection() const { return HasResetDATProtection; } - - // Return true if the target has the processor-activity-instrumentation - // facility. - bool hasProcessorActivityInstrumentation() const { - return HasProcessorActivityInstrumentation; - } - - // Return true if soft float should be used. - bool hasSoftFloat() const { return HasSoftFloat; } +// Getters for SubtargetFeatures defined in tablegen. +#define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \ + bool GETTER() const { return ATTRIBUTE; } +#include "SystemZGenSubtargetInfo.inc" // Return true if GV can be accessed using LARL for reloc model RM // and code model CM. diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp index 31f8ee2f894d..787c51645de1 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp @@ -9,10 +9,10 @@ #include "SystemZTargetMachine.h" #include "MCTargetDesc/SystemZMCTargetDesc.h" #include "SystemZ.h" +#include "SystemZMachineFunctionInfo.h" #include "SystemZMachineScheduler.h" #include "SystemZTargetTransformInfo.h" #include "TargetInfo/SystemZTargetInfo.h" -#include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -25,6 +25,7 @@ #include "llvm/Support/CodeGen.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Transforms/Scalar.h" +#include <optional> #include <string> using namespace llvm; @@ -40,39 +41,10 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSystemZTarget() { initializeSystemZShortenInstPass(PR); initializeSystemZPostRewritePass(PR); initializeSystemZTDCPassPass(PR); + initializeSystemZDAGToDAGISelPass(PR); } -// Determine whether we use the vector ABI. -static bool UsesVectorABI(StringRef CPU, StringRef FS) { - // We use the vector ABI whenever the vector facility is avaiable. - // This is the case by default if CPU is z13 or later, and can be - // overridden via "[+-]vector" feature string elements. - bool VectorABI = true; - bool SoftFloat = false; - if (CPU.empty() || CPU == "generic" || - CPU == "z10" || CPU == "z196" || CPU == "zEC12" || - CPU == "arch8" || CPU == "arch9" || CPU == "arch10") - VectorABI = false; - - SmallVector<StringRef, 3> Features; - FS.split(Features, ',', -1, false /* KeepEmpty */); - for (auto &Feature : Features) { - if (Feature == "vector" || Feature == "+vector") - VectorABI = true; - if (Feature == "-vector") - VectorABI = false; - if (Feature == "soft-float" || Feature == "+soft-float") - SoftFloat = true; - if (Feature == "-soft-float") - SoftFloat = false; - } - - return VectorABI && !SoftFloat; -} - -static std::string computeDataLayout(const Triple &TT, StringRef CPU, - StringRef FS) { - bool VectorABI = UsesVectorABI(CPU, FS); +static std::string computeDataLayout(const Triple &TT) { std::string Ret; // Big endian. @@ -92,10 +64,9 @@ static std::string computeDataLayout(const Triple &TT, StringRef CPU, // 128-bit floats are aligned only to 64 bits. Ret += "-f128:64"; - // When using the vector ABI on Linux, 128-bit vectors are also aligned to 64 - // bits. On z/OS, vector types are always aligned to 64 bits. - if (VectorABI || TT.isOSzOS()) - Ret += "-v128:64"; + // The DataLayout string always holds a vector alignment of 64 bits, see + // comment in clang/lib/Basic/Targets/SystemZ.h. + Ret += "-v128:64"; // We prefer 16 bits of aligned for all globals; see above. Ret += "-a:8:16"; @@ -115,7 +86,7 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { return std::make_unique<TargetLoweringObjectFileELF>(); } -static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) { +static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) { // Static code is suitable for use in a dynamic executable; there is no // separate DynamicNoPIC model. if (!RM || *RM == Reloc::DynamicNoPIC) @@ -153,8 +124,8 @@ static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) { // of copy relocs, so locally-binding data symbols might not be in // the range of LARL. We need the Medium model in that case. static CodeModel::Model -getEffectiveSystemZCodeModel(Optional<CodeModel::Model> CM, Reloc::Model RM, - bool JIT) { +getEffectiveSystemZCodeModel(std::optional<CodeModel::Model> CM, + Reloc::Model RM, bool JIT) { if (CM) { if (*CM == CodeModel::Tiny) report_fatal_error("Target does not support the tiny CodeModel", false); @@ -170,11 +141,11 @@ getEffectiveSystemZCodeModel(Optional<CodeModel::Model> CM, Reloc::Model RM, SystemZTargetMachine::SystemZTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, - Optional<Reloc::Model> RM, - Optional<CodeModel::Model> CM, + std::optional<Reloc::Model> RM, + std::optional<CodeModel::Model> CM, CodeGenOpt::Level OL, bool JIT) : LLVMTargetMachine( - T, computeDataLayout(TT, CPU, FS), TT, CPU, FS, Options, + T, computeDataLayout(TT), TT, CPU, FS, Options, getEffectiveRelocModel(RM), getEffectiveSystemZCodeModel(CM, getEffectiveRelocModel(RM), JIT), OL), @@ -341,3 +312,10 @@ TargetTransformInfo SystemZTargetMachine::getTargetTransformInfo(const Function &F) const { return TargetTransformInfo(SystemZTTIImpl(this, F)); } + +MachineFunctionInfo *SystemZTargetMachine::createMachineFunctionInfo( + BumpPtrAllocator &Allocator, const Function &F, + const TargetSubtargetInfo *STI) const { + return SystemZMachineFunctionInfo::create<SystemZMachineFunctionInfo>( + Allocator, F, STI); +} diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h index 2cdb33a5064b..20d68fff170a 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.h @@ -15,12 +15,12 @@ #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZTARGETMACHINE_H #include "SystemZSubtarget.h" -#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Support/CodeGen.h" #include "llvm/Target/TargetMachine.h" #include <memory> +#include <optional> namespace llvm { @@ -32,8 +32,9 @@ class SystemZTargetMachine : public LLVMTargetMachine { public: SystemZTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, - Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM, - CodeGenOpt::Level OL, bool JIT); + std::optional<Reloc::Model> RM, + std::optional<CodeModel::Model> CM, CodeGenOpt::Level OL, + bool JIT); ~SystemZTargetMachine() override; const SystemZSubtarget *getSubtargetImpl(const Function &) const override; @@ -50,6 +51,10 @@ public: return TLOF.get(); } + MachineFunctionInfo * + createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, + const TargetSubtargetInfo *STI) const override; + bool targetSchedulesPostRAScheduling() const override { return true; }; }; diff --git a/llvm/lib/Target/SystemZ/SystemZTargetStreamer.h b/llvm/lib/Target/SystemZ/SystemZTargetStreamer.h index 1b4e93ebe39b..884082cfa196 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetStreamer.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetStreamer.h @@ -48,7 +48,7 @@ public: void emitConstantPools() override; - virtual void emitMachine(StringRef CPU) = 0; + virtual void emitMachine(StringRef CPU) {}; }; } // end namespace llvm diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp index 69914049a00c..821efc1b758b 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp @@ -300,8 +300,8 @@ void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, } if (isa<StoreInst>(&I)) { Type *MemAccessTy = I.getOperand(0)->getType(); - NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, None, 0, - TTI::TCK_RecipThroughput); + NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, + std::nullopt, 0, TTI::TCK_RecipThroughput); } } @@ -419,16 +419,14 @@ static unsigned getNumVectorRegs(Type *Ty) { InstructionCost SystemZTTIImpl::getArithmeticInstrCost( unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, - TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info, - TTI::OperandValueProperties Opd1PropInfo, - TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, + TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info, + ArrayRef<const Value *> Args, const Instruction *CxtI) { // TODO: Handle more cost kinds. if (CostKind != TTI::TCK_RecipThroughput) return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, - Op2Info, Opd1PropInfo, - Opd2PropInfo, Args, CxtI); + Op2Info, Args, CxtI); // TODO: return a good value for BB-VECTORIZER that includes the // immediate loads, which we do not want to count for the loop @@ -534,7 +532,8 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost( return (NumVectors * (SignedDivRem ? SDivPow2Cost : 1)); if (DivRemConst) { SmallVector<Type *> Tys(Args.size(), Ty); - return VF * DivMulSeqCost + getScalarizationOverhead(VTy, Args, Tys); + return VF * DivMulSeqCost + + getScalarizationOverhead(VTy, Args, Tys, CostKind); } if ((SignedDivRem || UnsignedDivRem) && VF > 4) // Temporary hack: disable high vectorization factors with integer @@ -560,7 +559,8 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost( getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind); SmallVector<Type *> Tys(Args.size(), Ty); InstructionCost Cost = - (VF * ScalarCost) + getScalarizationOverhead(VTy, Args, Tys); + (VF * ScalarCost) + + getScalarizationOverhead(VTy, Args, Tys, CostKind); // FIXME: VF 2 for these FP operations are currently just as // expensive as for VF 4. if (VF == 2) @@ -578,8 +578,8 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost( // There is no native support for FRem. if (Opcode == Instruction::FRem) { SmallVector<Type *> Tys(Args.size(), Ty); - InstructionCost Cost = - (VF * LIBCALL_COST) + getScalarizationOverhead(VTy, Args, Tys); + InstructionCost Cost = (VF * LIBCALL_COST) + + getScalarizationOverhead(VTy, Args, Tys, CostKind); // FIXME: VF 2 for float is currently just as expensive as for VF 4. if (VF == 2 && ScalarBits == 32) Cost *= 2; @@ -589,13 +589,14 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost( // Fallback to the default implementation. return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info, - Opd1PropInfo, Opd2PropInfo, Args, CxtI); + Args, CxtI); } InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef<int> Mask, int Index, - VectorType *SubTp, + ArrayRef<int> Mask, + TTI::TargetCostKind CostKind, + int Index, VectorType *SubTp, ArrayRef<const Value *> Args) { Kind = improveShuffleKindFromMask(Kind, Mask); if (ST->hasVector()) { @@ -630,7 +631,7 @@ InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, } } - return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp); + return BaseT::getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp); } // Return the log2 difference of the element sizes of the two vector types. @@ -648,8 +649,8 @@ static unsigned getElSizeLog2Diff(Type *Ty0, Type *Ty1) { unsigned SystemZTTIImpl:: getVectorTruncCost(Type *SrcTy, Type *DstTy) { assert (SrcTy->isVectorTy() && DstTy->isVectorTy()); - assert(SrcTy->getPrimitiveSizeInBits().getFixedSize() > - DstTy->getPrimitiveSizeInBits().getFixedSize() && + assert(SrcTy->getPrimitiveSizeInBits().getFixedValue() > + DstTy->getPrimitiveSizeInBits().getFixedValue() && "Packing must reduce size of vector type."); assert(cast<FixedVectorType>(SrcTy)->getNumElements() == cast<FixedVectorType>(DstTy)->getNumElements() && @@ -866,8 +867,10 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, (Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI)) NeedsExtracts = false; - TotCost += getScalarizationOverhead(SrcVecTy, false, NeedsExtracts); - TotCost += getScalarizationOverhead(DstVecTy, NeedsInserts, false); + TotCost += getScalarizationOverhead(SrcVecTy, /*Insert*/ false, + NeedsExtracts, CostKind); + TotCost += getScalarizationOverhead(DstVecTy, NeedsInserts, + /*Extract*/ false, CostKind); // FIXME: VF 2 for float<->i32 is currently just as expensive as for VF 4. if (VF == 2 && SrcScalarBits == 32 && DstScalarBits == 32) @@ -879,7 +882,8 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, if (Opcode == Instruction::FPTrunc) { if (SrcScalarBits == 128) // fp128 -> double/float + inserts of elements. return VF /*ldxbr/lexbr*/ + - getScalarizationOverhead(DstVecTy, true, false); + getScalarizationOverhead(DstVecTy, /*Insert*/ true, + /*Extract*/ false, CostKind); else // double -> float return VF / 2 /*vledb*/ + std::max(1U, VF / 4 /*vperm*/); } @@ -892,7 +896,8 @@ InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, return VF * 2; } // -> fp128. VF * lxdb/lxeb + extraction of elements. - return VF + getScalarizationOverhead(SrcVecTy, false, true); + return VF + getScalarizationOverhead(SrcVecTy, /*Insert*/ false, + /*Extract*/ true, CostKind); } } @@ -997,7 +1002,9 @@ InstructionCost SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, } InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index) { + TTI::TargetCostKind CostKind, + unsigned Index, Value *Op0, + Value *Op1) { // vlvgp will insert two grs into a vector register, so only count half the // number of instructions. if (Opcode == Instruction::InsertElement && Val->isIntOrIntVectorTy(64)) @@ -1013,7 +1020,7 @@ InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, return Cost; } - return BaseT::getVectorInstrCost(Opcode, Val, Index); + return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); } // Check if a load may be folded as a memory operand in its user. @@ -1056,7 +1063,7 @@ isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) { case Instruction::ICmp: if (LoadedBits == 32 && ZExtBits == 64) return true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case Instruction::Mul: // SE: 16->32, 32->64, z14:16->64 if (UserI->getOpcode() != Instruction::ICmp) { if (LoadedBits == 16 && @@ -1066,11 +1073,11 @@ isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) { if (LoadOrTruncBits == 16) return true; } - LLVM_FALLTHROUGH; + [[fallthrough]]; case Instruction::SDiv:// SE: 32->64 if (LoadedBits == 32 && SExtBits == 64) return true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case Instruction::UDiv: case Instruction::And: case Instruction::Or: @@ -1109,6 +1116,7 @@ InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, + TTI::OperandValueInfo OpInfo, const Instruction *I) { assert(!Src->isVoidTy() && "Invalid type"); diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h index 33317e799eab..1c82e6940033 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -85,16 +85,15 @@ public: InstructionCost getArithmeticInstrCost( unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, - TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue, - TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue, - TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None, - TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, + TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None}, + TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None}, ArrayRef<const Value *> Args = ArrayRef<const Value *>(), const Instruction *CxtI = nullptr); InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, - ArrayRef<int> Mask, int Index, + ArrayRef<int> Mask, + TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, - ArrayRef<const Value *> Args = None); + ArrayRef<const Value *> Args = std::nullopt); unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy); unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy); unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst, @@ -107,13 +106,16 @@ public: CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, - unsigned Index); + TTI::TargetCostKind CostKind, + unsigned Index, Value *Op0, Value *Op1); bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue); - InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr); + InstructionCost + getMemoryOpCost(unsigned Opcode, Type *Src, MaybeAlign Alignment, + unsigned AddressSpace, TTI::TargetCostKind CostKind, + TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None}, + const Instruction *I = nullptr); InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, |