diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2023-07-26 19:03:47 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2023-07-26 19:04:23 +0000 |
commit | 7fa27ce4a07f19b07799a767fc29416f3b625afb (patch) | |
tree | 27825c83636c4de341eb09a74f49f5d38a15d165 /llvm/lib/Passes/PassBuilder.cpp | |
parent | e3b557809604d036af6e00c60f012c2025b59a5e (diff) | |
download | src-7fa27ce4a07f19b07799a767fc29416f3b625afb.tar.gz src-7fa27ce4a07f19b07799a767fc29416f3b625afb.zip |
Vendor import of llvm-project main llvmorg-17-init-19304-gd0b54bb50e51,vendor/llvm-project/llvmorg-17-init-19304-gd0b54bb50e51
the last commit before the upstream release/17.x branch was created.
Diffstat (limited to 'llvm/lib/Passes/PassBuilder.cpp')
-rw-r--r-- | llvm/lib/Passes/PassBuilder.cpp | 265 |
1 files changed, 231 insertions, 34 deletions
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index e251d56463a3..d0cbbcc0e310 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -34,7 +34,6 @@ #include "llvm/Analysis/Delinearization.h" #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/DependenceAnalysis.h" -#include "llvm/Analysis/DivergenceAnalysis.h" #include "llvm/Analysis/DomPrinter.h" #include "llvm/Analysis/DominanceFrontier.h" #include "llvm/Analysis/FunctionPropertiesAnalysis.h" @@ -46,7 +45,6 @@ #include "llvm/Analysis/InstCount.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/LazyValueInfo.h" -#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/Analysis/Lint.h" #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/LoopCacheAnalysis.h" @@ -73,6 +71,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/Analysis/UniformityAnalysis.h" +#include "llvm/CodeGen/HardwareLoops.h" #include "llvm/CodeGen/TypePromotion.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Dominators.h" @@ -81,6 +80,7 @@ #include "llvm/IR/SafepointIRVerifier.h" #include "llvm/IR/Verifier.h" #include "llvm/IRPrinter/IRPrintingPasses.h" +#include "llvm/Passes/OptimizationLevel.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -103,6 +103,7 @@ #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" #include "llvm/Transforms/IPO/ElimAvailExtern.h" +#include "llvm/Transforms/IPO/EmbedBitcodePass.h" #include "llvm/Transforms/IPO/ForceFunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/IPO/FunctionImport.h" @@ -116,6 +117,7 @@ #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/LoopExtractor.h" #include "llvm/Transforms/IPO/LowerTypeTests.h" +#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h" #include "llvm/Transforms/IPO/MergeFunctions.h" #include "llvm/Transforms/IPO/ModuleInliner.h" #include "llvm/Transforms/IPO/OpenMPOpt.h" @@ -205,6 +207,7 @@ #include "llvm/Transforms/Scalar/NaryReassociate.h" #include "llvm/Transforms/Scalar/NewGVN.h" #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h" +#include "llvm/Transforms/Scalar/PlaceSafepoints.h" #include "llvm/Transforms/Scalar/Reassociate.h" #include "llvm/Transforms/Scalar/Reg2Mem.h" #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h" @@ -227,6 +230,7 @@ #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h" +#include "llvm/Transforms/Utils/CountVisits.h" #include "llvm/Transforms/Utils/Debugify.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/FixIrreducible.h" @@ -243,6 +247,7 @@ #include "llvm/Transforms/Utils/LowerSwitch.h" #include "llvm/Transforms/Utils/Mem2Reg.h" #include "llvm/Transforms/Utils/MetaRenamer.h" +#include "llvm/Transforms/Utils/MoveAutoInit.h" #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/PredicateInfo.h" #include "llvm/Transforms/Utils/RelLookupTableConverter.h" @@ -483,6 +488,28 @@ static std::optional<int> parseRepeatPassName(StringRef Name) { return Count; } +static std::optional<std::pair<bool, bool>> +parseFunctionPipelineName(StringRef Name) { + std::pair<bool, bool> Params; + if (!Name.consume_front("function")) + return std::nullopt; + if (Name.empty()) + return Params; + if (!Name.consume_front("<") || !Name.consume_back(">")) + return std::nullopt; + while (!Name.empty()) { + auto [Front, Back] = Name.split(';'); + Name = Back; + if (Front == "eager-inv") + Params.first = true; + else if (Front == "no-rerun") + Params.second = true; + else + return std::nullopt; + } + return Params; +} + static std::optional<int> parseDevirtPassName(StringRef Name) { if (!Name.consume_front("devirt<") || !Name.consume_back(">")) return std::nullopt; @@ -501,6 +528,17 @@ static bool checkParametrizedPassName(StringRef Name, StringRef PassName) { return Name.startswith("<") && Name.endswith(">"); } +static std::optional<OptimizationLevel> parseOptLevel(StringRef S) { + return StringSwitch<std::optional<OptimizationLevel>>(S) + .Case("O0", OptimizationLevel::O0) + .Case("O1", OptimizationLevel::O1) + .Case("O2", OptimizationLevel::O2) + .Case("O3", OptimizationLevel::O3) + .Case("Os", OptimizationLevel::Os) + .Case("Oz", OptimizationLevel::Oz) + .Default(std::nullopt); +} + namespace { /// This performs customized parsing of pass name with parameters. @@ -539,20 +577,58 @@ auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name, return Result; } +/// Parser of parameters for HardwareLoops pass. +Expected<HardwareLoopOptions> parseHardwareLoopOptions(StringRef Params) { + HardwareLoopOptions HardwareLoopOpts; + + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + if (ParamName.consume_front("hardware-loop-decrement=")) { + int Count; + if (ParamName.getAsInteger(0, Count)) + return make_error<StringError>( + formatv("invalid HardwareLoopPass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + HardwareLoopOpts.setDecrement(Count); + continue; + } + if (ParamName.consume_front("hardware-loop-counter-bitwidth=")) { + int Count; + if (ParamName.getAsInteger(0, Count)) + return make_error<StringError>( + formatv("invalid HardwareLoopPass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + HardwareLoopOpts.setCounterBitwidth(Count); + continue; + } + if (ParamName == "force-hardware-loops") { + HardwareLoopOpts.setForce(true); + } else if (ParamName == "force-hardware-loop-phi") { + HardwareLoopOpts.setForcePhi(true); + } else if (ParamName == "force-nested-hardware-loop") { + HardwareLoopOpts.setForceNested(true); + } else if (ParamName == "force-hardware-loop-guard") { + HardwareLoopOpts.setForceGuard(true); + } else { + return make_error<StringError>( + formatv("invalid HardwarePass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + } + return HardwareLoopOpts; +} + /// Parser of parameters for LoopUnroll pass. Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) { LoopUnrollOptions UnrollOpts; while (!Params.empty()) { StringRef ParamName; std::tie(ParamName, Params) = Params.split(';'); - int OptLevel = StringSwitch<int>(ParamName) - .Case("O0", 0) - .Case("O1", 1) - .Case("O2", 2) - .Case("O3", 3) - .Default(-1); - if (OptLevel >= 0) { - UnrollOpts.setOptLevel(OptLevel); + std::optional<OptimizationLevel> OptLevel = parseOptLevel(ParamName); + // Don't accept -Os/-Oz. + if (OptLevel && !OptLevel->isOptimizingForSize()) { + UnrollOpts.setOptLevel(OptLevel->getSpeedupLevel()); continue; } if (ParamName.consume_front("full-unroll-max=")) { @@ -604,6 +680,10 @@ Expected<bool> parseSinglePassOption(StringRef Params, StringRef OptionName, return Result; } +Expected<bool> parseGlobalDCEPassOptions(StringRef Params) { + return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); +} + Expected<bool> parseInlinerPassOptions(StringRef Params) { return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); } @@ -612,6 +692,11 @@ Expected<bool> parseCoroSplitPassOptions(StringRef Params) { return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); } +Expected<bool> parsePostOrderFunctionAttrsPassOptions(StringRef Params) { + return parseSinglePassOption(Params, "skip-non-recursive", + "PostOrderFunctionAttrs"); +} + Expected<bool> parseEarlyCSEPassOptions(StringRef Params) { return parseSinglePassOption(Params, "memssa", "EarlyCSE"); } @@ -666,6 +751,26 @@ Expected<HWAddressSanitizerOptions> parseHWASanPassOptions(StringRef Params) { return Result; } +Expected<EmbedBitcodeOptions> parseEmbedBitcodePassOptions(StringRef Params) { + EmbedBitcodeOptions Result; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName == "thinlto") { + Result.IsThinLTO = true; + } else if (ParamName == "emit-summary") { + Result.EmitLTOSummary = true; + } else { + return make_error<StringError>( + formatv("invalid EmbedBitcode pass parameter '{0}' ", ParamName) + .str(), + inconvertibleErrorCode()); + } + } + return Result; +} + Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) { MemorySanitizerOptions Result; while (!Params.empty()) { @@ -704,7 +809,11 @@ Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) { std::tie(ParamName, Params) = Params.split(';'); bool Enable = !ParamName.consume_front("no-"); - if (ParamName == "forward-switch-cond") { + if (ParamName == "speculate-blocks") { + Result.speculateBlocks(Enable); + } else if (ParamName == "simplify-cond-branch") { + Result.setSimplifyCondBranch(Enable); + } else if (ParamName == "forward-switch-cond") { Result.forwardSwitchCondToPhi(Enable); } else if (ParamName == "switch-range-to-icmp") { Result.convertSwitchRangeToICmp(Enable); @@ -734,6 +843,33 @@ Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) { return Result; } +Expected<InstCombineOptions> parseInstCombineOptions(StringRef Params) { + InstCombineOptions Result; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + bool Enable = !ParamName.consume_front("no-"); + if (ParamName == "use-loop-info") { + Result.setUseLoopInfo(Enable); + } else if (Enable && ParamName.consume_front("max-iterations=")) { + APInt MaxIterations; + if (ParamName.getAsInteger(0, MaxIterations)) + return make_error<StringError>( + formatv("invalid argument to InstCombine pass max-iterations " + "parameter: '{0}' ", + ParamName).str(), + inconvertibleErrorCode()); + Result.setMaxIterations((unsigned)MaxIterations.getZExtValue()); + } else { + return make_error<StringError>( + formatv("invalid InstCombine pass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + } + return Result; +} + /// Parser of parameters for LoopVectorize pass. Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) { LoopVectorizeOptions Opts; @@ -794,6 +930,26 @@ Expected<LICMOptions> parseLICMOptions(StringRef Params) { return Result; } +Expected<std::pair<bool, bool>> parseLoopRotateOptions(StringRef Params) { + std::pair<bool, bool> Result = {true, false}; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + bool Enable = !ParamName.consume_front("no-"); + if (ParamName == "header-duplication") { + Result.first = Enable; + } else if (ParamName == "prepare-for-lto") { + Result.second = Enable; + } else { + return make_error<StringError>( + formatv("invalid LoopRotate pass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + } + return Result; +} + Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) { bool Result = false; while (!Params.empty()) { @@ -893,6 +1049,45 @@ Expected<bool> parseDependenceAnalysisPrinterOptions(StringRef Params) { "DependenceAnalysisPrinter"); } +Expected<bool> parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { + return parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); +} + +Expected<OptimizationLevel> +parseFunctionSimplificationPipelineOptions(StringRef Params) { + std::optional<OptimizationLevel> L = parseOptLevel(Params); + if (!L || *L == OptimizationLevel::O0) { + return make_error<StringError>( + formatv("invalid function-simplification parameter '{0}' ", Params) + .str(), + inconvertibleErrorCode()); + }; + return *L; +} + +Expected<bool> parseMemorySSAPrinterPassOptions(StringRef Params) { + return parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); +} + +Expected<std::string> parseMemProfUsePassOptions(StringRef Params) { + std::string Result; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName.consume_front("profile-filename=")) { + Result = ParamName.str(); + } else { + return make_error<StringError>( + formatv("invalid MemProfUse pass parameter '{0}' ", ParamName).str(), + inconvertibleErrorCode()); + } + } + return Result; +} + } // namespace /// Tests whether a pass name starts with a valid prefix for a default pipeline @@ -927,12 +1122,14 @@ static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { if (startsWithDefaultPipelineAliasPrefix(Name)) return DefaultAliasRegex.match(Name); + StringRef NameNoBracket = Name.take_until([](char C) { return C == '<'; }); + // Explicitly handle pass manager names. if (Name == "module") return true; if (Name == "cgscc") return true; - if (Name == "function" || Name == "function<eager-inv>") + if (NameNoBracket == "function") return true; if (Name == "coro-cond") return true; @@ -958,9 +1155,10 @@ static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { template <typename CallbacksT> static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. + StringRef NameNoBracket = Name.take_until([](char C) { return C == '<'; }); if (Name == "cgscc") return true; - if (Name == "function" || Name == "function<eager-inv>") + if (NameNoBracket == "function") return true; // Explicitly handle custom-parsed pass names. @@ -986,7 +1184,8 @@ static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { template <typename CallbacksT> static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. - if (Name == "function" || Name == "function<eager-inv>") + StringRef NameNoBracket = Name.take_until([](char C) { return C == '<'; }); + if (NameNoBracket == "function") return true; if (Name == "loop" || Name == "loop-mssa") return true; @@ -1144,12 +1343,16 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM, MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); return Error::success(); } - if (Name == "function" || Name == "function<eager-inv>") { + if (auto Params = parseFunctionPipelineName(Name)) { + if (Params->second) + return make_error<StringError>( + "cannot have a no-rerun module to function adaptor", + inconvertibleErrorCode()); FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), - Name != "function")); + MPM.addPass( + createModuleToFunctionPassAdaptor(std::move(FPM), Params->first)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { @@ -1181,19 +1384,7 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM, assert(Matches.size() == 3 && "Must capture two matched strings!"); - OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2]) - .Case("O0", OptimizationLevel::O0) - .Case("O1", OptimizationLevel::O1) - .Case("O2", OptimizationLevel::O2) - .Case("O3", OptimizationLevel::O3) - .Case("Os", OptimizationLevel::Os) - .Case("Oz", OptimizationLevel::Oz); - if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" && - Matches[1] != "lto") { - MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" || - Matches[1] == "lto-pre-link")); - return Error::success(); - } + OptimizationLevel L = *parseOptLevel(Matches[2]); // This is consistent with old pass manager invoked via opt, but // inconsistent with clang. Clang doesn't enable loop vectorization @@ -1210,7 +1401,13 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM, } else if (Matches[1] == "thinlto") { MPM.addPass(buildThinLTODefaultPipeline(L, nullptr)); } else if (Matches[1] == "lto-pre-link") { - MPM.addPass(buildLTOPreLinkDefaultPipeline(L)); + if (PTO.UnifiedLTO) + // When UnifiedLTO is enabled, use the ThinLTO pre-link pipeline. This + // avoids compile-time performance regressions and keeps the pre-link + // LTO pipeline "unified" for both LTO modes. + MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L)); + else + MPM.addPass(buildLTOPreLinkDefaultPipeline(L)); } else { assert(Matches[1] == "lto" && "Not one of the matched options!"); MPM.addPass(buildLTODefaultPipeline(L, nullptr)); @@ -1318,13 +1515,13 @@ Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, CGPM.addPass(std::move(NestedCGPM)); return Error::success(); } - if (Name == "function" || Name == "function<eager-inv>") { + if (auto Params = parseFunctionPipelineName(Name)) { FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; // Add the nested pass manager with the appropriate adaptor. - CGPM.addPass( - createCGSCCToFunctionPassAdaptor(std::move(FPM), Name != "function")); + CGPM.addPass(createCGSCCToFunctionPassAdaptor( + std::move(FPM), Params->first, Params->second)); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { |