diff options
Diffstat (limited to 'llvm/lib/Transforms/IPO/SampleProfile.cpp')
-rw-r--r-- | llvm/lib/Transforms/IPO/SampleProfile.cpp | 315 |
1 files changed, 241 insertions, 74 deletions
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index b6871e260532..264ac4065e8c 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -43,6 +43,7 @@ #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/Analysis/ReplayInlineAdvisor.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/BasicBlock.h" @@ -75,10 +76,11 @@ #include "llvm/Support/GenericDomTree.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/SampleContextTracker.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Utils/CallPromotionUtils.h" #include "llvm/Transforms/Utils/Cloning.h" -#include "llvm/Transforms/Utils/MisExpect.h" #include <algorithm> #include <cassert> #include <cstdint> @@ -102,6 +104,9 @@ STATISTIC(NumCSInlined, "Number of functions inlined with context sensitive profile"); STATISTIC(NumCSNotInlined, "Number of functions not inlined with context sensitive profile"); +STATISTIC(NumMismatchedProfile, + "Number of functions with CFG mismatched profile"); +STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile"); // Command line option to specify the file to read samples from. This is // mainly used for debugging. @@ -170,6 +175,13 @@ static cl::opt<int> SampleColdCallSiteThreshold( "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites")); +static cl::opt<std::string> ProfileInlineReplayFile( + "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), + cl::desc( + "Optimization remarks file containing inline remarks to be replayed " + "by inlining from sample profile loader."), + cl::Hidden); + namespace { using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>; @@ -309,17 +321,16 @@ private: class SampleProfileLoader { public: SampleProfileLoader( - StringRef Name, StringRef RemapName, bool IsThinLTOPreLink, + StringRef Name, StringRef RemapName, ThinOrFullLTOPhase LTOPhase, std::function<AssumptionCache &(Function &)> GetAssumptionCache, std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo, std::function<const TargetLibraryInfo &(Function &)> GetTLI) : GetAC(std::move(GetAssumptionCache)), GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)), CoverageTracker(*this), Filename(std::string(Name)), - RemappingFilename(std::string(RemapName)), - IsThinLTOPreLink(IsThinLTOPreLink) {} + RemappingFilename(std::string(RemapName)), LTOPhase(LTOPhase) {} - bool doInitialization(Module &M); + bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr); bool runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummaryInfo *_PSI, CallGraph *CG); @@ -332,6 +343,7 @@ protected: unsigned getFunctionLoc(Function &F); bool emitAnnotations(Function &F); ErrorOr<uint64_t> getInstWeight(const Instruction &I); + ErrorOr<uint64_t> getProbeWeight(const Instruction &I); ErrorOr<uint64_t> getBlockWeight(const BasicBlock *BB); const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const; std::vector<const FunctionSamples *> @@ -417,6 +429,9 @@ protected: /// Profile reader object. std::unique_ptr<SampleProfileReader> Reader; + /// Profile tracker for different context. + std::unique_ptr<SampleContextTracker> ContextTracker; + /// Samples collected for the body of this function. FunctionSamples *Samples = nullptr; @@ -429,11 +444,15 @@ protected: /// Flag indicating whether the profile input loaded successfully. bool ProfileIsValid = false; - /// Flag indicating if the pass is invoked in ThinLTO compile phase. + /// Flag indicating whether input profile is context-sensitive + bool ProfileIsCS = false; + + /// Flag indicating which LTO/ThinLTO phase the pass is invoked in. /// - /// In this phase, in annotation, we should not promote indirect calls. - /// Instead, we will mark GUIDs that needs to be annotated to the function. - bool IsThinLTOPreLink; + /// We need to know the LTO phase because for example in ThinLTOPrelink + /// phase, in annotation, we should not promote indirect calls. Instead, + /// we will mark GUIDs that needs to be annotated to the function. + ThinOrFullLTOPhase LTOPhase; /// Profile Summary Info computed from sample profile. ProfileSummaryInfo *PSI = nullptr; @@ -473,6 +492,12 @@ protected: // overriden by -profile-sample-accurate or profile-sample-accurate // attribute. bool ProfAccForSymsInList; + + // External inline advisor used to replay inline decision from remarks. + std::unique_ptr<ReplayInlineAdvisor> ExternalInlineAdvisor; + + // A pseudo probe helper to correlate the imported sample counts. + std::unique_ptr<PseudoProbeManager> ProbeManager; }; class SampleProfileLoaderLegacyPass : public ModulePass { @@ -480,10 +505,11 @@ public: // Class identification, replacement for typeinfo static char ID; - SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile, - bool IsThinLTOPreLink = false) + SampleProfileLoaderLegacyPass( + StringRef Name = SampleProfileFile, + ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None) : ModulePass(ID), SampleLoader( - Name, SampleProfileRemappingFile, IsThinLTOPreLink, + Name, SampleProfileRemappingFile, LTOPhase, [&](Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }, @@ -705,6 +731,9 @@ void SampleProfileLoader::printBlockWeight(raw_ostream &OS, /// /// \returns the weight of \p Inst. ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) { + if (FunctionSamples::ProfileIsProbeBased) + return getProbeWeight(Inst); + const DebugLoc &DLoc = Inst.getDebugLoc(); if (!DLoc) return std::error_code(); @@ -723,9 +752,10 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) { // (findCalleeFunctionSamples returns non-empty result), but not inlined here, // it means that the inlined callsite has no sample, thus the call // instruction should have 0 count. - if (auto *CB = dyn_cast<CallBase>(&Inst)) - if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) - return 0; + if (!ProfileIsCS) + if (const auto *CB = dyn_cast<CallBase>(&Inst)) + if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) + return 0; const DILocation *DIL = DLoc; uint32_t LineOffset = FunctionSamples::getOffset(DIL); @@ -757,6 +787,47 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) { return R; } +ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) { + assert(FunctionSamples::ProfileIsProbeBased && + "Profile is not pseudo probe based"); + Optional<PseudoProbe> Probe = extractProbe(Inst); + if (!Probe) + return std::error_code(); + + const FunctionSamples *FS = findFunctionSamples(Inst); + if (!FS) + return std::error_code(); + + // If a direct call/invoke instruction is inlined in profile + // (findCalleeFunctionSamples returns non-empty result), but not inlined here, + // it means that the inlined callsite has no sample, thus the call + // instruction should have 0 count. + if (const auto *CB = dyn_cast<CallBase>(&Inst)) + if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) + return 0; + + const ErrorOr<uint64_t> &R = FS->findSamplesAt(Probe->Id, 0); + if (R) { + uint64_t Samples = R.get(); + bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples); + if (FirstMark) { + ORE->emit([&]() { + OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "AppliedSamples", &Inst); + Remark << "Applied " << ore::NV("NumSamples", Samples); + Remark << " samples from profile (ProbeId="; + Remark << ore::NV("ProbeId", Probe->Id); + Remark << ")"; + return Remark; + }); + } + + LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Inst + << " - weight: " << R.get() << ")\n"); + return Samples; + } + return R; +} + /// Compute the weight of a basic block. /// /// The weight of basic block \p BB is the maximum weight of all the @@ -820,17 +891,18 @@ SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const { } StringRef CalleeName; - if (const CallInst *CI = dyn_cast<CallInst>(&Inst)) - if (Function *Callee = CI->getCalledFunction()) - CalleeName = Callee->getName(); + if (Function *Callee = Inst.getCalledFunction()) + CalleeName = FunctionSamples::getCanonicalFnName(*Callee); + + if (ProfileIsCS) + return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName); const FunctionSamples *FS = findFunctionSamples(Inst); if (FS == nullptr) return nullptr; - return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL), - DIL->getBaseDiscriminator()), - CalleeName); + return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL), + CalleeName, Reader->getRemapper()); } /// Returns a vector of FunctionSamples that are the indirect call targets @@ -850,16 +922,13 @@ SampleProfileLoader::findIndirectCallFunctionSamples( if (FS == nullptr) return R; - uint32_t LineOffset = FunctionSamples::getOffset(DIL); - uint32_t Discriminator = DIL->getBaseDiscriminator(); - - auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); + auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + auto T = FS->findCallTargetMapAt(CallSite); Sum = 0; if (T) for (const auto &T_C : T.get()) Sum += T_C.second; - if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(LineLocation( - FunctionSamples::getOffset(DIL), DIL->getBaseDiscriminator()))) { + if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) { if (M->empty()) return R; for (const auto &NameFS : *M) { @@ -887,17 +956,38 @@ SampleProfileLoader::findIndirectCallFunctionSamples( /// \returns the FunctionSamples pointer to the inlined instance. const FunctionSamples * SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const { + if (FunctionSamples::ProfileIsProbeBased) { + Optional<PseudoProbe> Probe = extractProbe(Inst); + if (!Probe) + return nullptr; + } + const DILocation *DIL = Inst.getDebugLoc(); if (!DIL) return Samples; auto it = DILocation2SampleMap.try_emplace(DIL,nullptr); - if (it.second) - it.first->second = Samples->findFunctionSamples(DIL); + if (it.second) { + if (ProfileIsCS) + it.first->second = ContextTracker->getContextSamplesFor(DIL); + else + it.first->second = + Samples->findFunctionSamples(DIL, Reader->getRemapper()); + } return it.first->second; } bool SampleProfileLoader::inlineCallInstruction(CallBase &CB) { + if (ExternalInlineAdvisor) { + auto Advice = ExternalInlineAdvisor->getAdvice(CB); + if (!Advice->isInliningRecommended()) { + Advice->recordUnattemptedInlining(); + return false; + } + // Dummy record, we don't use it for replay. + Advice->recordInlining(); + } + Function *CalledFunction = CB.getCalledFunction(); assert(CalledFunction); DebugLoc DLoc = CB.getDebugLoc(); @@ -938,6 +1028,12 @@ bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) { InlineCost Cost = getInlineCost(CallInst, getInlineParams(), GetTTI(*Callee), GetAC, GetTLI); + if (Cost.isNever()) + return false; + + if (Cost.isAlways()) + return true; + return Cost.getCost() <= SampleColdCallSiteThreshold; } @@ -995,8 +1091,10 @@ bool SampleProfileLoader::inlineHotFunctions( const FunctionSamples *FS = nullptr; if (auto *CB = dyn_cast<CallBase>(&I)) { if (!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(*CB))) { + assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && + "GUIDToFuncNameMap has to be populated"); AllCandidates.push_back(CB); - if (FS->getEntrySamples() > 0) + if (FS->getEntrySamples() > 0 || ProfileIsCS) localNotInlinedCallSites.try_emplace(CB, FS); if (callsiteIsHot(FS, PSI)) Hot = true; @@ -1005,7 +1103,7 @@ bool SampleProfileLoader::inlineHotFunctions( } } } - if (Hot) { + if (Hot || ExternalInlineAdvisor) { CIS.insert(CIS.begin(), AllCandidates.begin(), AllCandidates.end()); emitOptimizationRemarksForInlineCandidates(AllCandidates, F, true); } else { @@ -1023,29 +1121,28 @@ bool SampleProfileLoader::inlineHotFunctions( continue; uint64_t Sum; for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) { - if (IsThinLTOPreLink) { + if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { FS->findInlinedFunctions(InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); continue; } - auto CalleeFunctionName = FS->getFuncName(); + if (!callsiteIsHot(FS, PSI)) + continue; + + const char *Reason = "Callee function not available"; + // R->getValue() != &F is to prevent promoting a recursive call. // If it is a recursive call, we do not inline it as it could bloat // the code exponentially. There is way to better handle this, e.g. // clone the caller first, and inline the cloned caller if it is // recursive. As llvm does not inline recursive calls, we will // simply ignore it instead of handling it explicitly. - if (CalleeFunctionName == F.getName()) - continue; - - if (!callsiteIsHot(FS, PSI)) - continue; - - const char *Reason = "Callee function not available"; + auto CalleeFunctionName = FS->getFuncName(); auto R = SymbolMap.find(CalleeFunctionName); if (R != SymbolMap.end() && R->getValue() && !R->getValue()->isDeclaration() && R->getValue()->getSubprogram() && R->getValue()->hasFnAttribute("use-sample-profile") && + R->getValue() != &F && isLegalToPromote(*I, R->getValue(), &Reason)) { uint64_t C = FS->getEntrySamples(); auto &DI = @@ -1055,6 +1152,8 @@ bool SampleProfileLoader::inlineHotFunctions( // If profile mismatches, we should not attempt to inline DI. if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) && inlineCallInstruction(cast<CallBase>(DI))) { + if (ProfileIsCS) + ContextTracker->markContextSamplesInlined(FS); localNotInlinedCallSites.erase(I); LocalChanged = true; ++NumCSInlined; @@ -1068,11 +1167,14 @@ bool SampleProfileLoader::inlineHotFunctions( } else if (CalledFunction && CalledFunction->getSubprogram() && !CalledFunction->isDeclaration()) { if (inlineCallInstruction(*I)) { + if (ProfileIsCS) + ContextTracker->markContextSamplesInlined( + localNotInlinedCallSites[I]); localNotInlinedCallSites.erase(I); LocalChanged = true; ++NumCSInlined; } - } else if (IsThinLTOPreLink) { + } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) { findCalleeFunctionSamples(*I)->findInlinedFunctions( InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold()); } @@ -1104,16 +1206,23 @@ bool SampleProfileLoader::inlineHotFunctions( } if (ProfileMergeInlinee) { - // Use entry samples as head samples during the merge, as inlinees - // don't have head samples. - assert(FS->getHeadSamples() == 0 && "Expect 0 head sample for inlinee"); - const_cast<FunctionSamples *>(FS)->addHeadSamples(FS->getEntrySamples()); - - // Note that we have to do the merge right after processing function. - // This allows OutlineFS's profile to be used for annotation during - // top-down processing of functions' annotation. - FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee); - OutlineFS->merge(*FS); + // A function call can be replicated by optimizations like callsite + // splitting or jump threading and the replicates end up sharing the + // sample nested callee profile instead of slicing the original inlinee's + // profile. We want to do merge exactly once by filtering out callee + // profiles with a non-zero head sample count. + if (FS->getHeadSamples() == 0) { + // Use entry samples as head samples during the merge, as inlinees + // don't have head samples. + const_cast<FunctionSamples *>(FS)->addHeadSamples( + FS->getEntrySamples()); + + // Note that we have to do the merge right after processing function. + // This allows OutlineFS's profile to be used for annotation during + // top-down processing of functions' annotation. + FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee); + OutlineFS->merge(*FS); + } } else { auto pair = notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0}); @@ -1538,13 +1647,11 @@ void SampleProfileLoader::propagateWeights(Function &F) { if (!DLoc) continue; const DILocation *DIL = DLoc; - uint32_t LineOffset = FunctionSamples::getOffset(DIL); - uint32_t Discriminator = DIL->getBaseDiscriminator(); - const FunctionSamples *FS = findFunctionSamples(I); if (!FS) continue; - auto T = FS->findCallTargetMapAt(LineOffset, Discriminator); + auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); + auto T = FS->findCallTargetMapAt(CallSite); if (!T || T.get().empty()) continue; SmallVector<InstrProfValueData, 2> SortedCallTargets = @@ -1598,8 +1705,6 @@ void SampleProfileLoader::propagateWeights(Function &F) { } } - misexpect::verifyMisExpect(TI, Weights, TI->getContext()); - uint64_t TempWeight; // Only set weights if there is at least one non-zero weight. // In any other case, let the analyzer set weights. @@ -1710,11 +1815,22 @@ void SampleProfileLoader::computeDominanceAndLoopInfo(Function &F) { bool SampleProfileLoader::emitAnnotations(Function &F) { bool Changed = false; - if (getFunctionLoc(F) == 0) - return false; + if (FunctionSamples::ProfileIsProbeBased) { + if (!ProbeManager->profileIsValid(F, *Samples)) { + LLVM_DEBUG( + dbgs() << "Profile is invalid due to CFG mismatch for Function " + << F.getName()); + ++NumMismatchedProfile; + return false; + } + ++NumMatchedProfile; + } else { + if (getFunctionLoc(F) == 0) + return false; - LLVM_DEBUG(dbgs() << "Line number for the first instruction in " - << F.getName() << ": " << getFunctionLoc(F) << "\n"); + LLVM_DEBUG(dbgs() << "Line number for the first instruction in " + << F.getName() << ": " << getFunctionLoc(F) << "\n"); + } DenseSet<GlobalValue::GUID> InlinedGUIDs; Changed |= inlineHotFunctions(F, InlinedGUIDs); @@ -1818,10 +1934,10 @@ SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) { return FunctionOrderList; } -bool SampleProfileLoader::doInitialization(Module &M) { +bool SampleProfileLoader::doInitialization(Module &M, + FunctionAnalysisManager *FAM) { auto &Ctx = M.getContext(); - std::unique_ptr<SampleProfileReaderItaniumRemapper> RemapReader; auto ReaderOrErr = SampleProfileReader::create(Filename, Ctx, RemappingFilename); if (std::error_code EC = ReaderOrErr.getError()) { @@ -1830,8 +1946,14 @@ bool SampleProfileLoader::doInitialization(Module &M) { return false; } Reader = std::move(ReaderOrErr.get()); + Reader->setSkipFlatProf(LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink); Reader->collectFuncsFrom(M); - ProfileIsValid = (Reader->read() == sampleprof_error::success); + if (std::error_code EC = Reader->read()) { + std::string Msg = "profile reading failed: " + EC.message(); + Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); + return false; + } + PSL = Reader->getProfileSymbolList(); // While profile-sample-accurate is on, ignore symbol list. @@ -1843,6 +1965,35 @@ bool SampleProfileLoader::doInitialization(Module &M) { NamesInProfile.insert(NameTable->begin(), NameTable->end()); } + if (FAM && !ProfileInlineReplayFile.empty()) { + ExternalInlineAdvisor = std::make_unique<ReplayInlineAdvisor>( + M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr, ProfileInlineReplayFile, + /*EmitRemarks=*/false); + if (!ExternalInlineAdvisor->areReplayRemarksLoaded()) + ExternalInlineAdvisor.reset(); + } + + // Apply tweaks if context-sensitive profile is available. + if (Reader->profileIsCS()) { + ProfileIsCS = true; + FunctionSamples::ProfileIsCS = true; + + // Tracker for profiles under different context + ContextTracker = + std::make_unique<SampleContextTracker>(Reader->getProfiles()); + } + + // Load pseudo probe descriptors for probe-based function samples. + if (Reader->profileIsProbeBased()) { + ProbeManager = std::make_unique<PseudoProbeManager>(M); + if (!ProbeManager->moduleIsProbed(M)) { + const char *Msg = + "Pseudo-probe-based profile requires SampleProfileProbePass"; + Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg)); + return false; + } + } + return true; } @@ -1856,8 +2007,6 @@ ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) { bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummaryInfo *_PSI, CallGraph *CG) { - if (!ProfileIsValid) - return false; GUIDToFuncNameMapper Mapper(M, *Reader, GUIDToFuncNameMap); PSI = _PSI; @@ -1870,6 +2019,7 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, for (const auto &I : Reader->getProfiles()) TotalCollectedSamples += I.second.getTotalSamples(); + auto Remapper = Reader->getRemapper(); // Populate the symbol map. for (const auto &N_F : M.getValueSymbolTable()) { StringRef OrigName = N_F.getKey(); @@ -1887,6 +2037,15 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, // to nullptr to avoid confusion. if (!r.second) r.first->second = nullptr; + OrigName = NewName; + } + // Insert the remapped names into SymbolMap. + if (Remapper) { + if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) { + if (*MapName == OrigName) + continue; + SymbolMap.insert(std::make_pair(*MapName, F)); + } } } @@ -1898,9 +2057,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, } // Account for cold calls not inlined.... - for (const std::pair<Function *, NotInlinedProfileInfo> &pair : - notInlinedCallInfo) - updateProfileCallee(pair.first, pair.second.entryCount); + if (!ProfileIsCS) + for (const std::pair<Function *, NotInlinedProfileInfo> &pair : + notInlinedCallInfo) + updateProfileCallee(pair.first, pair.second.entryCount); return retval; } @@ -1915,7 +2075,6 @@ bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) { } bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) { - DILocation2SampleMap.clear(); // By default the entry count is initialized to -1, which will be treated // conservatively by getEntryCount as the same as unknown (None). This is @@ -1957,7 +2116,10 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) initialEntryCount = -1; } - F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real)); + // Initialize entry count when the function has no existing entry + // count value. + if (!F.getEntryCount().hasValue()) + F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real)); std::unique_ptr<OptimizationRemarkEmitter> OwnedORE; if (AM) { auto &FAM = @@ -1968,7 +2130,12 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F); ORE = OwnedORE.get(); } - Samples = Reader->getSamplesFor(F); + + if (ProfileIsCS) + Samples = ContextTracker->getBaseSamplesFor(F); + else + Samples = Reader->getSamplesFor(F); + if (Samples && !Samples->empty()) return emitAnnotations(F); return false; @@ -1993,9 +2160,9 @@ PreservedAnalyses SampleProfileLoaderPass::run(Module &M, ProfileFileName.empty() ? SampleProfileFile : ProfileFileName, ProfileRemappingFileName.empty() ? SampleProfileRemappingFile : ProfileRemappingFileName, - IsThinLTOPreLink, GetAssumptionCache, GetTTI, GetTLI); + LTOPhase, GetAssumptionCache, GetTTI, GetTLI); - if (!SampleLoader.doInitialization(M)) + if (!SampleLoader.doInitialization(M, &FAM)) return PreservedAnalyses::all(); ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M); |