19 #include "llvm/IR/Intrinsics.h"
20 #include "llvm/IR/MDBuilder.h"
21 #include "llvm/ProfileData/InstrProfReader.h"
22 #include "llvm/Support/Endian.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MD5.h"
26 using namespace clang;
27 using namespace CodeGen;
29 void CodeGenPGO::setFuncName(StringRef Name,
30 llvm::GlobalValue::LinkageTypes
Linkage) {
31 StringRef RawFuncName = Name;
36 if (RawFuncName[0] ==
'\1')
37 RawFuncName = RawFuncName.substr(1);
39 FuncName = RawFuncName;
40 if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
46 FuncName = FuncName.insert(0,
"<unknown>:");
53 createFuncNameVar(Linkage);
56 void CodeGenPGO::setFuncName(llvm::Function *Fn) {
57 setFuncName(Fn->getName(), Fn->getLinkage());
60 void CodeGenPGO::createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage) {
64 if (Linkage == llvm::GlobalValue::ExternalWeakLinkage)
65 Linkage = llvm::GlobalValue::LinkOnceAnyLinkage;
66 else if (Linkage == llvm::GlobalValue::AvailableExternallyLinkage)
67 Linkage = llvm::GlobalValue::LinkOnceODRLinkage;
70 Linkage = llvm::GlobalValue::PrivateLinkage;
73 llvm::ConstantDataArray::getString(CGM.
getLLVMContext(), FuncName,
false);
76 Value,
"__llvm_profile_name_" + FuncName);
79 if (!llvm::GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
99 static const int NumBitsPerType = 6;
100 static const unsigned NumTypesPerWord =
sizeof(uint64_t) * 8 / NumBitsPerType;
101 static const unsigned TooBig = 1u << NumBitsPerType;
111 enum HashType :
unsigned char {
133 static_assert(LastHashType <= TooBig,
"Too many types in HashType");
137 PGOHash() : Working(0),
Count(0) {}
138 void combine(HashType
Type);
141 const int PGOHash::NumBitsPerType;
142 const unsigned PGOHash::NumTypesPerWord;
143 const unsigned PGOHash::TooBig;
148 unsigned NextCounter;
152 llvm::DenseMap<const Stmt *, unsigned> &CounterMap;
154 MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap)
155 : NextCounter(0), CounterMap(CounterMap) {}
159 bool TraverseBlockExpr(
BlockExpr *BE) {
return true; }
160 bool TraverseLambdaBody(
LambdaExpr *LE) {
return true; }
161 bool TraverseCapturedStmt(
CapturedStmt *CS) {
return true; }
163 bool VisitDecl(
const Decl *D) {
168 case Decl::CXXMethod:
169 case Decl::CXXConstructor:
170 case Decl::CXXDestructor:
171 case Decl::CXXConversion:
172 case Decl::ObjCMethod:
175 CounterMap[D->
getBody()] = NextCounter++;
181 bool VisitStmt(
const Stmt *
S) {
182 auto Type = getHashType(S);
186 CounterMap[
S] = NextCounter++;
190 PGOHash::HashType getHashType(
const Stmt *S) {
191 switch (S->getStmtClass()) {
194 case Stmt::LabelStmtClass:
195 return PGOHash::LabelStmt;
196 case Stmt::WhileStmtClass:
197 return PGOHash::WhileStmt;
198 case Stmt::DoStmtClass:
199 return PGOHash::DoStmt;
200 case Stmt::ForStmtClass:
201 return PGOHash::ForStmt;
202 case Stmt::CXXForRangeStmtClass:
203 return PGOHash::CXXForRangeStmt;
204 case Stmt::ObjCForCollectionStmtClass:
205 return PGOHash::ObjCForCollectionStmt;
206 case Stmt::SwitchStmtClass:
207 return PGOHash::SwitchStmt;
208 case Stmt::CaseStmtClass:
209 return PGOHash::CaseStmt;
210 case Stmt::DefaultStmtClass:
211 return PGOHash::DefaultStmt;
212 case Stmt::IfStmtClass:
213 return PGOHash::IfStmt;
214 case Stmt::CXXTryStmtClass:
215 return PGOHash::CXXTryStmt;
216 case Stmt::CXXCatchStmtClass:
217 return PGOHash::CXXCatchStmt;
218 case Stmt::ConditionalOperatorClass:
219 return PGOHash::ConditionalOperator;
220 case Stmt::BinaryConditionalOperatorClass:
221 return PGOHash::BinaryConditionalOperator;
222 case Stmt::BinaryOperatorClass: {
225 return PGOHash::BinaryOperatorLAnd;
227 return PGOHash::BinaryOperatorLOr;
243 bool RecordNextStmtCount;
246 uint64_t CurrentCount;
249 llvm::DenseMap<const Stmt *, uint64_t> &
CountMap;
252 struct BreakContinue {
254 uint64_t ContinueCount;
255 BreakContinue() : BreakCount(0), ContinueCount(0) {}
259 ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &
CountMap,
261 : PGO(PGO), RecordNextStmtCount(
false), CountMap(CountMap) {}
263 void RecordStmtCount(
const Stmt *S) {
264 if (RecordNextStmtCount) {
266 RecordNextStmtCount =
false;
271 uint64_t setCount(uint64_t
Count) {
272 CurrentCount =
Count;
276 void VisitStmt(
const Stmt *S) {
278 for (
const Stmt *Child : S->children())
285 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
297 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
304 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
309 void VisitBlockDecl(
const BlockDecl *D) {
311 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
321 RecordNextStmtCount =
true;
329 RecordNextStmtCount =
true;
332 void VisitGotoStmt(
const GotoStmt *S) {
335 RecordNextStmtCount =
true;
338 void VisitLabelStmt(
const LabelStmt *S) {
339 RecordNextStmtCount =
false;
341 uint64_t BlockCount = setCount(PGO.getRegionCount(S));
346 void VisitBreakStmt(
const BreakStmt *S) {
348 assert(!BreakContinueStack.empty() &&
"break not in a loop or switch!");
349 BreakContinueStack.back().BreakCount += CurrentCount;
351 RecordNextStmtCount =
true;
356 assert(!BreakContinueStack.empty() &&
"continue stmt not in a loop!");
357 BreakContinueStack.back().ContinueCount += CurrentCount;
359 RecordNextStmtCount =
true;
362 void VisitWhileStmt(
const WhileStmt *S) {
364 uint64_t ParentCount = CurrentCount;
366 BreakContinueStack.push_back(BreakContinue());
369 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
370 CountMap[S->
getBody()] = CurrentCount;
372 uint64_t BackedgeCount = CurrentCount;
378 BreakContinue BC = BreakContinueStack.pop_back_val();
380 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
381 CountMap[S->
getCond()] = CondCount;
383 setCount(BC.BreakCount + CondCount - BodyCount);
384 RecordNextStmtCount =
true;
387 void VisitDoStmt(
const DoStmt *S) {
389 uint64_t LoopCount = PGO.getRegionCount(S);
391 BreakContinueStack.push_back(BreakContinue());
393 uint64_t BodyCount = setCount(LoopCount + CurrentCount);
394 CountMap[S->
getBody()] = BodyCount;
396 uint64_t BackedgeCount = CurrentCount;
398 BreakContinue BC = BreakContinueStack.pop_back_val();
401 uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount);
402 CountMap[S->
getCond()] = CondCount;
404 setCount(BC.BreakCount + CondCount - LoopCount);
405 RecordNextStmtCount =
true;
408 void VisitForStmt(
const ForStmt *S) {
413 uint64_t ParentCount = CurrentCount;
415 BreakContinueStack.push_back(BreakContinue());
418 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
419 CountMap[S->
getBody()] = BodyCount;
421 uint64_t BackedgeCount = CurrentCount;
422 BreakContinue BC = BreakContinueStack.pop_back_val();
427 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
428 CountMap[S->
getInc()] = IncCount;
434 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
436 CountMap[S->
getCond()] = CondCount;
439 setCount(BC.BreakCount + CondCount - BodyCount);
440 RecordNextStmtCount =
true;
449 uint64_t ParentCount = CurrentCount;
450 BreakContinueStack.push_back(BreakContinue());
453 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
454 CountMap[S->
getBody()] = BodyCount;
456 uint64_t BackedgeCount = CurrentCount;
457 BreakContinue BC = BreakContinueStack.pop_back_val();
461 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
462 CountMap[S->
getInc()] = IncCount;
467 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
468 CountMap[S->
getCond()] = CondCount;
470 setCount(BC.BreakCount + CondCount - BodyCount);
471 RecordNextStmtCount =
true;
477 uint64_t ParentCount = CurrentCount;
478 BreakContinueStack.push_back(BreakContinue());
480 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
481 CountMap[S->
getBody()] = BodyCount;
483 uint64_t BackedgeCount = CurrentCount;
484 BreakContinue BC = BreakContinueStack.pop_back_val();
486 setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount -
488 RecordNextStmtCount =
true;
495 BreakContinueStack.push_back(BreakContinue());
498 BreakContinue BC = BreakContinueStack.pop_back_val();
499 if (!BreakContinueStack.empty())
500 BreakContinueStack.back().ContinueCount += BC.ContinueCount;
502 setCount(PGO.getRegionCount(S));
503 RecordNextStmtCount =
true;
507 RecordNextStmtCount =
false;
511 uint64_t CaseCount = PGO.getRegionCount(S);
512 setCount(CurrentCount + CaseCount);
515 CountMap[
S] = CaseCount;
516 RecordNextStmtCount =
true;
520 void VisitIfStmt(
const IfStmt *S) {
522 uint64_t ParentCount = CurrentCount;
527 uint64_t ThenCount = setCount(PGO.getRegionCount(S));
528 CountMap[S->
getThen()] = ThenCount;
530 uint64_t OutCount = CurrentCount;
532 uint64_t ElseCount = ParentCount - ThenCount;
535 CountMap[S->
getElse()] = ElseCount;
537 OutCount += CurrentCount;
539 OutCount += ElseCount;
541 RecordNextStmtCount =
true;
550 setCount(PGO.getRegionCount(S));
551 RecordNextStmtCount =
true;
555 RecordNextStmtCount =
false;
557 uint64_t CatchCount = setCount(PGO.getRegionCount(S));
558 CountMap[
S] = CatchCount;
564 uint64_t ParentCount = CurrentCount;
569 uint64_t TrueCount = setCount(PGO.getRegionCount(E));
572 uint64_t OutCount = CurrentCount;
574 uint64_t FalseCount = setCount(ParentCount - TrueCount);
577 OutCount += CurrentCount;
580 RecordNextStmtCount =
true;
585 uint64_t ParentCount = CurrentCount;
588 uint64_t RHSCount = setCount(PGO.getRegionCount(E));
589 CountMap[E->
getRHS()] = RHSCount;
591 setCount(ParentCount + RHSCount - CurrentCount);
592 RecordNextStmtCount =
true;
597 uint64_t ParentCount = CurrentCount;
600 uint64_t RHSCount = setCount(PGO.getRegionCount(E));
601 CountMap[E->
getRHS()] = RHSCount;
603 setCount(ParentCount + RHSCount - CurrentCount);
604 RecordNextStmtCount =
true;
609 void PGOHash::combine(HashType
Type) {
611 assert(Type &&
"Hash is invalid: unexpected type 0");
612 assert(
unsigned(Type) < TooBig &&
"Hash is invalid: too many types");
616 using namespace llvm::support;
617 uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working);
618 MD5.update(llvm::makeArrayRef((uint8_t *)&Swapped,
sizeof(Swapped)));
624 Working = Working << NumBitsPerType | Type;
627 uint64_t PGOHash::finalize() {
629 if (
Count <= NumTypesPerWord)
640 llvm::MD5::MD5Result
Result;
642 using namespace llvm::support;
643 return endian::read<uint64_t, little, unaligned>(Result);
652 if ((isa<CXXConstructorDecl>(GD.
getDecl()) &&
654 (isa<CXXDestructorDecl>(GD.
getDecl()) &&
656 SkipCoverageMapping =
true;
661 bool InstrumentRegions = CGM.
getCodeGenOpts().ProfileInstrGenerate;
662 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
663 if (!InstrumentRegions && !PGOReader)
670 mapRegionCounters(D);
672 emitCounterRegionMapping(D);
676 computeRegionCounts(D);
677 applyFunctionAttributes(PGOReader, Fn);
681 void CodeGenPGO::mapRegionCounters(
const Decl *D) {
682 RegionCounterMap.reset(
new llvm::DenseMap<const Stmt *, unsigned>);
683 MapRegionCounters Walker(*RegionCounterMap);
684 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
685 Walker.TraverseDecl(const_cast<FunctionDecl *>(FD));
686 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
687 Walker.TraverseDecl(const_cast<ObjCMethodDecl *>(MD));
688 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
689 Walker.TraverseDecl(const_cast<BlockDecl *>(BD));
690 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
691 Walker.TraverseDecl(const_cast<CapturedDecl *>(CD));
692 assert(Walker.NextCounter > 0 &&
"no entry counter mapped for decl");
693 NumRegionCounters = Walker.NextCounter;
694 FunctionHash = Walker.Hash.finalize();
697 void CodeGenPGO::emitCounterRegionMapping(
const Decl *D) {
698 if (SkipCoverageMapping)
701 auto Loc = D->
getBody()->getLocStart();
705 std::string CoverageMapping;
706 llvm::raw_string_ostream OS(CoverageMapping);
713 if (CoverageMapping.empty())
717 FuncNameVar, FuncName, FunctionHash, CoverageMapping);
722 llvm::GlobalValue::LinkageTypes Linkage) {
723 if (SkipCoverageMapping)
726 auto Loc = D->
getBody()->getLocStart();
730 std::string CoverageMapping;
731 llvm::raw_string_ostream OS(CoverageMapping);
738 if (CoverageMapping.empty())
741 setFuncName(Name, Linkage);
743 FuncNameVar, FuncName, FunctionHash, CoverageMapping);
746 void CodeGenPGO::computeRegionCounts(
const Decl *D) {
747 StmtCountMap.reset(
new llvm::DenseMap<const Stmt *, uint64_t>);
748 ComputeRegionCounts Walker(*StmtCountMap, *
this);
749 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
750 Walker.VisitFunctionDecl(FD);
751 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
752 Walker.VisitObjCMethodDecl(MD);
753 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
754 Walker.VisitBlockDecl(BD);
755 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
756 Walker.VisitCapturedDecl(const_cast<CapturedDecl *>(CD));
760 CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
761 llvm::Function *Fn) {
765 uint64_t MaxFunctionCount = PGOReader->getMaximumFunctionCount();
767 if (FunctionCount >= (uint64_t)(0.3 * (
double)MaxFunctionCount))
770 Fn->addFnAttr(llvm::Attribute::InlineHint);
771 else if (FunctionCount <= (uint64_t)(0.01 * (
double)MaxFunctionCount))
774 Fn->addFnAttr(llvm::Attribute::Cold);
776 Fn->setEntryCount(FunctionCount);
780 if (!CGM.
getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap)
782 if (!Builder.GetInsertPoint())
785 unsigned Counter = (*RegionCounterMap)[
S];
787 Builder.CreateCall(CGM.
getIntrinsic(llvm::Intrinsic::instrprof_increment),
788 {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
789 Builder.getInt64(FunctionHash),
790 Builder.getInt32(NumRegionCounters),
791 Builder.getInt32(Counter)});
794 void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
797 RegionCounts.clear();
798 if (std::error_code EC =
799 PGOReader->getFunctionCounts(FuncName, FunctionHash, RegionCounts)) {
800 if (EC == llvm::instrprof_error::unknown_function)
802 else if (EC == llvm::instrprof_error::hash_mismatch)
804 else if (EC == llvm::instrprof_error::malformed)
807 RegionCounts.clear();
816 return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1;
829 assert(Scale &&
"scale by 0?");
830 uint64_t Scaled = Weight / Scale + 1;
831 assert(Scaled <= UINT32_MAX &&
"overflow 32-bits");
835 llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount,
836 uint64_t FalseCount) {
838 if (!TrueCount && !FalseCount)
852 if (Weights.size() < 2)
856 uint64_t MaxWeight = *std::max_element(Weights.begin(), Weights.end());
864 ScaledWeights.reserve(Weights.size());
865 for (uint64_t W : Weights)
869 return MDHelper.createBranchWeights(ScaledWeights);
872 llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(
const Stmt *Cond,
873 uint64_t LoopCount) {
877 assert(CondCount.hasValue() &&
"missing expected loop condition count");
880 return createProfileWeights(LoopCount,
881 std::max(*CondCount, LoopCount) - LoopCount);
Optional< uint64_t > getStmtCount(const Stmt *S)
External linkage, which indicates that the entity can be referred to from other translation units...
CXXCtorType getCtorType() const
llvm::Module & getModule() const
static uint64_t calculateWeightScale(uint64_t MaxWeight)
Calculate what to divide by to scale weights.
llvm::LLVMContext & getLLVMContext()
void checkGlobalDecl(GlobalDecl GD)
Check if we need to emit coverage mapping for a given declaration.
CXXCatchStmt * getHandler(unsigned i)
void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S)
Organizes the per-function state that is used while generating code coverage mapping data...
const Stmt * getElse() const
llvm::ImmutableMap< CountKey, unsigned > CountMap
A C++ throw-expression (C++ [except.throw]).
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have...
Stmt * getBody() const override
const Decl * getDecl() const
Expr * getTrueExpr() const
Stmt * getHandlerBlock() const
A builtin binary operation expression such as "x + y" or "x <= y".
void addMissing(bool MainFile)
Record that a function we've visited has no profile data.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
A class that does preorder depth-first traversal on the entire Clang AST and visits each node...
This represents the body of a CapturedStmt, and serves as its DeclContext.
InstrProfStats & getPGOStats()
CXXDtorType getDtorType() const
Stmt * getBody() const override
bool haveRegionCounts() const
ASTContext & getContext() const
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
void addMismatched(bool MainFile)
Record that a function we've visited has mismatched profile data.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=None)
The result type of a method or function.
llvm::IRBuilder< PreserveNames, llvm::ConstantFolder, CGBuilderInserterTy > CGBuilderTy
void addVisited(bool MainFile)
Stmt * getBody(const FunctionDecl *&Definition) const
bool isInMainFile(SourceLocation Loc) const
Returns whether the PresumedLoc for a given SourceLocation is in the main file.
This captures a statement into a function. For example, the following pragma annotated compound state...
const Expr * getCond() const
Stmt * getBody() const override
Retrieve the body of this method, if it has one.
virtual Stmt * getBody() const
const CodeGenOptions & getCodeGenOpts() const
const LangOptions & getLangOpts() const
const Expr * getSubExpr() const
const Stmt * getBody() const
unsigned getNumHandlers() const
const Expr * getRetValue() const
const Stmt * getThen() const
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, llvm::GlobalValue::LinkageTypes Linkage)
Expr * getFalseExpr() const
DeclStmt * getBeginEndStmt()
Represents Objective-C's collection statement.
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
SourceManager & getSourceManager()
DeclStmt * getRangeStmt()
static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale)
Scale an individual branch weight (and add 1).
void emitCounterMapping(const Decl *D, llvm::raw_ostream &OS)
Emit the coverage mapping data which maps the regions of code to counters that will be used to find t...
BoundNodesTreeBuilder *const Builder
const Expr * getCond() const
void ClearUnusedCoverageMapping(const Decl *D)
Remove the deferred empty coverage mapping as this declaration is actually instrumented.
CompoundStmt * getTryBlock()
void addFunctionMappingRecord(llvm::GlobalVariable *FunctionName, StringRef FunctionNameValue, uint64_t FunctionHash, const std::string &CoverageMapping)
Add a function's coverage mapping record to the collection of the function mapping records...
uint64_t getRegionCount(const Stmt *S)
Return the region count for the counter at the given index.
CoverageMappingModuleGen * getCoverageMapping() const
DeclStmt * getLoopVarStmt()
SourceLocation getLocation() const
llvm::IndexedInstrProfReader * getPGOReader() const
void assignRegionCounters(const Decl *D, llvm::Function *Fn)
void emitEmptyMapping(const Decl *D, llvm::raw_ostream &OS)
Emit the coverage mapping data for an unused function. It creates mapping regions with the counter of...
This class handles loading and caching of source files into memory.