19 #include "llvm/IR/Intrinsics.h"
20 #include "llvm/IR/MDBuilder.h"
21 #include "llvm/ProfileData/InstrProfReader.h"
22 #include "llvm/Support/Endian.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MD5.h"
26 using namespace clang;
27 using namespace CodeGen;
29 void CodeGenPGO::setFuncName(StringRef
Name,
30 llvm::GlobalValue::LinkageTypes
Linkage) {
31 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
32 FuncName = llvm::getPGOFuncName(
34 PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version);
38 FuncNameVar = llvm::createPGOFuncNameVar(CGM.
getModule(),
Linkage, FuncName);
41 void CodeGenPGO::setFuncName(llvm::Function *Fn) {
42 setFuncName(Fn->getName(), Fn->getLinkage());
61 static const int NumBitsPerType = 6;
62 static const unsigned NumTypesPerWord =
sizeof(uint64_t) * 8 / NumBitsPerType;
63 static const unsigned TooBig = 1u << NumBitsPerType;
73 enum HashType :
unsigned char {
95 static_assert(LastHashType <= TooBig,
"Too many types in HashType");
99 PGOHash() : Working(0),
Count(0) {}
100 void combine(HashType
Type);
103 const int PGOHash::NumBitsPerType;
104 const unsigned PGOHash::NumTypesPerWord;
105 const unsigned PGOHash::TooBig;
110 unsigned NextCounter;
114 llvm::DenseMap<const Stmt *, unsigned> &CounterMap;
116 MapRegionCounters(llvm::DenseMap<const Stmt *, unsigned> &CounterMap)
117 : NextCounter(0), CounterMap(CounterMap) {}
121 bool TraverseBlockExpr(
BlockExpr *BE) {
return true; }
122 bool TraverseLambdaBody(
LambdaExpr *LE) {
return true; }
123 bool TraverseCapturedStmt(
CapturedStmt *CS) {
return true; }
125 bool VisitDecl(
const Decl *D) {
130 case Decl::CXXMethod:
131 case Decl::CXXConstructor:
132 case Decl::CXXDestructor:
133 case Decl::CXXConversion:
134 case Decl::ObjCMethod:
137 CounterMap[D->
getBody()] = NextCounter++;
143 bool VisitStmt(
const Stmt *
S) {
144 auto Type = getHashType(S);
148 CounterMap[
S] = NextCounter++;
152 PGOHash::HashType getHashType(
const Stmt *S) {
153 switch (S->getStmtClass()) {
156 case Stmt::LabelStmtClass:
157 return PGOHash::LabelStmt;
158 case Stmt::WhileStmtClass:
159 return PGOHash::WhileStmt;
160 case Stmt::DoStmtClass:
161 return PGOHash::DoStmt;
162 case Stmt::ForStmtClass:
163 return PGOHash::ForStmt;
164 case Stmt::CXXForRangeStmtClass:
165 return PGOHash::CXXForRangeStmt;
166 case Stmt::ObjCForCollectionStmtClass:
167 return PGOHash::ObjCForCollectionStmt;
168 case Stmt::SwitchStmtClass:
169 return PGOHash::SwitchStmt;
170 case Stmt::CaseStmtClass:
171 return PGOHash::CaseStmt;
172 case Stmt::DefaultStmtClass:
173 return PGOHash::DefaultStmt;
174 case Stmt::IfStmtClass:
175 return PGOHash::IfStmt;
176 case Stmt::CXXTryStmtClass:
177 return PGOHash::CXXTryStmt;
178 case Stmt::CXXCatchStmtClass:
179 return PGOHash::CXXCatchStmt;
180 case Stmt::ConditionalOperatorClass:
181 return PGOHash::ConditionalOperator;
182 case Stmt::BinaryConditionalOperatorClass:
183 return PGOHash::BinaryConditionalOperator;
184 case Stmt::BinaryOperatorClass: {
187 return PGOHash::BinaryOperatorLAnd;
189 return PGOHash::BinaryOperatorLOr;
205 bool RecordNextStmtCount;
208 uint64_t CurrentCount;
211 llvm::DenseMap<const Stmt *, uint64_t> &
CountMap;
214 struct BreakContinue {
216 uint64_t ContinueCount;
217 BreakContinue() : BreakCount(0), ContinueCount(0) {}
221 ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &
CountMap,
223 : PGO(PGO), RecordNextStmtCount(
false), CountMap(CountMap) {}
225 void RecordStmtCount(
const Stmt *S) {
226 if (RecordNextStmtCount) {
228 RecordNextStmtCount =
false;
233 uint64_t setCount(uint64_t
Count) {
234 CurrentCount =
Count;
238 void VisitStmt(
const Stmt *S) {
240 for (
const Stmt *Child : S->children())
247 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
259 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
266 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
271 void VisitBlockDecl(
const BlockDecl *D) {
273 uint64_t BodyCount = setCount(PGO.getRegionCount(D->
getBody()));
283 RecordNextStmtCount =
true;
291 RecordNextStmtCount =
true;
294 void VisitGotoStmt(
const GotoStmt *S) {
297 RecordNextStmtCount =
true;
300 void VisitLabelStmt(
const LabelStmt *S) {
301 RecordNextStmtCount =
false;
303 uint64_t BlockCount = setCount(PGO.getRegionCount(S));
308 void VisitBreakStmt(
const BreakStmt *S) {
310 assert(!BreakContinueStack.empty() &&
"break not in a loop or switch!");
311 BreakContinueStack.back().BreakCount += CurrentCount;
313 RecordNextStmtCount =
true;
318 assert(!BreakContinueStack.empty() &&
"continue stmt not in a loop!");
319 BreakContinueStack.back().ContinueCount += CurrentCount;
321 RecordNextStmtCount =
true;
324 void VisitWhileStmt(
const WhileStmt *S) {
326 uint64_t ParentCount = CurrentCount;
328 BreakContinueStack.push_back(BreakContinue());
331 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
332 CountMap[S->
getBody()] = CurrentCount;
334 uint64_t BackedgeCount = CurrentCount;
340 BreakContinue BC = BreakContinueStack.pop_back_val();
342 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
343 CountMap[S->
getCond()] = CondCount;
345 setCount(BC.BreakCount + CondCount - BodyCount);
346 RecordNextStmtCount =
true;
349 void VisitDoStmt(
const DoStmt *S) {
351 uint64_t LoopCount = PGO.getRegionCount(S);
353 BreakContinueStack.push_back(BreakContinue());
355 uint64_t BodyCount = setCount(LoopCount + CurrentCount);
356 CountMap[S->
getBody()] = BodyCount;
358 uint64_t BackedgeCount = CurrentCount;
360 BreakContinue BC = BreakContinueStack.pop_back_val();
363 uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount);
364 CountMap[S->
getCond()] = CondCount;
366 setCount(BC.BreakCount + CondCount - LoopCount);
367 RecordNextStmtCount =
true;
370 void VisitForStmt(
const ForStmt *S) {
375 uint64_t ParentCount = CurrentCount;
377 BreakContinueStack.push_back(BreakContinue());
380 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
381 CountMap[S->
getBody()] = BodyCount;
383 uint64_t BackedgeCount = CurrentCount;
384 BreakContinue BC = BreakContinueStack.pop_back_val();
389 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
390 CountMap[S->
getInc()] = IncCount;
396 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
398 CountMap[S->
getCond()] = CondCount;
401 setCount(BC.BreakCount + CondCount - BodyCount);
402 RecordNextStmtCount =
true;
411 uint64_t ParentCount = CurrentCount;
412 BreakContinueStack.push_back(BreakContinue());
415 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
416 CountMap[S->
getBody()] = BodyCount;
418 uint64_t BackedgeCount = CurrentCount;
419 BreakContinue BC = BreakContinueStack.pop_back_val();
423 uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount);
424 CountMap[S->
getInc()] = IncCount;
429 setCount(ParentCount + BackedgeCount + BC.ContinueCount);
430 CountMap[S->
getCond()] = CondCount;
432 setCount(BC.BreakCount + CondCount - BodyCount);
433 RecordNextStmtCount =
true;
439 uint64_t ParentCount = CurrentCount;
440 BreakContinueStack.push_back(BreakContinue());
442 uint64_t BodyCount = setCount(PGO.getRegionCount(S));
443 CountMap[S->
getBody()] = BodyCount;
445 uint64_t BackedgeCount = CurrentCount;
446 BreakContinue BC = BreakContinueStack.pop_back_val();
448 setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount -
450 RecordNextStmtCount =
true;
457 BreakContinueStack.push_back(BreakContinue());
460 BreakContinue BC = BreakContinueStack.pop_back_val();
461 if (!BreakContinueStack.empty())
462 BreakContinueStack.back().ContinueCount += BC.ContinueCount;
464 setCount(PGO.getRegionCount(S));
465 RecordNextStmtCount =
true;
469 RecordNextStmtCount =
false;
473 uint64_t CaseCount = PGO.getRegionCount(S);
474 setCount(CurrentCount + CaseCount);
477 CountMap[
S] = CaseCount;
478 RecordNextStmtCount =
true;
482 void VisitIfStmt(
const IfStmt *S) {
484 uint64_t ParentCount = CurrentCount;
489 uint64_t ThenCount = setCount(PGO.getRegionCount(S));
490 CountMap[S->
getThen()] = ThenCount;
492 uint64_t OutCount = CurrentCount;
494 uint64_t ElseCount = ParentCount - ThenCount;
497 CountMap[S->
getElse()] = ElseCount;
499 OutCount += CurrentCount;
501 OutCount += ElseCount;
503 RecordNextStmtCount =
true;
512 setCount(PGO.getRegionCount(S));
513 RecordNextStmtCount =
true;
517 RecordNextStmtCount =
false;
519 uint64_t CatchCount = setCount(PGO.getRegionCount(S));
520 CountMap[
S] = CatchCount;
526 uint64_t ParentCount = CurrentCount;
531 uint64_t TrueCount = setCount(PGO.getRegionCount(E));
534 uint64_t OutCount = CurrentCount;
536 uint64_t FalseCount = setCount(ParentCount - TrueCount);
539 OutCount += CurrentCount;
542 RecordNextStmtCount =
true;
547 uint64_t ParentCount = CurrentCount;
550 uint64_t RHSCount = setCount(PGO.getRegionCount(E));
551 CountMap[E->
getRHS()] = RHSCount;
553 setCount(ParentCount + RHSCount - CurrentCount);
554 RecordNextStmtCount =
true;
559 uint64_t ParentCount = CurrentCount;
562 uint64_t RHSCount = setCount(PGO.getRegionCount(E));
563 CountMap[E->
getRHS()] = RHSCount;
565 setCount(ParentCount + RHSCount - CurrentCount);
566 RecordNextStmtCount =
true;
571 void PGOHash::combine(HashType
Type) {
573 assert(Type &&
"Hash is invalid: unexpected type 0");
574 assert(
unsigned(Type) < TooBig &&
"Hash is invalid: too many types");
578 using namespace llvm::support;
579 uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working);
580 MD5.update(llvm::makeArrayRef((uint8_t *)&Swapped,
sizeof(Swapped)));
586 Working = Working << NumBitsPerType | Type;
589 uint64_t PGOHash::finalize() {
591 if (
Count <= NumTypesPerWord)
602 llvm::MD5::MD5Result
Result;
604 using namespace llvm::support;
605 return endian::read<uint64_t, little, unaligned>(Result);
610 bool InstrumentRegions = CGM.
getCodeGenOpts().ProfileInstrGenerate;
611 llvm::IndexedInstrProfReader *PGOReader = CGM.
getPGOReader();
612 if (!InstrumentRegions && !PGOReader)
620 ((isa<CXXConstructorDecl>(GD.
getDecl()) &&
622 (isa<CXXDestructorDecl>(GD.
getDecl()) &&
629 mapRegionCounters(D);
631 emitCounterRegionMapping(D);
635 computeRegionCounts(D);
636 applyFunctionAttributes(PGOReader, Fn);
640 void CodeGenPGO::mapRegionCounters(
const Decl *D) {
641 RegionCounterMap.reset(
new llvm::DenseMap<const Stmt *, unsigned>);
642 MapRegionCounters Walker(*RegionCounterMap);
643 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
644 Walker.TraverseDecl(const_cast<FunctionDecl *>(FD));
645 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
646 Walker.TraverseDecl(const_cast<ObjCMethodDecl *>(MD));
647 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
648 Walker.TraverseDecl(const_cast<BlockDecl *>(BD));
649 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
650 Walker.TraverseDecl(const_cast<CapturedDecl *>(CD));
651 assert(Walker.NextCounter > 0 &&
"no entry counter mapped for decl");
652 NumRegionCounters = Walker.NextCounter;
653 FunctionHash = Walker.Hash.finalize();
656 void CodeGenPGO::emitCounterRegionMapping(
const Decl *D) {
657 if (SkipCoverageMapping)
660 auto Loc = D->
getBody()->getLocStart();
664 std::string CoverageMapping;
665 llvm::raw_string_ostream OS(CoverageMapping);
672 if (CoverageMapping.empty())
676 FuncNameVar, FuncName, FunctionHash, CoverageMapping);
681 llvm::GlobalValue::LinkageTypes Linkage) {
682 if (SkipCoverageMapping)
685 auto Loc = D->
getBody()->getLocStart();
689 std::string CoverageMapping;
690 llvm::raw_string_ostream OS(CoverageMapping);
697 if (CoverageMapping.empty())
700 setFuncName(Name, Linkage);
702 FuncNameVar, FuncName, FunctionHash, CoverageMapping,
false);
705 void CodeGenPGO::computeRegionCounts(
const Decl *D) {
706 StmtCountMap.reset(
new llvm::DenseMap<const Stmt *, uint64_t>);
707 ComputeRegionCounts Walker(*StmtCountMap, *
this);
708 if (
const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
709 Walker.VisitFunctionDecl(FD);
710 else if (
const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D))
711 Walker.VisitObjCMethodDecl(MD);
712 else if (
const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D))
713 Walker.VisitBlockDecl(BD);
714 else if (
const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D))
715 Walker.VisitCapturedDecl(const_cast<CapturedDecl *>(CD));
719 CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
720 llvm::Function *Fn) {
725 Fn->setEntryCount(FunctionCount);
729 if (!CGM.
getCodeGenOpts().ProfileInstrGenerate || !RegionCounterMap)
731 if (!Builder.GetInsertBlock())
734 unsigned Counter = (*RegionCounterMap)[
S];
736 Builder.CreateCall(CGM.
getIntrinsic(llvm::Intrinsic::instrprof_increment),
737 {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
738 Builder.getInt64(FunctionHash),
739 Builder.getInt32(NumRegionCounters),
740 Builder.getInt32(Counter)});
743 void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
746 RegionCounts.clear();
747 if (std::error_code EC =
748 PGOReader->getFunctionCounts(FuncName, FunctionHash, RegionCounts)) {
749 if (EC == llvm::instrprof_error::unknown_function)
751 else if (EC == llvm::instrprof_error::hash_mismatch)
753 else if (EC == llvm::instrprof_error::malformed)
756 RegionCounts.clear();
765 return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1;
778 assert(Scale &&
"scale by 0?");
779 uint64_t Scaled = Weight / Scale + 1;
780 assert(Scaled <= UINT32_MAX &&
"overflow 32-bits");
784 llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount,
785 uint64_t FalseCount) {
787 if (!TrueCount && !FalseCount)
801 if (Weights.size() < 2)
805 uint64_t MaxWeight = *std::max_element(Weights.begin(), Weights.end());
813 ScaledWeights.reserve(Weights.size());
814 for (uint64_t W : Weights)
818 return MDHelper.createBranchWeights(ScaledWeights);
821 llvm::MDNode *CodeGenFunction::createProfileWeightsForLoop(
const Stmt *Cond,
822 uint64_t LoopCount) {
826 assert(CondCount.hasValue() &&
"missing expected loop condition count");
829 return createProfileWeights(LoopCount,
830 std::max(*CondCount, LoopCount) - LoopCount);
Optional< uint64_t > getStmtCount(const Stmt *S)
Check if an execution count is known for a given statement.
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
CXXCtorType getCtorType() const
ConstStmtVisitor - This class implements a simple visitor for Stmt subclasses.
llvm::Module & getModule() const
static uint64_t calculateWeightScale(uint64_t MaxWeight)
Calculate what to divide by to scale weights.
llvm::LLVMContext & getLLVMContext()
CXXCatchStmt * getHandler(unsigned i)
IfStmt - This represents an if/then/else.
void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S)
Organizes the per-function state that is used while generating code coverage mapping data...
Decl - This represents one declaration (or definition), e.g.
The base class of the type hierarchy.
const Stmt * getElse() const
ObjCMethodDecl - Represents an instance or class method declaration.
llvm::ImmutableMap< CountKey, unsigned > CountMap
A C++ throw-expression (C++ [except.throw]).
Linkage
Describes the different kinds of linkage (C++ [basic.link], C99 6.2.2) that an entity may have...
LabelStmt - Represents a label, which has a substatement.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
const Decl * getDecl() const
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
ForStmt - This represents a 'for (init;cond;inc)' stmt.
Expr * getTrueExpr() const
bool isImplicit() const
isImplicit - Indicates whether the declaration was implicitly generated by the implementation.
Stmt * getHandlerBlock() const
A builtin binary operation expression such as "x + y" or "x <= y".
CXXForRangeStmt - This represents C++0x [stmt.ranged]'s ranged for statement, represented as 'for (ra...
void addMissing(bool MainFile)
Record that a function we've visited has no profile data.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
A C++ lambda expression, which produces a function object (of unspecified type) that can be invoked l...
A class that does preorder depth-first traversal on the entire Clang AST and visits each node...
This represents the body of a CapturedStmt, and serves as its DeclContext.
detail::InMemoryDirectory::const_iterator I
InstrProfStats & getPGOStats()
ConditionalOperator - The ?: ternary operator.
const TargetInfo & getTarget() const
bool hasConstructorVariants() const
Does this ABI have different entrypoints for complete-object and base-subobject constructors?
BlockDecl - This represents a block literal declaration, which is like an unnamed FunctionDecl...
CXXDtorType getDtorType() const
BlockExpr - Adaptor class for mixing a BlockDecl with expressions.
bool haveRegionCounts() const
Whether or not we have PGO region data for the current function.
ASTContext & getContext() const
CXXTryStmt - A C++ try block, including all handlers.
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
void addMismatched(bool MainFile)
Record that a function we've visited has mismatched profile data.
ReturnStmt - This represents a return, optionally of an expression: return; return 4;...
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=None)
The result type of a method or function.
GlobalDecl - represents a global declaration.
DoStmt - This represents a 'do/while' stmt.
void addVisited(bool MainFile)
Record that we've visited a function and whether or not that function was in the main source file...
Stmt * getBody(const FunctionDecl *&Definition) const
getBody - Retrieve the body (definition) of the function.
bool isInMainFile(SourceLocation Loc) const
Returns whether the PresumedLoc for a given SourceLocation is in the main file.
This captures a statement into a function.
const Expr * getCond() const
Stmt * getBody() const override
Retrieve the body of this method, if it has one.
virtual Stmt * getBody() const
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
const CodeGenOptions & getCodeGenOpts() const
const LangOptions & getLangOpts() const
const Expr * getSubExpr() const
const Stmt * getBody() const
unsigned getNumHandlers() const
detail::InMemoryDirectory::const_iterator E
const Expr * getRetValue() const
const Stmt * getThen() const
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, llvm::GlobalValue::LinkageTypes Linkage)
Emit a coverage mapping range with a counter zero for an unused declaration.
SwitchStmt - This represents a 'switch' stmt.
Expr * getFalseExpr() const
DeclStmt * getBeginEndStmt()
Represents Objective-C's collection statement.
AbstractConditionalOperator - An abstract base class for ConditionalOperator and BinaryConditionalOpe...
SourceManager & getSourceManager()
DeclStmt * getRangeStmt()
GotoStmt - This represents a direct goto.
static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale)
Scale an individual branch weight (and add 1).
void emitCounterMapping(const Decl *D, llvm::raw_ostream &OS)
Emit the coverage mapping data which maps the regions of code to counters that will be used to find t...
BoundNodesTreeBuilder *const Builder
TargetCXXABI getCXXABI() const
Get the C++ ABI currently in use.
ContinueStmt - This represents a continue.
std::string MainFileName
The user provided name for the "main file", if non-empty.
BinaryConditionalOperator - The GNU extension to the conditional operator which allows the middle ope...
CXXCatchStmt - This represents a C++ catch block.
WhileStmt - This represents a 'while' stmt.
const Expr * getCond() const
void ClearUnusedCoverageMapping(const Decl *D)
Remove the deferred empty coverage mapping as this declaration is actually instrumented.
CompoundStmt * getTryBlock()
void addFunctionMappingRecord(llvm::GlobalVariable *FunctionName, StringRef FunctionNameValue, uint64_t FunctionHash, const std::string &CoverageMapping, bool isUsed=true)
Add a function's coverage mapping record to the collection of the function mapping records...
uint64_t getRegionCount(const Stmt *S)
Return the region count for the counter at the given index.
BreakStmt - This represents a break.
CoverageMappingModuleGen * getCoverageMapping() const
DeclStmt * getLoopVarStmt()
SourceLocation getLocation() const
llvm::IndexedInstrProfReader * getPGOReader() const
void emitEmptyMapping(const Decl *D, llvm::raw_ostream &OS)
Emit the coverage mapping data for an unused function.
This class handles loading and caching of source files into memory.