15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
29 #include "llvm/ADT/ArrayRef.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/IntrusiveRefCntPtr.h"
32 #include "llvm/ADT/SmallPtrSet.h"
33 #include "llvm/ADT/SmallVector.h"
34 #include "llvm/ADT/TinyPtrVector.h"
35 #include "llvm/Support/Allocator.h"
40 template<
unsigned InternalLen>
class SmallString;
46 class ExternalPreprocessorSource;
50 class PragmaNamespace;
56 class CodeCompletionHandler;
57 class DirectoryLookup;
58 class PreprocessingRecord;
60 class PreprocessorOptions;
70 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
71 assert(Kind != tok::identifier &&
72 "Identifiers should be created by TokenValue(IdentifierInfo *)");
103 std::unique_ptr<ScratchBuffer> ScratchBuf;
113 std::unique_ptr<PTHManager> PTH;
117 llvm::BumpPtrAllocator BP;
143 unsigned CounterValue;
147 MaxAllowedIncludeStackDepth = 200
151 bool KeepComments : 1;
152 bool KeepMacroComments : 1;
153 bool SuppressIncludeNotFoundError : 1;
156 bool InMacroArgs : 1;
159 bool OwnsHeaderSearch : 1;
162 bool DisableMacroExpansion : 1;
166 bool MacroExpansionInDirectivesOverride : 1;
171 mutable bool ReadMacrosFromExternalSource : 1;
174 bool PragmasEnabled : 1;
177 bool PreprocessedOutput : 1;
180 bool ParsingIfOrElifDirective;
183 bool InMacroArgPreExpansion;
205 std::unique_ptr<PragmaNamespace> PragmaHandlers;
209 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
213 std::vector<CommentHandler *> CommentHandlers;
217 bool IncrementalProcessing;
229 unsigned CodeCompletionOffset;
253 bool ModuleImportExpectsIdentifier;
264 bool CodeCompletionReached;
276 std::pair<int, bool> SkipMainFilePreamble;
282 std::unique_ptr<Lexer> CurLexer;
288 std::unique_ptr<PTHLexer> CurPTHLexer;
306 std::unique_ptr<TokenLexer> CurTokenLexer;
314 CLK_LexAfterModuleImport
324 struct IncludeStackInfo {
325 enum CurLexerKind CurLexerKind;
327 std::unique_ptr<Lexer> TheLexer;
328 std::unique_ptr<PTHLexer> ThePTHLexer;
330 std::unique_ptr<TokenLexer> TheTokenLexer;
335 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
336 std::unique_ptr<Lexer> &&TheLexer,
337 std::unique_ptr<PTHLexer> &&ThePTHLexer,
339 std::unique_ptr<TokenLexer> &&TheTokenLexer,
341 : CurLexerKind(std::move(CurLexerKind)),
342 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
343 ThePTHLexer(std::move(ThePTHLexer)),
344 ThePPLexer(std::move(ThePPLexer)),
345 TheTokenLexer(std::move(TheTokenLexer)),
346 TheDirLookup(std::move(TheDirLookup)) {}
347 IncludeStackInfo(IncludeStackInfo &&RHS)
348 : CurLexerKind(std::move(RHS.CurLexerKind)),
349 TheSubmodule(std::move(RHS.TheSubmodule)),
350 TheLexer(std::move(RHS.TheLexer)),
351 ThePTHLexer(std::move(RHS.ThePTHLexer)),
352 ThePPLexer(std::move(RHS.ThePPLexer)),
353 TheTokenLexer(std::move(RHS.TheTokenLexer)),
354 TheDirLookup(std::move(RHS.TheDirLookup)) {}
356 std::vector<IncludeStackInfo> IncludeMacroStack;
360 std::unique_ptr<PPCallbacks> Callbacks;
362 struct MacroExpandsInfo {
367 : Tok(Tok), MD(MD), Range(Range) { }
372 struct ModuleMacroInfo {
374 : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(
false) {}
379 llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
382 unsigned ActiveModuleMacrosGeneration;
386 llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
391 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
400 !PP.CurSubmoduleState->VisibleModules.getGeneration())
403 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
410 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
411 Info->ActiveModuleMacrosGeneration)
412 PP.updateModuleMacroInfo(II, *Info);
417 MacroState() : MacroState(
nullptr) {}
419 MacroState(MacroState &&O) LLVM_NOEXCEPT :
State(O.State) {
422 MacroState &operator=(MacroState &&O) LLVM_NOEXCEPT {
429 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
430 Info->~ModuleMacroInfo();
434 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
439 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
446 auto *Info = getModuleInfo(PP, II);
447 return Info ? Info->IsAmbiguous :
false;
451 if (
auto *Info = getModuleInfo(PP, II))
452 return Info->ActiveModuleMacros;
459 if (
auto *Latest = getLatest())
460 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
465 if (
auto *Info = getModuleInfo(PP, II)) {
466 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
467 Info->ActiveModuleMacros.begin(),
468 Info->ActiveModuleMacros.end());
469 Info->ActiveModuleMacros.clear();
470 Info->IsAmbiguous =
false;
474 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
475 return Info->OverriddenMacros;
480 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
482 if (Overrides.empty())
488 Info->OverriddenMacros.clear();
489 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
490 Overrides.begin(), Overrides.end());
491 Info->ActiveModuleMacrosGeneration = 0;
500 typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
504 struct SubmoduleState;
507 struct BuildingSubmoduleInfo {
509 SubmoduleState *OuterSubmoduleState)
510 : M(M), ImportLoc(ImportLoc), OuterSubmoduleState(OuterSubmoduleState) {
518 SubmoduleState *OuterSubmoduleState;
520 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
523 struct SubmoduleState {
527 VisibleModuleSet VisibleModules;
531 std::map<Module*, SubmoduleState> Submodules;
534 SubmoduleState NullSubmoduleState;
538 SubmoduleState *CurSubmoduleState;
541 llvm::FoldingSet<ModuleMacro> ModuleMacros;
545 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
557 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
558 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
567 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
570 unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
571 unsigned NumIf, NumElse, NumEndif;
572 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
573 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
574 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
579 std::string Predefines;
586 enum { TokenLexerCacheSize = 8 };
587 unsigned NumCachedTokenLexers;
588 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
597 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
618 CachedTokensTy::size_type CachedLexPos;
625 std::vector<CachedTokensTy::size_type> BacktrackPositions;
627 struct MacroInfoChain {
629 MacroInfoChain *
Next;
634 MacroInfoChain *MIChainHead;
636 struct DeserializedMacroInfoChain {
638 unsigned OwningModuleID;
640 DeserializedMacroInfoChain *
Next;
642 DeserializedMacroInfoChain *DeserialMIChainHead;
645 Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
646 DiagnosticsEngine &diags, LangOptions &opts,
647 SourceManager &
SM, HeaderSearch &Headers,
648 ModuleLoader &TheModuleLoader,
649 IdentifierInfoLookup *IILookup =
nullptr,
650 bool OwnsHeaderSearch =
false,
696 ExternalSource = Source;
700 return ExternalSource;
712 return ParsingIfOrElifDirective;
717 this->KeepComments = KeepComments | KeepMacroComments;
718 this->KeepMacroComments = KeepMacroComments;
727 SuppressIncludeNotFoundError = Suppress;
731 return SuppressIncludeNotFoundError;
737 PreprocessedOutput = IsPreprocessedOutput;
746 return CurPPLexer == L;
775 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
776 std::move(Callbacks));
777 Callbacks = std::move(C);
795 auto I = Submodules.find(M);
796 if (I == Submodules.end())
798 auto J = I->second.Macros.find(II);
799 if (J == I->second.Macros.end())
801 auto *MD = J->second.getLatest();
802 return MD && MD->isDefined();
809 MacroState &
S = CurSubmoduleState->Macros[II];
810 auto *MD = S.getLatest();
811 while (MD && isa<VisibilityMacroDirective>(MD))
812 MD = MD->getPrevious();
814 S.getActiveModuleMacros(*
this, II),
815 S.isAmbiguous(*
this, II));
823 MacroState &
S = CurSubmoduleState->Macros[II];
825 if (
auto *MD = S.getLatest())
829 S.getActiveModuleMacros(*
this, II),
830 S.isAmbiguous(*
this, II));
840 if (!MD || MD->getDefinition().isUndefined())
854 return MD.getMacroInfo();
887 auto I = LeafModuleMacros.find(II);
888 if (I != LeafModuleMacros.end())
900 llvm::iterator_range<macro_iterator>
901 macros(
bool IncludeExternalMacros =
true)
const {
902 return llvm::make_range(
macro_begin(IncludeExternalMacros),
923 return &Identifiers.
get(Name);
958 CodeComplete = &Handler;
968 CodeComplete =
nullptr;
1018 bool DisableMacroExpansion,
bool OwnsTokens);
1060 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1067 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1068 if (AllowMacroExpansion)
1073 AllowMacroExpansion);
1079 const char *DiagnosticTag,
1080 bool AllowMacroExpansion);
1090 while (Result.
getKind() == tok::comment);
1096 bool OldVal = DisableMacroExpansion;
1097 DisableMacroExpansion =
true;
1102 DisableMacroExpansion = OldVal;
1110 while (Result.
getKind() == tok::comment);
1120 DisableMacroExpansion =
true;
1121 MacroExpansionInDirectivesOverride =
true;
1135 return PeekAhead(N+1);
1145 "Should only be called when tokens are cached for backtracking");
1146 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1147 &&
"Should revert tokens up to the last backtrack position, not more");
1148 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1149 "Corrupted backtrack positions ?");
1158 EnterCachingLexMode();
1171 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1173 AnnotatePreviousCachedTokens(Tok);
1179 assert(CachedLexPos != 0);
1192 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1214 IncrementalProcessing = value;
1248 return CodeCompletionFileLoc;
1258 CodeCompletionReached =
true;
1268 return PragmaARCCFCodeAuditedLoc;
1274 PragmaARCCFCodeAuditedLoc = Loc;
1282 return PragmaAssumeNonNullLoc;
1288 PragmaAssumeNonNullLoc = Loc;
1304 SkipMainFilePreamble.first = Bytes;
1305 SkipMainFilePreamble.second = StartOfLine;
1312 return Diags->
Report(Loc, DiagID);
1328 bool *invalid =
nullptr)
const {
1357 bool *Invalid =
nullptr)
const {
1367 bool *Invalid =
nullptr)
const;
1372 bool IgnoreWhiteSpace =
false) {
1380 bool *Invalid =
nullptr)
const {
1381 assert(Tok.
is(tok::numeric_constant) &&
1382 Tok.
getLength() == 1 &&
"Called on unsupported token");
1383 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1464 unsigned Char)
const {
1474 ++NumFastTokenPaste;
1498 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1513 if(II->isPoisoned()) {
1524 *Ident___exception_code,
1525 *Ident_GetExceptionCode;
1528 *Ident___exception_info,
1529 *Ident_GetExceptionInfo;
1532 *Ident___abnormal_termination,
1533 *Ident_AbnormalTermination;
1535 const char *getCurLexerEndPos();
1596 unsigned SubModuleID);
1620 bool SkipCache =
false);
1653 bool *ShadowFlag =
nullptr);
1657 void PushIncludeMacroStack() {
1658 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1659 IncludeMacroStack.emplace_back(
1660 CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
1661 CurPPLexer, std::move(CurTokenLexer), CurDirLookup);
1662 CurPPLexer =
nullptr;
1665 void PopIncludeMacroStack() {
1666 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1667 CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1668 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1669 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1670 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1671 CurSubmodule = IncludeMacroStack.back().TheSubmodule;
1672 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1673 IncludeMacroStack.pop_back();
1676 void PropagateLineStartLeadingSpaceInfo(
Token &
Result);
1678 void EnterSubmodule(Module *M, SourceLocation ImportLoc);
1679 void LeaveSubmodule();
1683 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
1688 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1689 SourceLocation Loc);
1690 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1691 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1705 bool *ShadowFlag =
nullptr);
1711 bool ReadMacroDefinitionArgList(MacroInfo *MI,
Token& LastTok);
1721 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1722 bool FoundNonSkipPortion,
bool FoundElse,
1723 SourceLocation ElseLoc = SourceLocation());
1726 void PTHSkipExcludedConditionalBlock();
1732 bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1736 void RegisterBuiltinPragmas();
1739 void RegisterBuiltinMacros();
1744 bool HandleMacroExpandedIdentifier(
Token &Tok,
const MacroDefinition &MD);
1751 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1752 ArrayRef<Token> tokens);
1753 void removeCachedMacroExpandedTokensOfLastLexer();
1759 bool isNextPPTokenLParen();
1763 MacroArgs *ReadFunctionLikeMacroArgs(
Token &MacroName, MacroInfo *MI,
1764 SourceLocation &ExpansionEnd);
1768 void ExpandBuiltinMacro(
Token &Tok);
1773 void Handle_Pragma(
Token &Tok);
1777 void HandleMicrosoft__pragma(
Token &Tok);
1781 void EnterSourceFileWithLexer(Lexer *TheLexer,
const DirectoryLookup *Dir);
1785 void EnterSourceFileWithPTH(PTHLexer *PL,
const DirectoryLookup *Dir);
1788 void setPredefinesFileID(FileID FID) {
1789 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
1790 PredefinesFileID = FID;
1795 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
1796 return L ? !L->isPragmaLexer() : P !=
nullptr;
1799 static bool IsFileLexer(
const IncludeStackInfo& I) {
1800 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1803 bool IsFileLexer()
const {
1804 return IsFileLexer(CurLexer.get(), CurPPLexer);
1810 bool InCachingLexMode()
const {
1813 return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1814 !IncludeMacroStack.empty();
1816 void EnterCachingLexMode();
1817 void ExitCachingLexMode() {
1818 if (InCachingLexMode())
1821 const Token &PeekAhead(
unsigned N);
1822 void AnnotatePreviousCachedTokens(
const Token &Tok);
1828 void HandleLineDirective(
Token &Tok);
1829 void HandleDigitDirective(
Token &Tok);
1830 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
1831 void HandleIdentSCCSDirective(
Token &Tok);
1832 void HandleMacroPublicDirective(
Token &Tok);
1833 void HandleMacroPrivateDirective(
Token &Tok);
1836 void HandleIncludeDirective(SourceLocation HashLoc,
1838 const DirectoryLookup *LookupFrom =
nullptr,
1839 const FileEntry *LookupFromFile =
nullptr,
1840 bool isImport =
false);
1841 void HandleIncludeNextDirective(SourceLocation HashLoc,
Token &Tok);
1842 void HandleIncludeMacrosDirective(SourceLocation HashLoc,
Token &Tok);
1843 void HandleImportDirective(SourceLocation HashLoc,
Token &Tok);
1844 void HandleMicrosoftImportDirective(
Token &Tok);
1859 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterTopLevelIfndef);
1860 void HandleUndefDirective(
Token &Tok);
1863 void HandleIfdefDirective(
Token &Tok,
bool isIfndef,
1864 bool ReadAnyTokensBeforeDirective);
1865 void HandleIfDirective(
Token &Tok,
bool ReadAnyTokensBeforeDirective);
1866 void HandleEndifDirective(
Token &Tok);
1867 void HandleElseDirective(
Token &Tok);
1868 void HandleElifDirective(
Token &Tok);
1871 void HandlePragmaDirective(SourceLocation IntroducerLoc,
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens. If there are multiple macro...
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to. If the location is in a file ...
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ExternalPreprocessorSource * getExternalSource() const
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Module * getCurrentSubmodule() const
Return the submodule owning the file being lexed.
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
MacroMap::const_iterator macro_iterator
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value. Floating point literals and user defined li...
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
SourceLocation getLastCachedTokenLocation() const
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
This interface provides a way to observe the actions of the preprocessor as it does its thing...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
bool getPragmasEnabled() const
TokenValue(tok::TokenKind Kind)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
const LangOptions & getLangOpts() const
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Describes a module or submodule.
FileManager & getFileManager() const
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
bool getCommentRetentionState() const
const IdentifierTable & getIdentifierTable() const
void LexNonComment(Token &Result)
Lex a token. If it's a comment, keep lexing until we get something not a comment. ...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
HeaderSearch & getHeaderSearchInfo() const
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
bool hadModuleLoaderFatalFailure() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
tok::TokenKind getKind() const
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
const TargetInfo & getTargetInfo() const
void setPreprocessedOutput(bool IsPreprocessedOutput)
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
A little helper class used to produce diagnostics.
void HandlePragmaIncludeAlias(Token &Tok)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
void HandlePragmaOnce(Token &OnceTok)
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Abstract interface for external sources of preprocessor information.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
void EnableBacktrackAtThisPos()
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD. Return true if the token i...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
void HandleMicrosoftCommentPaste(Token &Tok)
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values. Only handles simple toke...
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin. An invalid location ends...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
void HandlePragmaDependency(Token &DependencyTok)
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
Preprocessor(IntrusiveRefCntPtr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
MacroInfo * AllocateDeserializedMacroInfo(SourceLocation L, unsigned SubModuleID)
Allocate a new MacroInfo object loaded from an AST file.
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. An invalid location...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
void EnterTokenStream(const Token *Toks, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens)
Add a "macro" context to the top of the include stack, which will cause the lexer to start returning ...
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
bool is(tok::TokenKind K) const
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Reads an AST files chain containing the contents of a translation unit.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Abstract interface for a module loader.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Defines the Diagnostic-related interfaces.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g. its tokens).
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g. as an operand of #incl...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
void DumpMacro(const MacroInfo &MI) const
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
void HandlePragmaPoison(Token &PoisonTok)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
void Initialize(const TargetInfo &Target)
Initialize the preprocessor using information about the target.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
The translation unit is a complete translation unit.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
bool isPreprocessedOutput() const
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment. In this case there should b...
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
PTHManager * getPTHManager()
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
const DefMacroDirective * getDirective() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const