15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/IntrusiveRefCntPtr.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/TinyPtrVector.h"
34 #include "llvm/Support/Allocator.h"
39 template<
unsigned InternalLen>
class SmallString;
45 class ExternalPreprocessorSource;
49 class PragmaNamespace;
55 class CodeCompletionHandler;
56 class DirectoryLookup;
57 class PreprocessingRecord;
60 class PreprocessorOptions;
70 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
71 assert(Kind != tok::identifier &&
72 "Identifiers should be created by TokenValue(IdentifierInfo *)");
104 std::unique_ptr<ScratchBuffer> ScratchBuf;
114 std::unique_ptr<PTHManager> PTH;
118 llvm::BumpPtrAllocator BP;
144 unsigned CounterValue;
148 MaxAllowedIncludeStackDepth = 200
152 bool KeepComments : 1;
153 bool KeepMacroComments : 1;
154 bool SuppressIncludeNotFoundError : 1;
157 bool InMacroArgs : 1;
160 bool OwnsHeaderSearch : 1;
163 bool DisableMacroExpansion : 1;
167 bool MacroExpansionInDirectivesOverride : 1;
172 mutable bool ReadMacrosFromExternalSource : 1;
175 bool PragmasEnabled : 1;
178 bool PreprocessedOutput : 1;
181 bool ParsingIfOrElifDirective;
184 bool InMacroArgPreExpansion;
206 std::unique_ptr<PragmaNamespace> PragmaHandlers;
210 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
214 std::vector<CommentHandler *> CommentHandlers;
218 bool IncrementalProcessing;
230 unsigned CodeCompletionOffset;
254 bool ModuleImportExpectsIdentifier;
265 bool CodeCompletionReached;
277 std::pair<int, bool> SkipMainFilePreamble;
283 std::unique_ptr<Lexer> CurLexer;
289 std::unique_ptr<PTHLexer> CurPTHLexer;
307 std::unique_ptr<TokenLexer> CurTokenLexer;
315 CLK_LexAfterModuleImport
325 struct IncludeStackInfo {
326 enum CurLexerKind CurLexerKind;
328 std::unique_ptr<Lexer> TheLexer;
329 std::unique_ptr<PTHLexer> ThePTHLexer;
331 std::unique_ptr<TokenLexer> TheTokenLexer;
336 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
337 std::unique_ptr<Lexer> &&TheLexer,
338 std::unique_ptr<PTHLexer> &&ThePTHLexer,
340 std::unique_ptr<TokenLexer> &&TheTokenLexer,
342 : CurLexerKind(std::move(CurLexerKind)),
343 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
344 ThePTHLexer(std::move(ThePTHLexer)),
345 ThePPLexer(std::move(ThePPLexer)),
346 TheTokenLexer(std::move(TheTokenLexer)),
347 TheDirLookup(std::move(TheDirLookup)) {}
348 IncludeStackInfo(IncludeStackInfo &&RHS)
349 : CurLexerKind(std::move(RHS.CurLexerKind)),
350 TheSubmodule(std::move(RHS.TheSubmodule)),
351 TheLexer(std::move(RHS.TheLexer)),
352 ThePTHLexer(std::move(RHS.ThePTHLexer)),
353 ThePPLexer(std::move(RHS.ThePPLexer)),
354 TheTokenLexer(std::move(RHS.TheTokenLexer)),
355 TheDirLookup(std::move(RHS.TheDirLookup)) {}
357 std::vector<IncludeStackInfo> IncludeMacroStack;
361 std::unique_ptr<PPCallbacks> Callbacks;
363 struct MacroExpandsInfo {
368 : Tok(Tok), MD(MD), Range(Range) { }
373 struct ModuleMacroInfo {
375 : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(
false) {}
380 llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
383 unsigned ActiveModuleMacrosGeneration;
387 llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
392 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
401 !PP.CurSubmoduleState->VisibleModules.getGeneration())
404 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
411 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
412 Info->ActiveModuleMacrosGeneration)
413 PP.updateModuleMacroInfo(II, *Info);
418 MacroState() : MacroState(
nullptr) {}
420 MacroState(MacroState &&O) LLVM_NOEXCEPT :
State(O.State) {
423 MacroState &operator=(MacroState &&O) LLVM_NOEXCEPT {
430 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
431 Info->~ModuleMacroInfo();
435 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
440 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
447 auto *Info = getModuleInfo(PP, II);
448 return Info ? Info->IsAmbiguous :
false;
452 if (
auto *Info = getModuleInfo(PP, II))
453 return Info->ActiveModuleMacros;
460 if (
auto *Latest = getLatest())
461 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
466 if (
auto *Info = getModuleInfo(PP, II)) {
467 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
468 Info->ActiveModuleMacros.begin(),
469 Info->ActiveModuleMacros.end());
470 Info->ActiveModuleMacros.clear();
471 Info->IsAmbiguous =
false;
475 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
476 return Info->OverriddenMacros;
481 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
483 if (Overrides.empty())
489 Info->OverriddenMacros.clear();
490 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
491 Overrides.begin(), Overrides.end());
492 Info->ActiveModuleMacrosGeneration = 0;
501 typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
505 struct SubmoduleState;
508 struct BuildingSubmoduleInfo {
510 SubmoduleState *OuterSubmoduleState)
511 : M(M), ImportLoc(ImportLoc), OuterSubmoduleState(OuterSubmoduleState) {
519 SubmoduleState *OuterSubmoduleState;
521 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
524 struct SubmoduleState {
528 VisibleModuleSet VisibleModules;
532 std::map<Module*, SubmoduleState> Submodules;
535 SubmoduleState NullSubmoduleState;
539 SubmoduleState *CurSubmoduleState;
542 llvm::FoldingSet<ModuleMacro> ModuleMacros;
546 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
558 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
559 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
568 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
571 unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
572 unsigned NumIf, NumElse, NumEndif;
573 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
574 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
575 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
580 std::string Predefines;
587 enum { TokenLexerCacheSize = 8 };
588 unsigned NumCachedTokenLexers;
589 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
598 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
619 CachedTokensTy::size_type CachedLexPos;
626 std::vector<CachedTokensTy::size_type> BacktrackPositions;
628 struct MacroInfoChain {
630 MacroInfoChain *
Next;
635 MacroInfoChain *MIChainHead;
637 struct DeserializedMacroInfoChain {
639 unsigned OwningModuleID;
641 DeserializedMacroInfoChain *
Next;
643 DeserializedMacroInfoChain *DeserialMIChainHead;
646 Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
647 DiagnosticsEngine &diags, LangOptions &opts,
648 SourceManager &
SM, HeaderSearch &Headers,
649 ModuleLoader &TheModuleLoader,
650 IdentifierInfoLookup *IILookup =
nullptr,
651 bool OwnsHeaderSearch =
false,
663 const TargetInfo *AuxTarget =
nullptr);
701 ExternalSource = Source;
705 return ExternalSource;
717 return ParsingIfOrElifDirective;
722 this->KeepComments = KeepComments | KeepMacroComments;
723 this->KeepMacroComments = KeepMacroComments;
732 SuppressIncludeNotFoundError = Suppress;
736 return SuppressIncludeNotFoundError;
742 PreprocessedOutput = IsPreprocessedOutput;
751 return CurPPLexer == L;
780 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
781 std::move(Callbacks));
782 Callbacks = std::move(C);
800 auto I = Submodules.find(M);
801 if (
I == Submodules.end())
803 auto J =
I->second.Macros.find(II);
804 if (J ==
I->second.Macros.end())
806 auto *MD = J->second.getLatest();
807 return MD && MD->isDefined();
814 MacroState &
S = CurSubmoduleState->Macros[II];
815 auto *MD = S.getLatest();
816 while (MD && isa<VisibilityMacroDirective>(MD))
817 MD = MD->getPrevious();
819 S.getActiveModuleMacros(*
this, II),
820 S.isAmbiguous(*
this, II));
828 MacroState &
S = CurSubmoduleState->Macros[II];
830 if (
auto *MD = S.getLatest())
834 S.getActiveModuleMacros(*
this, II),
835 S.isAmbiguous(*
this, II));
845 if (!MD || MD->getDefinition().isUndefined())
859 return MD.getMacroInfo();
892 auto I = LeafModuleMacros.find(II);
893 if (
I != LeafModuleMacros.end())
905 llvm::iterator_range<macro_iterator>
906 macros(
bool IncludeExternalMacros =
true)
const {
907 return llvm::make_range(
macro_begin(IncludeExternalMacros),
928 return &Identifiers.
get(Name);
963 CodeComplete = &Handler;
973 CodeComplete =
nullptr;
1023 bool DisableMacroExpansion,
bool OwnsTokens);
1065 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1072 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1073 if (AllowMacroExpansion)
1078 AllowMacroExpansion);
1084 const char *DiagnosticTag,
1085 bool AllowMacroExpansion);
1095 while (Result.
getKind() == tok::comment);
1101 bool OldVal = DisableMacroExpansion;
1102 DisableMacroExpansion =
true;
1107 DisableMacroExpansion = OldVal;
1115 while (Result.
getKind() == tok::comment);
1125 DisableMacroExpansion =
true;
1126 MacroExpansionInDirectivesOverride =
true;
1140 return PeekAhead(N+1);
1150 "Should only be called when tokens are cached for backtracking");
1151 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1152 &&
"Should revert tokens up to the last backtrack position, not more");
1153 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1154 "Corrupted backtrack positions ?");
1163 EnterCachingLexMode();
1176 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1178 AnnotatePreviousCachedTokens(Tok);
1184 assert(CachedLexPos != 0);
1197 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1219 IncrementalProcessing = value;
1253 return CodeCompletionFileLoc;
1263 CodeCompletionReached =
true;
1273 return PragmaARCCFCodeAuditedLoc;
1279 PragmaARCCFCodeAuditedLoc = Loc;
1287 return PragmaAssumeNonNullLoc;
1293 PragmaAssumeNonNullLoc = Loc;
1309 SkipMainFilePreamble.first = Bytes;
1310 SkipMainFilePreamble.second = StartOfLine;
1317 return Diags->
Report(Loc, DiagID);
1333 bool *invalid =
nullptr)
const {
1362 bool *Invalid =
nullptr)
const {
1372 bool *Invalid =
nullptr)
const;
1377 bool IgnoreWhiteSpace =
false) {
1385 bool *Invalid =
nullptr)
const {
1386 assert(Tok.
is(tok::numeric_constant) &&
1387 Tok.
getLength() == 1 &&
"Called on unsupported token");
1388 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1469 unsigned Char)
const {
1479 ++NumFastTokenPaste;
1503 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1518 if(II->isPoisoned()) {
1529 *Ident___exception_code,
1530 *Ident_GetExceptionCode;
1533 *Ident___exception_info,
1534 *Ident_GetExceptionInfo;
1537 *Ident___abnormal_termination,
1538 *Ident_AbnormalTermination;
1540 const char *getCurLexerEndPos();
1601 unsigned SubModuleID);
1625 bool SkipCache =
false);
1658 bool *ShadowFlag =
nullptr);
1662 void PushIncludeMacroStack() {
1663 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1664 IncludeMacroStack.emplace_back(
1665 CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
1666 CurPPLexer, std::move(CurTokenLexer), CurDirLookup);
1667 CurPPLexer =
nullptr;
1670 void PopIncludeMacroStack() {
1671 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1672 CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1673 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1674 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1675 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1676 CurSubmodule = IncludeMacroStack.back().TheSubmodule;
1677 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1678 IncludeMacroStack.pop_back();
1681 void PropagateLineStartLeadingSpaceInfo(
Token &
Result);
1683 void EnterSubmodule(Module *M, SourceLocation ImportLoc);
1684 void LeaveSubmodule();
1688 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
1693 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1694 SourceLocation Loc);
1695 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1696 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1710 bool *ShadowFlag =
nullptr);
1716 bool ReadMacroDefinitionArgList(MacroInfo *MI,
Token& LastTok);
1726 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1727 bool FoundNonSkipPortion,
bool FoundElse,
1728 SourceLocation ElseLoc = SourceLocation());
1731 void PTHSkipExcludedConditionalBlock();
1737 bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1741 void RegisterBuiltinPragmas();
1744 void RegisterBuiltinMacros();
1749 bool HandleMacroExpandedIdentifier(
Token &Tok,
const MacroDefinition &MD);
1756 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1757 ArrayRef<Token> tokens);
1758 void removeCachedMacroExpandedTokensOfLastLexer();
1764 bool isNextPPTokenLParen();
1768 MacroArgs *ReadFunctionLikeMacroArgs(
Token &MacroName, MacroInfo *MI,
1769 SourceLocation &ExpansionEnd);
1773 void ExpandBuiltinMacro(
Token &Tok);
1778 void Handle_Pragma(
Token &Tok);
1782 void HandleMicrosoft__pragma(
Token &Tok);
1786 void EnterSourceFileWithLexer(Lexer *TheLexer,
const DirectoryLookup *Dir);
1790 void EnterSourceFileWithPTH(PTHLexer *PL,
const DirectoryLookup *Dir);
1793 void setPredefinesFileID(FileID FID) {
1794 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
1795 PredefinesFileID = FID;
1800 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
1801 return L ? !L->isPragmaLexer() : P !=
nullptr;
1804 static bool IsFileLexer(
const IncludeStackInfo&
I) {
1805 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1808 bool IsFileLexer()
const {
1809 return IsFileLexer(CurLexer.get(), CurPPLexer);
1815 bool InCachingLexMode()
const {
1818 return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1819 !IncludeMacroStack.empty();
1821 void EnterCachingLexMode();
1822 void ExitCachingLexMode() {
1823 if (InCachingLexMode())
1826 const Token &PeekAhead(
unsigned N);
1827 void AnnotatePreviousCachedTokens(
const Token &Tok);
1833 void HandleLineDirective(
Token &Tok);
1834 void HandleDigitDirective(
Token &Tok);
1835 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
1836 void HandleIdentSCCSDirective(
Token &Tok);
1837 void HandleMacroPublicDirective(
Token &Tok);
1838 void HandleMacroPrivateDirective(
Token &Tok);
1841 void HandleIncludeDirective(SourceLocation HashLoc,
1843 const DirectoryLookup *LookupFrom =
nullptr,
1844 const FileEntry *LookupFromFile =
nullptr,
1845 bool isImport =
false);
1846 void HandleIncludeNextDirective(SourceLocation HashLoc,
Token &Tok);
1847 void HandleIncludeMacrosDirective(SourceLocation HashLoc,
Token &Tok);
1848 void HandleImportDirective(SourceLocation HashLoc,
Token &Tok);
1849 void HandleMicrosoftImportDirective(
Token &Tok);
1864 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterTopLevelIfndef);
1865 void HandleUndefDirective(
Token &Tok);
1868 void HandleIfdefDirective(
Token &Tok,
bool isIfndef,
1869 bool ReadAnyTokensBeforeDirective);
1870 void HandleIfDirective(
Token &Tok,
bool ReadAnyTokensBeforeDirective);
1871 void HandleEndifDirective(
Token &Tok);
1872 void HandleElseDirective(
Token &Tok);
1873 void HandleElifDirective(
Token &Tok);
1876 void HandlePragmaDirective(SourceLocation IntroducerLoc,
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer...
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ExternalPreprocessorSource * getExternalSource() const
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Module * getCurrentSubmodule() const
Return the submodule owning the file being lexed.
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
std::unique_ptr< llvm::MemoryBuffer > Buffer
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
MacroMap::const_iterator macro_iterator
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
This interface provides a way to observe the actions of the preprocessor as it does its thing...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool getPragmasEnabled() const
TokenValue(tok::TokenKind Kind)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Describes a module or submodule.
FileManager & getFileManager() const
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
bool getCommentRetentionState() const
const IdentifierTable & getIdentifierTable() const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
void LexNonComment(Token &Result)
Lex a token.
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
HeaderSearch & getHeaderSearchInfo() const
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
bool hadModuleLoaderFatalFailure() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
tok::TokenKind getKind() const
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
const TargetInfo & getTargetInfo() const
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token...
detail::InMemoryDirectory::const_iterator I
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
A little helper class used to produce diagnostics.
void HandlePragmaIncludeAlias(Token &Tok)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once.
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Abstract interface for external sources of preprocessor information.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
Preprocessor(IntrusiveRefCntPtr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
MacroInfo * AllocateDeserializedMacroInfo(SourceLocation L, unsigned SubModuleID)
Allocate a new MacroInfo object loaded from an AST file.
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
void EnterTokenStream(const Token *Toks, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens)
Add a "macro" context to the top of the include stack, which will cause the lexer to start returning ...
const TargetInfo * getAuxTargetInfo() const
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Reads an AST files chain containing the contents of a translation unit.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Abstract interface for a module loader.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Defines the Diagnostic-related interfaces.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
void DumpMacro(const MacroInfo &MI) const
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
void HandlePragmaPoison(Token &PoisonTok)
HandlePragmaPoison - Handle #pragma GCC poison.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
The translation unit is a complete translation unit.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
PTHManager * getPTHManager()
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
const DefMacroDirective * getDirective() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const