clang  3.7.0
Preprocessor.h
Go to the documentation of this file.
1 //===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Defines the clang::Preprocessor interface.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 
18 #include "clang/Basic/Builtins.h"
19 #include "clang/Basic/Diagnostic.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/ModuleMap.h"
25 #include "clang/Lex/PPCallbacks.h"
26 #include "clang/Lex/PTHLexer.h"
27 #include "clang/Lex/PTHManager.h"
28 #include "clang/Lex/TokenLexer.h"
29 #include "llvm/ADT/ArrayRef.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/IntrusiveRefCntPtr.h"
32 #include "llvm/ADT/SmallPtrSet.h"
33 #include "llvm/ADT/SmallVector.h"
34 #include "llvm/ADT/TinyPtrVector.h"
35 #include "llvm/Support/Allocator.h"
36 #include <memory>
37 #include <vector>
38 
39 namespace llvm {
40  template<unsigned InternalLen> class SmallString;
41 }
42 
43 namespace clang {
44 
45 class SourceManager;
46 class ExternalPreprocessorSource;
47 class FileManager;
48 class FileEntry;
49 class HeaderSearch;
50 class PragmaNamespace;
51 class PragmaHandler;
52 class CommentHandler;
53 class ScratchBuffer;
54 class TargetInfo;
55 class PPCallbacks;
56 class CodeCompletionHandler;
57 class DirectoryLookup;
58 class PreprocessingRecord;
59 class ModuleLoader;
60 class PreprocessorOptions;
61 
62 /// \brief Stores token information for comparing actual tokens with
63 /// predefined values. Only handles simple tokens and identifiers.
64 class TokenValue {
66  IdentifierInfo *II;
67 
68 public:
69  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
70  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
71  assert(Kind != tok::identifier &&
72  "Identifiers should be created by TokenValue(IdentifierInfo *)");
73  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
74  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
75  }
76  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
77  bool operator==(const Token &Tok) const {
78  return Tok.getKind() == Kind &&
79  (!II || II == Tok.getIdentifierInfo());
80  }
81 };
82 
83 /// \brief Context in which macro name is used.
84 enum MacroUse {
85  MU_Other = 0, // other than #define or #undef
86  MU_Define = 1, // macro name specified in #define
87  MU_Undef = 2 // macro name specified in #undef
88 };
89 
90 /// \brief Engages in a tight little dance with the lexer to efficiently
91 /// preprocess tokens.
92 ///
93 /// Lexers know only about tokens within a single source file, and don't
94 /// know anything about preprocessor-level issues like the \#include stack,
95 /// token expansion, etc.
96 class Preprocessor : public RefCountedBase<Preprocessor> {
98  DiagnosticsEngine *Diags;
99  LangOptions &LangOpts;
100  const TargetInfo *Target;
101  FileManager &FileMgr;
102  SourceManager &SourceMgr;
103  std::unique_ptr<ScratchBuffer> ScratchBuf;
104  HeaderSearch &HeaderInfo;
105  ModuleLoader &TheModuleLoader;
106 
107  /// \brief External source of macros.
108  ExternalPreprocessorSource *ExternalSource;
109 
110 
111  /// An optional PTHManager object used for getting tokens from
112  /// a token cache rather than lexing the original source file.
113  std::unique_ptr<PTHManager> PTH;
114 
115  /// A BumpPtrAllocator object used to quickly allocate and release
116  /// objects internal to the Preprocessor.
117  llvm::BumpPtrAllocator BP;
118 
119  /// Identifiers for builtin macros and other builtins.
120  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
121  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
122  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
123  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
124  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
125  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
126  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
127  IdentifierInfo *Ident__identifier; // __identifier
128  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
129  IdentifierInfo *Ident__has_feature; // __has_feature
130  IdentifierInfo *Ident__has_extension; // __has_extension
131  IdentifierInfo *Ident__has_builtin; // __has_builtin
132  IdentifierInfo *Ident__has_attribute; // __has_attribute
133  IdentifierInfo *Ident__has_include; // __has_include
134  IdentifierInfo *Ident__has_include_next; // __has_include_next
135  IdentifierInfo *Ident__has_warning; // __has_warning
136  IdentifierInfo *Ident__is_identifier; // __is_identifier
137  IdentifierInfo *Ident__building_module; // __building_module
138  IdentifierInfo *Ident__MODULE__; // __MODULE__
139  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
140  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
141 
142  SourceLocation DATELoc, TIMELoc;
143  unsigned CounterValue; // Next __COUNTER__ value.
144 
145  enum {
146  /// \brief Maximum depth of \#includes.
147  MaxAllowedIncludeStackDepth = 200
148  };
149 
150  // State that is set before the preprocessor begins.
151  bool KeepComments : 1;
152  bool KeepMacroComments : 1;
153  bool SuppressIncludeNotFoundError : 1;
154 
155  // State that changes while the preprocessor runs:
156  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
157 
158  /// Whether the preprocessor owns the header search object.
159  bool OwnsHeaderSearch : 1;
160 
161  /// True if macro expansion is disabled.
162  bool DisableMacroExpansion : 1;
163 
164  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
165  /// when parsing preprocessor directives.
166  bool MacroExpansionInDirectivesOverride : 1;
167 
169 
170  /// \brief Whether we have already loaded macros from the external source.
171  mutable bool ReadMacrosFromExternalSource : 1;
172 
173  /// \brief True if pragmas are enabled.
174  bool PragmasEnabled : 1;
175 
176  /// \brief True if the current build action is a preprocessing action.
177  bool PreprocessedOutput : 1;
178 
179  /// \brief True if we are currently preprocessing a #if or #elif directive
180  bool ParsingIfOrElifDirective;
181 
182  /// \brief True if we are pre-expanding macro arguments.
183  bool InMacroArgPreExpansion;
184 
185  /// \brief Mapping/lookup information for all identifiers in
186  /// the program, including program keywords.
187  mutable IdentifierTable Identifiers;
188 
189  /// \brief This table contains all the selectors in the program.
190  ///
191  /// Unlike IdentifierTable above, this table *isn't* populated by the
192  /// preprocessor. It is declared/expanded here because its role/lifetime is
193  /// conceptually similar to the IdentifierTable. In addition, the current
194  /// control flow (in clang::ParseAST()), make it convenient to put here.
195  ///
196  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
197  /// the lifetime of the preprocessor.
198  SelectorTable Selectors;
199 
200  /// \brief Information about builtins.
202 
203  /// \brief Tracks all of the pragmas that the client registered
204  /// with this preprocessor.
205  std::unique_ptr<PragmaNamespace> PragmaHandlers;
206 
207  /// \brief Pragma handlers of the original source is stored here during the
208  /// parsing of a model file.
209  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
210 
211  /// \brief Tracks all of the comment handlers that the client registered
212  /// with this preprocessor.
213  std::vector<CommentHandler *> CommentHandlers;
214 
215  /// \brief True if we want to ignore EOF token and continue later on (thus
216  /// avoid tearing the Lexer and etc. down).
217  bool IncrementalProcessing;
218 
219  /// The kind of translation unit we are processing.
220  TranslationUnitKind TUKind;
221 
222  /// \brief The code-completion handler.
223  CodeCompletionHandler *CodeComplete;
224 
225  /// \brief The file that we're performing code-completion for, if any.
226  const FileEntry *CodeCompletionFile;
227 
228  /// \brief The offset in file for the code-completion point.
229  unsigned CodeCompletionOffset;
230 
231  /// \brief The location for the code-completion point. This gets instantiated
232  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
233  SourceLocation CodeCompletionLoc;
234 
235  /// \brief The start location for the file of the code-completion point.
236  ///
237  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
238  /// for preprocessing.
239  SourceLocation CodeCompletionFileLoc;
240 
241  /// \brief The source location of the \c import contextual keyword we just
242  /// lexed, if any.
243  SourceLocation ModuleImportLoc;
244 
245  /// \brief The module import path that we're currently processing.
247 
248  /// \brief Whether the last token we lexed was an '@'.
249  bool LastTokenWasAt;
250 
251  /// \brief Whether the module import expects an identifier next. Otherwise,
252  /// it expects a '.' or ';'.
253  bool ModuleImportExpectsIdentifier;
254 
255  /// \brief The source location of the currently-active
256  /// \#pragma clang arc_cf_code_audited begin.
257  SourceLocation PragmaARCCFCodeAuditedLoc;
258 
259  /// \brief The source location of the currently-active
260  /// \#pragma clang assume_nonnull begin.
261  SourceLocation PragmaAssumeNonNullLoc;
262 
263  /// \brief True if we hit the code-completion point.
264  bool CodeCompletionReached;
265 
266  /// \brief The directory that the main file should be considered to occupy,
267  /// if it does not correspond to a real file (as happens when building a
268  /// module).
269  const DirectoryEntry *MainFileDir;
270 
271  /// \brief The number of bytes that we will initially skip when entering the
272  /// main file, along with a flag that indicates whether skipping this number
273  /// of bytes will place the lexer at the start of a line.
274  ///
275  /// This is used when loading a precompiled preamble.
276  std::pair<int, bool> SkipMainFilePreamble;
277 
278  /// \brief The current top of the stack that we're lexing from if
279  /// not expanding a macro and we are lexing directly from source code.
280  ///
281  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
282  std::unique_ptr<Lexer> CurLexer;
283 
284  /// \brief The current top of stack that we're lexing from if
285  /// not expanding from a macro and we are lexing from a PTH cache.
286  ///
287  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
288  std::unique_ptr<PTHLexer> CurPTHLexer;
289 
290  /// \brief The current top of the stack what we're lexing from
291  /// if not expanding a macro.
292  ///
293  /// This is an alias for either CurLexer or CurPTHLexer.
294  PreprocessorLexer *CurPPLexer;
295 
296  /// \brief Used to find the current FileEntry, if CurLexer is non-null
297  /// and if applicable.
298  ///
299  /// This allows us to implement \#include_next and find directory-specific
300  /// properties.
301  const DirectoryLookup *CurDirLookup;
302 
303  /// \brief The current macro we are expanding, if we are expanding a macro.
304  ///
305  /// One of CurLexer and CurTokenLexer must be null.
306  std::unique_ptr<TokenLexer> CurTokenLexer;
307 
308  /// \brief The kind of lexer we're currently working with.
309  enum CurLexerKind {
310  CLK_Lexer,
311  CLK_PTHLexer,
312  CLK_TokenLexer,
313  CLK_CachingLexer,
314  CLK_LexAfterModuleImport
315  } CurLexerKind;
316 
317  /// \brief If the current lexer is for a submodule that is being built, this
318  /// is that submodule.
319  Module *CurSubmodule;
320 
321  /// \brief Keeps track of the stack of files currently
322  /// \#included, and macros currently being expanded from, not counting
323  /// CurLexer/CurTokenLexer.
324  struct IncludeStackInfo {
325  enum CurLexerKind CurLexerKind;
326  Module *TheSubmodule;
327  std::unique_ptr<Lexer> TheLexer;
328  std::unique_ptr<PTHLexer> ThePTHLexer;
329  PreprocessorLexer *ThePPLexer;
330  std::unique_ptr<TokenLexer> TheTokenLexer;
331  const DirectoryLookup *TheDirLookup;
332 
333  // The following constructors are completely useless copies of the default
334  // versions, only needed to pacify MSVC.
335  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
336  std::unique_ptr<Lexer> &&TheLexer,
337  std::unique_ptr<PTHLexer> &&ThePTHLexer,
338  PreprocessorLexer *ThePPLexer,
339  std::unique_ptr<TokenLexer> &&TheTokenLexer,
340  const DirectoryLookup *TheDirLookup)
341  : CurLexerKind(std::move(CurLexerKind)),
342  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
343  ThePTHLexer(std::move(ThePTHLexer)),
344  ThePPLexer(std::move(ThePPLexer)),
345  TheTokenLexer(std::move(TheTokenLexer)),
346  TheDirLookup(std::move(TheDirLookup)) {}
347  IncludeStackInfo(IncludeStackInfo &&RHS)
348  : CurLexerKind(std::move(RHS.CurLexerKind)),
349  TheSubmodule(std::move(RHS.TheSubmodule)),
350  TheLexer(std::move(RHS.TheLexer)),
351  ThePTHLexer(std::move(RHS.ThePTHLexer)),
352  ThePPLexer(std::move(RHS.ThePPLexer)),
353  TheTokenLexer(std::move(RHS.TheTokenLexer)),
354  TheDirLookup(std::move(RHS.TheDirLookup)) {}
355  };
356  std::vector<IncludeStackInfo> IncludeMacroStack;
357 
358  /// \brief Actions invoked when some preprocessor activity is
359  /// encountered (e.g. a file is \#included, etc).
360  std::unique_ptr<PPCallbacks> Callbacks;
361 
362  struct MacroExpandsInfo {
363  Token Tok;
364  MacroDefinition MD;
365  SourceRange Range;
366  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
367  : Tok(Tok), MD(MD), Range(Range) { }
368  };
369  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
370 
371  /// Information about a name that has been used to define a module macro.
372  struct ModuleMacroInfo {
373  ModuleMacroInfo(MacroDirective *MD)
374  : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(false) {}
375 
376  /// The most recent macro directive for this identifier.
377  MacroDirective *MD;
378  /// The active module macros for this identifier.
379  llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
380  /// The generation number at which we last updated ActiveModuleMacros.
381  /// \see Preprocessor::VisibleModules.
382  unsigned ActiveModuleMacrosGeneration;
383  /// Whether this macro name is ambiguous.
384  bool IsAmbiguous;
385  /// The module macros that are overridden by this macro.
386  llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
387  };
388 
389  /// The state of a macro for an identifier.
390  class MacroState {
391  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
392 
393  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
394  const IdentifierInfo *II) const {
395  // FIXME: Find a spare bit on IdentifierInfo and store a
396  // HasModuleMacros flag.
397  if (!II->hasMacroDefinition() ||
398  (!PP.getLangOpts().Modules &&
399  !PP.getLangOpts().ModulesLocalVisibility) ||
400  !PP.CurSubmoduleState->VisibleModules.getGeneration())
401  return nullptr;
402 
403  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
404  if (!Info) {
405  Info = new (PP.getPreprocessorAllocator())
406  ModuleMacroInfo(State.get<MacroDirective *>());
407  State = Info;
408  }
409 
410  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
411  Info->ActiveModuleMacrosGeneration)
412  PP.updateModuleMacroInfo(II, *Info);
413  return Info;
414  }
415 
416  public:
417  MacroState() : MacroState(nullptr) {}
418  MacroState(MacroDirective *MD) : State(MD) {}
419  MacroState(MacroState &&O) LLVM_NOEXCEPT : State(O.State) {
420  O.State = (MacroDirective *)nullptr;
421  }
422  MacroState &operator=(MacroState &&O) LLVM_NOEXCEPT {
423  auto S = O.State;
424  O.State = (MacroDirective *)nullptr;
425  State = S;
426  return *this;
427  }
428  ~MacroState() {
429  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
430  Info->~ModuleMacroInfo();
431  }
432 
433  MacroDirective *getLatest() const {
434  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
435  return Info->MD;
436  return State.get<MacroDirective*>();
437  }
438  void setLatest(MacroDirective *MD) {
439  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
440  Info->MD = MD;
441  else
442  State = MD;
443  }
444 
445  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
446  auto *Info = getModuleInfo(PP, II);
447  return Info ? Info->IsAmbiguous : false;
448  }
450  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
451  if (auto *Info = getModuleInfo(PP, II))
452  return Info->ActiveModuleMacros;
453  return None;
454  }
455 
456  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
457  SourceManager &SourceMgr) const {
458  // FIXME: Incorporate module macros into the result of this.
459  if (auto *Latest = getLatest())
460  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
461  return MacroDirective::DefInfo();
462  }
463 
464  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
465  if (auto *Info = getModuleInfo(PP, II)) {
466  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
467  Info->ActiveModuleMacros.begin(),
468  Info->ActiveModuleMacros.end());
469  Info->ActiveModuleMacros.clear();
470  Info->IsAmbiguous = false;
471  }
472  }
473  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
474  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
475  return Info->OverriddenMacros;
476  return None;
477  }
478  void setOverriddenMacros(Preprocessor &PP,
479  ArrayRef<ModuleMacro *> Overrides) {
480  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
481  if (!Info) {
482  if (Overrides.empty())
483  return;
484  Info = new (PP.getPreprocessorAllocator())
485  ModuleMacroInfo(State.get<MacroDirective *>());
486  State = Info;
487  }
488  Info->OverriddenMacros.clear();
489  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
490  Overrides.begin(), Overrides.end());
491  Info->ActiveModuleMacrosGeneration = 0;
492  }
493  };
494 
495  /// For each IdentifierInfo that was associated with a macro, we
496  /// keep a mapping to the history of all macro definitions and #undefs in
497  /// the reverse order (the latest one is in the head of the list).
498  ///
499  /// This mapping lives within the \p CurSubmoduleState.
500  typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
501 
502  friend class ASTReader;
503 
504  struct SubmoduleState;
505 
506  /// \brief Information about a submodule that we're currently building.
507  struct BuildingSubmoduleInfo {
508  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc,
509  SubmoduleState *OuterSubmoduleState)
510  : M(M), ImportLoc(ImportLoc), OuterSubmoduleState(OuterSubmoduleState) {
511  }
512 
513  /// The module that we are building.
514  Module *M;
515  /// The location at which the module was included.
516  SourceLocation ImportLoc;
517  /// The previous SubmoduleState.
518  SubmoduleState *OuterSubmoduleState;
519  };
520  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
521 
522  /// \brief Information about a submodule's preprocessor state.
523  struct SubmoduleState {
524  /// The macros for the submodule.
525  MacroMap Macros;
526  /// The set of modules that are visible within the submodule.
527  VisibleModuleSet VisibleModules;
528  // FIXME: CounterValue?
529  // FIXME: PragmaPushMacroInfo?
530  };
531  std::map<Module*, SubmoduleState> Submodules;
532 
533  /// The preprocessor state for preprocessing outside of any submodule.
534  SubmoduleState NullSubmoduleState;
535 
536  /// The current submodule state. Will be \p NullSubmoduleState if we're not
537  /// in a submodule.
538  SubmoduleState *CurSubmoduleState;
539 
540  /// The set of known macros exported from modules.
541  llvm::FoldingSet<ModuleMacro> ModuleMacros;
542 
543  /// The list of module macros, for each identifier, that are not overridden by
544  /// any other module macro.
545  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
546  LeafModuleMacros;
547 
548  /// \brief Macros that we want to warn because they are not used at the end
549  /// of the translation unit.
550  ///
551  /// We store just their SourceLocations instead of
552  /// something like MacroInfo*. The benefit of this is that when we are
553  /// deserializing from PCH, we don't need to deserialize identifier & macros
554  /// just so that we can report that they are unused, we just warn using
555  /// the SourceLocations of this set (that will be filled by the ASTReader).
556  /// We are using SmallPtrSet instead of a vector for faster removal.
557  typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
558  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
559 
560  /// \brief A "freelist" of MacroArg objects that can be
561  /// reused for quick allocation.
562  MacroArgs *MacroArgCache;
563  friend class MacroArgs;
564 
565  /// For each IdentifierInfo used in a \#pragma push_macro directive,
566  /// we keep a MacroInfo stack used to restore the previous macro value.
567  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
568 
569  // Various statistics we track for performance analysis.
570  unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
571  unsigned NumIf, NumElse, NumEndif;
572  unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
573  unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
574  unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
575  unsigned NumSkipped;
576 
577  /// \brief The predefined macros that preprocessor should use from the
578  /// command line etc.
579  std::string Predefines;
580 
581  /// \brief The file ID for the preprocessor predefines.
582  FileID PredefinesFileID;
583 
584  /// \{
585  /// \brief Cache of macro expanders to reduce malloc traffic.
586  enum { TokenLexerCacheSize = 8 };
587  unsigned NumCachedTokenLexers;
588  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
589  /// \}
590 
591  /// \brief Keeps macro expanded tokens for TokenLexers.
592  //
593  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
594  /// going to lex in the cache and when it finishes the tokens are removed
595  /// from the end of the cache.
596  SmallVector<Token, 16> MacroExpandedTokens;
597  std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
598 
599  /// \brief A record of the macro definitions and expansions that
600  /// occurred during preprocessing.
601  ///
602  /// This is an optional side structure that can be enabled with
603  /// \c createPreprocessingRecord() prior to preprocessing.
604  PreprocessingRecord *Record;
605 
606  /// Cached tokens state.
607  typedef SmallVector<Token, 1> CachedTokensTy;
608 
609  /// \brief Cached tokens are stored here when we do backtracking or
610  /// lookahead. They are "lexed" by the CachingLex() method.
611  CachedTokensTy CachedTokens;
612 
613  /// \brief The position of the cached token that CachingLex() should
614  /// "lex" next.
615  ///
616  /// If it points beyond the CachedTokens vector, it means that a normal
617  /// Lex() should be invoked.
618  CachedTokensTy::size_type CachedLexPos;
619 
620  /// \brief Stack of backtrack positions, allowing nested backtracks.
621  ///
622  /// The EnableBacktrackAtThisPos() method pushes a position to
623  /// indicate where CachedLexPos should be set when the BackTrack() method is
624  /// invoked (at which point the last position is popped).
625  std::vector<CachedTokensTy::size_type> BacktrackPositions;
626 
627  struct MacroInfoChain {
628  MacroInfo MI;
629  MacroInfoChain *Next;
630  };
631 
632  /// MacroInfos are managed as a chain for easy disposal. This is the head
633  /// of that list.
634  MacroInfoChain *MIChainHead;
635 
636  struct DeserializedMacroInfoChain {
637  MacroInfo MI;
638  unsigned OwningModuleID; // MUST be immediately after the MacroInfo object
639  // so it can be accessed by MacroInfo::getOwningModuleID().
640  DeserializedMacroInfoChain *Next;
641  };
642  DeserializedMacroInfoChain *DeserialMIChainHead;
643 
644 public:
645  Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
646  DiagnosticsEngine &diags, LangOptions &opts,
647  SourceManager &SM, HeaderSearch &Headers,
648  ModuleLoader &TheModuleLoader,
649  IdentifierInfoLookup *IILookup = nullptr,
650  bool OwnsHeaderSearch = false,
652 
653  ~Preprocessor();
654 
655  /// \brief Initialize the preprocessor using information about the target.
656  ///
657  /// \param Target is owned by the caller and must remain valid for the
658  /// lifetime of the preprocessor.
659  void Initialize(const TargetInfo &Target);
660 
661  /// \brief Initialize the preprocessor to parse a model file
662  ///
663  /// To parse model files the preprocessor of the original source is reused to
664  /// preserver the identifier table. However to avoid some duplicate
665  /// information in the preprocessor some cleanup is needed before it is used
666  /// to parse model files. This method does that cleanup.
667  void InitializeForModelFile();
668 
669  /// \brief Cleanup after model file parsing
670  void FinalizeForModelFile();
671 
672  /// \brief Retrieve the preprocessor options used to initialize this
673  /// preprocessor.
674  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
675 
676  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
677  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
678 
679  const LangOptions &getLangOpts() const { return LangOpts; }
680  const TargetInfo &getTargetInfo() const { return *Target; }
681  FileManager &getFileManager() const { return FileMgr; }
682  SourceManager &getSourceManager() const { return SourceMgr; }
683  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
684 
685  IdentifierTable &getIdentifierTable() { return Identifiers; }
686  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
687  SelectorTable &getSelectorTable() { return Selectors; }
688  Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
689  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
690 
691  void setPTHManager(PTHManager* pm);
692 
693  PTHManager *getPTHManager() { return PTH.get(); }
694 
696  ExternalSource = Source;
697  }
698 
700  return ExternalSource;
701  }
702 
703  /// \brief Retrieve the module loader associated with this preprocessor.
704  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
705 
707  return TheModuleLoader.HadFatalFailure;
708  }
709 
710  /// \brief True if we are currently preprocessing a #if or #elif directive
712  return ParsingIfOrElifDirective;
713  }
714 
715  /// \brief Control whether the preprocessor retains comments in output.
716  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
717  this->KeepComments = KeepComments | KeepMacroComments;
718  this->KeepMacroComments = KeepMacroComments;
719  }
720 
721  bool getCommentRetentionState() const { return KeepComments; }
722 
723  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
724  bool getPragmasEnabled() const { return PragmasEnabled; }
725 
726  void SetSuppressIncludeNotFoundError(bool Suppress) {
727  SuppressIncludeNotFoundError = Suppress;
728  }
729 
731  return SuppressIncludeNotFoundError;
732  }
733 
734  /// Sets whether the preprocessor is responsible for producing output or if
735  /// it is producing tokens to be consumed by Parse and Sema.
736  void setPreprocessedOutput(bool IsPreprocessedOutput) {
737  PreprocessedOutput = IsPreprocessedOutput;
738  }
739 
740  /// Returns true if the preprocessor is responsible for generating output,
741  /// false if it is producing tokens to be consumed by Parse and Sema.
742  bool isPreprocessedOutput() const { return PreprocessedOutput; }
743 
744  /// \brief Return true if we are lexing directly from the specified lexer.
745  bool isCurrentLexer(const PreprocessorLexer *L) const {
746  return CurPPLexer == L;
747  }
748 
749  /// \brief Return the current lexer being lexed from.
750  ///
751  /// Note that this ignores any potentially active macro expansions and _Pragma
752  /// expansions going on at the time.
753  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
754 
755  /// \brief Return the current file lexer being lexed from.
756  ///
757  /// Note that this ignores any potentially active macro expansions and _Pragma
758  /// expansions going on at the time.
760 
761  /// \brief Return the submodule owning the file being lexed.
762  Module *getCurrentSubmodule() const { return CurSubmodule; }
763 
764  /// \brief Returns the FileID for the preprocessor predefines.
765  FileID getPredefinesFileID() const { return PredefinesFileID; }
766 
767  /// \{
768  /// \brief Accessors for preprocessor callbacks.
769  ///
770  /// Note that this class takes ownership of any PPCallbacks object given to
771  /// it.
772  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
773  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
774  if (Callbacks)
775  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
776  std::move(Callbacks));
777  Callbacks = std::move(C);
778  }
779  /// \}
780 
781  bool isMacroDefined(StringRef Id) {
782  return isMacroDefined(&Identifiers.get(Id));
783  }
784  bool isMacroDefined(const IdentifierInfo *II) {
785  return II->hasMacroDefinition() &&
786  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
787  }
788 
789  /// \brief Determine whether II is defined as a macro within the module M,
790  /// if that is a module that we've already preprocessed. Does not check for
791  /// macros imported into M.
793  if (!II->hasMacroDefinition())
794  return false;
795  auto I = Submodules.find(M);
796  if (I == Submodules.end())
797  return false;
798  auto J = I->second.Macros.find(II);
799  if (J == I->second.Macros.end())
800  return false;
801  auto *MD = J->second.getLatest();
802  return MD && MD->isDefined();
803  }
804 
806  if (!II->hasMacroDefinition())
807  return MacroDefinition();
808 
809  MacroState &S = CurSubmoduleState->Macros[II];
810  auto *MD = S.getLatest();
811  while (MD && isa<VisibilityMacroDirective>(MD))
812  MD = MD->getPrevious();
813  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
814  S.getActiveModuleMacros(*this, II),
815  S.isAmbiguous(*this, II));
816  }
817 
819  SourceLocation Loc) {
820  if (!II->hadMacroDefinition())
821  return MacroDefinition();
822 
823  MacroState &S = CurSubmoduleState->Macros[II];
825  if (auto *MD = S.getLatest())
826  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
827  // FIXME: Compute the set of active module macros at the specified location.
828  return MacroDefinition(DI.getDirective(),
829  S.getActiveModuleMacros(*this, II),
830  S.isAmbiguous(*this, II));
831  }
832 
833  /// \brief Given an identifier, return its latest non-imported MacroDirective
834  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
836  if (!II->hasMacroDefinition())
837  return nullptr;
838 
839  auto *MD = getLocalMacroDirectiveHistory(II);
840  if (!MD || MD->getDefinition().isUndefined())
841  return nullptr;
842 
843  return MD;
844  }
845 
846  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
847  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
848  }
849 
851  if (!II->hasMacroDefinition())
852  return nullptr;
853  if (auto MD = getMacroDefinition(II))
854  return MD.getMacroInfo();
855  return nullptr;
856  }
857 
858  /// \brief Given an identifier, return the latest non-imported macro
859  /// directive for that identifier.
860  ///
861  /// One can iterate over all previous macro directives from the most recent
862  /// one.
864 
865  /// \brief Add a directive to the macro directive history for this identifier.
868  SourceLocation Loc) {
869  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
870  appendMacroDirective(II, MD);
871  return MD;
872  }
874  MacroInfo *MI) {
875  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
876  }
877  /// \brief Set a MacroDirective that was loaded from a PCH file.
879 
880  /// \brief Register an exported macro for a module and identifier.
882  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
884 
885  /// \brief Get the list of leaf (non-overridden) module macros for a name.
887  auto I = LeafModuleMacros.find(II);
888  if (I != LeafModuleMacros.end())
889  return I->second;
890  return None;
891  }
892 
893  /// \{
894  /// Iterators for the macro history table. Currently defined macros have
895  /// IdentifierInfo::hasMacroDefinition() set and an empty
896  /// MacroInfo::getUndefLoc() at the head of the list.
897  typedef MacroMap::const_iterator macro_iterator;
898  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
899  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
900  llvm::iterator_range<macro_iterator>
901  macros(bool IncludeExternalMacros = true) const {
902  return llvm::make_range(macro_begin(IncludeExternalMacros),
903  macro_end(IncludeExternalMacros));
904  }
905  /// \}
906 
907  /// \brief Return the name of the macro defined before \p Loc that has
908  /// spelling \p Tokens. If there are multiple macros with same spelling,
909  /// return the last one defined.
912 
913  const std::string &getPredefines() const { return Predefines; }
914  /// \brief Set the predefines for this Preprocessor.
915  ///
916  /// These predefines are automatically injected when parsing the main file.
917  void setPredefines(const char *P) { Predefines = P; }
918  void setPredefines(StringRef P) { Predefines = P; }
919 
920  /// Return information about the specified preprocessor
921  /// identifier token.
922  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
923  return &Identifiers.get(Name);
924  }
925 
926  /// \brief Add the specified pragma handler to this preprocessor.
927  ///
928  /// If \p Namespace is non-null, then it is a token required to exist on the
929  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
930  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
932  AddPragmaHandler(StringRef(), Handler);
933  }
934 
935  /// \brief Remove the specific pragma handler from this preprocessor.
936  ///
937  /// If \p Namespace is non-null, then it should be the namespace that
938  /// \p Handler was added to. It is an error to remove a handler that
939  /// has not been registered.
940  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
942  RemovePragmaHandler(StringRef(), Handler);
943  }
944 
945  /// Install empty handlers for all pragmas (making them ignored).
946  void IgnorePragmas();
947 
948  /// \brief Add the specified comment handler to the preprocessor.
949  void addCommentHandler(CommentHandler *Handler);
950 
951  /// \brief Remove the specified comment handler.
952  ///
953  /// It is an error to remove a handler that has not been registered.
954  void removeCommentHandler(CommentHandler *Handler);
955 
956  /// \brief Set the code completion handler to the given object.
958  CodeComplete = &Handler;
959  }
960 
961  /// \brief Retrieve the current code-completion handler.
963  return CodeComplete;
964  }
965 
966  /// \brief Clear out the code completion handler.
968  CodeComplete = nullptr;
969  }
970 
971  /// \brief Hook used by the lexer to invoke the "natural language" code
972  /// completion point.
974 
975  /// \brief Retrieve the preprocessing record, or NULL if there is no
976  /// preprocessing record.
977  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
978 
979  /// \brief Create a new preprocessing record, which will keep track of
980  /// all macro expansions, macro definitions, etc.
982 
983  /// \brief Enter the specified FileID as the main source file,
984  /// which implicitly adds the builtin defines etc.
985  void EnterMainSourceFile();
986 
987  /// \brief Inform the preprocessor callbacks that processing is complete.
988  void EndSourceFile();
989 
990  /// \brief Add a source file to the top of the include stack and
991  /// start lexing tokens from it instead of the current buffer.
992  ///
993  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
994  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
995  SourceLocation Loc);
996 
997  /// \brief Add a Macro to the top of the include stack and start lexing
998  /// tokens from it instead of the current buffer.
999  ///
1000  /// \param Args specifies the tokens input to a function-like macro.
1001  /// \param ILEnd specifies the location of the ')' for a function-like macro
1002  /// or the identifier for an object-like macro.
1003  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1004  MacroArgs *Args);
1005 
1006  /// \brief Add a "macro" context to the top of the include stack,
1007  /// which will cause the lexer to start returning the specified tokens.
1008  ///
1009  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1010  /// will not be subject to further macro expansion. Otherwise, these tokens
1011  /// will be re-macro-expanded when/if expansion is enabled.
1012  ///
1013  /// If \p OwnsTokens is false, this method assumes that the specified stream
1014  /// of tokens has a permanent owner somewhere, so they do not need to be
1015  /// copied. If it is true, it assumes the array of tokens is allocated with
1016  /// \c new[] and must be freed.
1017  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1018  bool DisableMacroExpansion, bool OwnsTokens);
1019 
1020  /// \brief Pop the current lexer/macro exp off the top of the lexer stack.
1021  ///
1022  /// This should only be used in situations where the current state of the
1023  /// top-of-stack lexer is known.
1024  void RemoveTopOfLexerStack();
1025 
1026  /// From the point that this method is called, and until
1027  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1028  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1029  /// make the Preprocessor re-lex the same tokens.
1030  ///
1031  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1032  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1033  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1034  ///
1035  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1036  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1037  /// tokens will continue indefinitely.
1038  ///
1039  void EnableBacktrackAtThisPos();
1040 
1041  /// \brief Disable the last EnableBacktrackAtThisPos call.
1042  void CommitBacktrackedTokens();
1043 
1044  /// \brief Make Preprocessor re-lex the tokens that were lexed since
1045  /// EnableBacktrackAtThisPos() was previously called.
1046  void Backtrack();
1047 
1048  /// \brief True if EnableBacktrackAtThisPos() was called and
1049  /// caching of tokens is on.
1050  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1051 
1052  /// \brief Lex the next token for this preprocessor.
1053  void Lex(Token &Result);
1054 
1056 
1057  void makeModuleVisible(Module *M, SourceLocation Loc);
1058 
1060  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1061  }
1062 
1063  /// \brief Lex a string literal, which may be the concatenation of multiple
1064  /// string literals and may even come from macro expansion.
1065  /// \returns true on success, false if a error diagnostic has been generated.
1066  bool LexStringLiteral(Token &Result, std::string &String,
1067  const char *DiagnosticTag, bool AllowMacroExpansion) {
1068  if (AllowMacroExpansion)
1069  Lex(Result);
1070  else
1071  LexUnexpandedToken(Result);
1072  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1073  AllowMacroExpansion);
1074  }
1075 
1076  /// \brief Complete the lexing of a string literal where the first token has
1077  /// already been lexed (see LexStringLiteral).
1078  bool FinishLexStringLiteral(Token &Result, std::string &String,
1079  const char *DiagnosticTag,
1080  bool AllowMacroExpansion);
1081 
1082  /// \brief Lex a token. If it's a comment, keep lexing until we get
1083  /// something not a comment.
1084  ///
1085  /// This is useful in -E -C mode where comments would foul up preprocessor
1086  /// directive handling.
1088  do
1089  Lex(Result);
1090  while (Result.getKind() == tok::comment);
1091  }
1092 
1093  /// \brief Just like Lex, but disables macro expansion of identifier tokens.
1095  // Disable macro expansion.
1096  bool OldVal = DisableMacroExpansion;
1097  DisableMacroExpansion = true;
1098  // Lex the token.
1099  Lex(Result);
1100 
1101  // Reenable it.
1102  DisableMacroExpansion = OldVal;
1103  }
1104 
1105  /// \brief Like LexNonComment, but this disables macro expansion of
1106  /// identifier tokens.
1108  do
1109  LexUnexpandedToken(Result);
1110  while (Result.getKind() == tok::comment);
1111  }
1112 
1113  /// \brief Parses a simple integer literal to get its numeric value. Floating
1114  /// point literals and user defined literals are rejected. Used primarily to
1115  /// handle pragmas that accept integer arguments.
1116  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1117 
1118  /// Disables macro expansion everywhere except for preprocessor directives.
1120  DisableMacroExpansion = true;
1121  MacroExpansionInDirectivesOverride = true;
1122  }
1123 
1124  /// \brief Peeks ahead N tokens and returns that token without consuming any
1125  /// tokens.
1126  ///
1127  /// LookAhead(0) returns the next token that would be returned by Lex(),
1128  /// LookAhead(1) returns the token after it, etc. This returns normal
1129  /// tokens after phase 5. As such, it is equivalent to using
1130  /// 'Lex', not 'LexUnexpandedToken'.
1131  const Token &LookAhead(unsigned N) {
1132  if (CachedLexPos + N < CachedTokens.size())
1133  return CachedTokens[CachedLexPos+N];
1134  else
1135  return PeekAhead(N+1);
1136  }
1137 
1138  /// \brief When backtracking is enabled and tokens are cached,
1139  /// this allows to revert a specific number of tokens.
1140  ///
1141  /// Note that the number of tokens being reverted should be up to the last
1142  /// backtrack position, not more.
1143  void RevertCachedTokens(unsigned N) {
1144  assert(isBacktrackEnabled() &&
1145  "Should only be called when tokens are cached for backtracking");
1146  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1147  && "Should revert tokens up to the last backtrack position, not more");
1148  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1149  "Corrupted backtrack positions ?");
1150  CachedLexPos -= N;
1151  }
1152 
1153  /// \brief Enters a token in the token stream to be lexed next.
1154  ///
1155  /// If BackTrack() is called afterwards, the token will remain at the
1156  /// insertion point.
1157  void EnterToken(const Token &Tok) {
1158  EnterCachingLexMode();
1159  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1160  }
1161 
1162  /// We notify the Preprocessor that if it is caching tokens (because
1163  /// backtrack is enabled) it should replace the most recent cached tokens
1164  /// with the given annotation token. This function has no effect if
1165  /// backtracking is not enabled.
1166  ///
1167  /// Note that the use of this function is just for optimization, so that the
1168  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1169  /// invoked.
1170  void AnnotateCachedTokens(const Token &Tok) {
1171  assert(Tok.isAnnotation() && "Expected annotation token");
1172  if (CachedLexPos != 0 && isBacktrackEnabled())
1173  AnnotatePreviousCachedTokens(Tok);
1174  }
1175 
1176  /// Get the location of the last cached token, suitable for setting the end
1177  /// location of an annotation token.
1179  assert(CachedLexPos != 0);
1180  return CachedTokens[CachedLexPos-1].getLastLoc();
1181  }
1182 
1183  /// \brief Replace the last token with an annotation token.
1184  ///
1185  /// Like AnnotateCachedTokens(), this routine replaces an
1186  /// already-parsed (and resolved) token with an annotation
1187  /// token. However, this routine only replaces the last token with
1188  /// the annotation token; it does not affect any other cached
1189  /// tokens. This function has no effect if backtracking is not
1190  /// enabled.
1192  assert(Tok.isAnnotation() && "Expected annotation token");
1193  if (CachedLexPos != 0 && isBacktrackEnabled())
1194  CachedTokens[CachedLexPos-1] = Tok;
1195  }
1196 
1197  /// Update the current token to represent the provided
1198  /// identifier, in order to cache an action performed by typo correction.
1199  void TypoCorrectToken(const Token &Tok) {
1200  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1201  if (CachedLexPos != 0 && isBacktrackEnabled())
1202  CachedTokens[CachedLexPos-1] = Tok;
1203  }
1204 
1205  /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1206  /// CurTokenLexer pointers.
1207  void recomputeCurLexerKind();
1208 
1209  /// \brief Returns true if incremental processing is enabled
1210  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1211 
1212  /// \brief Enables the incremental processing
1213  void enableIncrementalProcessing(bool value = true) {
1214  IncrementalProcessing = value;
1215  }
1216 
1217  /// \brief Specify the point at which code-completion will be performed.
1218  ///
1219  /// \param File the file in which code completion should occur. If
1220  /// this file is included multiple times, code-completion will
1221  /// perform completion the first time it is included. If NULL, this
1222  /// function clears out the code-completion point.
1223  ///
1224  /// \param Line the line at which code completion should occur
1225  /// (1-based).
1226  ///
1227  /// \param Column the column at which code completion should occur
1228  /// (1-based).
1229  ///
1230  /// \returns true if an error occurred, false otherwise.
1231  bool SetCodeCompletionPoint(const FileEntry *File,
1232  unsigned Line, unsigned Column);
1233 
1234  /// \brief Determine if we are performing code completion.
1235  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1236 
1237  /// \brief Returns the location of the code-completion point.
1238  ///
1239  /// Returns an invalid location if code-completion is not enabled or the file
1240  /// containing the code-completion point has not been lexed yet.
1241  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1242 
1243  /// \brief Returns the start location of the file of code-completion point.
1244  ///
1245  /// Returns an invalid location if code-completion is not enabled or the file
1246  /// containing the code-completion point has not been lexed yet.
1248  return CodeCompletionFileLoc;
1249  }
1250 
1251  /// \brief Returns true if code-completion is enabled and we have hit the
1252  /// code-completion point.
1253  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1254 
1255  /// \brief Note that we hit the code-completion point.
1257  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1258  CodeCompletionReached = true;
1259  // Silence any diagnostics that occur after we hit the code-completion.
1261  }
1262 
1263  /// \brief The location of the currently-active \#pragma clang
1264  /// arc_cf_code_audited begin.
1265  ///
1266  /// Returns an invalid location if there is no such pragma active.
1268  return PragmaARCCFCodeAuditedLoc;
1269  }
1270 
1271  /// \brief Set the location of the currently-active \#pragma clang
1272  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1274  PragmaARCCFCodeAuditedLoc = Loc;
1275  }
1276 
1277  /// \brief The location of the currently-active \#pragma clang
1278  /// assume_nonnull begin.
1279  ///
1280  /// Returns an invalid location if there is no such pragma active.
1282  return PragmaAssumeNonNullLoc;
1283  }
1284 
1285  /// \brief Set the location of the currently-active \#pragma clang
1286  /// assume_nonnull begin. An invalid location ends the pragma.
1288  PragmaAssumeNonNullLoc = Loc;
1289  }
1290 
1291  /// \brief Set the directory in which the main file should be considered
1292  /// to have been found, if it is not a real file.
1293  void setMainFileDir(const DirectoryEntry *Dir) {
1294  MainFileDir = Dir;
1295  }
1296 
1297  /// \brief Instruct the preprocessor to skip part of the main source file.
1298  ///
1299  /// \param Bytes The number of bytes in the preamble to skip.
1300  ///
1301  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1302  /// start of a line.
1303  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1304  SkipMainFilePreamble.first = Bytes;
1305  SkipMainFilePreamble.second = StartOfLine;
1306  }
1307 
1308  /// Forwarding function for diagnostics. This emits a diagnostic at
1309  /// the specified Token's location, translating the token's start
1310  /// position in the current buffer into a SourcePosition object for rendering.
1311  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1312  return Diags->Report(Loc, DiagID);
1313  }
1314 
1315  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1316  return Diags->Report(Tok.getLocation(), DiagID);
1317  }
1318 
1319  /// Return the 'spelling' of the token at the given
1320  /// location; does not go up to the spelling location or down to the
1321  /// expansion location.
1322  ///
1323  /// \param buffer A buffer which will be used only if the token requires
1324  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1325  /// \param invalid If non-null, will be set \c true if an error occurs.
1327  SmallVectorImpl<char> &buffer,
1328  bool *invalid = nullptr) const {
1329  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1330  }
1331 
1332  /// \brief Return the 'spelling' of the Tok token.
1333  ///
1334  /// The spelling of a token is the characters used to represent the token in
1335  /// the source file after trigraph expansion and escaped-newline folding. In
1336  /// particular, this wants to get the true, uncanonicalized, spelling of
1337  /// things like digraphs, UCNs, etc.
1338  ///
1339  /// \param Invalid If non-null, will be set \c true if an error occurs.
1340  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1341  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1342  }
1343 
1344  /// \brief Get the spelling of a token into a preallocated buffer, instead
1345  /// of as an std::string.
1346  ///
1347  /// The caller is required to allocate enough space for the token, which is
1348  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1349  /// actual result is returned.
1350  ///
1351  /// Note that this method may do two possible things: it may either fill in
1352  /// the buffer specified with characters, or it may *change the input pointer*
1353  /// to point to a constant buffer with the data already in it (avoiding a
1354  /// copy). The caller is not allowed to modify the returned buffer pointer
1355  /// if an internal buffer is returned.
1356  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1357  bool *Invalid = nullptr) const {
1358  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1359  }
1360 
1361  /// \brief Get the spelling of a token into a SmallVector.
1362  ///
1363  /// Note that the returned StringRef may not point to the
1364  /// supplied buffer if a copy can be avoided.
1365  StringRef getSpelling(const Token &Tok,
1366  SmallVectorImpl<char> &Buffer,
1367  bool *Invalid = nullptr) const;
1368 
1369  /// \brief Relex the token at the specified location.
1370  /// \returns true if there was a failure, false on success.
1372  bool IgnoreWhiteSpace = false) {
1373  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1374  }
1375 
1376  /// \brief Given a Token \p Tok that is a numeric constant with length 1,
1377  /// return the character.
1378  char
1380  bool *Invalid = nullptr) const {
1381  assert(Tok.is(tok::numeric_constant) &&
1382  Tok.getLength() == 1 && "Called on unsupported token");
1383  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1384 
1385  // If the token is carrying a literal data pointer, just use it.
1386  if (const char *D = Tok.getLiteralData())
1387  return *D;
1388 
1389  // Otherwise, fall back on getCharacterData, which is slower, but always
1390  // works.
1391  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1392  }
1393 
1394  /// \brief Retrieve the name of the immediate macro expansion.
1395  ///
1396  /// This routine starts from a source location, and finds the name of the
1397  /// macro responsible for its immediate expansion. It looks through any
1398  /// intervening macro argument expansions to compute this. It returns a
1399  /// StringRef that refers to the SourceManager-owned buffer of the source
1400  /// where that macro name is spelled. Thus, the result shouldn't out-live
1401  /// the SourceManager.
1403  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1404  }
1405 
1406  /// \brief Plop the specified string into a scratch buffer and set the
1407  /// specified token's location and length to it.
1408  ///
1409  /// If specified, the source location provides a location of the expansion
1410  /// point of the token.
1411  void CreateString(StringRef Str, Token &Tok,
1412  SourceLocation ExpansionLocStart = SourceLocation(),
1413  SourceLocation ExpansionLocEnd = SourceLocation());
1414 
1415  /// \brief Computes the source location just past the end of the
1416  /// token at this source location.
1417  ///
1418  /// This routine can be used to produce a source location that
1419  /// points just past the end of the token referenced by \p Loc, and
1420  /// is generally used when a diagnostic needs to point just after a
1421  /// token where it expected something different that it received. If
1422  /// the returned source location would not be meaningful (e.g., if
1423  /// it points into a macro), this routine returns an invalid
1424  /// source location.
1425  ///
1426  /// \param Offset an offset from the end of the token, where the source
1427  /// location should refer to. The default offset (0) produces a source
1428  /// location pointing just past the end of the token; an offset of 1 produces
1429  /// a source location pointing to the last character in the token, etc.
1431  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1432  }
1433 
1434  /// \brief Returns true if the given MacroID location points at the first
1435  /// token of the macro expansion.
1436  ///
1437  /// \param MacroBegin If non-null and function returns true, it is set to
1438  /// begin location of the macro.
1440  SourceLocation *MacroBegin = nullptr) const {
1441  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1442  MacroBegin);
1443  }
1444 
1445  /// \brief Returns true if the given MacroID location points at the last
1446  /// token of the macro expansion.
1447  ///
1448  /// \param MacroEnd If non-null and function returns true, it is set to
1449  /// end location of the macro.
1451  SourceLocation *MacroEnd = nullptr) const {
1452  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1453  }
1454 
1455  /// \brief Print the token to stderr, used for debugging.
1456  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1457  void DumpLocation(SourceLocation Loc) const;
1458  void DumpMacro(const MacroInfo &MI) const;
1459  void dumpMacroInfo(const IdentifierInfo *II);
1460 
1461  /// \brief Given a location that specifies the start of a
1462  /// token, return a new location that specifies a character within the token.
1464  unsigned Char) const {
1465  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1466  }
1467 
1468  /// \brief Increment the counters for the number of token paste operations
1469  /// performed.
1470  ///
1471  /// If fast was specified, this is a 'fast paste' case we handled.
1472  void IncrementPasteCounter(bool isFast) {
1473  if (isFast)
1474  ++NumFastTokenPaste;
1475  else
1476  ++NumTokenPaste;
1477  }
1478 
1479  void PrintStats();
1480 
1481  size_t getTotalMemory() const;
1482 
1483  /// When the macro expander pastes together a comment (/##/) in Microsoft
1484  /// mode, this method handles updating the current state, returning the
1485  /// token on the next source line.
1487 
1488  //===--------------------------------------------------------------------===//
1489  // Preprocessor callback methods. These are invoked by a lexer as various
1490  // directives and events are found.
1491 
1492  /// Given a tok::raw_identifier token, look up the
1493  /// identifier information for the token and install it into the token,
1494  /// updating the token kind accordingly.
1495  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1496 
1497 private:
1498  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1499 
1500 public:
1501 
1502  /// \brief Specifies the reason for poisoning an identifier.
1503  ///
1504  /// If that identifier is accessed while poisoned, then this reason will be
1505  /// used instead of the default "poisoned" diagnostic.
1506  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1507 
1508  /// \brief Display reason for poisoned identifier.
1509  void HandlePoisonedIdentifier(Token & Tok);
1510 
1512  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1513  if(II->isPoisoned()) {
1514  HandlePoisonedIdentifier(Identifier);
1515  }
1516  }
1517  }
1518 
1519 private:
1520  /// Identifiers used for SEH handling in Borland. These are only
1521  /// allowed in particular circumstances
1522  // __except block
1523  IdentifierInfo *Ident__exception_code,
1524  *Ident___exception_code,
1525  *Ident_GetExceptionCode;
1526  // __except filter expression
1527  IdentifierInfo *Ident__exception_info,
1528  *Ident___exception_info,
1529  *Ident_GetExceptionInfo;
1530  // __finally
1531  IdentifierInfo *Ident__abnormal_termination,
1532  *Ident___abnormal_termination,
1533  *Ident_AbnormalTermination;
1534 
1535  const char *getCurLexerEndPos();
1536 
1537 public:
1538  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1539 
1540  /// \brief Callback invoked when the lexer reads an identifier and has
1541  /// filled in the tokens IdentifierInfo member.
1542  ///
1543  /// This callback potentially macro expands it or turns it into a named
1544  /// token (like 'for').
1545  ///
1546  /// \returns true if we actually computed a token, false if we need to
1547  /// lex again.
1548  bool HandleIdentifier(Token &Identifier);
1549 
1550 
1551  /// \brief Callback invoked when the lexer hits the end of the current file.
1552  ///
1553  /// This either returns the EOF token and returns true, or
1554  /// pops a level off the include stack and returns false, at which point the
1555  /// client should call lex again.
1556  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1557 
1558  /// \brief Callback invoked when the current TokenLexer hits the end of its
1559  /// token stream.
1561 
1562  /// \brief Callback invoked when the lexer sees a # token at the start of a
1563  /// line.
1564  ///
1565  /// This consumes the directive, modifies the lexer/preprocessor state, and
1566  /// advances the lexer(s) so that the next token read is the correct one.
1567  void HandleDirective(Token &Result);
1568 
1569  /// \brief Ensure that the next token is a tok::eod token.
1570  ///
1571  /// If not, emit a diagnostic and consume up until the eod.
1572  /// If \p EnableMacros is true, then we consider macros that expand to zero
1573  /// tokens as being ok.
1574  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1575 
1576  /// \brief Read and discard all tokens remaining on the current line until
1577  /// the tok::eod token is found.
1579 
1580  /// \brief Returns true if the preprocessor has seen a use of
1581  /// __DATE__ or __TIME__ in the file so far.
1582  bool SawDateOrTime() const {
1583  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1584  }
1585  unsigned getCounterValue() const { return CounterValue; }
1586  void setCounterValue(unsigned V) { CounterValue = V; }
1587 
1588  /// \brief Retrieves the module that we're currently building, if any.
1590 
1591  /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1592  MacroInfo *AllocateMacroInfo(SourceLocation L);
1593 
1594  /// \brief Allocate a new MacroInfo object loaded from an AST file.
1596  unsigned SubModuleID);
1597 
1598  /// \brief Turn the specified lexer token into a fully checked and spelled
1599  /// filename, e.g. as an operand of \#include.
1600  ///
1601  /// The caller is expected to provide a buffer that is large enough to hold
1602  /// the spelling of the filename, but is also expected to handle the case
1603  /// when this method decides to use a different buffer.
1604  ///
1605  /// \returns true if the input filename was in <>'s or false if it was
1606  /// in ""'s.
1607  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1608 
1609  /// \brief Given a "foo" or <foo> reference, look up the indicated file.
1610  ///
1611  /// Returns null on failure. \p isAngled indicates whether the file
1612  /// reference is for system \#include's or not (i.e. using <> instead of "").
1613  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1614  bool isAngled, const DirectoryLookup *FromDir,
1615  const FileEntry *FromFile,
1616  const DirectoryLookup *&CurDir,
1617  SmallVectorImpl<char> *SearchPath,
1618  SmallVectorImpl<char> *RelativePath,
1619  ModuleMap::KnownHeader *SuggestedModule,
1620  bool SkipCache = false);
1621 
1622  /// \brief Get the DirectoryLookup structure used to find the current
1623  /// FileEntry, if CurLexer is non-null and if applicable.
1624  ///
1625  /// This allows us to implement \#include_next and find directory-specific
1626  /// properties.
1627  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1628 
1629  /// \brief Return true if we're in the top-level file, not in a \#include.
1630  bool isInPrimaryFile() const;
1631 
1632  /// \brief Handle cases where the \#include name is expanded
1633  /// from a macro as multiple tokens, which need to be glued together.
1634  ///
1635  /// This occurs for code like:
1636  /// \code
1637  /// \#define FOO <x/y.h>
1638  /// \#include FOO
1639  /// \endcode
1640  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1641  ///
1642  /// This code concatenates and consumes tokens up to the '>' token. It
1643  /// returns false if the > was found, otherwise it returns true if it finds
1644  /// and consumes the EOD marker.
1645  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1646  SourceLocation &End);
1647 
1648  /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1649  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1650  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1651 
1652  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1653  bool *ShadowFlag = nullptr);
1654 
1655 private:
1656 
1657  void PushIncludeMacroStack() {
1658  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1659  IncludeMacroStack.emplace_back(
1660  CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer),
1661  CurPPLexer, std::move(CurTokenLexer), CurDirLookup);
1662  CurPPLexer = nullptr;
1663  }
1664 
1665  void PopIncludeMacroStack() {
1666  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1667  CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1668  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1669  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1670  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1671  CurSubmodule = IncludeMacroStack.back().TheSubmodule;
1672  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1673  IncludeMacroStack.pop_back();
1674  }
1675 
1676  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1677 
1678  void EnterSubmodule(Module *M, SourceLocation ImportLoc);
1679  void LeaveSubmodule();
1680 
1681  /// Update the set of active module macros and ambiguity flag for a module
1682  /// macro name.
1683  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1684 
1685  /// \brief Allocate a new MacroInfo object.
1686  MacroInfo *AllocateMacroInfo();
1687 
1688  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1689  SourceLocation Loc);
1690  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1691  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1692  bool isPublic);
1693 
1694  /// \brief Lex and validate a macro name, which occurs after a
1695  /// \#define or \#undef.
1696  ///
1697  /// \param MacroNameTok Token that represents the name defined or undefined.
1698  /// \param IsDefineUndef Kind if preprocessor directive.
1699  /// \param ShadowFlag Points to flag that is set if macro name shadows
1700  /// a keyword.
1701  ///
1702  /// This emits a diagnostic, sets the token kind to eod,
1703  /// and discards the rest of the macro line if the macro name is invalid.
1704  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1705  bool *ShadowFlag = nullptr);
1706 
1707  /// The ( starting an argument list of a macro definition has just been read.
1708  /// Lex the rest of the arguments and the closing ), updating \p MI with
1709  /// what we learn and saving in \p LastTok the last token read.
1710  /// Return true if an error occurs parsing the arg list.
1711  bool ReadMacroDefinitionArgList(MacroInfo *MI, Token& LastTok);
1712 
1713  /// We just read a \#if or related directive and decided that the
1714  /// subsequent tokens are in the \#if'd out portion of the
1715  /// file. Lex the rest of the file, until we see an \#endif. If \p
1716  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1717  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1718  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1719  /// already seen one so a \#else directive is a duplicate. When this returns,
1720  /// the caller can lex the first valid token.
1721  void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1722  bool FoundNonSkipPortion, bool FoundElse,
1723  SourceLocation ElseLoc = SourceLocation());
1724 
1725  /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1726  void PTHSkipExcludedConditionalBlock();
1727 
1728  /// \brief Evaluate an integer constant expression that may occur after a
1729  /// \#if or \#elif directive and return it as a bool.
1730  ///
1731  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1732  bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1733 
1734  /// \brief Install the standard preprocessor pragmas:
1735  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1736  void RegisterBuiltinPragmas();
1737 
1738  /// \brief Register builtin macros such as __LINE__ with the identifier table.
1739  void RegisterBuiltinMacros();
1740 
1741  /// If an identifier token is read that is to be expanded as a macro, handle
1742  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1743  /// otherwise the caller should lex again.
1744  bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1745 
1746  /// \brief Cache macro expanded tokens for TokenLexers.
1747  //
1748  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1749  /// going to lex in the cache and when it finishes the tokens are removed
1750  /// from the end of the cache.
1751  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1752  ArrayRef<Token> tokens);
1753  void removeCachedMacroExpandedTokensOfLastLexer();
1755 
1756  /// Determine whether the next preprocessor token to be
1757  /// lexed is a '('. If so, consume the token and return true, if not, this
1758  /// method should have no observable side-effect on the lexed tokens.
1759  bool isNextPPTokenLParen();
1760 
1761  /// After reading "MACRO(", this method is invoked to read all of the formal
1762  /// arguments specified for the macro invocation. Returns null on error.
1763  MacroArgs *ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI,
1764  SourceLocation &ExpansionEnd);
1765 
1766  /// \brief If an identifier token is read that is to be expanded
1767  /// as a builtin macro, handle it and return the next token as 'Tok'.
1768  void ExpandBuiltinMacro(Token &Tok);
1769 
1770  /// \brief Read a \c _Pragma directive, slice it up, process it, then
1771  /// return the first token after the directive.
1772  /// This assumes that the \c _Pragma token has just been read into \p Tok.
1773  void Handle_Pragma(Token &Tok);
1774 
1775  /// \brief Like Handle_Pragma except the pragma text is not enclosed within
1776  /// a string literal.
1777  void HandleMicrosoft__pragma(Token &Tok);
1778 
1779  /// \brief Add a lexer to the top of the include stack and
1780  /// start lexing tokens from it instead of the current buffer.
1781  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
1782 
1783  /// \brief Add a lexer to the top of the include stack and
1784  /// start getting tokens from it using the PTH cache.
1785  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
1786 
1787  /// \brief Set the FileID for the preprocessor predefines.
1788  void setPredefinesFileID(FileID FID) {
1789  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
1790  PredefinesFileID = FID;
1791  }
1792 
1793  /// \brief Returns true if we are lexing from a file and not a
1794  /// pragma or a macro.
1795  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
1796  return L ? !L->isPragmaLexer() : P != nullptr;
1797  }
1798 
1799  static bool IsFileLexer(const IncludeStackInfo& I) {
1800  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1801  }
1802 
1803  bool IsFileLexer() const {
1804  return IsFileLexer(CurLexer.get(), CurPPLexer);
1805  }
1806 
1807  //===--------------------------------------------------------------------===//
1808  // Caching stuff.
1809  void CachingLex(Token &Result);
1810  bool InCachingLexMode() const {
1811  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
1812  // that we are past EOF, not that we are in CachingLex mode.
1813  return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1814  !IncludeMacroStack.empty();
1815  }
1816  void EnterCachingLexMode();
1817  void ExitCachingLexMode() {
1818  if (InCachingLexMode())
1820  }
1821  const Token &PeekAhead(unsigned N);
1822  void AnnotatePreviousCachedTokens(const Token &Tok);
1823 
1824  //===--------------------------------------------------------------------===//
1825  /// Handle*Directive - implement the various preprocessor directives. These
1826  /// should side-effect the current preprocessor object so that the next call
1827  /// to Lex() will return the appropriate token next.
1828  void HandleLineDirective(Token &Tok);
1829  void HandleDigitDirective(Token &Tok);
1830  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
1831  void HandleIdentSCCSDirective(Token &Tok);
1832  void HandleMacroPublicDirective(Token &Tok);
1833  void HandleMacroPrivateDirective(Token &Tok);
1834 
1835  // File inclusion.
1836  void HandleIncludeDirective(SourceLocation HashLoc,
1837  Token &Tok,
1838  const DirectoryLookup *LookupFrom = nullptr,
1839  const FileEntry *LookupFromFile = nullptr,
1840  bool isImport = false);
1841  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
1842  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
1843  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
1844  void HandleMicrosoftImportDirective(Token &Tok);
1845 
1846 public:
1847  // Module inclusion testing.
1848  /// \brief Find the module that owns the source or header file that
1849  /// \p Loc points to. If the location is in a file that was included
1850  /// into a module, or is outside any module, returns nullptr.
1851  Module *getModuleForLocation(SourceLocation Loc);
1852 
1853  /// \brief Find the module that contains the specified location, either
1854  /// directly or indirectly.
1855  Module *getModuleContainingLocation(SourceLocation Loc);
1856 
1857 private:
1858  // Macro handling.
1859  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
1860  void HandleUndefDirective(Token &Tok);
1861 
1862  // Conditional Inclusion.
1863  void HandleIfdefDirective(Token &Tok, bool isIfndef,
1864  bool ReadAnyTokensBeforeDirective);
1865  void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective);
1866  void HandleEndifDirective(Token &Tok);
1867  void HandleElseDirective(Token &Tok);
1868  void HandleElifDirective(Token &Tok);
1869 
1870  // Pragmas.
1871  void HandlePragmaDirective(SourceLocation IntroducerLoc,
1872  PragmaIntroducerKind Introducer);
1873 public:
1874  void HandlePragmaOnce(Token &OnceTok);
1875  void HandlePragmaMark();
1876  void HandlePragmaPoison(Token &PoisonTok);
1877  void HandlePragmaSystemHeader(Token &SysHeaderTok);
1878  void HandlePragmaDependency(Token &DependencyTok);
1879  void HandlePragmaPushMacro(Token &Tok);
1880  void HandlePragmaPopMacro(Token &Tok);
1881  void HandlePragmaIncludeAlias(Token &Tok);
1882  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
1883 
1884  // Return true and store the first token only if any CommentHandler
1885  // has inserted some tokens and getCommentRetentionState() is false.
1886  bool HandleComment(Token &Token, SourceRange Comment);
1887 
1888  /// \brief A macro is used, update information about macros that need unused
1889  /// warnings.
1890  void markMacroAsUsed(MacroInfo *MI);
1891 };
1892 
1893 /// \brief Abstract base class that describes a handler that will receive
1894 /// source ranges for each of the comments encountered in the source file.
1896 public:
1897  virtual ~CommentHandler();
1898 
1899  // The handler shall return true if it has pushed any tokens
1900  // to be read using e.g. EnterToken or EnterTokenStream.
1901  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
1902 };
1903 
1904 } // end namespace clang
1905 
1906 #endif
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens. If there are multiple macro...
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:682
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
Definition: Lexer.cpp:358
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Definition: Preprocessor.h:901
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to. If the location is in a file ...
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:689
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:699
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:687
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:66
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:115
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:725
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
Module * getCurrentSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:762
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Definition: Preprocessor.h:922
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:22
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
Definition: Preprocessor.h:913
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1118
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:280
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:897
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value. Floating point literals and user defined li...
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
Definition: Lexer.cpp:781
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
Definition: Preprocessor.h:962
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:1467
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:120
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
SourceLocation getLastCachedTokenLocation() const
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:688
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
Definition: Preprocessor.h:917
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:38
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
bool getPragmasEnabled() const
Definition: Preprocessor.h:724
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:69
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Definition: Preprocessor.h:867
Represents a macro directive exported by a module.
Definition: MacroInfo.h:498
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:419
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:559
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:846
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:679
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:716
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
Describes a module or submodule.
Definition: Basic/Module.h:49
FileManager & getFileManager() const
Definition: Preprocessor.h:681
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:711
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:726
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:84
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
uint32_t Offset
Definition: CacheTokens.cpp:43
bool getCommentRetentionState() const
Definition: Preprocessor.h:721
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:686
void LexNonComment(Token &Result)
Lex a token. If it's a comment, keep lexing until we get something not a comment. ...
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:417
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:135
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:683
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:784
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Definition: Preprocessor.h:792
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:706
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
tok::TokenKind getKind() const
Definition: Token.h:90
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
Definition: Preprocessor.h:957
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:680
void setPreprocessedOutput(bool IsPreprocessedOutput)
Definition: Preprocessor.h:736
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
Definition: Lexer.cpp:700
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:32
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:151
AnnotatingParser & P
friend class MacroArgs
Definition: Preprocessor.h:563
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:866
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:617
void clearCodeCompletionHandler()
Clear out the code completion handler.
Definition: Preprocessor.h:967
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:40
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
SourceManager & SM
void HandlePragmaOnce(Token &OnceTok)
Definition: Pragma.cpp:352
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:803
Abstract interface for external sources of preprocessor information.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:759
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
void EnableBacktrackAtThisPos()
Definition: PPCaching.cpp:26
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD. Return true if the token i...
Definition: Pragma.cpp:778
SourceManager & SourceMgr
Definition: Format.cpp:1205
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:77
#define bool
Definition: stdbool.h:31
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:756
void HandleMicrosoftCommentPaste(Token &Tok)
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:704
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
FormatToken * Token
Stores token information for comparing actual tokens with predefined values. Only handles simple toke...
Definition: Preprocessor.h:64
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin. An invalid location ends...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
void HandlePragmaDependency(Token &DependencyTok)
Definition: Pragma.cpp:450
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:76
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
Preprocessor(IntrusiveRefCntPtr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
Definition: Token.h:215
MacroInfo * AllocateDeserializedMacroInfo(SourceLocation L, unsigned SubModuleID)
Allocate a new MacroInfo object loaded from an AST file.
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:308
Kind
void TypoCorrectToken(const Token &Tok)
SmallVectorImpl< AnnotatedLine * >::const_iterator Next
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:695
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:753
AnnotatedLine & Line
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:95
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. An invalid location...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:805
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
Definition: Preprocessor.h:835
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:685
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:723
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:53
void EnterTokenStream(const Token *Toks, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens)
Add a "macro" context to the top of the include stack, which will cause the lexer to start returning ...
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:87
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:818
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
Definition: Preprocessor.h:772
SmallVector< FormatToken *, 16 > Tokens
Definition: Format.cpp:1214
bool is(tok::TokenKind K) const
Definition: Token.h:95
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:676
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:956
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:765
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:781
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1080
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:582
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:302
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Definition: Preprocessor.h:977
Abstract interface for a module loader.
Definition: ModuleLoader.h:56
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Defines the Diagnostic-related interfaces.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g. its tokens).
Definition: MacroInfo.h:34
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:49
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g. as an operand of #incl...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool isInvalid() const
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
Definition: Pragma.cpp:504
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:40
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
Definition: Preprocessor.h:873
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:745
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:674
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:465
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:941
void DumpMacro(const MacroInfo &MI) const
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
void HandlePragmaPoison(Token &PoisonTok)
Definition: Pragma.cpp:374
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:163
void Initialize(const TargetInfo &Target)
Initialize the preprocessor using information about the target.
void setPredefines(StringRef P)
Definition: Preprocessor.h:918
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
Definition: Preprocessor.h:886
The translation unit is a complete translation unit.
Definition: LangOptions.h:165
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void HandlePragmaSystemHeader(Token &SysHeaderTok)
Definition: Pragma.cpp:416
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
Definition: Token.h:127
bool isPreprocessedOutput() const
Definition: Preprocessor.h:742
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment. In this case there should b...
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
void HandlePragmaMark()
Definition: Pragma.cpp:363
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:730
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:850
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:90
PTHManager * getPTHManager()
Definition: Preprocessor.h:693
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:773
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
unsigned Column
Definition: Format.cpp:1202
void AddPragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:931
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:96
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:365
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:677
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177