clang  3.7.0
PPLexerChange.cpp
Go to the documentation of this file.
1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // current lexer stack.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
18 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/MacroInfo.h"
21 #include "llvm/ADT/StringSwitch.h"
22 #include "llvm/Support/FileSystem.h"
23 #include "llvm/Support/MemoryBuffer.h"
24 #include "llvm/Support/Path.h"
25 using namespace clang;
26 
28 
29 //===----------------------------------------------------------------------===//
30 // Miscellaneous Methods.
31 //===----------------------------------------------------------------------===//
32 
33 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
34 /// \#include. This looks through macro expansions and active _Pragma lexers.
36  if (IsFileLexer())
37  return IncludeMacroStack.empty();
38 
39  // If there are any stacked lexers, we're in a #include.
40  assert(IsFileLexer(IncludeMacroStack[0]) &&
41  "Top level include stack isn't our primary lexer?");
42  for (unsigned i = 1, e = IncludeMacroStack.size(); i != e; ++i)
43  if (IsFileLexer(IncludeMacroStack[i]))
44  return false;
45  return true;
46 }
47 
48 /// getCurrentLexer - Return the current file lexer being lexed from. Note
49 /// that this ignores any potentially active macro expansions and _Pragma
50 /// expansions going on at the time.
52  if (IsFileLexer())
53  return CurPPLexer;
54 
55  // Look for a stacked lexer.
56  for (unsigned i = IncludeMacroStack.size(); i != 0; --i) {
57  const IncludeStackInfo& ISI = IncludeMacroStack[i-1];
58  if (IsFileLexer(ISI))
59  return ISI.ThePPLexer;
60  }
61  return nullptr;
62 }
63 
64 
65 //===----------------------------------------------------------------------===//
66 // Methods for Entering and Callbacks for leaving various contexts
67 //===----------------------------------------------------------------------===//
68 
69 /// EnterSourceFile - Add a source file to the top of the include stack and
70 /// start lexing tokens from it instead of the current buffer.
72  SourceLocation Loc) {
73  assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
74  ++NumEnteredSourceFiles;
75 
76  if (MaxIncludeStackDepth < IncludeMacroStack.size())
77  MaxIncludeStackDepth = IncludeMacroStack.size();
78 
79  if (PTH) {
80  if (PTHLexer *PL = PTH->CreateLexer(FID)) {
81  EnterSourceFileWithPTH(PL, CurDir);
82  return false;
83  }
84  }
85 
86  // Get the MemoryBuffer for this FID, if it fails, we fail.
87  bool Invalid = false;
88  const llvm::MemoryBuffer *InputFile =
89  getSourceManager().getBuffer(FID, Loc, &Invalid);
90  if (Invalid) {
91  SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
92  Diag(Loc, diag::err_pp_error_opening_file)
93  << std::string(SourceMgr.getBufferName(FileStart)) << "";
94  return true;
95  }
96 
98  SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
99  CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
100  CodeCompletionLoc =
101  CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
102  }
103 
104  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
105  return false;
106 }
107 
108 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
109 /// and start lexing tokens from it instead of the current buffer.
110 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
111  const DirectoryLookup *CurDir) {
112 
113  // Add the current lexer to the include stack.
114  if (CurPPLexer || CurTokenLexer)
115  PushIncludeMacroStack();
116 
117  CurLexer.reset(TheLexer);
118  CurPPLexer = TheLexer;
119  CurDirLookup = CurDir;
120  CurSubmodule = nullptr;
121  if (CurLexerKind != CLK_LexAfterModuleImport)
122  CurLexerKind = CLK_Lexer;
123 
124  // Notify the client, if desired, that we are in a new source file.
125  if (Callbacks && !CurLexer->Is_PragmaLexer) {
126  SrcMgr::CharacteristicKind FileType =
127  SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
128 
129  Callbacks->FileChanged(CurLexer->getFileLoc(),
130  PPCallbacks::EnterFile, FileType);
131  }
132 }
133 
134 /// EnterSourceFileWithPTH - Add a source file to the top of the include stack
135 /// and start getting tokens from it using the PTH cache.
136 void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
137  const DirectoryLookup *CurDir) {
138 
139  if (CurPPLexer || CurTokenLexer)
140  PushIncludeMacroStack();
141 
142  CurDirLookup = CurDir;
143  CurPTHLexer.reset(PL);
144  CurPPLexer = CurPTHLexer.get();
145  CurSubmodule = nullptr;
146  if (CurLexerKind != CLK_LexAfterModuleImport)
147  CurLexerKind = CLK_PTHLexer;
148 
149  // Notify the client, if desired, that we are in a new source file.
150  if (Callbacks) {
151  FileID FID = CurPPLexer->getFileID();
152  SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
153  SrcMgr::CharacteristicKind FileType =
154  SourceMgr.getFileCharacteristic(EnterLoc);
155  Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
156  }
157 }
158 
159 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
160 /// tokens from it instead of the current buffer.
162  MacroInfo *Macro, MacroArgs *Args) {
163  std::unique_ptr<TokenLexer> TokLexer;
164  if (NumCachedTokenLexers == 0) {
165  TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
166  } else {
167  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
168  TokLexer->Init(Tok, ILEnd, Macro, Args);
169  }
170 
171  PushIncludeMacroStack();
172  CurDirLookup = nullptr;
173  CurTokenLexer = std::move(TokLexer);
174  if (CurLexerKind != CLK_LexAfterModuleImport)
175  CurLexerKind = CLK_TokenLexer;
176 }
177 
178 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
179 /// which will cause the lexer to start returning the specified tokens.
180 ///
181 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
182 /// not be subject to further macro expansion. Otherwise, these tokens will
183 /// be re-macro-expanded when/if expansion is enabled.
184 ///
185 /// If OwnsTokens is false, this method assumes that the specified stream of
186 /// tokens has a permanent owner somewhere, so they do not need to be copied.
187 /// If it is true, it assumes the array of tokens is allocated with new[] and
188 /// must be freed.
189 ///
190 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
191  bool DisableMacroExpansion,
192  bool OwnsTokens) {
193  if (CurLexerKind == CLK_CachingLexer) {
194  if (CachedLexPos < CachedTokens.size()) {
195  // We're entering tokens into the middle of our cached token stream. We
196  // can't represent that, so just insert the tokens into the buffer.
197  CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
198  Toks, Toks + NumToks);
199  if (OwnsTokens)
200  delete [] Toks;
201  return;
202  }
203 
204  // New tokens are at the end of the cached token sequnece; insert the
205  // token stream underneath the caching lexer.
206  ExitCachingLexMode();
207  EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
208  EnterCachingLexMode();
209  return;
210  }
211 
212  // Create a macro expander to expand from the specified token stream.
213  std::unique_ptr<TokenLexer> TokLexer;
214  if (NumCachedTokenLexers == 0) {
215  TokLexer = llvm::make_unique<TokenLexer>(
216  Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
217  } else {
218  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
219  TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
220  }
221 
222  // Save our current state.
223  PushIncludeMacroStack();
224  CurDirLookup = nullptr;
225  CurTokenLexer = std::move(TokLexer);
226  if (CurLexerKind != CLK_LexAfterModuleImport)
227  CurLexerKind = CLK_TokenLexer;
228 }
229 
230 /// \brief Compute the relative path that names the given file relative to
231 /// the given directory.
232 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
233  const FileEntry *File,
234  SmallString<128> &Result) {
235  Result.clear();
236 
237  StringRef FilePath = File->getDir()->getName();
238  StringRef Path = FilePath;
239  while (!Path.empty()) {
240  if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
241  if (CurDir == Dir) {
242  Result = FilePath.substr(Path.size());
243  llvm::sys::path::append(Result,
244  llvm::sys::path::filename(File->getName()));
245  return;
246  }
247  }
248 
249  Path = llvm::sys::path::parent_path(Path);
250  }
251 
252  Result = File->getName();
253 }
254 
255 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
256  if (CurTokenLexer) {
257  CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
258  return;
259  }
260  if (CurLexer) {
261  CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
262  return;
263  }
264  // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
265  // but it might if they're empty?
266 }
267 
268 /// \brief Determine the location to use as the end of the buffer for a lexer.
269 ///
270 /// If the file ends with a newline, form the EOF token on the newline itself,
271 /// rather than "on the line following it", which doesn't exist. This makes
272 /// diagnostics relating to the end of file include the last file that the user
273 /// actually typed, which is goodness.
274 const char *Preprocessor::getCurLexerEndPos() {
275  const char *EndPos = CurLexer->BufferEnd;
276  if (EndPos != CurLexer->BufferStart &&
277  (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
278  --EndPos;
279 
280  // Handle \n\r and \r\n:
281  if (EndPos != CurLexer->BufferStart &&
282  (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
283  EndPos[-1] != EndPos[0])
284  --EndPos;
285  }
286 
287  return EndPos;
288 }
289 
290 
291 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
292 /// the current file. This either returns the EOF token or pops a level off
293 /// the include stack and keeps going.
294 bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
295  assert(!CurTokenLexer &&
296  "Ending a file when currently in a macro!");
297 
298  // See if this file had a controlling macro.
299  if (CurPPLexer) { // Not ending a macro, ignore it.
300  if (const IdentifierInfo *ControllingMacro =
301  CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
302  // Okay, this has a controlling macro, remember in HeaderFileInfo.
303  if (const FileEntry *FE =
304  SourceMgr.getFileEntryForID(CurPPLexer->getFileID())) {
305  HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
306  if (MacroInfo *MI =
307  getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro))) {
308  MI->UsedForHeaderGuard = true;
309  }
310  if (const IdentifierInfo *DefinedMacro =
311  CurPPLexer->MIOpt.GetDefinedMacro()) {
312  if (!isMacroDefined(ControllingMacro) &&
313  DefinedMacro != ControllingMacro &&
314  HeaderInfo.FirstTimeLexingFile(FE)) {
315 
316  // If the edit distance between the two macros is more than 50%,
317  // DefinedMacro may not be header guard, or can be header guard of
318  // another header file. Therefore, it maybe defining something
319  // completely different. This can be observed in the wild when
320  // handling feature macros or header guards in different files.
321 
322  const StringRef ControllingMacroName = ControllingMacro->getName();
323  const StringRef DefinedMacroName = DefinedMacro->getName();
324  const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
325  DefinedMacroName.size()) / 2;
326  const unsigned ED = ControllingMacroName.edit_distance(
327  DefinedMacroName, true, MaxHalfLength);
328  if (ED <= MaxHalfLength) {
329  // Emit a warning for a bad header guard.
330  Diag(CurPPLexer->MIOpt.GetMacroLocation(),
331  diag::warn_header_guard)
332  << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
333  Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
334  diag::note_header_guard)
335  << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
336  << ControllingMacro
338  CurPPLexer->MIOpt.GetDefinedLocation(),
339  ControllingMacro->getName());
340  }
341  }
342  }
343  }
344  }
345  }
346 
347  // Complain about reaching a true EOF within arc_cf_code_audited.
348  // We don't want to complain about reaching the end of a macro
349  // instantiation or a _Pragma.
350  if (PragmaARCCFCodeAuditedLoc.isValid() &&
351  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
352  Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
353 
354  // Recover by leaving immediately.
355  PragmaARCCFCodeAuditedLoc = SourceLocation();
356  }
357 
358  // Complain about reaching a true EOF within assume_nonnull.
359  // We don't want to complain about reaching the end of a macro
360  // instantiation or a _Pragma.
361  if (PragmaAssumeNonNullLoc.isValid() &&
362  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
363  Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
364 
365  // Recover by leaving immediately.
366  PragmaAssumeNonNullLoc = SourceLocation();
367  }
368 
369  // If this is a #include'd file, pop it off the include stack and continue
370  // lexing the #includer file.
371  if (!IncludeMacroStack.empty()) {
372 
373  // If we lexed the code-completion file, act as if we reached EOF.
374  if (isCodeCompletionEnabled() && CurPPLexer &&
375  SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
376  CodeCompletionFileLoc) {
377  if (CurLexer) {
378  Result.startToken();
379  CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
380  CurLexer.reset();
381  } else {
382  assert(CurPTHLexer && "Got EOF but no current lexer set!");
383  CurPTHLexer->getEOF(Result);
384  CurPTHLexer.reset();
385  }
386 
387  CurPPLexer = nullptr;
388  return true;
389  }
390 
391  if (!isEndOfMacro && CurPPLexer &&
392  SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
393  // Notify SourceManager to record the number of FileIDs that were created
394  // during lexing of the #include'd file.
395  unsigned NumFIDs =
396  SourceMgr.local_sloc_entry_size() -
397  CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
398  SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
399  }
400 
401  FileID ExitedFID;
402  if (Callbacks && !isEndOfMacro && CurPPLexer)
403  ExitedFID = CurPPLexer->getFileID();
404 
405  bool LeavingSubmodule = CurSubmodule && CurLexer;
406  if (LeavingSubmodule) {
407  // Notify the parser that we've left the module.
408  const char *EndPos = getCurLexerEndPos();
409  Result.startToken();
410  CurLexer->BufferPtr = EndPos;
411  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
412  Result.setAnnotationEndLoc(Result.getLocation());
413  Result.setAnnotationValue(CurSubmodule);
414 
415  // We're done with this submodule.
416  LeaveSubmodule();
417  }
418 
419  // We're done with the #included file.
421 
422  // Propagate info about start-of-line/leading white-space/etc.
423  PropagateLineStartLeadingSpaceInfo(Result);
424 
425  // Notify the client, if desired, that we are in a new source file.
426  if (Callbacks && !isEndOfMacro && CurPPLexer) {
427  SrcMgr::CharacteristicKind FileType =
428  SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
429  Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
430  PPCallbacks::ExitFile, FileType, ExitedFID);
431  }
432 
433  // Client should lex another token unless we generated an EOM.
434  return LeavingSubmodule;
435  }
436 
437  // If this is the end of the main file, form an EOF token.
438  if (CurLexer) {
439  const char *EndPos = getCurLexerEndPos();
440  Result.startToken();
441  CurLexer->BufferPtr = EndPos;
442  CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
443 
444  if (isCodeCompletionEnabled()) {
445  // Inserting the code-completion point increases the source buffer by 1,
446  // but the main FileID was created before inserting the point.
447  // Compensate by reducing the EOF location by 1, otherwise the location
448  // will point to the next FileID.
449  // FIXME: This is hacky, the code-completion point should probably be
450  // inserted before the main FileID is created.
451  if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
452  Result.setLocation(Result.getLocation().getLocWithOffset(-1));
453  }
454 
456  // We're done with lexing.
457  CurLexer.reset();
458  } else {
459  assert(CurPTHLexer && "Got EOF but no current lexer set!");
460  CurPTHLexer->getEOF(Result);
461  CurPTHLexer.reset();
462  }
463 
465  CurPPLexer = nullptr;
466 
467  if (TUKind == TU_Complete) {
468  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
469  // collected all macro locations that we need to warn because they are not
470  // used.
471  for (WarnUnusedMacroLocsTy::iterator
472  I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
473  I!=E; ++I)
474  Diag(*I, diag::pp_macro_not_used);
475  }
476 
477  // If we are building a module that has an umbrella header, make sure that
478  // each of the headers within the directory covered by the umbrella header
479  // was actually included by the umbrella header.
480  if (Module *Mod = getCurrentModule()) {
481  if (Mod->getUmbrellaHeader()) {
482  SourceLocation StartLoc
483  = SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
484 
485  if (!getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
486  StartLoc)) {
488  const DirectoryEntry *Dir = Mod->getUmbrellaDir().Entry;
489  vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
490  std::error_code EC;
491  for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End;
492  Entry != End && !EC; Entry.increment(EC)) {
493  using llvm::StringSwitch;
494 
495  // Check whether this entry has an extension typically associated with
496  // headers.
497  if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
498  .Cases(".h", ".H", ".hh", ".hpp", true)
499  .Default(false))
500  continue;
501 
502  if (const FileEntry *Header =
503  getFileManager().getFile(Entry->getName()))
504  if (!getSourceManager().hasFileInfo(Header)) {
505  if (!ModMap.isHeaderInUnavailableModule(Header)) {
506  // Find the relative path that would access this header.
507  SmallString<128> RelativePath;
508  computeRelativePath(FileMgr, Dir, Header, RelativePath);
509  Diag(StartLoc, diag::warn_uncovered_module_header)
510  << Mod->getFullModuleName() << RelativePath;
511  }
512  }
513  }
514  }
515  }
516  }
517 
518  return true;
519 }
520 
521 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
522 /// hits the end of its token stream.
524  assert(CurTokenLexer && !CurPPLexer &&
525  "Ending a macro when currently in a #include file!");
526 
527  if (!MacroExpandingLexersStack.empty() &&
528  MacroExpandingLexersStack.back().first == CurTokenLexer.get())
529  removeCachedMacroExpandedTokensOfLastLexer();
530 
531  // Delete or cache the now-dead macro expander.
532  if (NumCachedTokenLexers == TokenLexerCacheSize)
533  CurTokenLexer.reset();
534  else
535  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
536 
537  // Handle this like a #include file being popped off the stack.
538  return HandleEndOfFile(Result, true);
539 }
540 
541 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
542 /// lexer stack. This should only be used in situations where the current
543 /// state of the top-of-stack lexer is unknown.
545  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
546 
547  if (CurTokenLexer) {
548  // Delete or cache the now-dead macro expander.
549  if (NumCachedTokenLexers == TokenLexerCacheSize)
550  CurTokenLexer.reset();
551  else
552  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
553  }
554 
555  PopIncludeMacroStack();
556 }
557 
558 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
559 /// comment (/##/) in microsoft mode, this method handles updating the current
560 /// state, returning the token on the next source line.
562  assert(CurTokenLexer && !CurPPLexer &&
563  "Pasted comment can only be formed from macro");
564 
565  // We handle this by scanning for the closest real lexer, switching it to
566  // raw mode and preprocessor mode. This will cause it to return \n as an
567  // explicit EOD token.
568  PreprocessorLexer *FoundLexer = nullptr;
569  bool LexerWasInPPMode = false;
570  for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
571  IncludeStackInfo &ISI = *(IncludeMacroStack.end()-i-1);
572  if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
573 
574  // Once we find a real lexer, mark it as raw mode (disabling macro
575  // expansions) and preprocessor mode (return EOD). We know that the lexer
576  // was *not* in raw mode before, because the macro that the comment came
577  // from was expanded. However, it could have already been in preprocessor
578  // mode (#if COMMENT) in which case we have to return it to that mode and
579  // return EOD.
580  FoundLexer = ISI.ThePPLexer;
581  FoundLexer->LexingRawMode = true;
582  LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
583  FoundLexer->ParsingPreprocessorDirective = true;
584  break;
585  }
586 
587  // Okay, we either found and switched over the lexer, or we didn't find a
588  // lexer. In either case, finish off the macro the comment came from, getting
589  // the next token.
590  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
591 
592  // Discarding comments as long as we don't have EOF or EOD. This 'comments
593  // out' the rest of the line, including any tokens that came from other macros
594  // that were active, as in:
595  // #define submacro a COMMENT b
596  // submacro c
597  // which should lex to 'a' only: 'b' and 'c' should be removed.
598  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
599  Lex(Tok);
600 
601  // If we got an eod token, then we successfully found the end of the line.
602  if (Tok.is(tok::eod)) {
603  assert(FoundLexer && "Can't get end of line without an active lexer");
604  // Restore the lexer back to normal mode instead of raw mode.
605  FoundLexer->LexingRawMode = false;
606 
607  // If the lexer was already in preprocessor mode, just return the EOD token
608  // to finish the preprocessor line.
609  if (LexerWasInPPMode) return;
610 
611  // Otherwise, switch out of PP mode and return the next lexed token.
612  FoundLexer->ParsingPreprocessorDirective = false;
613  return Lex(Tok);
614  }
615 
616  // If we got an EOF token, then we reached the end of the token stream but
617  // didn't find an explicit \n. This can only happen if there was no lexer
618  // active (an active lexer would return EOD at EOF if there was no \n in
619  // preprocessor directive mode), so just return EOF as our token.
620  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
621 }
622 
623 void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc) {
624  if (!getLangOpts().ModulesLocalVisibility) {
625  // Just track that we entered this submodule.
626  BuildingSubmoduleStack.push_back(
627  BuildingSubmoduleInfo(M, ImportLoc, CurSubmoduleState));
628  return;
629  }
630 
631  // Resolve as much of the module definition as we can now, before we enter
632  // one of its headers.
633  // FIXME: Can we enable Complain here?
634  // FIXME: Can we do this when local visibility is disabled?
636  ModMap.resolveExports(M, /*Complain=*/false);
637  ModMap.resolveUses(M, /*Complain=*/false);
638  ModMap.resolveConflicts(M, /*Complain=*/false);
639 
640  // If this is the first time we've entered this module, set up its state.
641  auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
642  auto &State = R.first->second;
643  bool FirstTime = R.second;
644  if (FirstTime) {
645  // Determine the set of starting macros for this submodule; take these
646  // from the "null" module (the predefines buffer).
647  //
648  // FIXME: If we have local visibility but not modules enabled, the
649  // NullSubmoduleState is polluted by #defines in the top-level source
650  // file.
651  auto &StartingMacros = NullSubmoduleState.Macros;
652 
653  // Restore to the starting state.
654  // FIXME: Do this lazily, when each macro name is first referenced.
655  for (auto &Macro : StartingMacros) {
656  // Skip uninteresting macros.
657  if (!Macro.second.getLatest() &&
658  Macro.second.getOverriddenMacros().empty())
659  continue;
660 
661  MacroState MS(Macro.second.getLatest());
662  MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
663  State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
664  }
665  }
666 
667  // Track that we entered this module.
668  BuildingSubmoduleStack.push_back(
669  BuildingSubmoduleInfo(M, ImportLoc, CurSubmoduleState));
670 
671  // Switch to this submodule as the current submodule.
672  CurSubmoduleState = &State;
673 
674  // This module is visible to itself.
675  if (FirstTime)
676  makeModuleVisible(M, ImportLoc);
677 }
678 
679 void Preprocessor::LeaveSubmodule() {
680  auto &Info = BuildingSubmoduleStack.back();
681 
682  Module *LeavingMod = Info.M;
683  SourceLocation ImportLoc = Info.ImportLoc;
684 
685  // Create ModuleMacros for any macros defined in this submodule.
686  for (auto &Macro : CurSubmoduleState->Macros) {
687  auto *II = const_cast<IdentifierInfo*>(Macro.first);
688 
689  // Find the starting point for the MacroDirective chain in this submodule.
690  MacroDirective *OldMD = nullptr;
691  if (getLangOpts().ModulesLocalVisibility) {
692  // FIXME: It'd be better to start at the state from when we most recently
693  // entered this submodule, but it doesn't really matter.
694  auto &PredefMacros = NullSubmoduleState.Macros;
695  auto PredefMacroIt = PredefMacros.find(Macro.first);
696  if (PredefMacroIt == PredefMacros.end())
697  OldMD = nullptr;
698  else
699  OldMD = PredefMacroIt->second.getLatest();
700  }
701 
702  // This module may have exported a new macro. If so, create a ModuleMacro
703  // representing that fact.
704  bool ExplicitlyPublic = false;
705  for (auto *MD = Macro.second.getLatest(); MD != OldMD;
706  MD = MD->getPrevious()) {
707  assert(MD && "broken macro directive chain");
708 
709  // Stop on macros defined in other submodules we #included along the way.
710  // There's no point doing this if we're tracking local submodule
711  // visibility, since there can be no such directives in our list.
712  if (!getLangOpts().ModulesLocalVisibility) {
713  Module *Mod = getModuleContainingLocation(MD->getLocation());
714  if (Mod != LeavingMod)
715  break;
716  }
717 
718  if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
719  // The latest visibility directive for a name in a submodule affects
720  // all the directives that come before it.
721  if (VisMD->isPublic())
722  ExplicitlyPublic = true;
723  else if (!ExplicitlyPublic)
724  // Private with no following public directive: not exported.
725  break;
726  } else {
727  MacroInfo *Def = nullptr;
728  if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
729  Def = DefMD->getInfo();
730 
731  // FIXME: Issue a warning if multiple headers for the same submodule
732  // define a macro, rather than silently ignoring all but the first.
733  bool IsNew;
734  // Don't bother creating a module macro if it would represent a #undef
735  // that doesn't override anything.
736  if (Def || !Macro.second.getOverriddenMacros().empty())
737  addModuleMacro(LeavingMod, II, Def,
738  Macro.second.getOverriddenMacros(), IsNew);
739  break;
740  }
741  }
742  }
743 
744  // FIXME: Before we leave this submodule, we should parse all the other
745  // headers within it. Otherwise, we're left with an inconsistent state
746  // where we've made the module visible but don't yet have its complete
747  // contents.
748 
749  // Put back the outer module's state, if we're tracking it.
750  if (getLangOpts().ModulesLocalVisibility)
751  CurSubmoduleState = Info.OuterSubmoduleState;
752 
753  BuildingSubmoduleStack.pop_back();
754 
755  // A nested #include makes the included submodule visible.
756  makeModuleVisible(LeavingMod, ImportLoc);
757 }
SourceManager & getSourceManager() const
Definition: Preprocessor.h:682
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:115
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
Defines the clang::FileManager interface and associated types.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Definition: ModuleMap.cpp:434
Defines the SourceManager interface.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:866
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:78
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:585
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
SourceLocation GetDefinedLocation() const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasFileInfo(const FileEntry *File) const
Module * getModuleContainingLocation(SourceLocation Loc)
Find the module that contains the specified location, either directly or indirectly.
The virtual file system interface.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:419
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:879
LineState State
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:846
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:679
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
Describes a module or submodule.
Definition: Basic/Module.h:49
FileManager & getFileManager() const
Definition: Preprocessor.h:681
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
IntrusiveRefCntPtr< vfs::FileSystem > getVirtualFileSystem() const
Definition: FileManager.h:223
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:683
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setAnnotationValue(void *val)
Definition: Token.h:228
bool LexingRawMode
True if in raw mode.
void makeModuleVisible(Module *M, SourceLocation Loc)
Defines the clang::Preprocessor interface.
const char * getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
void HandleMicrosoftCommentPaste(Token &Tok)
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:853
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location...
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:308
const char * getName() const
Definition: FileManager.h:84
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
bool isValid() const
Return true if this is a valid SourceLocation object.
void setAnnotationEndLoc(SourceLocation L)
Definition: Token.h:142
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:53
void EnterTokenStream(const Token *Toks, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens)
Add a "macro" context to the top of the include stack, which will cause the lexer to start returning ...
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
FileID getMainFileID() const
Returns the FileID of the main source file.
bool is(tok::TokenKind K) const
Definition: Token.h:95
bool isIgnored(unsigned DiagID, SourceLocation Loc) const
Determine whether the diagnostic is known to be ignored.
Definition: Diagnostic.h:645
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:676
const char * getName() const
Definition: FileManager.h:45
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:781
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined...
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Encapsulates the data about a macro definition (e.g. its tokens).
Definition: MacroInfo.h:34
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:40
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
Definition: Diagnostic.h:115
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID...
The translation unit is a complete translation unit.
Definition: LangOptions.h:165
SourceLocation GetMacroLocation() const
bool FirstTimeLexingFile(const FileEntry *File)
Return true if this is the first time encountering this header.
Definition: HeaderSearch.h:469
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
Definition: Token.h:132
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:93
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:343
void SetFileControllingMacro(const FileEntry *File, const IdentifierInfo *ControllingMacro)
Mark the specified file as having a controlling macro.
Definition: HeaderSearch.h:463
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...