20 #include "llvm/ADT/SmallString.h"
21 using namespace clang;
40 NextTokGetsSpace =
false;
43 DisableMacroExpansion =
false;
51 assert(Tokens[0].getLocation().isValid());
52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
53 "Macro defined in macro?");
54 assert(ExpandLocStart.
isValid());
71 ExpandFunctionArguments();
84 bool disableMacroExpansion,
bool ownsTokens) {
92 OwnsTokens = ownsTokens;
93 DisableMacroExpansion = disableMacroExpansion;
97 AtStartOfLine =
false;
98 HasLeadingSpace =
false;
99 NextTokGetsSpace =
false;
111 void TokenLexer::destroy() {
121 if (ActualArgs) ActualArgs->
destroy(PP);
124 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
134 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
147 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
151 if (HasPasteOperator)
152 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
155 ResultToks.pop_back();
161 if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
162 ResultToks.pop_back();
165 NextTokGetsSpace =
false;
171 void TokenLexer::ExpandFunctionArguments() {
178 bool MadeChange =
false;
180 for (
unsigned i = 0, e = NumTokens; i != e; ++i) {
184 const Token &CurTok = Tokens[i];
185 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
186 NextTokGetsSpace =
true;
188 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
189 int ArgNo = Macro->
getArgumentNum(Tokens[i+1].getIdentifierInfo());
190 assert(ArgNo != -1 &&
"Token following # is not an argument?");
193 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
195 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
198 if (CurTok.
is(tok::hash))
213 if (NextTokGetsSpace)
216 ResultToks.push_back(Res);
219 NextTokGetsSpace =
false;
224 bool NonEmptyPasteBefore =
225 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
226 bool PasteBefore = i != 0 && Tokens[i-1].
is(tok::hashhash);
227 bool PasteAfter = i+1 != e && Tokens[i+1].
is(tok::hashhash);
228 assert(!NonEmptyPasteBefore || PasteBefore);
236 ResultToks.push_back(CurTok);
238 if (NextTokGetsSpace) {
240 NextTokGetsSpace =
false;
241 }
else if (PasteBefore && !NonEmptyPasteBefore)
256 MaybeRemoveCommaBeforeVaArgs(ResultToks,
264 if (!PasteBefore && !PasteAfter) {
265 const Token *ResultArgToks;
273 ResultArgToks = ArgTok;
277 unsigned FirstResult = ResultToks.size();
279 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
286 ResultToks.back().is(tok::comma))
291 for (
unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
292 Token &Tok = ResultToks[i];
293 if (Tok.
is(tok::hashhash))
299 ResultToks.begin()+FirstResult,
308 NextTokGetsSpace =
false;
322 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
323 ResultToks[ResultToks.size()-2].is(tok::comma) &&
327 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
330 ResultToks.append(ArgToks, ArgToks+NumToks);
334 for (
unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
336 Token &Tok = ResultToks[i];
337 if (Tok.
is(tok::hashhash))
341 if (ExpandLocStart.
isValid()) {
343 ResultToks.end()-NumToks, ResultToks.end());
356 if (NextTokGetsSpace)
359 NextTokGetsSpace =
false;
378 if (NonEmptyPasteBefore) {
379 assert(ResultToks.back().is(tok::hashhash));
380 ResultToks.pop_back();
388 MaybeRemoveCommaBeforeVaArgs(ResultToks,
397 assert(!OwnsTokens &&
"This would leak if we already own the token list");
399 NumTokens = ResultToks.size();
402 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
411 const Token &SecondTok) {
412 return FirstTok.
is(tok::identifier) &&
438 bool isFirstToken = CurToken == 0;
441 Tok = Tokens[CurToken++];
443 bool TokenIsFromPaste =
false;
447 if (!isAtEnd() && Macro &&
448 (Tokens[CurToken].is(tok::hashhash) ||
456 if (PasteTokens(Tok))
459 TokenIsFromPaste =
true;
467 if (ExpandLocStart.
isValid() &&
471 if (Tok.
is(tok::comment)) {
477 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
494 AtStartOfLine =
false;
495 HasLeadingSpace =
false;
523 bool TokenLexer::PasteTokens(
Token &Tok) {
527 if (PP.
getLangOpts().MicrosoftExt && (CurToken >= 2) &&
528 Tokens[CurToken - 2].is(tok::hashhash))
532 const char *ResultTokStrPtr =
nullptr;
538 if (Tokens[CurToken].is(tok::hashhash))
540 assert(!isAtEnd() &&
"No token on the RHS of a paste operator!");
543 const Token &RHS = Tokens[CurToken];
550 const char *BufPtr = &Buffer[0];
551 bool Invalid =
false;
552 unsigned LHSLen = PP.
getSpelling(Tok, BufPtr, &Invalid);
553 if (BufPtr != &Buffer[0])
554 memcpy(&Buffer[0], BufPtr, LHSLen);
558 BufPtr = Buffer.data() + LHSLen;
559 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
562 if (RHSLen && BufPtr != &Buffer[LHSLen])
564 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
567 Buffer.resize(LHSLen+RHSLen);
576 ResultTokTmp.
setKind(tok::string_literal);
589 Result.
setKind(tok::raw_identifier);
597 "Should be a raw location into scratch buffer");
599 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
601 bool Invalid =
false;
602 const char *ScratchBufStart
603 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
609 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
611 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
617 bool isInvalid = !TL.LexFromRawLexer(Result);
636 RHS.
is(tok::slash)) {
637 HandleMicrosoftCommentPaste(Tok, Loc);
647 : diag::err_pp_bad_paste)
657 if (Result.
is(tok::hashhash))
668 }
while (!isAtEnd() && Tokens[CurToken].
is(tok::hashhash));
679 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
681 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
683 while (SM.
getFileID(StartLoc) != MacroFID)
694 if (Tok.
is(tok::raw_identifier)) {
709 return Tokens[CurToken].
is(tok::l_paren);
715 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
724 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
731 assert(Macro &&
"Token streams can't paste comments");
742 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
743 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
744 "Not appropriate for token streams");
748 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
749 "Expected loc to come from the macro definition");
751 unsigned relativeOffset = 0;
767 Token *&begin_tokens,
768 Token * end_tokens) {
769 assert(begin_tokens < end_tokens);
784 Token *NextTok = begin_tokens + 1;
785 for (; NextTok < end_tokens; ++NextTok) {
795 if (RelOffs < 0 || RelOffs > 50)
802 Token &LastConsecutiveTok = *(NextTok-1);
806 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
814 for (; begin_tokens < NextTok; ++begin_tokens) {
815 Token &Tok = *begin_tokens;
828 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
834 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
836 while (begin_tokens < end_tokens) {
838 if (end_tokens - begin_tokens == 1) {
839 Token &Tok = *begin_tokens;
850 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
bool isPoisoned() const
Return true if this token has been poisoned.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Defines the SourceManager interface.
unsigned getNextLocalOffset() const
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setRawIdentifierData(const char *Ptr)
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const std::vector< Token > & getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
tokens_iterator tokens_begin() const
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been 'stringified...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
unsigned getNumArgs() const
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
Defines the clang::Preprocessor interface.
int getArgumentNum(const IdentifierInfo *Arg) const
Return the argument number of the specified identifier, or -1 if the identifier is not a formal argum...
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
unsigned getDefinitionLength(SourceManager &SM) const
Get length in characters of the macro definition.
bool isNot(tok::TokenKind K) const
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
Encodes a location in the source.
void setLength(unsigned Len)
bool isValid() const
Return true if this is a valid SourceLocation object.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isFunctionLike() const
Encapsulates the data about a macro definition (e.g.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
tokens_iterator tokens_end() const
static bool isInvalid(SourceLocation Loc, bool *Invalid)
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
unsigned getLength() const
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const