20 #include "llvm/ADT/SmallString.h"
21 using namespace clang;
40 NextTokGetsSpace =
false;
43 DisableMacroExpansion =
false;
51 assert(Tokens[0].getLocation().isValid());
52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
53 "Macro defined in macro?");
54 assert(ExpandLocStart.
isValid());
71 ExpandFunctionArguments();
84 bool disableMacroExpansion,
bool ownsTokens) {
92 OwnsTokens = ownsTokens;
93 DisableMacroExpansion = disableMacroExpansion;
97 AtStartOfLine =
false;
98 HasLeadingSpace =
false;
99 NextTokGetsSpace =
false;
111 void TokenLexer::destroy() {
121 if (ActualArgs) ActualArgs->
destroy(PP);
124 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
134 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
147 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
151 if (HasPasteOperator)
152 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
155 ResultToks.pop_back();
161 if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
162 ResultToks.pop_back();
165 NextTokGetsSpace =
false;
171 void TokenLexer::ExpandFunctionArguments() {
178 bool MadeChange =
false;
180 for (
unsigned i = 0, e = NumTokens; i != e; ++i) {
184 const Token &CurTok = Tokens[i];
185 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
186 NextTokGetsSpace =
true;
188 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
189 int ArgNo = Macro->
getArgumentNum(Tokens[i+1].getIdentifierInfo());
190 assert(ArgNo != -1 &&
"Token following # is not an argument?");
193 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
195 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
198 if (CurTok.
is(tok::hash))
213 if (NextTokGetsSpace)
216 ResultToks.push_back(Res);
219 NextTokGetsSpace =
false;
224 bool NonEmptyPasteBefore =
225 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
226 bool PasteBefore = i != 0 && Tokens[i-1].
is(tok::hashhash);
227 bool PasteAfter = i+1 != e && Tokens[i+1].
is(tok::hashhash);
228 assert(!NonEmptyPasteBefore || PasteBefore);
236 ResultToks.push_back(CurTok);
238 if (NextTokGetsSpace) {
240 NextTokGetsSpace =
false;
241 }
else if (PasteBefore && !NonEmptyPasteBefore)
256 MaybeRemoveCommaBeforeVaArgs(ResultToks,
264 if (!PasteBefore && !PasteAfter) {
265 const Token *ResultArgToks;
273 ResultArgToks = ArgTok;
277 unsigned FirstResult = ResultToks.size();
279 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
286 ResultToks.back().is(tok::comma))
291 for (
unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
292 Token &Tok = ResultToks[i];
293 if (Tok.
is(tok::hashhash))
299 ResultToks.begin()+FirstResult,
308 NextTokGetsSpace =
false;
322 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
323 ResultToks[ResultToks.size()-2].is(tok::comma) &&
327 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
330 ResultToks.append(ArgToks, ArgToks+NumToks);
334 for (
unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
336 Token &Tok = ResultToks[i];
337 if (Tok.
is(tok::hashhash))
341 if (ExpandLocStart.
isValid()) {
343 ResultToks.end()-NumToks, ResultToks.end());
356 if (NextTokGetsSpace)
359 NextTokGetsSpace =
false;
378 if (NonEmptyPasteBefore) {
379 assert(ResultToks.back().is(tok::hashhash));
380 ResultToks.pop_back();
388 MaybeRemoveCommaBeforeVaArgs(ResultToks,
397 assert(!OwnsTokens &&
"This would leak if we already own the token list");
399 NumTokens = ResultToks.size();
402 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
411 const Token &SecondTok) {
412 return FirstTok.
is(tok::identifier) &&
438 bool isFirstToken = CurToken == 0;
441 Tok = Tokens[CurToken++];
443 bool TokenIsFromPaste =
false;
447 if (!isAtEnd() && Macro &&
448 (Tokens[CurToken].is(tok::hashhash) ||
456 if (PasteTokens(Tok))
459 TokenIsFromPaste =
true;
467 if (ExpandLocStart.
isValid() &&
471 if (Tok.
is(tok::comment)) {
477 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
494 AtStartOfLine =
false;
495 HasLeadingSpace =
false;
523 bool TokenLexer::PasteTokens(
Token &Tok) {
527 if (PP.
getLangOpts().MicrosoftExt && (CurToken >= 2) &&
528 Tokens[CurToken - 2].is(tok::hashhash))
532 const char *ResultTokStrPtr =
nullptr;
538 if (Tokens[CurToken].is(tok::hashhash))
540 assert(!isAtEnd() &&
"No token on the RHS of a paste operator!");
543 const Token &RHS = Tokens[CurToken];
550 const char *BufPtr = &Buffer[0];
551 bool Invalid =
false;
552 unsigned LHSLen = PP.
getSpelling(Tok, BufPtr, &Invalid);
553 if (BufPtr != &Buffer[0])
554 memcpy(&Buffer[0], BufPtr, LHSLen);
558 BufPtr = Buffer.data() + LHSLen;
559 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
562 if (RHSLen && BufPtr != &Buffer[LHSLen])
564 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
567 Buffer.resize(LHSLen+RHSLen);
576 ResultTokTmp.
setKind(tok::string_literal);
589 Result.
setKind(tok::raw_identifier);
597 "Should be a raw location into scratch buffer");
599 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
601 bool Invalid =
false;
602 const char *ScratchBufStart
603 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
609 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
611 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
617 bool isInvalid = !TL.LexFromRawLexer(Result);
630 RHS.
is(tok::slash)) {
631 HandleMicrosoftCommentPaste(Tok);
646 : diag::err_pp_bad_paste)
656 if (Result.
is(tok::hashhash))
667 }
while (!isAtEnd() && Tokens[CurToken].
is(tok::hashhash));
678 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
680 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
682 while (SM.
getFileID(StartLoc) != MacroFID)
693 if (Tok.
is(tok::raw_identifier)) {
708 return Tokens[CurToken].
is(tok::l_paren);
714 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
722 void TokenLexer::HandleMicrosoftCommentPaste(
Token &Tok) {
728 assert(Macro &&
"Token streams can't paste comments");
739 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
740 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
741 "Not appropriate for token streams");
745 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
746 "Expected loc to come from the macro definition");
748 unsigned relativeOffset = 0;
764 Token *&begin_tokens,
765 Token * end_tokens) {
766 assert(begin_tokens < end_tokens);
781 Token *NextTok = begin_tokens + 1;
782 for (; NextTok < end_tokens; ++NextTok) {
792 if (RelOffs < 0 || RelOffs > 50)
799 Token &LastConsecutiveTok = *(NextTok-1);
803 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
811 for (; begin_tokens < NextTok; ++begin_tokens) {
812 Token &Tok = *begin_tokens;
825 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
831 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
833 while (begin_tokens < end_tokens) {
835 if (end_tokens - begin_tokens == 1) {
836 Token &Tok = *begin_tokens;
847 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
bool isAtStartOfLine() const
SourceManager & getSourceManager() const
bool isPoisoned() const
Return true if this token has been poisoned.
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
unsigned isNextTokenLParen() const
Defines the SourceManager interface.
unsigned getNextLocalOffset() const
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry. e.g for assert(foo == bar); There will be a single SLocEntry for the "foo == bar" chunk and locations for the 'foo', '==', 'bar' tokens will point inside that chunk.
bool isParsingPreprocessorDirective() const
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool stringifiedInMacro() const
void setRawIdentifierData(const char *Ptr)
const LangOptions & getLangOpts() const
void setKind(tok::TokenKind K)
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
const std::vector< Token > & getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP)
void destroy(Preprocessor &PP)
const Token * getUnexpArgument(unsigned Arg) const
tokens_iterator tokens_begin() const
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
unsigned getNumArgs() const
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
Defines the clang::Preprocessor interface.
int getArgumentNum(const IdentifierInfo *Arg) const
Return the argument number of the specified identifier, or -1 if the identifier is not a formal argum...
void HandleMicrosoftCommentPaste(Token &Tok)
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
unsigned getDefinitionLength(SourceManager &SM) const
Get length in characters of the macro definition.
bool isNot(tok::TokenKind K) const
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
const char * getLiteralData() const
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
void setLength(unsigned Len)
bool isValid() const
Return true if this is a valid SourceLocation object.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
bool is(tok::TokenKind K) const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
static unsigned getArgLength(const Token *ArgPtr)
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isFunctionLike() const
Encapsulates the data about a macro definition (e.g. its tokens).
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
tokens_iterator tokens_end() const
static bool isInvalid(SourceLocation Loc, bool *Invalid)
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
unsigned getLength() const
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const