17 #include "llvm/Support/ErrorHandling.h"
18 using namespace clang;
26 (CPlusPlus11 && (Str[0] ==
'u' || Str[0] ==
'U' || Str[0] ==
'R'))) {
33 if (Str[1] ==
'R' && Str[0] !=
'R' && Str.size() == 2 &&
CPlusPlus11)
37 if (Str[0] ==
'u' && Str[1] ==
'8') {
38 if (Str.size() == 2)
return true;
39 if (Str.size() == 3 && Str[2] ==
'R')
return true;
48 bool TokenConcatenation::IsIdentifierStringPrefix(
const Token &Tok)
const {
57 LangOpts.CPlusPlus11);
62 const char *TokPtr = Buffer;
64 return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus11);
71 memset(TokenInfo, 0,
sizeof(TokenInfo));
74 TokenInfo[tok::identifier ] |= aci_custom;
75 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
76 TokenInfo[tok::period ] |= aci_custom_firstchar;
77 TokenInfo[tok::amp ] |= aci_custom_firstchar;
78 TokenInfo[tok::plus ] |= aci_custom_firstchar;
79 TokenInfo[tok::minus ] |= aci_custom_firstchar;
80 TokenInfo[tok::slash ] |= aci_custom_firstchar;
81 TokenInfo[tok::less ] |= aci_custom_firstchar;
82 TokenInfo[tok::greater ] |= aci_custom_firstchar;
83 TokenInfo[tok::pipe ] |= aci_custom_firstchar;
84 TokenInfo[tok::percent ] |= aci_custom_firstchar;
85 TokenInfo[tok::colon ] |= aci_custom_firstchar;
86 TokenInfo[tok::hash ] |= aci_custom_firstchar;
87 TokenInfo[tok::arrow ] |= aci_custom_firstchar;
91 TokenInfo[tok::string_literal ] |= aci_custom;
92 TokenInfo[tok::wide_string_literal ] |= aci_custom;
93 TokenInfo[tok::utf8_string_literal ] |= aci_custom;
94 TokenInfo[tok::utf16_string_literal] |= aci_custom;
95 TokenInfo[tok::utf32_string_literal] |= aci_custom;
96 TokenInfo[tok::char_constant ] |= aci_custom;
97 TokenInfo[tok::wide_char_constant ] |= aci_custom;
98 TokenInfo[tok::utf16_char_constant ] |= aci_custom;
99 TokenInfo[tok::utf32_char_constant ] |= aci_custom;
104 TokenInfo[tok::utf8_char_constant] |= aci_custom;
107 TokenInfo[tok::amp ] |= aci_avoid_equal;
108 TokenInfo[tok::plus ] |= aci_avoid_equal;
109 TokenInfo[tok::minus ] |= aci_avoid_equal;
110 TokenInfo[tok::slash ] |= aci_avoid_equal;
111 TokenInfo[tok::less ] |= aci_avoid_equal;
112 TokenInfo[tok::greater ] |= aci_avoid_equal;
113 TokenInfo[tok::pipe ] |= aci_avoid_equal;
114 TokenInfo[tok::percent ] |= aci_avoid_equal;
115 TokenInfo[tok::star ] |= aci_avoid_equal;
116 TokenInfo[tok::exclaim ] |= aci_avoid_equal;
117 TokenInfo[tok::lessless ] |= aci_avoid_equal;
118 TokenInfo[tok::greatergreater] |= aci_avoid_equal;
119 TokenInfo[tok::caret ] |= aci_avoid_equal;
120 TokenInfo[tok::equal ] |= aci_avoid_equal;
128 return II->getNameStart()[0];
138 const char *TokPtr = Buffer;
158 const Token &PrevTok,
159 const Token &Tok)
const {
171 PrevKind = tok::identifier;
174 unsigned ConcatInfo = TokenInfo[PrevKind];
177 if (ConcatInfo == 0)
return false;
179 if (ConcatInfo & aci_avoid_equal) {
181 if (Tok.
isOneOf(tok::equal, tok::equalequal))
183 ConcatInfo &= ~aci_avoid_equal;
187 assert(Tok.
isOneOf(tok::annot_module_include, tok::annot_module_begin,
188 tok::annot_module_end) &&
189 "unexpected annotation in AvoidConcat");
200 if (ConcatInfo & aci_custom) {
208 llvm_unreachable(
"InitAvoidConcatTokenInfo built wrong");
210 case tok::raw_identifier:
211 llvm_unreachable(
"tok::raw_identifier in non-raw lexing mode!");
213 case tok::string_literal:
214 case tok::wide_string_literal:
215 case tok::utf8_string_literal:
216 case tok::utf16_string_literal:
217 case tok::utf32_string_literal:
218 case tok::char_constant:
219 case tok::wide_char_constant:
220 case tok::utf8_char_constant:
221 case tok::utf16_char_constant:
222 case tok::utf32_char_constant:
236 case tok::identifier:
238 if (Tok.
is(tok::numeric_constant))
242 Tok.
isOneOf(tok::wide_string_literal, tok::utf8_string_literal,
243 tok::utf16_string_literal, tok::utf32_string_literal,
244 tok::wide_char_constant, tok::utf8_char_constant,
245 tok::utf16_char_constant, tok::utf32_char_constant))
249 if (Tok.
isNot(tok::char_constant) && Tok.
isNot(tok::string_literal))
254 return IsIdentifierStringPrefix(PrevTok);
256 case tok::numeric_constant:
258 FirstChar ==
'+' || FirstChar ==
'-';
260 return (FirstChar ==
'.' && PrevPrevTok.
is(tok::period)) ||
264 return FirstChar ==
'&';
266 return FirstChar ==
'+';
268 return FirstChar ==
'-' || FirstChar ==
'>';
270 return FirstChar ==
'*' || FirstChar ==
'/';
272 return FirstChar ==
'<' || FirstChar ==
':' || FirstChar ==
'%';
274 return FirstChar ==
'>';
276 return FirstChar ==
'|';
278 return FirstChar ==
'>' || FirstChar ==
':';
280 return FirstChar ==
'>' ||
283 return FirstChar ==
'#' || FirstChar ==
'@' || FirstChar ==
'%';
285 return PP.
getLangOpts().CPlusPlus && FirstChar ==
'*';
SourceManager & getSourceManager() const
static LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) const
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
static LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
const LangOptions & getLangOpts() const
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getKind() const
Defines the clang::Preprocessor interface.
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
bool isNot(tok::TokenKind K) const
const char * getLiteralData() const
TokenConcatenation(Preprocessor &PP)
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
bool is(tok::TokenKind K) const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
static bool IsStringPrefix(StringRef Str, bool CPlusPlus11)
static char GetFirstChar(Preprocessor &PP, const Token &Tok)
unsigned getLength() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const