clang  3.7.0
CommentLexer.cpp
Go to the documentation of this file.
4 #include "clang/Basic/CharInfo.h"
5 #include "llvm/ADT/StringExtras.h"
6 #include "llvm/ADT/StringSwitch.h"
7 #include "llvm/Support/ConvertUTF.h"
8 #include "llvm/Support/ErrorHandling.h"
9 
10 namespace clang {
11 namespace comments {
12 
13 void Token::dump(const Lexer &L, const SourceManager &SM) const {
14  llvm::errs() << "comments::Token Kind=" << Kind << " ";
15  Loc.dump(SM);
16  llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
17 }
18 
19 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
20  return isLetter(C);
21 }
22 
23 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
24  return isDigit(C);
25 }
26 
27 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
28  return isHexDigit(C);
29 }
30 
31 static inline StringRef convertCodePointToUTF8(
32  llvm::BumpPtrAllocator &Allocator,
33  unsigned CodePoint) {
34  char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
35  char *ResolvedPtr = Resolved;
36  if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
37  return StringRef(Resolved, ResolvedPtr - Resolved);
38  else
39  return StringRef();
40 }
41 
42 namespace {
43 
44 #include "clang/AST/CommentHTMLTags.inc"
45 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
46 
47 } // unnamed namespace
48 
49 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
50  // Fast path, first check a few most widely used named character references.
51  return llvm::StringSwitch<StringRef>(Name)
52  .Case("amp", "&")
53  .Case("lt", "<")
54  .Case("gt", ">")
55  .Case("quot", "\"")
56  .Case("apos", "\'")
57  // Slow path.
58  .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
59 }
60 
61 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
62  unsigned CodePoint = 0;
63  for (unsigned i = 0, e = Name.size(); i != e; ++i) {
65  CodePoint *= 10;
66  CodePoint += Name[i] - '0';
67  }
68  return convertCodePointToUTF8(Allocator, CodePoint);
69 }
70 
71 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
72  unsigned CodePoint = 0;
73  for (unsigned i = 0, e = Name.size(); i != e; ++i) {
74  CodePoint *= 16;
75  const char C = Name[i];
77  CodePoint += llvm::hexDigitValue(C);
78  }
79  return convertCodePointToUTF8(Allocator, CodePoint);
80 }
81 
82 void Lexer::skipLineStartingDecorations() {
83  // This function should be called only for C comments
84  assert(CommentState == LCS_InsideCComment);
85 
86  if (BufferPtr == CommentEnd)
87  return;
88 
89  switch (*BufferPtr) {
90  case ' ':
91  case '\t':
92  case '\f':
93  case '\v': {
94  const char *NewBufferPtr = BufferPtr;
95  NewBufferPtr++;
96  if (NewBufferPtr == CommentEnd)
97  return;
98 
99  char C = *NewBufferPtr;
100  while (isHorizontalWhitespace(C)) {
101  NewBufferPtr++;
102  if (NewBufferPtr == CommentEnd)
103  return;
104  C = *NewBufferPtr;
105  }
106  if (C == '*')
107  BufferPtr = NewBufferPtr + 1;
108  break;
109  }
110  case '*':
111  BufferPtr++;
112  break;
113  }
114 }
115 
116 namespace {
117 /// Returns pointer to the first newline character in the string.
118 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
119  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
120  if (isVerticalWhitespace(*BufferPtr))
121  return BufferPtr;
122  }
123  return BufferEnd;
124 }
125 
126 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
127  if (BufferPtr == BufferEnd)
128  return BufferPtr;
129 
130  if (*BufferPtr == '\n')
131  BufferPtr++;
132  else {
133  assert(*BufferPtr == '\r');
134  BufferPtr++;
135  if (BufferPtr != BufferEnd && *BufferPtr == '\n')
136  BufferPtr++;
137  }
138  return BufferPtr;
139 }
140 
141 const char *skipNamedCharacterReference(const char *BufferPtr,
142  const char *BufferEnd) {
143  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
145  return BufferPtr;
146  }
147  return BufferEnd;
148 }
149 
150 const char *skipDecimalCharacterReference(const char *BufferPtr,
151  const char *BufferEnd) {
152  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
154  return BufferPtr;
155  }
156  return BufferEnd;
157 }
158 
159 const char *skipHexCharacterReference(const char *BufferPtr,
160  const char *BufferEnd) {
161  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162  if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
163  return BufferPtr;
164  }
165  return BufferEnd;
166 }
167 
168 bool isHTMLIdentifierStartingCharacter(char C) {
169  return isLetter(C);
170 }
171 
172 bool isHTMLIdentifierCharacter(char C) {
173  return isAlphanumeric(C);
174 }
175 
176 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
177  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
178  if (!isHTMLIdentifierCharacter(*BufferPtr))
179  return BufferPtr;
180  }
181  return BufferEnd;
182 }
183 
184 /// Skip HTML string quoted in single or double quotes. Escaping quotes inside
185 /// string allowed.
186 ///
187 /// Returns pointer to closing quote.
188 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
189 {
190  const char Quote = *BufferPtr;
191  assert(Quote == '\"' || Quote == '\'');
192 
193  BufferPtr++;
194  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
195  const char C = *BufferPtr;
196  if (C == Quote && BufferPtr[-1] != '\\')
197  return BufferPtr;
198  }
199  return BufferEnd;
200 }
201 
202 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
203  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204  if (!isWhitespace(*BufferPtr))
205  return BufferPtr;
206  }
207  return BufferEnd;
208 }
209 
210 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
211  return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
212 }
213 
214 bool isCommandNameStartCharacter(char C) {
215  return isLetter(C);
216 }
217 
218 bool isCommandNameCharacter(char C) {
219  return isAlphanumeric(C);
220 }
221 
222 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
223  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
224  if (!isCommandNameCharacter(*BufferPtr))
225  return BufferPtr;
226  }
227  return BufferEnd;
228 }
229 
230 /// Return the one past end pointer for BCPL comments.
231 /// Handles newlines escaped with backslash or trigraph for backslahs.
232 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
233  const char *CurPtr = BufferPtr;
234  while (CurPtr != BufferEnd) {
235  while (!isVerticalWhitespace(*CurPtr)) {
236  CurPtr++;
237  if (CurPtr == BufferEnd)
238  return BufferEnd;
239  }
240  // We found a newline, check if it is escaped.
241  const char *EscapePtr = CurPtr - 1;
242  while(isHorizontalWhitespace(*EscapePtr))
243  EscapePtr--;
244 
245  if (*EscapePtr == '\\' ||
246  (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
247  EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
248  // We found an escaped newline.
249  CurPtr = skipNewline(CurPtr, BufferEnd);
250  } else
251  return CurPtr; // Not an escaped newline.
252  }
253  return BufferEnd;
254 }
255 
256 /// Return the one past end pointer for C comments.
257 /// Very dumb, does not handle escaped newlines or trigraphs.
258 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
259  for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
260  if (*BufferPtr == '*') {
261  assert(BufferPtr + 1 != BufferEnd);
262  if (*(BufferPtr + 1) == '/')
263  return BufferPtr;
264  }
265  }
266  llvm_unreachable("buffer end hit before '*/' was seen");
267 }
268 
269 } // unnamed namespace
270 
271 void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
273  const unsigned TokLen = TokEnd - BufferPtr;
274  Result.setLocation(getSourceLocation(BufferPtr));
275  Result.setKind(Kind);
276  Result.setLength(TokLen);
277 #ifndef NDEBUG
278  Result.TextPtr = "<UNSET>";
279  Result.IntVal = 7;
280 #endif
281  BufferPtr = TokEnd;
282 }
283 
284 void Lexer::lexCommentText(Token &T) {
285  assert(CommentState == LCS_InsideBCPLComment ||
286  CommentState == LCS_InsideCComment);
287 
288  switch (State) {
289  case LS_Normal:
290  break;
291  case LS_VerbatimBlockFirstLine:
292  lexVerbatimBlockFirstLine(T);
293  return;
294  case LS_VerbatimBlockBody:
295  lexVerbatimBlockBody(T);
296  return;
297  case LS_VerbatimLineText:
298  lexVerbatimLineText(T);
299  return;
300  case LS_HTMLStartTag:
301  lexHTMLStartTag(T);
302  return;
303  case LS_HTMLEndTag:
304  lexHTMLEndTag(T);
305  return;
306  }
307 
308  assert(State == LS_Normal);
309 
310  const char *TokenPtr = BufferPtr;
311  assert(TokenPtr < CommentEnd);
312  while (TokenPtr != CommentEnd) {
313  switch(*TokenPtr) {
314  case '\\':
315  case '@': {
316  // Commands that start with a backslash and commands that start with
317  // 'at' have equivalent semantics. But we keep information about the
318  // exact syntax in AST for comments.
319  tok::TokenKind CommandKind =
320  (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
321  TokenPtr++;
322  if (TokenPtr == CommentEnd) {
323  formTextToken(T, TokenPtr);
324  return;
325  }
326  char C = *TokenPtr;
327  switch (C) {
328  default:
329  break;
330 
331  case '\\': case '@': case '&': case '$':
332  case '#': case '<': case '>': case '%':
333  case '\"': case '.': case ':':
334  // This is one of \\ \@ \& \$ etc escape sequences.
335  TokenPtr++;
336  if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
337  // This is the \:: escape sequence.
338  TokenPtr++;
339  }
340  StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
341  formTokenWithChars(T, TokenPtr, tok::text);
342  T.setText(UnescapedText);
343  return;
344  }
345 
346  // Don't make zero-length commands.
347  if (!isCommandNameStartCharacter(*TokenPtr)) {
348  formTextToken(T, TokenPtr);
349  return;
350  }
351 
352  TokenPtr = skipCommandName(TokenPtr, CommentEnd);
353  unsigned Length = TokenPtr - (BufferPtr + 1);
354 
355  // Hardcoded support for lexing LaTeX formula commands
356  // \f$ \f[ \f] \f{ \f} as a single command.
357  if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
358  C = *TokenPtr;
359  if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
360  TokenPtr++;
361  Length++;
362  }
363  }
364 
365  StringRef CommandName(BufferPtr + 1, Length);
366 
367  const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
368  if (!Info) {
369  if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
370  StringRef CorrectedName = Info->Name;
371  SourceLocation Loc = getSourceLocation(BufferPtr);
372  SourceRange CommandRange(Loc.getLocWithOffset(1),
373  getSourceLocation(TokenPtr));
374  Diag(Loc, diag::warn_correct_comment_command_name)
375  << CommandName << CorrectedName
376  << FixItHint::CreateReplacement(CommandRange, CorrectedName);
377  } else {
378  formTokenWithChars(T, TokenPtr, tok::unknown_command);
379  T.setUnknownCommandName(CommandName);
380  Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
381  return;
382  }
383  }
384  if (Info->IsVerbatimBlockCommand) {
385  setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
386  return;
387  }
388  if (Info->IsVerbatimLineCommand) {
389  setupAndLexVerbatimLine(T, TokenPtr, Info);
390  return;
391  }
392  formTokenWithChars(T, TokenPtr, CommandKind);
393  T.setCommandID(Info->getID());
394  return;
395  }
396 
397  case '&':
398  lexHTMLCharacterReference(T);
399  return;
400 
401  case '<': {
402  TokenPtr++;
403  if (TokenPtr == CommentEnd) {
404  formTextToken(T, TokenPtr);
405  return;
406  }
407  const char C = *TokenPtr;
408  if (isHTMLIdentifierStartingCharacter(C))
409  setupAndLexHTMLStartTag(T);
410  else if (C == '/')
411  setupAndLexHTMLEndTag(T);
412  else
413  formTextToken(T, TokenPtr);
414 
415  return;
416  }
417 
418  case '\n':
419  case '\r':
420  TokenPtr = skipNewline(TokenPtr, CommentEnd);
421  formTokenWithChars(T, TokenPtr, tok::newline);
422 
423  if (CommentState == LCS_InsideCComment)
424  skipLineStartingDecorations();
425  return;
426 
427  default: {
428  size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
429  find_first_of("\n\r\\@&<");
430  if (End != StringRef::npos)
431  TokenPtr += End;
432  else
433  TokenPtr = CommentEnd;
434  formTextToken(T, TokenPtr);
435  return;
436  }
437  }
438  }
439 }
440 
441 void Lexer::setupAndLexVerbatimBlock(Token &T,
442  const char *TextBegin,
443  char Marker, const CommandInfo *Info) {
444  assert(Info->IsVerbatimBlockCommand);
445 
446  VerbatimBlockEndCommandName.clear();
447  VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
448  VerbatimBlockEndCommandName.append(Info->EndCommandName);
449 
450  formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
451  T.setVerbatimBlockID(Info->getID());
452 
453  // If there is a newline following the verbatim opening command, skip the
454  // newline so that we don't create an tok::verbatim_block_line with empty
455  // text content.
456  if (BufferPtr != CommentEnd &&
457  isVerticalWhitespace(*BufferPtr)) {
458  BufferPtr = skipNewline(BufferPtr, CommentEnd);
459  State = LS_VerbatimBlockBody;
460  return;
461  }
462 
463  State = LS_VerbatimBlockFirstLine;
464 }
465 
466 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
467 again:
468  assert(BufferPtr < CommentEnd);
469 
470  // FIXME: It would be better to scan the text once, finding either the block
471  // end command or newline.
472  //
473  // Extract current line.
474  const char *Newline = findNewline(BufferPtr, CommentEnd);
475  StringRef Line(BufferPtr, Newline - BufferPtr);
476 
477  // Look for end command in current line.
478  size_t Pos = Line.find(VerbatimBlockEndCommandName);
479  const char *TextEnd;
480  const char *NextLine;
481  if (Pos == StringRef::npos) {
482  // Current line is completely verbatim.
483  TextEnd = Newline;
484  NextLine = skipNewline(Newline, CommentEnd);
485  } else if (Pos == 0) {
486  // Current line contains just an end command.
487  const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
488  StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
489  formTokenWithChars(T, End, tok::verbatim_block_end);
490  T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
491  State = LS_Normal;
492  return;
493  } else {
494  // There is some text, followed by end command. Extract text first.
495  TextEnd = BufferPtr + Pos;
496  NextLine = TextEnd;
497  // If there is only whitespace before end command, skip whitespace.
498  if (isWhitespace(BufferPtr, TextEnd)) {
499  BufferPtr = TextEnd;
500  goto again;
501  }
502  }
503 
504  StringRef Text(BufferPtr, TextEnd - BufferPtr);
505  formTokenWithChars(T, NextLine, tok::verbatim_block_line);
506  T.setVerbatimBlockText(Text);
507 
508  State = LS_VerbatimBlockBody;
509 }
510 
511 void Lexer::lexVerbatimBlockBody(Token &T) {
512  assert(State == LS_VerbatimBlockBody);
513 
514  if (CommentState == LCS_InsideCComment)
515  skipLineStartingDecorations();
516 
517  if (BufferPtr == CommentEnd) {
518  formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
519  T.setVerbatimBlockText("");
520  return;
521  }
522 
523  lexVerbatimBlockFirstLine(T);
524 }
525 
526 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
527  const CommandInfo *Info) {
528  assert(Info->IsVerbatimLineCommand);
529  formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
530  T.setVerbatimLineID(Info->getID());
531 
532  State = LS_VerbatimLineText;
533 }
534 
535 void Lexer::lexVerbatimLineText(Token &T) {
536  assert(State == LS_VerbatimLineText);
537 
538  // Extract current line.
539  const char *Newline = findNewline(BufferPtr, CommentEnd);
540  StringRef Text(BufferPtr, Newline - BufferPtr);
541  formTokenWithChars(T, Newline, tok::verbatim_line_text);
542  T.setVerbatimLineText(Text);
543 
544  State = LS_Normal;
545 }
546 
547 void Lexer::lexHTMLCharacterReference(Token &T) {
548  const char *TokenPtr = BufferPtr;
549  assert(*TokenPtr == '&');
550  TokenPtr++;
551  if (TokenPtr == CommentEnd) {
552  formTextToken(T, TokenPtr);
553  return;
554  }
555  const char *NamePtr;
556  bool isNamed = false;
557  bool isDecimal = false;
558  char C = *TokenPtr;
560  NamePtr = TokenPtr;
561  TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
562  isNamed = true;
563  } else if (C == '#') {
564  TokenPtr++;
565  if (TokenPtr == CommentEnd) {
566  formTextToken(T, TokenPtr);
567  return;
568  }
569  C = *TokenPtr;
571  NamePtr = TokenPtr;
572  TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
573  isDecimal = true;
574  } else if (C == 'x' || C == 'X') {
575  TokenPtr++;
576  NamePtr = TokenPtr;
577  TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
578  } else {
579  formTextToken(T, TokenPtr);
580  return;
581  }
582  } else {
583  formTextToken(T, TokenPtr);
584  return;
585  }
586  if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
587  *TokenPtr != ';') {
588  formTextToken(T, TokenPtr);
589  return;
590  }
591  StringRef Name(NamePtr, TokenPtr - NamePtr);
592  TokenPtr++; // Skip semicolon.
593  StringRef Resolved;
594  if (isNamed)
595  Resolved = resolveHTMLNamedCharacterReference(Name);
596  else if (isDecimal)
597  Resolved = resolveHTMLDecimalCharacterReference(Name);
598  else
599  Resolved = resolveHTMLHexCharacterReference(Name);
600 
601  if (Resolved.empty()) {
602  formTextToken(T, TokenPtr);
603  return;
604  }
605  formTokenWithChars(T, TokenPtr, tok::text);
606  T.setText(Resolved);
607  return;
608 }
609 
610 void Lexer::setupAndLexHTMLStartTag(Token &T) {
611  assert(BufferPtr[0] == '<' &&
612  isHTMLIdentifierStartingCharacter(BufferPtr[1]));
613  const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
614  StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
615  if (!isHTMLTagName(Name)) {
616  formTextToken(T, TagNameEnd);
617  return;
618  }
619 
620  formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
621  T.setHTMLTagStartName(Name);
622 
623  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
624 
625  const char C = *BufferPtr;
626  if (BufferPtr != CommentEnd &&
627  (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
628  State = LS_HTMLStartTag;
629 }
630 
631 void Lexer::lexHTMLStartTag(Token &T) {
632  assert(State == LS_HTMLStartTag);
633 
634  const char *TokenPtr = BufferPtr;
635  char C = *TokenPtr;
636  if (isHTMLIdentifierCharacter(C)) {
637  TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
638  StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
639  formTokenWithChars(T, TokenPtr, tok::html_ident);
640  T.setHTMLIdent(Ident);
641  } else {
642  switch (C) {
643  case '=':
644  TokenPtr++;
645  formTokenWithChars(T, TokenPtr, tok::html_equals);
646  break;
647  case '\"':
648  case '\'': {
649  const char *OpenQuote = TokenPtr;
650  TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
651  const char *ClosingQuote = TokenPtr;
652  if (TokenPtr != CommentEnd) // Skip closing quote.
653  TokenPtr++;
654  formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
655  T.setHTMLQuotedString(StringRef(OpenQuote + 1,
656  ClosingQuote - (OpenQuote + 1)));
657  break;
658  }
659  case '>':
660  TokenPtr++;
661  formTokenWithChars(T, TokenPtr, tok::html_greater);
662  State = LS_Normal;
663  return;
664  case '/':
665  TokenPtr++;
666  if (TokenPtr != CommentEnd && *TokenPtr == '>') {
667  TokenPtr++;
668  formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
669  } else
670  formTextToken(T, TokenPtr);
671 
672  State = LS_Normal;
673  return;
674  }
675  }
676 
677  // Now look ahead and return to normal state if we don't see any HTML tokens
678  // ahead.
679  BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
680  if (BufferPtr == CommentEnd) {
681  State = LS_Normal;
682  return;
683  }
684 
685  C = *BufferPtr;
686  if (!isHTMLIdentifierStartingCharacter(C) &&
687  C != '=' && C != '\"' && C != '\'' && C != '>') {
688  State = LS_Normal;
689  return;
690  }
691 }
692 
693 void Lexer::setupAndLexHTMLEndTag(Token &T) {
694  assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
695 
696  const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
697  const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
698  StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
699  if (!isHTMLTagName(Name)) {
700  formTextToken(T, TagNameEnd);
701  return;
702  }
703 
704  const char *End = skipWhitespace(TagNameEnd, CommentEnd);
705 
706  formTokenWithChars(T, End, tok::html_end_tag);
707  T.setHTMLTagEndName(Name);
708 
709  if (BufferPtr != CommentEnd && *BufferPtr == '>')
710  State = LS_HTMLEndTag;
711 }
712 
713 void Lexer::lexHTMLEndTag(Token &T) {
714  assert(BufferPtr != CommentEnd && *BufferPtr == '>');
715 
716  formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
717  State = LS_Normal;
718 }
719 
720 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
721  const CommandTraits &Traits,
722  SourceLocation FileLoc,
723  const char *BufferStart, const char *BufferEnd):
724  Allocator(Allocator), Diags(Diags), Traits(Traits),
725  BufferStart(BufferStart), BufferEnd(BufferEnd),
726  FileLoc(FileLoc), BufferPtr(BufferStart),
727  CommentState(LCS_BeforeComment), State(LS_Normal) {
728 }
729 
730 void Lexer::lex(Token &T) {
731 again:
732  switch (CommentState) {
733  case LCS_BeforeComment:
734  if (BufferPtr == BufferEnd) {
735  formTokenWithChars(T, BufferPtr, tok::eof);
736  return;
737  }
738 
739  assert(*BufferPtr == '/');
740  BufferPtr++; // Skip first slash.
741  switch(*BufferPtr) {
742  case '/': { // BCPL comment.
743  BufferPtr++; // Skip second slash.
744 
745  if (BufferPtr != BufferEnd) {
746  // Skip Doxygen magic marker, if it is present.
747  // It might be missing because of a typo //< or /*<, or because we
748  // merged this non-Doxygen comment into a bunch of Doxygen comments
749  // around it: /** ... */ /* ... */ /** ... */
750  const char C = *BufferPtr;
751  if (C == '/' || C == '!')
752  BufferPtr++;
753  }
754 
755  // Skip less-than symbol that marks trailing comments.
756  // Skip it even if the comment is not a Doxygen one, because //< and /*<
757  // are frequent typos.
758  if (BufferPtr != BufferEnd && *BufferPtr == '<')
759  BufferPtr++;
760 
761  CommentState = LCS_InsideBCPLComment;
762  if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
763  State = LS_Normal;
764  CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
765  goto again;
766  }
767  case '*': { // C comment.
768  BufferPtr++; // Skip star.
769 
770  // Skip Doxygen magic marker.
771  const char C = *BufferPtr;
772  if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
773  BufferPtr++;
774 
775  // Skip less-than symbol that marks trailing comments.
776  if (BufferPtr != BufferEnd && *BufferPtr == '<')
777  BufferPtr++;
778 
779  CommentState = LCS_InsideCComment;
780  State = LS_Normal;
781  CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
782  goto again;
783  }
784  default:
785  llvm_unreachable("second character of comment should be '/' or '*'");
786  }
787 
788  case LCS_BetweenComments: {
789  // Consecutive comments are extracted only if there is only whitespace
790  // between them. So we can search for the start of the next comment.
791  const char *EndWhitespace = BufferPtr;
792  while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
793  EndWhitespace++;
794 
795  // Turn any whitespace between comments (and there is only whitespace
796  // between them -- guaranteed by comment extraction) into a newline. We
797  // have two newlines between C comments in total (first one was synthesized
798  // after a comment).
799  formTokenWithChars(T, EndWhitespace, tok::newline);
800 
801  CommentState = LCS_BeforeComment;
802  break;
803  }
804 
805  case LCS_InsideBCPLComment:
806  case LCS_InsideCComment:
807  if (BufferPtr != CommentEnd) {
808  lexCommentText(T);
809  break;
810  } else {
811  // Skip C comment closing sequence.
812  if (CommentState == LCS_InsideCComment) {
813  assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
814  BufferPtr += 2;
815  assert(BufferPtr <= BufferEnd);
816 
817  // Synthenize newline just after the C comment, regardless if there is
818  // actually a newline.
819  formTokenWithChars(T, BufferPtr, tok::newline);
820 
821  CommentState = LCS_BetweenComments;
822  break;
823  } else {
824  // Don't synthesized a newline after BCPL comment.
825  CommentState = LCS_BetweenComments;
826  goto again;
827  }
828  }
829  }
830 }
831 
832 StringRef Lexer::getSpelling(const Token &Tok,
833  const SourceManager &SourceMgr,
834  bool *Invalid) const {
835  SourceLocation Loc = Tok.getLocation();
836  std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
837 
838  bool InvalidTemp = false;
839  StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
840  if (InvalidTemp) {
841  *Invalid = true;
842  return StringRef();
843  }
844 
845  const char *Begin = File.data() + LocInfo.second;
846  return StringRef(Begin, Tok.getLength());
847 }
848 
849 } // end namespace comments
850 } // end namespace clang
851 
static LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
Definition: CharInfo.h:94
static StringRef convertCodePointToUTF8(llvm::BumpPtrAllocator &Allocator, unsigned CodePoint)
unsigned getLength() const LLVM_READONLY
Definition: CommentLexer.h:97
static LLVM_READONLY bool isWhitespace(unsigned char c)
Definition: CharInfo.h:88
static LLVM_READONLY bool isLetter(unsigned char c)
Return true if this character is an ASCII letter: [a-zA-Z].
Definition: CharInfo.h:112
const CommandInfo * getCommandInfo(StringRef Name) const
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
LineState State
static LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)
Definition: CharInfo.h:71
void dump(const Lexer &L, const SourceManager &SM) const
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:135
void dump(const SourceManager &SM) const
llvm::SpecificBumpPtrAllocator< FormatToken > Allocator
Definition: Format.cpp:1211
static unsigned skipWhitespace(unsigned Idx, StringRef Str, unsigned Length)
Skip over whitespace in the string, starting at the given index.
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
SourceManager & SM
static bool isHTMLHexCharacterReferenceCharacter(char C)
SourceManager & SourceMgr
Definition: Format.cpp:1205
const CommandInfo * getCommandInfoOrNULL(StringRef Name) const
FormatToken * Token
const CommandInfo * getTypoCorrectCommandInfo(StringRef Typo) const
static bool isHTMLNamedCharacterReferenceCharacter(char C)
SourceLocation getLocation() const LLVM_READONLY
Definition: CommentLexer.h:82
static bool isNamed(const NamedDecl *ND, const char(&Str)[Len])
Definition: Decl.cpp:2429
Kind
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
AnnotatedLine & Line
static LLVM_READONLY bool isAlphanumeric(unsigned char c)
Return true if this character is an ASCII letter or digit: [a-zA-Z0-9].
Definition: CharInfo.h:118
Comment lexer.
Definition: CommentLexer.h:222
static bool isHTMLDecimalCharacterReferenceCharacter(char C)
Comment token.
Definition: CommentLexer.h:57
StringRef getSpelling(const Token &Tok, const SourceManager &SourceMgr, bool *Invalid=nullptr) const
static LLVM_READONLY bool isVerticalWhitespace(unsigned char c)
Definition: CharInfo.h:79
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
Definition: Diagnostic.h:115
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
This class handles loading and caching of source files into memory.
static LLVM_READONLY bool isHexDigit(unsigned char c)
Return true if this character is an ASCII hex digit: [0-9a-fA-F].
Definition: CharInfo.h:124