clang  3.7.0
TokenAnnotator.cpp
Go to the documentation of this file.
1 //===--- TokenAnnotator.cpp - Format C++ code -----------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file implements a token annotator, i.e. creates
12 /// \c AnnotatedTokens out of \c FormatTokens with required extra information.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "TokenAnnotator.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/Support/Debug.h"
20 
21 #define DEBUG_TYPE "format-token-annotator"
22 
23 namespace clang {
24 namespace format {
25 
26 namespace {
27 
28 /// \brief A parser that gathers additional information about tokens.
29 ///
30 /// The \c TokenAnnotator tries to match parenthesis and square brakets and
31 /// store a parenthesis levels. It also tries to resolve matching "<" and ">"
32 /// into template parameter lists.
33 class AnnotatingParser {
34 public:
35  AnnotatingParser(const FormatStyle &Style, AnnotatedLine &Line,
36  const AdditionalKeywords &Keywords)
37  : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
38  Keywords(Keywords) {
39  Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
40  resetTokenMetadata(CurrentToken);
41  }
42 
43 private:
44  bool parseAngle() {
45  if (!CurrentToken)
46  return false;
47  FormatToken *Left = CurrentToken->Previous;
48  Left->ParentBracket = Contexts.back().ContextKind;
49  ScopedContextCreator ContextCreator(*this, tok::less, 10);
50 
51  // If this angle is in the context of an expression, we need to be more
52  // hesitant to detect it as opening template parameters.
53  bool InExprContext = Contexts.back().IsExpression;
54 
55  Contexts.back().IsExpression = false;
56  // If there's a template keyword before the opening angle bracket, this is a
57  // template parameter, not an argument.
58  Contexts.back().InTemplateArgument =
59  Left->Previous && Left->Previous->Tok.isNot(tok::kw_template);
60 
61  if (Style.Language == FormatStyle::LK_Java &&
62  CurrentToken->is(tok::question))
63  next();
64 
65  while (CurrentToken) {
66  if (CurrentToken->is(tok::greater)) {
67  Left->MatchingParen = CurrentToken;
68  CurrentToken->MatchingParen = Left;
69  CurrentToken->Type = TT_TemplateCloser;
70  next();
71  return true;
72  }
73  if (CurrentToken->is(tok::question) &&
74  Style.Language == FormatStyle::LK_Java) {
75  next();
76  continue;
77  }
78  if (CurrentToken->isOneOf(tok::r_paren, tok::r_square, tok::r_brace) ||
79  (CurrentToken->isOneOf(tok::colon, tok::question) && InExprContext))
80  return false;
81  // If a && or || is found and interpreted as a binary operator, this set
82  // of angles is likely part of something like "a < b && c > d". If the
83  // angles are inside an expression, the ||/&& might also be a binary
84  // operator that was misinterpreted because we are parsing template
85  // parameters.
86  // FIXME: This is getting out of hand, write a decent parser.
87  if (CurrentToken->Previous->isOneOf(tok::pipepipe, tok::ampamp) &&
88  CurrentToken->Previous->is(TT_BinaryOperator) &&
89  Contexts[Contexts.size() - 2].IsExpression &&
90  !Line.startsWith(tok::kw_template))
91  return false;
92  updateParameterCount(Left, CurrentToken);
93  if (!consumeToken())
94  return false;
95  }
96  return false;
97  }
98 
99  bool parseParens(bool LookForDecls = false) {
100  if (!CurrentToken)
101  return false;
102  FormatToken *Left = CurrentToken->Previous;
103  Left->ParentBracket = Contexts.back().ContextKind;
104  ScopedContextCreator ContextCreator(*this, tok::l_paren, 1);
105 
106  // FIXME: This is a bit of a hack. Do better.
107  Contexts.back().ColonIsForRangeExpr =
108  Contexts.size() == 2 && Contexts[0].ColonIsForRangeExpr;
109 
110  bool StartsObjCMethodExpr = false;
111  if (CurrentToken->is(tok::caret)) {
112  // (^ can start a block type.
113  Left->Type = TT_ObjCBlockLParen;
114  } else if (FormatToken *MaybeSel = Left->Previous) {
115  // @selector( starts a selector.
116  if (MaybeSel->isObjCAtKeyword(tok::objc_selector) && MaybeSel->Previous &&
117  MaybeSel->Previous->is(tok::at)) {
118  StartsObjCMethodExpr = true;
119  }
120  }
121 
122  if (Left->Previous &&
123  (Left->Previous->isOneOf(tok::kw_static_assert, tok::kw_if,
124  tok::kw_while, tok::l_paren, tok::comma) ||
125  Left->Previous->is(TT_BinaryOperator))) {
126  // static_assert, if and while usually contain expressions.
127  Contexts.back().IsExpression = true;
128  } else if (Left->Previous && Left->Previous->is(tok::r_square) &&
129  Left->Previous->MatchingParen &&
130  Left->Previous->MatchingParen->is(TT_LambdaLSquare)) {
131  // This is a parameter list of a lambda expression.
132  Contexts.back().IsExpression = false;
133  } else if (Line.InPPDirective &&
134  (!Left->Previous ||
135  !Left->Previous->isOneOf(tok::identifier,
136  TT_OverloadedOperator))) {
137  Contexts.back().IsExpression = true;
138  } else if (Contexts[Contexts.size() - 2].CaretFound) {
139  // This is the parameter list of an ObjC block.
140  Contexts.back().IsExpression = false;
141  } else if (Left->Previous && Left->Previous->is(tok::kw___attribute)) {
142  Left->Type = TT_AttributeParen;
143  } else if (Left->Previous && Left->Previous->is(TT_ForEachMacro)) {
144  // The first argument to a foreach macro is a declaration.
145  Contexts.back().IsForEachMacro = true;
146  Contexts.back().IsExpression = false;
147  } else if (Left->Previous && Left->Previous->MatchingParen &&
148  Left->Previous->MatchingParen->is(TT_ObjCBlockLParen)) {
149  Contexts.back().IsExpression = false;
150  }
151 
152  if (StartsObjCMethodExpr) {
153  Contexts.back().ColonIsObjCMethodExpr = true;
154  Left->Type = TT_ObjCMethodExpr;
155  }
156 
157  bool MightBeFunctionType = CurrentToken->is(tok::star);
158  bool HasMultipleLines = false;
159  bool HasMultipleParametersOnALine = false;
160  bool MightBeObjCForRangeLoop =
161  Left->Previous && Left->Previous->is(tok::kw_for);
162  while (CurrentToken) {
163  // LookForDecls is set when "if (" has been seen. Check for
164  // 'identifier' '*' 'identifier' followed by not '=' -- this
165  // '*' has to be a binary operator but determineStarAmpUsage() will
166  // categorize it as an unary operator, so set the right type here.
167  if (LookForDecls && CurrentToken->Next) {
168  FormatToken *Prev = CurrentToken->getPreviousNonComment();
169  if (Prev) {
170  FormatToken *PrevPrev = Prev->getPreviousNonComment();
171  FormatToken *Next = CurrentToken->Next;
172  if (PrevPrev && PrevPrev->is(tok::identifier) &&
173  Prev->isOneOf(tok::star, tok::amp, tok::ampamp) &&
174  CurrentToken->is(tok::identifier) && Next->isNot(tok::equal)) {
175  Prev->Type = TT_BinaryOperator;
176  LookForDecls = false;
177  }
178  }
179  }
180 
181  if (CurrentToken->Previous->is(TT_PointerOrReference) &&
182  CurrentToken->Previous->Previous->isOneOf(tok::l_paren,
183  tok::coloncolon))
184  MightBeFunctionType = true;
185  if (CurrentToken->Previous->is(TT_BinaryOperator))
186  Contexts.back().IsExpression = true;
187  if (CurrentToken->is(tok::r_paren)) {
188  if (MightBeFunctionType && CurrentToken->Next &&
189  (CurrentToken->Next->is(tok::l_paren) ||
190  (CurrentToken->Next->is(tok::l_square) &&
191  !Contexts.back().IsExpression)))
192  Left->Type = TT_FunctionTypeLParen;
193  Left->MatchingParen = CurrentToken;
194  CurrentToken->MatchingParen = Left;
195 
196  if (StartsObjCMethodExpr) {
197  CurrentToken->Type = TT_ObjCMethodExpr;
198  if (Contexts.back().FirstObjCSelectorName) {
199  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
200  Contexts.back().LongestObjCSelectorName;
201  }
202  }
203 
204  if (Left->is(TT_AttributeParen))
205  CurrentToken->Type = TT_AttributeParen;
206  if (Left->Previous && Left->Previous->is(TT_JavaAnnotation))
207  CurrentToken->Type = TT_JavaAnnotation;
208  if (Left->Previous && Left->Previous->is(TT_LeadingJavaAnnotation))
209  CurrentToken->Type = TT_LeadingJavaAnnotation;
210 
211  if (!HasMultipleLines)
212  Left->PackingKind = PPK_Inconclusive;
213  else if (HasMultipleParametersOnALine)
214  Left->PackingKind = PPK_BinPacked;
215  else
216  Left->PackingKind = PPK_OnePerLine;
217 
218  next();
219  return true;
220  }
221  if (CurrentToken->isOneOf(tok::r_square, tok::r_brace))
222  return false;
223 
224  if (CurrentToken->is(tok::l_brace))
225  Left->Type = TT_Unknown; // Not TT_ObjCBlockLParen
226  if (CurrentToken->is(tok::comma) && CurrentToken->Next &&
227  !CurrentToken->Next->HasUnescapedNewline &&
228  !CurrentToken->Next->isTrailingComment())
229  HasMultipleParametersOnALine = true;
230  if (CurrentToken->isOneOf(tok::kw_const, tok::kw_auto) ||
231  CurrentToken->isSimpleTypeSpecifier())
232  Contexts.back().IsExpression = false;
233  if (CurrentToken->isOneOf(tok::semi, tok::colon))
234  MightBeObjCForRangeLoop = false;
235  if (MightBeObjCForRangeLoop && CurrentToken->is(Keywords.kw_in))
236  CurrentToken->Type = TT_ObjCForIn;
237  // When we discover a 'new', we set CanBeExpression to 'false' in order to
238  // parse the type correctly. Reset that after a comma.
239  if (CurrentToken->is(tok::comma))
240  Contexts.back().CanBeExpression = true;
241 
242  FormatToken *Tok = CurrentToken;
243  if (!consumeToken())
244  return false;
245  updateParameterCount(Left, Tok);
246  if (CurrentToken && CurrentToken->HasUnescapedNewline)
247  HasMultipleLines = true;
248  }
249  return false;
250  }
251 
252  bool parseSquare() {
253  if (!CurrentToken)
254  return false;
255 
256  // A '[' could be an index subscript (after an identifier or after
257  // ')' or ']'), it could be the start of an Objective-C method
258  // expression, or it could the start of an Objective-C array literal.
259  FormatToken *Left = CurrentToken->Previous;
260  Left->ParentBracket = Contexts.back().ContextKind;
261  FormatToken *Parent = Left->getPreviousNonComment();
262  bool StartsObjCMethodExpr =
263  Style.Language == FormatStyle::LK_Cpp &&
264  Contexts.back().CanBeExpression && Left->isNot(TT_LambdaLSquare) &&
265  CurrentToken->isNot(tok::l_brace) &&
266  (!Parent ||
267  Parent->isOneOf(tok::colon, tok::l_square, tok::l_paren,
268  tok::kw_return, tok::kw_throw) ||
269  Parent->isUnaryOperator() ||
270  Parent->isOneOf(TT_ObjCForIn, TT_CastRParen) ||
271  getBinOpPrecedence(Parent->Tok.getKind(), true, true) > prec::Unknown);
272  bool ColonFound = false;
273 
274  unsigned BindingIncrease = 1;
275  if (Left->is(TT_Unknown)) {
276  if (StartsObjCMethodExpr) {
277  Left->Type = TT_ObjCMethodExpr;
278  } else if (Style.Language == FormatStyle::LK_JavaScript && Parent &&
279  Contexts.back().ContextKind == tok::l_brace &&
280  Parent->isOneOf(tok::l_brace, tok::comma)) {
281  Left->Type = TT_JsComputedPropertyName;
282  } else if (Parent &&
283  Parent->isOneOf(tok::at, tok::equal, tok::comma, tok::l_paren,
284  tok::l_square, tok::question, tok::colon,
285  tok::kw_return)) {
286  Left->Type = TT_ArrayInitializerLSquare;
287  } else {
288  BindingIncrease = 10;
289  Left->Type = TT_ArraySubscriptLSquare;
290  }
291  }
292 
293  ScopedContextCreator ContextCreator(*this, tok::l_square, BindingIncrease);
294  Contexts.back().IsExpression = true;
295  Contexts.back().ColonIsObjCMethodExpr = StartsObjCMethodExpr;
296 
297  while (CurrentToken) {
298  if (CurrentToken->is(tok::r_square)) {
299  if (CurrentToken->Next && CurrentToken->Next->is(tok::l_paren) &&
300  Left->is(TT_ObjCMethodExpr)) {
301  // An ObjC method call is rarely followed by an open parenthesis.
302  // FIXME: Do we incorrectly label ":" with this?
303  StartsObjCMethodExpr = false;
304  Left->Type = TT_Unknown;
305  }
306  if (StartsObjCMethodExpr && CurrentToken->Previous != Left) {
307  CurrentToken->Type = TT_ObjCMethodExpr;
308  // determineStarAmpUsage() thinks that '*' '[' is allocating an
309  // array of pointers, but if '[' starts a selector then '*' is a
310  // binary operator.
311  if (Parent && Parent->is(TT_PointerOrReference))
312  Parent->Type = TT_BinaryOperator;
313  }
314  Left->MatchingParen = CurrentToken;
315  CurrentToken->MatchingParen = Left;
316  if (Contexts.back().FirstObjCSelectorName) {
317  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
318  Contexts.back().LongestObjCSelectorName;
319  if (Left->BlockParameterCount > 1)
320  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName = 0;
321  }
322  next();
323  return true;
324  }
325  if (CurrentToken->isOneOf(tok::r_paren, tok::r_brace))
326  return false;
327  if (CurrentToken->is(tok::colon)) {
328  if (Left->is(TT_ArraySubscriptLSquare)) {
329  Left->Type = TT_ObjCMethodExpr;
330  StartsObjCMethodExpr = true;
331  Contexts.back().ColonIsObjCMethodExpr = true;
332  if (Parent && Parent->is(tok::r_paren))
333  Parent->Type = TT_CastRParen;
334  }
335  ColonFound = true;
336  }
337  if (CurrentToken->is(tok::comma) && Left->is(TT_ObjCMethodExpr) &&
338  !ColonFound)
339  Left->Type = TT_ArrayInitializerLSquare;
340  FormatToken *Tok = CurrentToken;
341  if (!consumeToken())
342  return false;
343  updateParameterCount(Left, Tok);
344  }
345  return false;
346  }
347 
348  bool parseBrace() {
349  if (CurrentToken) {
350  FormatToken *Left = CurrentToken->Previous;
351  Left->ParentBracket = Contexts.back().ContextKind;
352 
353  if (Contexts.back().CaretFound)
354  Left->Type = TT_ObjCBlockLBrace;
355  Contexts.back().CaretFound = false;
356 
357  ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
358  Contexts.back().ColonIsDictLiteral = true;
359  if (Left->BlockKind == BK_BracedInit)
360  Contexts.back().IsExpression = true;
361 
362  while (CurrentToken) {
363  if (CurrentToken->is(tok::r_brace)) {
364  Left->MatchingParen = CurrentToken;
365  CurrentToken->MatchingParen = Left;
366  next();
367  return true;
368  }
369  if (CurrentToken->isOneOf(tok::r_paren, tok::r_square))
370  return false;
371  updateParameterCount(Left, CurrentToken);
372  if (CurrentToken->isOneOf(tok::colon, tok::l_brace)) {
373  FormatToken *Previous = CurrentToken->getPreviousNonComment();
374  if ((CurrentToken->is(tok::colon) ||
375  Style.Language == FormatStyle::LK_Proto) &&
376  Previous->is(tok::identifier))
377  Previous->Type = TT_SelectorName;
378  if (CurrentToken->is(tok::colon) ||
379  Style.Language == FormatStyle::LK_JavaScript)
380  Left->Type = TT_DictLiteral;
381  }
382  if (!consumeToken())
383  return false;
384  }
385  }
386  return true;
387  }
388 
389  void updateParameterCount(FormatToken *Left, FormatToken *Current) {
390  if (Current->is(TT_LambdaLSquare) ||
391  (Current->is(tok::caret) && Current->is(TT_UnaryOperator)) ||
392  (Style.Language == FormatStyle::LK_JavaScript &&
393  Current->is(Keywords.kw_function))) {
394  ++Left->BlockParameterCount;
395  }
396  if (Current->is(tok::comma)) {
397  ++Left->ParameterCount;
398  if (!Left->Role)
399  Left->Role.reset(new CommaSeparatedList(Style));
400  Left->Role->CommaFound(Current);
401  } else if (Left->ParameterCount == 0 && Current->isNot(tok::comment)) {
402  Left->ParameterCount = 1;
403  }
404  }
405 
406  bool parseConditional() {
407  while (CurrentToken) {
408  if (CurrentToken->is(tok::colon)) {
409  CurrentToken->Type = TT_ConditionalExpr;
410  next();
411  return true;
412  }
413  if (!consumeToken())
414  return false;
415  }
416  return false;
417  }
418 
419  bool parseTemplateDeclaration() {
420  if (CurrentToken && CurrentToken->is(tok::less)) {
421  CurrentToken->Type = TT_TemplateOpener;
422  next();
423  if (!parseAngle())
424  return false;
425  if (CurrentToken)
426  CurrentToken->Previous->ClosesTemplateDeclaration = true;
427  return true;
428  }
429  return false;
430  }
431 
432  bool consumeToken() {
433  FormatToken *Tok = CurrentToken;
434  next();
435  switch (Tok->Tok.getKind()) {
436  case tok::plus:
437  case tok::minus:
438  if (!Tok->Previous && Line.MustBeDeclaration)
439  Tok->Type = TT_ObjCMethodSpecifier;
440  break;
441  case tok::colon:
442  if (!Tok->Previous)
443  return false;
444  // Colons from ?: are handled in parseConditional().
445  if (Style.Language == FormatStyle::LK_JavaScript) {
446  if (Contexts.back().ColonIsForRangeExpr || // colon in for loop
447  (Contexts.size() == 1 && // switch/case labels
448  !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) ||
449  Contexts.back().ContextKind == tok::l_paren || // function params
450  Contexts.back().ContextKind == tok::l_square || // array type
451  (Contexts.size() == 1 &&
452  Line.MustBeDeclaration)) { // method/property declaration
453  Tok->Type = TT_JsTypeColon;
454  break;
455  }
456  }
457  if (Contexts.back().ColonIsDictLiteral) {
458  Tok->Type = TT_DictLiteral;
459  } else if (Contexts.back().ColonIsObjCMethodExpr ||
460  Line.startsWith(TT_ObjCMethodSpecifier)) {
461  Tok->Type = TT_ObjCMethodExpr;
462  Tok->Previous->Type = TT_SelectorName;
463  if (Tok->Previous->ColumnWidth >
464  Contexts.back().LongestObjCSelectorName) {
465  Contexts.back().LongestObjCSelectorName = Tok->Previous->ColumnWidth;
466  }
467  if (!Contexts.back().FirstObjCSelectorName)
468  Contexts.back().FirstObjCSelectorName = Tok->Previous;
469  } else if (Contexts.back().ColonIsForRangeExpr) {
470  Tok->Type = TT_RangeBasedForLoopColon;
471  } else if (CurrentToken && CurrentToken->is(tok::numeric_constant)) {
472  Tok->Type = TT_BitFieldColon;
473  } else if (Contexts.size() == 1 &&
474  !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) {
475  if (Tok->Previous->is(tok::r_paren))
476  Tok->Type = TT_CtorInitializerColon;
477  else
478  Tok->Type = TT_InheritanceColon;
479  } else if (Tok->Previous->is(tok::identifier) && Tok->Next &&
480  Tok->Next->isOneOf(tok::r_paren, tok::comma)) {
481  // This handles a special macro in ObjC code where selectors including
482  // the colon are passed as macro arguments.
483  Tok->Type = TT_ObjCMethodExpr;
484  } else if (Contexts.back().ContextKind == tok::l_paren) {
485  Tok->Type = TT_InlineASMColon;
486  }
487  break;
488  case tok::kw_if:
489  case tok::kw_while:
490  if (CurrentToken && CurrentToken->is(tok::l_paren)) {
491  next();
492  if (!parseParens(/*LookForDecls=*/true))
493  return false;
494  }
495  break;
496  case tok::kw_for:
497  Contexts.back().ColonIsForRangeExpr = true;
498  next();
499  if (!parseParens())
500  return false;
501  break;
502  case tok::l_paren:
503  if (!parseParens())
504  return false;
505  if (Line.MustBeDeclaration && Contexts.size() == 1 &&
506  !Contexts.back().IsExpression && !Line.startsWith(TT_ObjCProperty) &&
507  (!Tok->Previous ||
508  !Tok->Previous->isOneOf(tok::kw_decltype, tok::kw___attribute,
509  TT_LeadingJavaAnnotation)))
510  Line.MightBeFunctionDecl = true;
511  break;
512  case tok::l_square:
513  if (!parseSquare())
514  return false;
515  break;
516  case tok::l_brace:
517  if (!parseBrace())
518  return false;
519  break;
520  case tok::less:
521  if (!NonTemplateLess.count(Tok) &&
522  (!Tok->Previous ||
523  (!Tok->Previous->Tok.isLiteral() &&
524  !(Tok->Previous->is(tok::r_paren) && Contexts.size() > 1))) &&
525  parseAngle()) {
526  Tok->Type = TT_TemplateOpener;
527  } else {
528  Tok->Type = TT_BinaryOperator;
529  NonTemplateLess.insert(Tok);
530  CurrentToken = Tok;
531  next();
532  }
533  break;
534  case tok::r_paren:
535  case tok::r_square:
536  return false;
537  case tok::r_brace:
538  // Lines can start with '}'.
539  if (Tok->Previous)
540  return false;
541  break;
542  case tok::greater:
543  Tok->Type = TT_BinaryOperator;
544  break;
545  case tok::kw_operator:
546  while (CurrentToken &&
547  !CurrentToken->isOneOf(tok::l_paren, tok::semi, tok::r_paren)) {
548  if (CurrentToken->isOneOf(tok::star, tok::amp))
549  CurrentToken->Type = TT_PointerOrReference;
550  consumeToken();
551  if (CurrentToken && CurrentToken->Previous->is(TT_BinaryOperator))
552  CurrentToken->Previous->Type = TT_OverloadedOperator;
553  }
554  if (CurrentToken) {
555  CurrentToken->Type = TT_OverloadedOperatorLParen;
556  if (CurrentToken->Previous->is(TT_BinaryOperator))
557  CurrentToken->Previous->Type = TT_OverloadedOperator;
558  }
559  break;
560  case tok::question:
561  if (Style.Language == FormatStyle::LK_JavaScript && Tok->Next &&
562  Tok->Next->isOneOf(tok::semi, tok::comma, tok::colon, tok::r_paren,
563  tok::r_brace)) {
564  // Question marks before semicolons, colons, etc. indicate optional
565  // types (fields, parameters), e.g.
566  // function(x?: string, y?) {...}
567  // class X { y?; }
568  Tok->Type = TT_JsTypeOptionalQuestion;
569  break;
570  }
571  // Declarations cannot be conditional expressions, this can only be part
572  // of a type declaration.
573  if (Line.MustBeDeclaration &&
574  Style.Language == FormatStyle::LK_JavaScript)
575  break;
576  parseConditional();
577  break;
578  case tok::kw_template:
579  parseTemplateDeclaration();
580  break;
581  case tok::comma:
582  if (Contexts.back().InCtorInitializer)
583  Tok->Type = TT_CtorInitializerComma;
584  else if (Contexts.back().FirstStartOfName &&
585  (Contexts.size() == 1 || Line.startsWith(tok::kw_for))) {
586  Contexts.back().FirstStartOfName->PartOfMultiVariableDeclStmt = true;
587  Line.IsMultiVariableDeclStmt = true;
588  }
589  if (Contexts.back().IsForEachMacro)
590  Contexts.back().IsExpression = true;
591  break;
592  default:
593  break;
594  }
595  return true;
596  }
597 
598  void parseIncludeDirective() {
599  if (CurrentToken && CurrentToken->is(tok::less)) {
600  next();
601  while (CurrentToken) {
602  if (CurrentToken->isNot(tok::comment) || CurrentToken->Next)
603  CurrentToken->Type = TT_ImplicitStringLiteral;
604  next();
605  }
606  }
607  }
608 
609  void parseWarningOrError() {
610  next();
611  // We still want to format the whitespace left of the first token of the
612  // warning or error.
613  next();
614  while (CurrentToken) {
615  CurrentToken->Type = TT_ImplicitStringLiteral;
616  next();
617  }
618  }
619 
620  void parsePragma() {
621  next(); // Consume "pragma".
622  if (CurrentToken &&
623  CurrentToken->isOneOf(Keywords.kw_mark, Keywords.kw_option)) {
624  bool IsMark = CurrentToken->is(Keywords.kw_mark);
625  next(); // Consume "mark".
626  next(); // Consume first token (so we fix leading whitespace).
627  while (CurrentToken) {
628  if (IsMark || CurrentToken->Previous->is(TT_BinaryOperator))
629  CurrentToken->Type = TT_ImplicitStringLiteral;
630  next();
631  }
632  }
633  }
634 
635  LineType parsePreprocessorDirective() {
637  next();
638  if (!CurrentToken)
639  return Type;
640  if (CurrentToken->Tok.is(tok::numeric_constant)) {
641  CurrentToken->SpacesRequiredBefore = 1;
642  return Type;
643  }
644  // Hashes in the middle of a line can lead to any strange token
645  // sequence.
646  if (!CurrentToken->Tok.getIdentifierInfo())
647  return Type;
648  switch (CurrentToken->Tok.getIdentifierInfo()->getPPKeywordID()) {
649  case tok::pp_include:
650  case tok::pp_include_next:
651  case tok::pp_import:
652  next();
653  parseIncludeDirective();
654  Type = LT_ImportStatement;
655  break;
656  case tok::pp_error:
657  case tok::pp_warning:
658  parseWarningOrError();
659  break;
660  case tok::pp_pragma:
661  parsePragma();
662  break;
663  case tok::pp_if:
664  case tok::pp_elif:
665  Contexts.back().IsExpression = true;
666  parseLine();
667  break;
668  default:
669  break;
670  }
671  while (CurrentToken)
672  next();
673  return Type;
674  }
675 
676 public:
677  LineType parseLine() {
678  NonTemplateLess.clear();
679  if (CurrentToken->is(tok::hash))
680  return parsePreprocessorDirective();
681 
682  // Directly allow to 'import <string-literal>' to support protocol buffer
683  // definitions (code.google.com/p/protobuf) or missing "#" (either way we
684  // should not break the line).
685  IdentifierInfo *Info = CurrentToken->Tok.getIdentifierInfo();
686  if ((Style.Language == FormatStyle::LK_Java &&
687  CurrentToken->is(Keywords.kw_package)) ||
688  (Info && Info->getPPKeywordID() == tok::pp_import &&
689  CurrentToken->Next &&
690  CurrentToken->Next->isOneOf(tok::string_literal, tok::identifier,
691  tok::kw_static))) {
692  next();
693  parseIncludeDirective();
694  return LT_ImportStatement;
695  }
696 
697  // If this line starts and ends in '<' and '>', respectively, it is likely
698  // part of "#define <a/b.h>".
699  if (CurrentToken->is(tok::less) && Line.Last->is(tok::greater)) {
700  parseIncludeDirective();
701  return LT_ImportStatement;
702  }
703 
704  // In .proto files, top-level options are very similar to import statements
705  // and should not be line-wrapped.
706  if (Style.Language == FormatStyle::LK_Proto && Line.Level == 0 &&
707  CurrentToken->is(Keywords.kw_option)) {
708  next();
709  if (CurrentToken && CurrentToken->is(tok::identifier))
710  return LT_ImportStatement;
711  }
712 
713  bool KeywordVirtualFound = false;
714  bool ImportStatement = false;
715  while (CurrentToken) {
716  if (CurrentToken->is(tok::kw_virtual))
717  KeywordVirtualFound = true;
718  if (IsImportStatement(*CurrentToken))
719  ImportStatement = true;
720  if (!consumeToken())
721  return LT_Invalid;
722  }
723  if (KeywordVirtualFound)
724  return LT_VirtualFunctionDecl;
725  if (ImportStatement)
726  return LT_ImportStatement;
727 
728  if (Line.startsWith(TT_ObjCMethodSpecifier)) {
729  if (Contexts.back().FirstObjCSelectorName)
730  Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
731  Contexts.back().LongestObjCSelectorName;
732  return LT_ObjCMethodDecl;
733  }
734 
735  return LT_Other;
736  }
737 
738 private:
739  bool IsImportStatement(const FormatToken &Tok) {
740  // FIXME: Closure-library specific stuff should not be hard-coded but be
741  // configurable.
742  return Style.Language == FormatStyle::LK_JavaScript &&
743  Tok.TokenText == "goog" && Tok.Next && Tok.Next->is(tok::period) &&
744  Tok.Next->Next && (Tok.Next->Next->TokenText == "module" ||
745  Tok.Next->Next->TokenText == "require" ||
746  Tok.Next->Next->TokenText == "provide") &&
747  Tok.Next->Next->Next && Tok.Next->Next->Next->is(tok::l_paren);
748  }
749 
750  void resetTokenMetadata(FormatToken *Token) {
751  if (!Token)
752  return;
753 
754  // Reset token type in case we have already looked at it and then
755  // recovered from an error (e.g. failure to find the matching >).
756  if (!CurrentToken->isOneOf(TT_LambdaLSquare, TT_ForEachMacro,
757  TT_FunctionLBrace, TT_ImplicitStringLiteral,
758  TT_InlineASMBrace, TT_JsFatArrow, TT_LambdaArrow,
759  TT_RegexLiteral))
760  CurrentToken->Type = TT_Unknown;
761  CurrentToken->Role.reset();
762  CurrentToken->MatchingParen = nullptr;
763  CurrentToken->FakeLParens.clear();
764  CurrentToken->FakeRParens = 0;
765  }
766 
767  void next() {
768  if (CurrentToken) {
769  CurrentToken->NestingLevel = Contexts.size() - 1;
770  CurrentToken->BindingStrength = Contexts.back().BindingStrength;
771  modifyContext(*CurrentToken);
772  determineTokenType(*CurrentToken);
773  CurrentToken = CurrentToken->Next;
774  }
775 
776  resetTokenMetadata(CurrentToken);
777  }
778 
779  /// \brief A struct to hold information valid in a specific context, e.g.
780  /// a pair of parenthesis.
781  struct Context {
783  bool IsExpression)
784  : ContextKind(ContextKind), BindingStrength(BindingStrength),
785  IsExpression(IsExpression) {}
786 
788  unsigned BindingStrength;
791  bool ColonIsForRangeExpr = false;
792  bool ColonIsDictLiteral = false;
793  bool ColonIsObjCMethodExpr = false;
794  FormatToken *FirstObjCSelectorName = nullptr;
795  FormatToken *FirstStartOfName = nullptr;
796  bool CanBeExpression = true;
797  bool InTemplateArgument = false;
798  bool InCtorInitializer = false;
799  bool CaretFound = false;
800  bool IsForEachMacro = false;
801  };
802 
803  /// \brief Puts a new \c Context onto the stack \c Contexts for the lifetime
804  /// of each instance.
805  struct ScopedContextCreator {
806  AnnotatingParser &P;
807 
808  ScopedContextCreator(AnnotatingParser &P, tok::TokenKind ContextKind,
809  unsigned Increase)
810  : P(P) {
811  P.Contexts.push_back(Context(ContextKind,
812  P.Contexts.back().BindingStrength + Increase,
813  P.Contexts.back().IsExpression));
814  }
815 
816  ~ScopedContextCreator() { P.Contexts.pop_back(); }
817  };
818 
819  void modifyContext(const FormatToken &Current) {
820  if (Current.getPrecedence() == prec::Assignment &&
821  !Line.First->isOneOf(tok::kw_template, tok::kw_using) &&
822  (!Current.Previous || Current.Previous->isNot(tok::kw_operator))) {
823  Contexts.back().IsExpression = true;
824  if (!Line.startsWith(TT_UnaryOperator)) {
825  for (FormatToken *Previous = Current.Previous;
826  Previous && !Previous->isOneOf(tok::comma, tok::semi);
827  Previous = Previous->Previous) {
828  if (Previous->isOneOf(tok::r_square, tok::r_paren)) {
829  Previous = Previous->MatchingParen;
830  if (!Previous)
831  break;
832  }
833  if (Previous->opensScope())
834  break;
835  if (Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator) &&
836  Previous->isOneOf(tok::star, tok::amp, tok::ampamp) &&
837  Previous->Previous && Previous->Previous->isNot(tok::equal))
838  Previous->Type = TT_PointerOrReference;
839  }
840  }
841  } else if (Current.is(tok::lessless) &&
842  (!Current.Previous || !Current.Previous->is(tok::kw_operator))) {
843  Contexts.back().IsExpression = true;
844  } else if (Current.isOneOf(tok::kw_return, tok::kw_throw)) {
845  Contexts.back().IsExpression = true;
846  } else if (Current.is(TT_TrailingReturnArrow)) {
847  Contexts.back().IsExpression = false;
848  } else if (Current.is(TT_LambdaArrow)) {
849  Contexts.back().IsExpression = Style.Language == FormatStyle::LK_Java;
850  } else if (Current.is(tok::l_paren) && !Line.MustBeDeclaration &&
851  !Line.InPPDirective &&
852  (!Current.Previous ||
853  Current.Previous->isNot(tok::kw_decltype))) {
854  bool ParametersOfFunctionType =
855  Current.Previous && Current.Previous->is(tok::r_paren) &&
856  Current.Previous->MatchingParen &&
857  Current.Previous->MatchingParen->is(TT_FunctionTypeLParen);
858  bool IsForOrCatch = Current.Previous &&
859  Current.Previous->isOneOf(tok::kw_for, tok::kw_catch);
860  Contexts.back().IsExpression = !ParametersOfFunctionType && !IsForOrCatch;
861  } else if (Current.isOneOf(tok::r_paren, tok::greater, tok::comma)) {
862  for (FormatToken *Previous = Current.Previous;
863  Previous && Previous->isOneOf(tok::star, tok::amp);
864  Previous = Previous->Previous)
865  Previous->Type = TT_PointerOrReference;
866  if (Line.MustBeDeclaration)
867  Contexts.back().IsExpression = Contexts.front().InCtorInitializer;
868  } else if (Current.Previous &&
869  Current.Previous->is(TT_CtorInitializerColon)) {
870  Contexts.back().IsExpression = true;
871  Contexts.back().InCtorInitializer = true;
872  } else if (Current.is(tok::kw_new)) {
873  Contexts.back().CanBeExpression = false;
874  } else if (Current.isOneOf(tok::semi, tok::exclaim)) {
875  // This should be the condition or increment in a for-loop.
876  Contexts.back().IsExpression = true;
877  }
878  }
879 
880  void determineTokenType(FormatToken &Current) {
881  if (!Current.is(TT_Unknown))
882  // The token type is already known.
883  return;
884 
885  // Line.MightBeFunctionDecl can only be true after the parentheses of a
886  // function declaration have been found. In this case, 'Current' is a
887  // trailing token of this declaration and thus cannot be a name.
888  if (Current.is(Keywords.kw_instanceof)) {
889  Current.Type = TT_BinaryOperator;
890  } else if (isStartOfName(Current) &&
891  (!Line.MightBeFunctionDecl || Current.NestingLevel != 0)) {
892  Contexts.back().FirstStartOfName = &Current;
893  Current.Type = TT_StartOfName;
894  } else if (Current.is(tok::kw_auto)) {
895  AutoFound = true;
896  } else if (Current.is(tok::arrow) &&
897  Style.Language == FormatStyle::LK_Java) {
898  Current.Type = TT_LambdaArrow;
899  } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
900  Current.NestingLevel == 0) {
901  Current.Type = TT_TrailingReturnArrow;
902  } else if (Current.isOneOf(tok::star, tok::amp, tok::ampamp)) {
903  Current.Type =
904  determineStarAmpUsage(Current, Contexts.back().CanBeExpression &&
905  Contexts.back().IsExpression,
906  Contexts.back().InTemplateArgument);
907  } else if (Current.isOneOf(tok::minus, tok::plus, tok::caret)) {
908  Current.Type = determinePlusMinusCaretUsage(Current);
909  if (Current.is(TT_UnaryOperator) && Current.is(tok::caret))
910  Contexts.back().CaretFound = true;
911  } else if (Current.isOneOf(tok::minusminus, tok::plusplus)) {
912  Current.Type = determineIncrementUsage(Current);
913  } else if (Current.isOneOf(tok::exclaim, tok::tilde)) {
914  Current.Type = TT_UnaryOperator;
915  } else if (Current.is(tok::question)) {
916  if (Style.Language == FormatStyle::LK_JavaScript &&
917  Line.MustBeDeclaration) {
918  // In JavaScript, `interface X { foo?(): bar; }` is an optional method
919  // on the interface, not a ternary expression.
920  Current.Type = TT_JsTypeOptionalQuestion;
921  } else {
922  Current.Type = TT_ConditionalExpr;
923  }
924  } else if (Current.isBinaryOperator() &&
925  (!Current.Previous || Current.Previous->isNot(tok::l_square))) {
926  Current.Type = TT_BinaryOperator;
927  } else if (Current.is(tok::comment)) {
928  if (Current.TokenText.startswith("/*")) {
929  if (Current.TokenText.endswith("*/"))
930  Current.Type = TT_BlockComment;
931  else
932  // The lexer has for some reason determined a comment here. But we
933  // cannot really handle it, if it isn't properly terminated.
934  Current.Tok.setKind(tok::unknown);
935  } else {
936  Current.Type = TT_LineComment;
937  }
938  } else if (Current.is(tok::r_paren)) {
939  if (rParenEndsCast(Current))
940  Current.Type = TT_CastRParen;
941  if (Current.MatchingParen && Current.Next &&
942  !Current.Next->isBinaryOperator() &&
943  !Current.Next->isOneOf(tok::semi, tok::colon, tok::l_brace))
944  if (FormatToken *BeforeParen = Current.MatchingParen->Previous)
945  if (BeforeParen->is(tok::identifier) &&
946  BeforeParen->TokenText == BeforeParen->TokenText.upper() &&
947  (!BeforeParen->Previous ||
948  BeforeParen->Previous->ClosesTemplateDeclaration))
949  Current.Type = TT_FunctionAnnotationRParen;
950  } else if (Current.is(tok::at) && Current.Next) {
951  if (Current.Next->isStringLiteral()) {
952  Current.Type = TT_ObjCStringLiteral;
953  } else {
954  switch (Current.Next->Tok.getObjCKeywordID()) {
955  case tok::objc_interface:
956  case tok::objc_implementation:
957  case tok::objc_protocol:
958  Current.Type = TT_ObjCDecl;
959  break;
960  case tok::objc_property:
961  Current.Type = TT_ObjCProperty;
962  break;
963  default:
964  break;
965  }
966  }
967  } else if (Current.is(tok::period)) {
968  FormatToken *PreviousNoComment = Current.getPreviousNonComment();
969  if (PreviousNoComment &&
970  PreviousNoComment->isOneOf(tok::comma, tok::l_brace))
971  Current.Type = TT_DesignatedInitializerPeriod;
972  else if (Style.Language == FormatStyle::LK_Java && Current.Previous &&
973  Current.Previous->isOneOf(TT_JavaAnnotation,
974  TT_LeadingJavaAnnotation)) {
975  Current.Type = Current.Previous->Type;
976  }
977  } else if (Current.isOneOf(tok::identifier, tok::kw_const) &&
978  Current.Previous &&
979  !Current.Previous->isOneOf(tok::equal, tok::at) &&
980  Line.MightBeFunctionDecl && Contexts.size() == 1) {
981  // Line.MightBeFunctionDecl can only be true after the parentheses of a
982  // function declaration have been found.
983  Current.Type = TT_TrailingAnnotation;
984  } else if ((Style.Language == FormatStyle::LK_Java ||
985  Style.Language == FormatStyle::LK_JavaScript) &&
986  Current.Previous) {
987  if (Current.Previous->is(tok::at) &&
988  Current.isNot(Keywords.kw_interface)) {
989  const FormatToken &AtToken = *Current.Previous;
990  const FormatToken *Previous = AtToken.getPreviousNonComment();
991  if (!Previous || Previous->is(TT_LeadingJavaAnnotation))
992  Current.Type = TT_LeadingJavaAnnotation;
993  else
994  Current.Type = TT_JavaAnnotation;
995  } else if (Current.Previous->is(tok::period) &&
996  Current.Previous->isOneOf(TT_JavaAnnotation,
997  TT_LeadingJavaAnnotation)) {
998  Current.Type = Current.Previous->Type;
999  }
1000  }
1001  }
1002 
1003  /// \brief Take a guess at whether \p Tok starts a name of a function or
1004  /// variable declaration.
1005  ///
1006  /// This is a heuristic based on whether \p Tok is an identifier following
1007  /// something that is likely a type.
1008  bool isStartOfName(const FormatToken &Tok) {
1009  if (Tok.isNot(tok::identifier) || !Tok.Previous)
1010  return false;
1011 
1012  if (Tok.Previous->isOneOf(TT_LeadingJavaAnnotation, Keywords.kw_instanceof))
1013  return false;
1014 
1015  // Skip "const" as it does not have an influence on whether this is a name.
1016  FormatToken *PreviousNotConst = Tok.Previous;
1017  while (PreviousNotConst && PreviousNotConst->is(tok::kw_const))
1018  PreviousNotConst = PreviousNotConst->Previous;
1019 
1020  if (!PreviousNotConst)
1021  return false;
1022 
1023  bool IsPPKeyword = PreviousNotConst->is(tok::identifier) &&
1024  PreviousNotConst->Previous &&
1025  PreviousNotConst->Previous->is(tok::hash);
1026 
1027  if (PreviousNotConst->is(TT_TemplateCloser))
1028  return PreviousNotConst && PreviousNotConst->MatchingParen &&
1029  PreviousNotConst->MatchingParen->Previous &&
1030  PreviousNotConst->MatchingParen->Previous->isNot(tok::period) &&
1031  PreviousNotConst->MatchingParen->Previous->isNot(tok::kw_template);
1032 
1033  if (PreviousNotConst->is(tok::r_paren) && PreviousNotConst->MatchingParen &&
1034  PreviousNotConst->MatchingParen->Previous &&
1035  PreviousNotConst->MatchingParen->Previous->is(tok::kw_decltype))
1036  return true;
1037 
1038  return (!IsPPKeyword && PreviousNotConst->is(tok::identifier)) ||
1039  PreviousNotConst->is(TT_PointerOrReference) ||
1040  PreviousNotConst->isSimpleTypeSpecifier();
1041  }
1042 
1043  /// \brief Determine whether ')' is ending a cast.
1044  bool rParenEndsCast(const FormatToken &Tok) {
1045  FormatToken *LeftOfParens = nullptr;
1046  if (Tok.MatchingParen)
1047  LeftOfParens = Tok.MatchingParen->getPreviousNonComment();
1048  if (LeftOfParens && LeftOfParens->is(tok::r_paren) &&
1049  LeftOfParens->MatchingParen)
1050  LeftOfParens = LeftOfParens->MatchingParen->Previous;
1051  if (LeftOfParens && LeftOfParens->is(tok::r_square) &&
1052  LeftOfParens->MatchingParen &&
1053  LeftOfParens->MatchingParen->is(TT_LambdaLSquare))
1054  return false;
1055  if (Tok.Next) {
1056  if (Tok.Next->is(tok::question))
1057  return false;
1058  if (Style.Language == FormatStyle::LK_JavaScript &&
1059  Tok.Next->is(Keywords.kw_in))
1060  return false;
1061  if (Style.Language == FormatStyle::LK_Java && Tok.Next->is(tok::l_paren))
1062  return true;
1063  }
1064  bool IsCast = false;
1065  bool ParensAreEmpty = Tok.Previous == Tok.MatchingParen;
1066  bool ParensAreType =
1067  !Tok.Previous ||
1068  Tok.Previous->isOneOf(TT_PointerOrReference, TT_TemplateCloser) ||
1069  Tok.Previous->isSimpleTypeSpecifier();
1070  bool ParensCouldEndDecl =
1071  Tok.Next && Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace);
1072  bool IsSizeOfOrAlignOf =
1073  LeftOfParens && LeftOfParens->isOneOf(tok::kw_sizeof, tok::kw_alignof);
1074  if (ParensAreType && !ParensCouldEndDecl && !IsSizeOfOrAlignOf &&
1075  (Contexts.size() > 1 && Contexts[Contexts.size() - 2].IsExpression))
1076  IsCast = true;
1077  else if (Tok.Next && Tok.Next->isNot(tok::string_literal) &&
1078  (Tok.Next->Tok.isLiteral() ||
1079  Tok.Next->isOneOf(tok::kw_sizeof, tok::kw_alignof)))
1080  IsCast = true;
1081  // If there is an identifier after the (), it is likely a cast, unless
1082  // there is also an identifier before the ().
1083  else if (LeftOfParens && Tok.Next &&
1084  (LeftOfParens->Tok.getIdentifierInfo() == nullptr ||
1085  LeftOfParens->isOneOf(tok::kw_return, tok::kw_case)) &&
1086  !LeftOfParens->isOneOf(TT_OverloadedOperator, tok::at,
1087  TT_TemplateCloser)) {
1088  if (Tok.Next->isOneOf(tok::identifier, tok::numeric_constant)) {
1089  IsCast = true;
1090  } else {
1091  // Use heuristics to recognize c style casting.
1092  FormatToken *Prev = Tok.Previous;
1093  if (Prev && Prev->isOneOf(tok::amp, tok::star))
1094  Prev = Prev->Previous;
1095 
1096  if (Prev && Tok.Next && Tok.Next->Next) {
1097  bool NextIsUnary = Tok.Next->isUnaryOperator() ||
1098  Tok.Next->isOneOf(tok::amp, tok::star);
1099  IsCast =
1100  NextIsUnary && !Tok.Next->is(tok::plus) &&
1101  Tok.Next->Next->isOneOf(tok::identifier, tok::numeric_constant);
1102  }
1103 
1104  for (; Prev != Tok.MatchingParen; Prev = Prev->Previous) {
1105  if (!Prev ||
1106  !Prev->isOneOf(tok::kw_const, tok::identifier, tok::coloncolon)) {
1107  IsCast = false;
1108  break;
1109  }
1110  }
1111  }
1112  }
1113  return IsCast && !ParensAreEmpty;
1114  }
1115 
1116  /// \brief Return the type of the given token assuming it is * or &.
1117  TokenType determineStarAmpUsage(const FormatToken &Tok, bool IsExpression,
1118  bool InTemplateArgument) {
1119  if (Style.Language == FormatStyle::LK_JavaScript)
1120  return TT_BinaryOperator;
1121 
1122  const FormatToken *PrevToken = Tok.getPreviousNonComment();
1123  if (!PrevToken)
1124  return TT_UnaryOperator;
1125 
1126  const FormatToken *NextToken = Tok.getNextNonComment();
1127  if (!NextToken || NextToken->is(tok::arrow) ||
1128  (NextToken->is(tok::l_brace) && !NextToken->getNextNonComment()))
1129  return TT_Unknown;
1130 
1131  if (PrevToken->is(tok::coloncolon))
1132  return TT_PointerOrReference;
1133 
1134  if (PrevToken->isOneOf(tok::l_paren, tok::l_square, tok::l_brace,
1135  tok::comma, tok::semi, tok::kw_return, tok::colon,
1136  tok::equal, tok::kw_delete, tok::kw_sizeof) ||
1137  PrevToken->isOneOf(TT_BinaryOperator, TT_ConditionalExpr,
1138  TT_UnaryOperator, TT_CastRParen))
1139  return TT_UnaryOperator;
1140 
1141  if (NextToken->is(tok::l_square) && NextToken->isNot(TT_LambdaLSquare))
1142  return TT_PointerOrReference;
1143  if (NextToken->isOneOf(tok::kw_operator, tok::comma, tok::semi))
1144  return TT_PointerOrReference;
1145 
1146  if (PrevToken->is(tok::r_paren) && PrevToken->MatchingParen &&
1147  PrevToken->MatchingParen->Previous &&
1148  PrevToken->MatchingParen->Previous->isOneOf(tok::kw_typeof,
1149  tok::kw_decltype))
1150  return TT_PointerOrReference;
1151 
1152  if (PrevToken->Tok.isLiteral() ||
1153  PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::kw_true,
1154  tok::kw_false, tok::r_brace) ||
1155  NextToken->Tok.isLiteral() ||
1156  NextToken->isOneOf(tok::kw_true, tok::kw_false) ||
1157  NextToken->isUnaryOperator() ||
1158  // If we know we're in a template argument, there are no named
1159  // declarations. Thus, having an identifier on the right-hand side
1160  // indicates a binary operator.
1161  (InTemplateArgument && NextToken->Tok.isAnyIdentifier()))
1162  return TT_BinaryOperator;
1163 
1164  // "&&(" is quite unlikely to be two successive unary "&".
1165  if (Tok.is(tok::ampamp) && NextToken && NextToken->is(tok::l_paren))
1166  return TT_BinaryOperator;
1167 
1168  // This catches some cases where evaluation order is used as control flow:
1169  // aaa && aaa->f();
1170  const FormatToken *NextNextToken = NextToken->getNextNonComment();
1171  if (NextNextToken && NextNextToken->is(tok::arrow))
1172  return TT_BinaryOperator;
1173 
1174  // It is very unlikely that we are going to find a pointer or reference type
1175  // definition on the RHS of an assignment.
1176  if (IsExpression && !Contexts.back().CaretFound)
1177  return TT_BinaryOperator;
1178 
1179  return TT_PointerOrReference;
1180  }
1181 
1182  TokenType determinePlusMinusCaretUsage(const FormatToken &Tok) {
1183  const FormatToken *PrevToken = Tok.getPreviousNonComment();
1184  if (!PrevToken || PrevToken->is(TT_CastRParen))
1185  return TT_UnaryOperator;
1186 
1187  // Use heuristics to recognize unary operators.
1188  if (PrevToken->isOneOf(tok::equal, tok::l_paren, tok::comma, tok::l_square,
1189  tok::question, tok::colon, tok::kw_return,
1190  tok::kw_case, tok::at, tok::l_brace))
1191  return TT_UnaryOperator;
1192 
1193  // There can't be two consecutive binary operators.
1194  if (PrevToken->is(TT_BinaryOperator))
1195  return TT_UnaryOperator;
1196 
1197  // Fall back to marking the token as binary operator.
1198  return TT_BinaryOperator;
1199  }
1200 
1201  /// \brief Determine whether ++/-- are pre- or post-increments/-decrements.
1202  TokenType determineIncrementUsage(const FormatToken &Tok) {
1203  const FormatToken *PrevToken = Tok.getPreviousNonComment();
1204  if (!PrevToken || PrevToken->is(TT_CastRParen))
1205  return TT_UnaryOperator;
1206  if (PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::identifier))
1207  return TT_TrailingUnaryOperator;
1208 
1209  return TT_UnaryOperator;
1210  }
1211 
1212  SmallVector<Context, 8> Contexts;
1213 
1214  const FormatStyle &Style;
1215  AnnotatedLine &Line;
1216  FormatToken *CurrentToken;
1218  const AdditionalKeywords &Keywords;
1219 
1220  // Set of "<" tokens that do not open a template parameter list. If parseAngle
1221  // determines that a specific token can't be a template opener, it will make
1222  // same decision irrespective of the decisions for tokens leading up to it.
1223  // Store this information to prevent this from causing exponential runtime.
1224  llvm::SmallPtrSet<FormatToken *, 16> NonTemplateLess;
1225 };
1226 
1227 static const int PrecedenceUnaryOperator = prec::PointerToMember + 1;
1228 static const int PrecedenceArrowAndPeriod = prec::PointerToMember + 2;
1229 
1230 /// \brief Parses binary expressions by inserting fake parenthesis based on
1231 /// operator precedence.
1232 class ExpressionParser {
1233 public:
1234  ExpressionParser(const FormatStyle &Style, const AdditionalKeywords &Keywords,
1235  AnnotatedLine &Line)
1236  : Style(Style), Keywords(Keywords), Current(Line.First) {}
1237 
1238  /// \brief Parse expressions with the given operatore precedence.
1239  void parse(int Precedence = 0) {
1240  // Skip 'return' and ObjC selector colons as they are not part of a binary
1241  // expression.
1242  while (Current && (Current->is(tok::kw_return) ||
1243  (Current->is(tok::colon) &&
1244  Current->isOneOf(TT_ObjCMethodExpr, TT_DictLiteral))))
1245  next();
1246 
1247  if (!Current || Precedence > PrecedenceArrowAndPeriod)
1248  return;
1249 
1250  // Conditional expressions need to be parsed separately for proper nesting.
1251  if (Precedence == prec::Conditional) {
1252  parseConditionalExpr();
1253  return;
1254  }
1255 
1256  // Parse unary operators, which all have a higher precedence than binary
1257  // operators.
1258  if (Precedence == PrecedenceUnaryOperator) {
1259  parseUnaryOperator();
1260  return;
1261  }
1262 
1263  FormatToken *Start = Current;
1264  FormatToken *LatestOperator = nullptr;
1265  unsigned OperatorIndex = 0;
1266 
1267  while (Current) {
1268  // Consume operators with higher precedence.
1269  parse(Precedence + 1);
1270 
1271  int CurrentPrecedence = getCurrentPrecedence();
1272 
1273  if (Current && Current->is(TT_SelectorName) &&
1274  Precedence == CurrentPrecedence) {
1275  if (LatestOperator)
1276  addFakeParenthesis(Start, prec::Level(Precedence));
1277  Start = Current;
1278  }
1279 
1280  // At the end of the line or when an operator with higher precedence is
1281  // found, insert fake parenthesis and return.
1282  if (!Current || (Current->closesScope() && Current->MatchingParen) ||
1283  (CurrentPrecedence != -1 && CurrentPrecedence < Precedence) ||
1284  (CurrentPrecedence == prec::Conditional &&
1285  Precedence == prec::Assignment && Current->is(tok::colon))) {
1286  break;
1287  }
1288 
1289  // Consume scopes: (), [], <> and {}
1290  if (Current->opensScope()) {
1291  while (Current && !Current->closesScope()) {
1292  next();
1293  parse();
1294  }
1295  next();
1296  } else {
1297  // Operator found.
1298  if (CurrentPrecedence == Precedence) {
1299  LatestOperator = Current;
1300  Current->OperatorIndex = OperatorIndex;
1301  ++OperatorIndex;
1302  }
1303  next(/*SkipPastLeadingComments=*/Precedence > 0);
1304  }
1305  }
1306 
1307  if (LatestOperator && (Current || Precedence > 0)) {
1308  LatestOperator->LastOperator = true;
1309  if (Precedence == PrecedenceArrowAndPeriod) {
1310  // Call expressions don't have a binary operator precedence.
1311  addFakeParenthesis(Start, prec::Unknown);
1312  } else {
1313  addFakeParenthesis(Start, prec::Level(Precedence));
1314  }
1315  }
1316  }
1317 
1318 private:
1319  /// \brief Gets the precedence (+1) of the given token for binary operators
1320  /// and other tokens that we treat like binary operators.
1321  int getCurrentPrecedence() {
1322  if (Current) {
1323  const FormatToken *NextNonComment = Current->getNextNonComment();
1324  if (Current->is(TT_ConditionalExpr))
1325  return prec::Conditional;
1326  if (NextNonComment && NextNonComment->is(tok::colon) &&
1327  NextNonComment->is(TT_DictLiteral))
1328  return prec::Comma;
1329  if (Current->is(TT_LambdaArrow))
1330  return prec::Comma;
1331  if (Current->is(TT_JsFatArrow))
1332  return prec::Assignment;
1333  if (Current->isOneOf(tok::semi, TT_InlineASMColon, TT_SelectorName,
1334  TT_JsComputedPropertyName) ||
1335  (Current->is(tok::comment) && NextNonComment &&
1336  NextNonComment->is(TT_SelectorName)))
1337  return 0;
1338  if (Current->is(TT_RangeBasedForLoopColon))
1339  return prec::Comma;
1340  if ((Style.Language == FormatStyle::LK_Java ||
1341  Style.Language == FormatStyle::LK_JavaScript) &&
1342  Current->is(Keywords.kw_instanceof))
1343  return prec::Relational;
1344  if (Current->is(TT_BinaryOperator) || Current->is(tok::comma))
1345  return Current->getPrecedence();
1346  if (Current->isOneOf(tok::period, tok::arrow))
1347  return PrecedenceArrowAndPeriod;
1348  if (Style.Language == FormatStyle::LK_Java &&
1349  Current->isOneOf(Keywords.kw_extends, Keywords.kw_implements,
1350  Keywords.kw_throws))
1351  return 0;
1352  }
1353  return -1;
1354  }
1355 
1356  void addFakeParenthesis(FormatToken *Start, prec::Level Precedence) {
1357  Start->FakeLParens.push_back(Precedence);
1358  if (Precedence > prec::Unknown)
1359  Start->StartsBinaryExpression = true;
1360  if (Current) {
1361  FormatToken *Previous = Current->Previous;
1362  while (Previous->is(tok::comment) && Previous->Previous)
1363  Previous = Previous->Previous;
1364  ++Previous->FakeRParens;
1365  if (Precedence > prec::Unknown)
1366  Previous->EndsBinaryExpression = true;
1367  }
1368  }
1369 
1370  /// \brief Parse unary operator expressions and surround them with fake
1371  /// parentheses if appropriate.
1372  void parseUnaryOperator() {
1373  if (!Current || Current->isNot(TT_UnaryOperator)) {
1374  parse(PrecedenceArrowAndPeriod);
1375  return;
1376  }
1377 
1378  FormatToken *Start = Current;
1379  next();
1380  parseUnaryOperator();
1381 
1382  // The actual precedence doesn't matter.
1383  addFakeParenthesis(Start, prec::Unknown);
1384  }
1385 
1386  void parseConditionalExpr() {
1387  while (Current && Current->isTrailingComment()) {
1388  next();
1389  }
1390  FormatToken *Start = Current;
1391  parse(prec::LogicalOr);
1392  if (!Current || !Current->is(tok::question))
1393  return;
1394  next();
1395  parse(prec::Assignment);
1396  if (!Current || Current->isNot(TT_ConditionalExpr))
1397  return;
1398  next();
1399  parse(prec::Assignment);
1400  addFakeParenthesis(Start, prec::Conditional);
1401  }
1402 
1403  void next(bool SkipPastLeadingComments = true) {
1404  if (Current)
1405  Current = Current->Next;
1406  while (Current &&
1407  (Current->NewlinesBefore == 0 || SkipPastLeadingComments) &&
1408  Current->isTrailingComment())
1409  Current = Current->Next;
1410  }
1411 
1412  const FormatStyle &Style;
1413  const AdditionalKeywords &Keywords;
1414  FormatToken *Current;
1415 };
1416 
1417 } // end anonymous namespace
1418 
1421  const AnnotatedLine *NextNonCommentLine = nullptr;
1423  E = Lines.rend();
1424  I != E; ++I) {
1425  if (NextNonCommentLine && (*I)->First->is(tok::comment) &&
1426  (*I)->First->Next == nullptr)
1427  (*I)->Level = NextNonCommentLine->Level;
1428  else
1429  NextNonCommentLine = (*I)->First->isNot(tok::r_brace) ? (*I) : nullptr;
1430 
1431  setCommentLineLevels((*I)->Children);
1432  }
1433 }
1434 
1437  E = Line.Children.end();
1438  I != E; ++I) {
1439  annotate(**I);
1440  }
1441  AnnotatingParser Parser(Style, Line, Keywords);
1442  Line.Type = Parser.parseLine();
1443  if (Line.Type == LT_Invalid)
1444  return;
1445 
1446  ExpressionParser ExprParser(Style, Keywords, Line);
1447  ExprParser.parse();
1448 
1449  if (Line.startsWith(TT_ObjCMethodSpecifier))
1450  Line.Type = LT_ObjCMethodDecl;
1451  else if (Line.startsWith(TT_ObjCDecl))
1452  Line.Type = LT_ObjCDecl;
1453  else if (Line.startsWith(TT_ObjCProperty))
1454  Line.Type = LT_ObjCProperty;
1455 
1456  Line.First->SpacesRequiredBefore = 1;
1457  Line.First->CanBreakBefore = Line.First->MustBreakBefore;
1458 }
1459 
1460 // This function heuristically determines whether 'Current' starts the name of a
1461 // function declaration.
1462 static bool isFunctionDeclarationName(const FormatToken &Current) {
1463  if (!Current.is(TT_StartOfName) || Current.NestingLevel != 0)
1464  return false;
1465  const FormatToken *Next = Current.Next;
1466  for (; Next; Next = Next->Next) {
1467  if (Next->is(TT_TemplateOpener)) {
1468  Next = Next->MatchingParen;
1469  } else if (Next->is(tok::coloncolon)) {
1470  Next = Next->Next;
1471  if (!Next || !Next->is(tok::identifier))
1472  return false;
1473  } else if (Next->is(tok::l_paren)) {
1474  break;
1475  } else {
1476  return false;
1477  }
1478  }
1479  if (!Next)
1480  return false;
1481  assert(Next->is(tok::l_paren));
1482  if (Next->Next == Next->MatchingParen)
1483  return true;
1484  for (const FormatToken *Tok = Next->Next; Tok && Tok != Next->MatchingParen;
1485  Tok = Tok->Next) {
1486  if (Tok->is(tok::kw_const) || Tok->isSimpleTypeSpecifier() ||
1487  Tok->isOneOf(TT_PointerOrReference, TT_StartOfName))
1488  return true;
1489  if (Tok->isOneOf(tok::l_brace, tok::string_literal, TT_ObjCMethodExpr) ||
1490  Tok->Tok.isLiteral())
1491  return false;
1492  }
1493  return false;
1494 }
1495 
1498  E = Line.Children.end();
1499  I != E; ++I) {
1501  }
1502 
1503  Line.First->TotalLength =
1504  Line.First->IsMultiline ? Style.ColumnLimit : Line.First->ColumnWidth;
1505  if (!Line.First->Next)
1506  return;
1507  FormatToken *Current = Line.First->Next;
1508  bool InFunctionDecl = Line.MightBeFunctionDecl;
1509  while (Current) {
1510  if (isFunctionDeclarationName(*Current))
1511  Current->Type = TT_FunctionDeclarationName;
1512  if (Current->is(TT_LineComment)) {
1513  if (Current->Previous->BlockKind == BK_BracedInit &&
1514  Current->Previous->opensScope())
1515  Current->SpacesRequiredBefore = Style.Cpp11BracedListStyle ? 0 : 1;
1516  else
1517  Current->SpacesRequiredBefore = Style.SpacesBeforeTrailingComments;
1518 
1519  // If we find a trailing comment, iterate backwards to determine whether
1520  // it seems to relate to a specific parameter. If so, break before that
1521  // parameter to avoid changing the comment's meaning. E.g. don't move 'b'
1522  // to the previous line in:
1523  // SomeFunction(a,
1524  // b, // comment
1525  // c);
1526  if (!Current->HasUnescapedNewline) {
1527  for (FormatToken *Parameter = Current->Previous; Parameter;
1528  Parameter = Parameter->Previous) {
1529  if (Parameter->isOneOf(tok::comment, tok::r_brace))
1530  break;
1531  if (Parameter->Previous && Parameter->Previous->is(tok::comma)) {
1532  if (!Parameter->Previous->is(TT_CtorInitializerComma) &&
1533  Parameter->HasUnescapedNewline)
1534  Parameter->MustBreakBefore = true;
1535  break;
1536  }
1537  }
1538  }
1539  } else if (Current->SpacesRequiredBefore == 0 &&
1540  spaceRequiredBefore(Line, *Current)) {
1541  Current->SpacesRequiredBefore = 1;
1542  }
1543 
1544  Current->MustBreakBefore =
1545  Current->MustBreakBefore || mustBreakBefore(Line, *Current);
1546 
1547  if ((Style.AlwaysBreakAfterDefinitionReturnType == FormatStyle::DRTBS_All ||
1548  (Style.AlwaysBreakAfterDefinitionReturnType ==
1550  Line.Level == 0)) &&
1551  InFunctionDecl && Current->is(TT_FunctionDeclarationName) &&
1552  !Line.Last->isOneOf(tok::semi, tok::comment)) // Only for definitions.
1553  // FIXME: Line.Last points to other characters than tok::semi
1554  // and tok::lbrace.
1555  Current->MustBreakBefore = true;
1556 
1557  Current->CanBreakBefore =
1558  Current->MustBreakBefore || canBreakBefore(Line, *Current);
1559  unsigned ChildSize = 0;
1560  if (Current->Previous->Children.size() == 1) {
1561  FormatToken &LastOfChild = *Current->Previous->Children[0]->Last;
1562  ChildSize = LastOfChild.isTrailingComment() ? Style.ColumnLimit
1563  : LastOfChild.TotalLength + 1;
1564  }
1565  const FormatToken *Prev = Current->Previous;
1566  if (Current->MustBreakBefore || Prev->Children.size() > 1 ||
1567  (Prev->Children.size() == 1 &&
1568  Prev->Children[0]->First->MustBreakBefore) ||
1569  Current->IsMultiline)
1570  Current->TotalLength = Prev->TotalLength + Style.ColumnLimit;
1571  else
1572  Current->TotalLength = Prev->TotalLength + Current->ColumnWidth +
1573  ChildSize + Current->SpacesRequiredBefore;
1574 
1575  if (Current->is(TT_CtorInitializerColon))
1576  InFunctionDecl = false;
1577 
1578  // FIXME: Only calculate this if CanBreakBefore is true once static
1579  // initializers etc. are sorted out.
1580  // FIXME: Move magic numbers to a better place.
1581  Current->SplitPenalty = 20 * Current->BindingStrength +
1582  splitPenalty(Line, *Current, InFunctionDecl);
1583 
1584  Current = Current->Next;
1585  }
1586 
1587  calculateUnbreakableTailLengths(Line);
1588  for (Current = Line.First; Current != nullptr; Current = Current->Next) {
1589  if (Current->Role)
1590  Current->Role->precomputeFormattingInfos(Current);
1591  }
1592 
1593  DEBUG({ printDebugInfo(Line); });
1594 }
1595 
1596 void TokenAnnotator::calculateUnbreakableTailLengths(AnnotatedLine &Line) {
1597  unsigned UnbreakableTailLength = 0;
1598  FormatToken *Current = Line.Last;
1599  while (Current) {
1600  Current->UnbreakableTailLength = UnbreakableTailLength;
1601  if (Current->CanBreakBefore ||
1602  Current->isOneOf(tok::comment, tok::string_literal)) {
1603  UnbreakableTailLength = 0;
1604  } else {
1605  UnbreakableTailLength +=
1606  Current->ColumnWidth + Current->SpacesRequiredBefore;
1607  }
1608  Current = Current->Previous;
1609  }
1610 }
1611 
1612 unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
1613  const FormatToken &Tok,
1614  bool InFunctionDecl) {
1615  const FormatToken &Left = *Tok.Previous;
1616  const FormatToken &Right = Tok;
1617 
1618  if (Left.is(tok::semi))
1619  return 0;
1620 
1621  if (Style.Language == FormatStyle::LK_Java) {
1622  if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_throws))
1623  return 1;
1624  if (Right.is(Keywords.kw_implements))
1625  return 2;
1626  if (Left.is(tok::comma) && Left.NestingLevel == 0)
1627  return 3;
1628  } else if (Style.Language == FormatStyle::LK_JavaScript) {
1629  if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
1630  return 100;
1631  if (Left.is(TT_JsTypeColon))
1632  return 100;
1633  }
1634 
1635  if (Left.is(tok::comma) || (Right.is(tok::identifier) && Right.Next &&
1636  Right.Next->is(TT_DictLiteral)))
1637  return 1;
1638  if (Right.is(tok::l_square)) {
1639  if (Style.Language == FormatStyle::LK_Proto)
1640  return 1;
1641  // Slightly prefer formatting local lambda definitions like functions.
1642  if (Right.is(TT_LambdaLSquare) && Left.is(tok::equal))
1643  return 50;
1644  if (!Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare,
1645  TT_ArrayInitializerLSquare))
1646  return 500;
1647  }
1648 
1649  if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
1650  Right.is(tok::kw_operator)) {
1651  if (Line.startsWith(tok::kw_for) && Right.PartOfMultiVariableDeclStmt)
1652  return 3;
1653  if (Left.is(TT_StartOfName))
1654  return 110;
1655  if (InFunctionDecl && Right.NestingLevel == 0)
1656  return Style.PenaltyReturnTypeOnItsOwnLine;
1657  return 200;
1658  }
1659  if (Right.is(TT_PointerOrReference))
1660  return 190;
1661  if (Right.is(TT_LambdaArrow))
1662  return 110;
1663  if (Left.is(tok::equal) && Right.is(tok::l_brace))
1664  return 150;
1665  if (Left.is(TT_CastRParen))
1666  return 100;
1667  if (Left.is(tok::coloncolon) ||
1668  (Right.is(tok::period) && Style.Language == FormatStyle::LK_Proto))
1669  return 500;
1670  if (Left.isOneOf(tok::kw_class, tok::kw_struct))
1671  return 5000;
1672 
1673  if (Left.isOneOf(TT_RangeBasedForLoopColon, TT_InheritanceColon))
1674  return 2;
1675 
1676  if (Right.isMemberAccess()) {
1677  if (Left.is(tok::r_paren) && Left.MatchingParen &&
1678  Left.MatchingParen->ParameterCount > 0)
1679  return 20; // Should be smaller than breaking at a nested comma.
1680  return 150;
1681  }
1682 
1683  if (Right.is(TT_TrailingAnnotation) &&
1684  (!Right.Next || Right.Next->isNot(tok::l_paren))) {
1685  // Moving trailing annotations to the next line is fine for ObjC method
1686  // declarations.
1687  if (Line.startsWith(TT_ObjCMethodSpecifier))
1688  return 10;
1689  // Generally, breaking before a trailing annotation is bad unless it is
1690  // function-like. It seems to be especially preferable to keep standard
1691  // annotations (i.e. "const", "final" and "override") on the same line.
1692  // Use a slightly higher penalty after ")" so that annotations like
1693  // "const override" are kept together.
1694  bool is_short_annotation = Right.TokenText.size() < 10;
1695  return (Left.is(tok::r_paren) ? 100 : 120) + (is_short_annotation ? 50 : 0);
1696  }
1697 
1698  // In for-loops, prefer breaking at ',' and ';'.
1699  if (Line.startsWith(tok::kw_for) && Left.is(tok::equal))
1700  return 4;
1701 
1702  // In Objective-C method expressions, prefer breaking before "param:" over
1703  // breaking after it.
1704  if (Right.is(TT_SelectorName))
1705  return 0;
1706  if (Left.is(tok::colon) && Left.is(TT_ObjCMethodExpr))
1707  return Line.MightBeFunctionDecl ? 50 : 500;
1708 
1709  if (Left.is(tok::l_paren) && InFunctionDecl && Style.AlignAfterOpenBracket)
1710  return 100;
1711  if (Left.is(tok::l_paren) && Left.Previous &&
1712  Left.Previous->isOneOf(tok::kw_if, tok::kw_for))
1713  return 1000;
1714  if (Left.is(tok::equal) && InFunctionDecl)
1715  return 110;
1716  if (Right.is(tok::r_brace))
1717  return 1;
1718  if (Left.is(TT_TemplateOpener))
1719  return 100;
1720  if (Left.opensScope()) {
1721  if (!Style.AlignAfterOpenBracket)
1722  return 0;
1723  return Left.ParameterCount > 1 ? Style.PenaltyBreakBeforeFirstCallParameter
1724  : 19;
1725  }
1726  if (Left.is(TT_JavaAnnotation))
1727  return 50;
1728 
1729  if (Right.is(tok::lessless)) {
1730  if (Left.is(tok::string_literal) &&
1731  (!Right.LastOperator || Right.OperatorIndex != 1)) {
1732  StringRef Content = Left.TokenText;
1733  if (Content.startswith("\""))
1734  Content = Content.drop_front(1);
1735  if (Content.endswith("\""))
1736  Content = Content.drop_back(1);
1737  Content = Content.trim();
1738  if (Content.size() > 1 &&
1739  (Content.back() == ':' || Content.back() == '='))
1740  return 25;
1741  }
1742  return 1; // Breaking at a << is really cheap.
1743  }
1744  if (Left.is(TT_ConditionalExpr))
1745  return prec::Conditional;
1746  prec::Level Level = Left.getPrecedence();
1747  if (Level != prec::Unknown)
1748  return Level;
1749  Level = Right.getPrecedence();
1750  if (Level != prec::Unknown)
1751  return Level;
1752 
1753  return 3;
1754 }
1755 
1756 bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
1757  const FormatToken &Left,
1758  const FormatToken &Right) {
1759  if (Left.is(tok::kw_return) && Right.isNot(tok::semi))
1760  return true;
1761  if (Style.ObjCSpaceAfterProperty && Line.Type == LT_ObjCProperty &&
1762  Left.Tok.getObjCKeywordID() == tok::objc_property)
1763  return true;
1764  if (Right.is(tok::hashhash))
1765  return Left.is(tok::hash);
1766  if (Left.isOneOf(tok::hashhash, tok::hash))
1767  return Right.is(tok::hash);
1768  if (Left.is(tok::l_paren) && Right.is(tok::r_paren))
1769  return Style.SpaceInEmptyParentheses;
1770  if (Left.is(tok::l_paren) || Right.is(tok::r_paren))
1771  return (Right.is(TT_CastRParen) ||
1772  (Left.MatchingParen && Left.MatchingParen->is(TT_CastRParen)))
1773  ? Style.SpacesInCStyleCastParentheses
1774  : Style.SpacesInParentheses;
1775  if (Right.isOneOf(tok::semi, tok::comma))
1776  return false;
1777  if (Right.is(tok::less) &&
1778  (Left.is(tok::kw_template) ||
1779  (Line.Type == LT_ObjCDecl && Style.ObjCSpaceBeforeProtocolList)))
1780  return true;
1781  if (Left.isOneOf(tok::exclaim, tok::tilde))
1782  return false;
1783  if (Left.is(tok::at) &&
1784  Right.isOneOf(tok::identifier, tok::string_literal, tok::char_constant,
1785  tok::numeric_constant, tok::l_paren, tok::l_brace,
1786  tok::kw_true, tok::kw_false))
1787  return false;
1788  if (Left.is(tok::coloncolon))
1789  return false;
1790  if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less))
1791  return false;
1792  if (Right.is(tok::ellipsis))
1793  return Left.Tok.isLiteral();
1794  if (Left.is(tok::l_square) && Right.is(tok::amp))
1795  return false;
1796  if (Right.is(TT_PointerOrReference))
1797  return !(Left.is(tok::r_paren) && Left.MatchingParen &&
1798  (Left.MatchingParen->is(TT_OverloadedOperatorLParen) ||
1799  (Left.MatchingParen->Previous &&
1800  Left.MatchingParen->Previous->is(
1801  TT_FunctionDeclarationName)))) &&
1802  (Left.Tok.isLiteral() ||
1803  (!Left.isOneOf(TT_PointerOrReference, tok::l_paren) &&
1804  (Style.PointerAlignment != FormatStyle::PAS_Left ||
1805  Line.IsMultiVariableDeclStmt)));
1806  if (Right.is(TT_FunctionTypeLParen) && Left.isNot(tok::l_paren) &&
1807  (!Left.is(TT_PointerOrReference) ||
1808  (Style.PointerAlignment != FormatStyle::PAS_Right &&
1809  !Line.IsMultiVariableDeclStmt)))
1810  return true;
1811  if (Left.is(TT_PointerOrReference))
1812  return Right.Tok.isLiteral() || Right.is(TT_BlockComment) ||
1813  (Right.is(tok::l_brace) && Right.BlockKind == BK_Block) ||
1814  (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
1815  tok::l_paren) &&
1816  (Style.PointerAlignment != FormatStyle::PAS_Right &&
1817  !Line.IsMultiVariableDeclStmt) &&
1818  Left.Previous &&
1819  !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon));
1820  if (Right.is(tok::star) && Left.is(tok::l_paren))
1821  return false;
1822  if (Left.is(tok::l_square))
1823  return (Left.is(TT_ArrayInitializerLSquare) &&
1824  Style.SpacesInContainerLiterals && Right.isNot(tok::r_square)) ||
1825  (Left.is(TT_ArraySubscriptLSquare) && Style.SpacesInSquareBrackets &&
1826  Right.isNot(tok::r_square));
1827  if (Right.is(tok::r_square))
1828  return Right.MatchingParen &&
1829  ((Style.SpacesInContainerLiterals &&
1830  Right.MatchingParen->is(TT_ArrayInitializerLSquare)) ||
1831  (Style.SpacesInSquareBrackets &&
1832  Right.MatchingParen->is(TT_ArraySubscriptLSquare)));
1833  if (Right.is(tok::l_square) &&
1834  !Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare) &&
1835  !Left.isOneOf(tok::numeric_constant, TT_DictLiteral))
1836  return false;
1837  if (Left.is(tok::colon))
1838  return !Left.is(TT_ObjCMethodExpr);
1839  if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
1840  return !Left.Children.empty(); // No spaces in "{}".
1841  if ((Left.is(tok::l_brace) && Left.BlockKind != BK_Block) ||
1842  (Right.is(tok::r_brace) && Right.MatchingParen &&
1843  Right.MatchingParen->BlockKind != BK_Block))
1844  return !Style.Cpp11BracedListStyle;
1845  if (Left.is(TT_BlockComment))
1846  return !Left.TokenText.endswith("=*/");
1847  if (Right.is(tok::l_paren)) {
1848  if (Left.is(tok::r_paren) && Left.is(TT_AttributeParen))
1849  return true;
1850  return Line.Type == LT_ObjCDecl || Left.is(tok::semi) ||
1851  (Style.SpaceBeforeParens != FormatStyle::SBPO_Never &&
1852  (Left.isOneOf(tok::kw_if, tok::kw_for, tok::kw_while,
1853  tok::kw_switch, tok::kw_case, TT_ForEachMacro) ||
1854  (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch,
1855  tok::kw_new, tok::kw_delete) &&
1856  (!Left.Previous || Left.Previous->isNot(tok::period))))) ||
1857  (Style.SpaceBeforeParens == FormatStyle::SBPO_Always &&
1858  (Left.is(tok::identifier) || Left.isFunctionLikeKeyword() ||
1859  Left.is(tok::r_paren)) &&
1860  Line.Type != LT_PreprocessorDirective);
1861  }
1862  if (Left.is(tok::at) && Right.Tok.getObjCKeywordID() != tok::objc_not_keyword)
1863  return false;
1864  if (Right.is(TT_UnaryOperator))
1865  return !Left.isOneOf(tok::l_paren, tok::l_square, tok::at) &&
1866  (Left.isNot(tok::colon) || Left.isNot(TT_ObjCMethodExpr));
1867  if ((Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
1868  tok::r_paren) ||
1869  Left.isSimpleTypeSpecifier()) &&
1870  Right.is(tok::l_brace) && Right.getNextNonComment() &&
1871  Right.BlockKind != BK_Block)
1872  return false;
1873  if (Left.is(tok::period) || Right.is(tok::period))
1874  return false;
1875  if (Right.is(tok::hash) && Left.is(tok::identifier) && Left.TokenText == "L")
1876  return false;
1877  if (Left.is(TT_TemplateCloser) && Left.MatchingParen &&
1878  Left.MatchingParen->Previous &&
1879  Left.MatchingParen->Previous->is(tok::period))
1880  // A.<B>DoSomething();
1881  return false;
1882  if (Left.is(TT_TemplateCloser) && Right.is(tok::l_square))
1883  return false;
1884  return true;
1885 }
1886 
1887 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
1888  const FormatToken &Right) {
1889  const FormatToken &Left = *Right.Previous;
1890  if (Right.Tok.getIdentifierInfo() && Left.Tok.getIdentifierInfo())
1891  return true; // Never ever merge two identifiers.
1892  if (Style.Language == FormatStyle::LK_Cpp) {
1893  if (Left.is(tok::kw_operator))
1894  return Right.is(tok::coloncolon);
1895  } else if (Style.Language == FormatStyle::LK_Proto) {
1896  if (Right.is(tok::period) &&
1897  Left.isOneOf(Keywords.kw_optional, Keywords.kw_required,
1898  Keywords.kw_repeated))
1899  return true;
1900  if (Right.is(tok::l_paren) &&
1901  Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
1902  return true;
1903  } else if (Style.Language == FormatStyle::LK_JavaScript) {
1904  if (Left.isOneOf(Keywords.kw_var, TT_JsFatArrow))
1905  return true;
1906  if (Right.isOneOf(TT_JsTypeColon, TT_JsTypeOptionalQuestion))
1907  return false;
1908  if ((Left.is(tok::l_brace) || Right.is(tok::r_brace)) &&
1909  Line.First->isOneOf(Keywords.kw_import, tok::kw_export))
1910  return false;
1911  if (Left.is(tok::ellipsis))
1912  return false;
1913  if (Left.is(TT_TemplateCloser) &&
1914  !Right.isOneOf(tok::equal, tok::l_brace, tok::comma, tok::l_square,
1915  Keywords.kw_implements, Keywords.kw_extends))
1916  // Type assertions ('<type>expr') are not followed by whitespace. Other
1917  // locations that should have whitespace following are identified by the
1918  // above set of follower tokens.
1919  return false;
1920  } else if (Style.Language == FormatStyle::LK_Java) {
1921  if (Left.is(tok::r_square) && Right.is(tok::l_brace))
1922  return true;
1923  if (Left.is(Keywords.kw_synchronized) && Right.is(tok::l_paren))
1924  return Style.SpaceBeforeParens != FormatStyle::SBPO_Never;
1925  if ((Left.isOneOf(tok::kw_static, tok::kw_public, tok::kw_private,
1926  tok::kw_protected) ||
1927  Left.isOneOf(Keywords.kw_final, Keywords.kw_abstract,
1928  Keywords.kw_native)) &&
1929  Right.is(TT_TemplateOpener))
1930  return true;
1931  }
1932  if (Left.is(TT_ImplicitStringLiteral))
1933  return Right.WhitespaceRange.getBegin() != Right.WhitespaceRange.getEnd();
1934  if (Line.Type == LT_ObjCMethodDecl) {
1935  if (Left.is(TT_ObjCMethodSpecifier))
1936  return true;
1937  if (Left.is(tok::r_paren) && Right.is(tok::identifier))
1938  // Don't space between ')' and <id>
1939  return false;
1940  }
1941  if (Line.Type == LT_ObjCProperty &&
1942  (Right.is(tok::equal) || Left.is(tok::equal)))
1943  return false;
1944 
1945  if (Right.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow) ||
1946  Left.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow))
1947  return true;
1948  if (Left.is(tok::comma))
1949  return true;
1950  if (Right.is(tok::comma))
1951  return false;
1952  if (Right.isOneOf(TT_CtorInitializerColon, TT_ObjCBlockLParen))
1953  return true;
1954  if (Right.is(TT_OverloadedOperatorLParen))
1955  return false;
1956  if (Right.is(tok::colon)) {
1957  if (Line.First->isOneOf(tok::kw_case, tok::kw_default) ||
1958  !Right.getNextNonComment() || Right.getNextNonComment()->is(tok::semi))
1959  return false;
1960  if (Right.is(TT_ObjCMethodExpr))
1961  return false;
1962  if (Left.is(tok::question))
1963  return false;
1964  if (Right.is(TT_InlineASMColon) && Left.is(tok::coloncolon))
1965  return false;
1966  if (Right.is(TT_DictLiteral))
1967  return Style.SpacesInContainerLiterals;
1968  return true;
1969  }
1970  if (Left.is(TT_UnaryOperator))
1971  return Right.is(TT_BinaryOperator);
1972 
1973  // If the next token is a binary operator or a selector name, we have
1974  // incorrectly classified the parenthesis as a cast. FIXME: Detect correctly.
1975  if (Left.is(TT_CastRParen))
1976  return Style.SpaceAfterCStyleCast ||
1977  Right.isOneOf(TT_BinaryOperator, TT_SelectorName);
1978 
1979  if (Left.is(tok::greater) && Right.is(tok::greater))
1980  return Right.is(TT_TemplateCloser) && Left.is(TT_TemplateCloser) &&
1981  (Style.Standard != FormatStyle::LS_Cpp11 || Style.SpacesInAngles);
1982  if (Right.isOneOf(tok::arrow, tok::period, tok::arrowstar, tok::periodstar) ||
1983  Left.isOneOf(tok::arrow, tok::period, tok::arrowstar, tok::periodstar))
1984  return false;
1985  if (!Style.SpaceBeforeAssignmentOperators &&
1986  Right.getPrecedence() == prec::Assignment)
1987  return false;
1988  if (Right.is(tok::coloncolon) && Left.isNot(tok::l_brace))
1989  return (Left.is(TT_TemplateOpener) &&
1990  Style.Standard == FormatStyle::LS_Cpp03) ||
1991  !(Left.isOneOf(tok::identifier, tok::l_paren, tok::r_paren) ||
1992  Left.isOneOf(TT_TemplateCloser, TT_TemplateOpener));
1993  if ((Left.is(TT_TemplateOpener)) != (Right.is(TT_TemplateCloser)))
1994  return Style.SpacesInAngles;
1995  if ((Right.is(TT_BinaryOperator) && !Left.is(tok::l_paren)) ||
1996  Left.isOneOf(TT_BinaryOperator, TT_ConditionalExpr))
1997  return true;
1998  if (Left.is(TT_TemplateCloser) && Right.is(tok::l_paren) &&
1999  Right.isNot(TT_FunctionTypeLParen))
2000  return Style.SpaceBeforeParens == FormatStyle::SBPO_Always;
2001  if (Right.is(TT_TemplateOpener) && Left.is(tok::r_paren) &&
2002  Left.MatchingParen && Left.MatchingParen->is(TT_OverloadedOperatorLParen))
2003  return false;
2004  if (Right.is(tok::less) && Left.isNot(tok::l_paren) &&
2005  Line.startsWith(tok::hash))
2006  return true;
2007  if (Right.is(TT_TrailingUnaryOperator))
2008  return false;
2009  if (Left.is(TT_RegexLiteral))
2010  return false;
2011  return spaceRequiredBetween(Line, Left, Right);
2012 }
2013 
2014 // Returns 'true' if 'Tok' is a brace we'd want to break before in Allman style.
2015 static bool isAllmanBrace(const FormatToken &Tok) {
2016  return Tok.is(tok::l_brace) && Tok.BlockKind == BK_Block &&
2017  !Tok.isOneOf(TT_ObjCBlockLBrace, TT_DictLiteral);
2018 }
2019 
2020 bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
2021  const FormatToken &Right) {
2022  const FormatToken &Left = *Right.Previous;
2023  if (Right.NewlinesBefore > 1)
2024  return true;
2025 
2026  if (Style.Language == FormatStyle::LK_JavaScript) {
2027  // FIXME: This might apply to other languages and token kinds.
2028  if (Right.is(tok::char_constant) && Left.is(tok::plus) && Left.Previous &&
2029  Left.Previous->is(tok::char_constant))
2030  return true;
2031  if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace) && Line.Level == 0 &&
2032  Left.Previous && Left.Previous->is(tok::equal) &&
2033  Line.First->isOneOf(tok::identifier, Keywords.kw_import, tok::kw_export,
2034  tok::kw_const) &&
2035  // kw_var is a pseudo-token that's a tok::identifier, so matches above.
2036  !Line.startsWith(Keywords.kw_var))
2037  // Object literals on the top level of a file are treated as "enum-style".
2038  // Each key/value pair is put on a separate line, instead of bin-packing.
2039  return true;
2040  if (Left.is(tok::l_brace) && Line.Level == 0 &&
2041  (Line.startsWith(tok::kw_enum) ||
2042  Line.startsWith(tok::kw_export, tok::kw_enum)))
2043  // JavaScript top-level enum key/value pairs are put on separate lines
2044  // instead of bin-packing.
2045  return true;
2046  if (Right.is(tok::r_brace) && Left.is(tok::l_brace) &&
2047  !Left.Children.empty())
2048  // Support AllowShortFunctionsOnASingleLine for JavaScript.
2049  return Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_None ||
2050  (Left.NestingLevel == 0 && Line.Level == 0 &&
2051  Style.AllowShortFunctionsOnASingleLine ==
2053  } else if (Style.Language == FormatStyle::LK_Java) {
2054  if (Right.is(tok::plus) && Left.is(tok::string_literal) && Right.Next &&
2055  Right.Next->is(tok::string_literal))
2056  return true;
2057  }
2058 
2059  // If the last token before a '}' is a comma or a trailing comment, the
2060  // intention is to insert a line break after it in order to make shuffling
2061  // around entries easier.
2062  const FormatToken *BeforeClosingBrace = nullptr;
2063  if (Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) &&
2064  Left.BlockKind != BK_Block && Left.MatchingParen)
2065  BeforeClosingBrace = Left.MatchingParen->Previous;
2066  else if (Right.MatchingParen &&
2067  Right.MatchingParen->isOneOf(tok::l_brace,
2068  TT_ArrayInitializerLSquare))
2069  BeforeClosingBrace = &Left;
2070  if (BeforeClosingBrace && (BeforeClosingBrace->is(tok::comma) ||
2071  BeforeClosingBrace->isTrailingComment()))
2072  return true;
2073 
2074  if (Right.is(tok::comment))
2075  return Left.BlockKind != BK_BracedInit &&
2076  Left.isNot(TT_CtorInitializerColon) &&
2077  (Right.NewlinesBefore > 0 && Right.HasUnescapedNewline);
2078  if (Left.isTrailingComment())
2079  return true;
2080  if (Left.isStringLiteral() &&
2081  (Right.isStringLiteral() || Right.is(TT_ObjCStringLiteral)))
2082  return true;
2083  if (Right.Previous->IsUnterminatedLiteral)
2084  return true;
2085  if (Right.is(tok::lessless) && Right.Next &&
2086  Right.Previous->is(tok::string_literal) &&
2087  Right.Next->is(tok::string_literal))
2088  return true;
2089  if (Right.Previous->ClosesTemplateDeclaration &&
2090  Right.Previous->MatchingParen &&
2091  Right.Previous->MatchingParen->NestingLevel == 0 &&
2092  Style.AlwaysBreakTemplateDeclarations)
2093  return true;
2094  if ((Right.isOneOf(TT_CtorInitializerComma, TT_CtorInitializerColon)) &&
2095  Style.BreakConstructorInitializersBeforeComma &&
2096  !Style.ConstructorInitializerAllOnOneLineOrOnePerLine)
2097  return true;
2098  if (Right.is(tok::string_literal) && Right.TokenText.startswith("R\""))
2099  // Raw string literals are special wrt. line breaks. The author has made a
2100  // deliberate choice and might have aligned the contents of the string
2101  // literal accordingly. Thus, we try keep existing line breaks.
2102  return Right.NewlinesBefore > 0;
2103  if (Right.Previous->is(tok::l_brace) && Right.NestingLevel == 1 &&
2104  Style.Language == FormatStyle::LK_Proto)
2105  // Don't put enums onto single lines in protocol buffers.
2106  return true;
2107  if (Right.is(TT_InlineASMBrace))
2108  return Right.HasUnescapedNewline;
2109  if (isAllmanBrace(Left) || isAllmanBrace(Right))
2110  return Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
2111  Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
2112  (Style.BreakBeforeBraces == FormatStyle::BS_Mozilla &&
2113  Line.startsWith(tok::kw_enum));
2114  if (Style.Language == FormatStyle::LK_Proto && Left.isNot(tok::l_brace) &&
2115  Right.is(TT_SelectorName))
2116  return true;
2117  if (Left.is(TT_ObjCBlockLBrace) && !Style.AllowShortBlocksOnASingleLine)
2118  return true;
2119 
2120  if ((Style.Language == FormatStyle::LK_Java ||
2121  Style.Language == FormatStyle::LK_JavaScript) &&
2122  Left.is(TT_LeadingJavaAnnotation) &&
2123  Right.isNot(TT_LeadingJavaAnnotation) && Right.isNot(tok::l_paren) &&
2124  Line.Last->is(tok::l_brace))
2125  return true;
2126 
2127  return false;
2128 }
2129 
2130 bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
2131  const FormatToken &Right) {
2132  const FormatToken &Left = *Right.Previous;
2133 
2134  // Language-specific stuff.
2135  if (Style.Language == FormatStyle::LK_Java) {
2136  if (Left.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
2137  Keywords.kw_implements))
2138  return false;
2139  if (Right.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
2140  Keywords.kw_implements))
2141  return true;
2142  } else if (Style.Language == FormatStyle::LK_JavaScript) {
2143  if (Left.is(TT_JsFatArrow) && Right.is(tok::l_brace))
2144  return false;
2145  if (Left.is(TT_JsTypeColon))
2146  return true;
2147  }
2148 
2149  if (Left.is(tok::at))
2150  return false;
2151  if (Left.Tok.getObjCKeywordID() == tok::objc_interface)
2152  return false;
2153  if (Left.isOneOf(TT_JavaAnnotation, TT_LeadingJavaAnnotation))
2154  return !Right.is(tok::l_paren);
2155  if (Right.is(TT_PointerOrReference))
2156  return Line.IsMultiVariableDeclStmt ||
2157  (Style.PointerAlignment == FormatStyle::PAS_Right &&
2158  (!Right.Next || Right.Next->isNot(TT_FunctionDeclarationName)));
2159  if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
2160  Right.is(tok::kw_operator))
2161  return true;
2162  if (Left.is(TT_PointerOrReference))
2163  return false;
2164  if (Right.isTrailingComment())
2165  // We rely on MustBreakBefore being set correctly here as we should not
2166  // change the "binding" behavior of a comment.
2167  // The first comment in a braced lists is always interpreted as belonging to
2168  // the first list element. Otherwise, it should be placed outside of the
2169  // list.
2170  return Left.BlockKind == BK_BracedInit;
2171  if (Left.is(tok::question) && Right.is(tok::colon))
2172  return false;
2173  if (Right.is(TT_ConditionalExpr) || Right.is(tok::question))
2174  return Style.BreakBeforeTernaryOperators;
2175  if (Left.is(TT_ConditionalExpr) || Left.is(tok::question))
2176  return !Style.BreakBeforeTernaryOperators;
2177  if (Right.is(TT_InheritanceColon))
2178  return true;
2179  if (Right.is(tok::colon) &&
2180  !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon))
2181  return false;
2182  if (Left.is(tok::colon) && (Left.isOneOf(TT_DictLiteral, TT_ObjCMethodExpr)))
2183  return true;
2184  if (Right.is(TT_SelectorName) || (Right.is(tok::identifier) && Right.Next &&
2185  Right.Next->is(TT_ObjCMethodExpr)))
2186  return Left.isNot(tok::period); // FIXME: Properly parse ObjC calls.
2187  if (Left.is(tok::r_paren) && Line.Type == LT_ObjCProperty)
2188  return true;
2189  if (Left.ClosesTemplateDeclaration)
2190  return true;
2191  if (Right.isOneOf(TT_RangeBasedForLoopColon, TT_OverloadedOperatorLParen,
2192  TT_OverloadedOperator))
2193  return false;
2194  if (Left.is(TT_RangeBasedForLoopColon))
2195  return true;
2196  if (Right.is(TT_RangeBasedForLoopColon))
2197  return false;
2198  if (Left.isOneOf(TT_TemplateCloser, TT_UnaryOperator) ||
2199  Left.is(tok::kw_operator))
2200  return false;
2201  if (Left.is(tok::equal) && !Right.isOneOf(tok::kw_default, tok::kw_delete) &&
2202  Line.Type == LT_VirtualFunctionDecl)
2203  return false;
2204  if (Left.is(tok::l_paren) && Left.is(TT_AttributeParen))
2205  return false;
2206  if (Left.is(tok::l_paren) && Left.Previous &&
2207  (Left.Previous->isOneOf(TT_BinaryOperator, TT_CastRParen)))
2208  return false;
2209  if (Right.is(TT_ImplicitStringLiteral))
2210  return false;
2211 
2212  if (Right.is(tok::r_paren) || Right.is(TT_TemplateCloser))
2213  return false;
2214  if (Right.is(tok::r_square) && Right.MatchingParen &&
2215  Right.MatchingParen->is(TT_LambdaLSquare))
2216  return false;
2217 
2218  // We only break before r_brace if there was a corresponding break before
2219  // the l_brace, which is tracked by BreakBeforeClosingBrace.
2220  if (Right.is(tok::r_brace))
2221  return Right.MatchingParen && Right.MatchingParen->BlockKind == BK_Block;
2222 
2223  // Allow breaking after a trailing annotation, e.g. after a method
2224  // declaration.
2225  if (Left.is(TT_TrailingAnnotation))
2226  return !Right.isOneOf(tok::l_brace, tok::semi, tok::equal, tok::l_paren,
2227  tok::less, tok::coloncolon);
2228 
2229  if (Right.is(tok::kw___attribute))
2230  return true;
2231 
2232  if (Left.is(tok::identifier) && Right.is(tok::string_literal))
2233  return true;
2234 
2235  if (Right.is(tok::identifier) && Right.Next && Right.Next->is(TT_DictLiteral))
2236  return true;
2237 
2238  if (Left.is(TT_CtorInitializerComma) &&
2239  Style.BreakConstructorInitializersBeforeComma)
2240  return false;
2241  if (Right.is(TT_CtorInitializerComma) &&
2242  Style.BreakConstructorInitializersBeforeComma)
2243  return true;
2244  if ((Left.is(tok::greater) && Right.is(tok::greater)) ||
2245  (Left.is(tok::less) && Right.is(tok::less)))
2246  return false;
2247  if (Right.is(TT_BinaryOperator) &&
2248  Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None &&
2249  (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_All ||
2250  Right.getPrecedence() != prec::Assignment))
2251  return true;
2252  if (Left.is(TT_ArrayInitializerLSquare))
2253  return true;
2254  if (Right.is(tok::kw_typename) && Left.isNot(tok::kw_const))
2255  return true;
2256  if ((Left.isBinaryOperator() || Left.is(TT_BinaryOperator)) &&
2257  !Left.isOneOf(tok::arrowstar, tok::lessless) &&
2258  Style.BreakBeforeBinaryOperators != FormatStyle::BOS_All &&
2259  (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_None ||
2260  Left.getPrecedence() == prec::Assignment))
2261  return true;
2262  return Left.isOneOf(tok::comma, tok::coloncolon, tok::semi, tok::l_brace,
2263  tok::kw_class, tok::kw_struct) ||
2264  Right.isMemberAccess() ||
2265  Right.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow, tok::lessless,
2266  tok::colon, tok::l_square, tok::at) ||
2267  (Left.is(tok::r_paren) &&
2268  Right.isOneOf(tok::identifier, tok::kw_const)) ||
2269  (Left.is(tok::l_paren) && !Right.is(tok::r_paren));
2270 }
2271 
2272 void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
2273  llvm::errs() << "AnnotatedTokens:\n";
2274  const FormatToken *Tok = Line.First;
2275  while (Tok) {
2276  llvm::errs() << " M=" << Tok->MustBreakBefore
2277  << " C=" << Tok->CanBreakBefore
2278  << " T=" << getTokenTypeName(Tok->Type)
2279  << " S=" << Tok->SpacesRequiredBefore
2280  << " B=" << Tok->BlockParameterCount
2281  << " P=" << Tok->SplitPenalty << " Name=" << Tok->Tok.getName()
2282  << " L=" << Tok->TotalLength << " PPK=" << Tok->PackingKind
2283  << " FakeLParens=";
2284  for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
2285  llvm::errs() << Tok->FakeLParens[i] << "/";
2286  llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";
2287  if (!Tok->Next)
2288  assert(Tok == Line.Last);
2289  Tok = Tok->Next;
2290  }
2291  llvm::errs() << "----\n";
2292 }
2293 
2294 } // namespace format
2295 } // namespace clang
unsigned NestingLevel
The nesting level of this token, i.e. the number of surrounding (), [], {} or <>. ...
Definition: FormatToken.h:220
bool AutoFound
Defines the SourceManager interface.
std::unique_ptr< TokenRole > Role
A token can have a special role that can carry extra information about the token's formatting...
Definition: FormatToken.h:196
bool isOneOf(A K1, B K2) const
Definition: FormatToken.h:286
Align pointer to the left.
Definition: Format.h:351
FormatToken * CurrentToken
Should be used for C, C++, ObjectiveC, ObjectiveC++.
Definition: Format.h:283
bool IsMultiline
Whether the token text contains newlines (escaped or not).
Definition: FormatToken.h:145
bool isNot(T Kind) const
Definition: FormatToken.h:293
void setCommentLineLevels(SmallVectorImpl< AnnotatedLine * > &Lines)
Adapts the indent levels of comment lines to the indent of the subsequent line.
unsigned TotalLength
The total length of the unwrapped line up to and including this token.
Definition: FormatToken.h:203
FormatToken * Next
The next token in the unwrapped line.
Definition: FormatToken.h:267
Break after operators.
Definition: Format.h:152
bool CanBeExpression
tok::TokenKind ContextKind
unsigned UnbreakableTailLength
The length of following tokens until the next natural split point, or the next token that can be brok...
Definition: FormatToken.h:211
unsigned SplitPenalty
Penalty for inserting a line break before this token.
Definition: FormatToken.h:223
bool ColonIsForRangeExpr
bool CanBreakBefore
true if it is allowed to break before this token.
Definition: FormatToken.h:174
Should be used for Java.
Definition: Format.h:285
FormatToken * Previous
The previous token in the unwrapped line.
Definition: FormatToken.h:264
This file implements a token annotator, i.e. creates AnnotatedTokens out of FormatTokens with require...
Always break after the return types of top level functions.
Definition: Format.h:123
unsigned SpacesRequiredBefore
The number of spaces that should be inserted before this token.
Definition: FormatToken.h:171
llvm::SmallPtrSet< FormatToken *, 16 > NonTemplateLess
Never merge functions into a single line.
Definition: Format.h:94
AnnotatingParser & P
ASTContext * Context
Should be used for JavaScript.
Definition: Format.h:287
StateNode * Previous
SmallVector< AnnotatedLine *, 0 > Children
A wrapper around a Token storing information about the whitespace characters preceding it...
Definition: FormatToken.h:112
FormatToken * Token
FormatToken * FirstStartOfName
void annotate(AnnotatedLine &Line)
bool startsWith(Ts...Tokens) const
const char * getTokenTypeName(TokenType Type)
Determines the name of a token type.
Definition: FormatToken.cpp:26
const FormatStyle & Style
bool isTrailingComment() const
Definition: FormatToken.h:355
SmallVectorImpl< AnnotatedLine * >::const_iterator Next
static bool isAllmanBrace(const FormatToken &Tok)
AnnotatedLine & Line
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
StringRef TokenText
The raw text of the token.
Definition: FormatToken.h:160
Never put a space before opening parentheses.
Definition: Format.h:370
Always break before braces.
Definition: Format.h:175
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
bool ColonIsObjCMethodExpr
unsigned ColumnWidth
The width of the non-whitespace parts of the token (or its first line for multi-line tokens) in colum...
Definition: FormatToken.h:138
const AdditionalKeywords & Keywords
Use C++03-compatible syntax.
Definition: Format.h:414
bool IsExpression
SmallVector< Context, 8 > Contexts
bool opensScope() const
Returns whether Tok is ([{ or a template opening <.
Definition: FormatToken.h:317
void calculateFormattingInformation(AnnotatedLine &Line)
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:281
Only merge functions defined inside a class. Implies "empty".
Definition: Format.h:98
unsigned BindingStrength
FormatToken * MatchingParen
If this is a bracket, this points to the matching one.
Definition: FormatToken.h:261
SmallVector< AnnotatedLine *, 1 > Children
If this token starts a block, this contains all the unwrapped lines in it.
Definition: FormatToken.h:271
FormatToken * Current
FormatToken * FirstObjCSelectorName
bool InTemplateArgument
bool ColonIsDictLiteral
The parameter type of a method or function.
Break before operators.
Definition: Format.h:156
unsigned LongestObjCSelectorName
bool MustBreakBefore
Whether there must be a line break before this token.
Definition: FormatToken.h:154
prec::Level getBinOpPrecedence(tok::TokenKind Kind, bool GreaterThanIsOperator, bool CPlusPlus11)
Return the precedence of the specified binary operator token.
Align pointer to the right.
Definition: Format.h:353
unsigned BindingStrength
The binding strength of a token. This is a combined value of operator precedence, parenthesis nesting...
Definition: FormatToken.h:216
bool CaretFound
bool InCtorInitializer
Always break after the return type.
Definition: Format.h:121
bool HasUnescapedNewline
Whether there is at least one unescaped newline before the Token.
Definition: FormatToken.h:126
BraceBlockKind BlockKind
Contains the kind of block if this token is a brace.
Definition: FormatToken.h:166
bool IsForEachMacro
static bool isFunctionDeclarationName(const FormatToken &Current)