clang  3.8.0
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Stmt.h"
19 #include "clang/AST/StmtOpenMP.h"
20 using namespace clang;
21 using namespace CodeGen;
22 
24  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
25  const RecordDecl *RD = S.getCapturedRecordDecl();
26  auto CurField = RD->field_begin();
27  auto CurCap = S.captures().begin();
29  E = S.capture_init_end();
30  I != E; ++I, ++CurField, ++CurCap) {
31  if (CurField->hasCapturedVLAType()) {
32  auto VAT = CurField->getCapturedVLAType();
33  auto *Val = VLASizeMap[VAT->getSizeExpr()];
34  CapturedVars.push_back(Val);
35  } else if (CurCap->capturesThis())
36  CapturedVars.push_back(CXXThisValue);
37  else if (CurCap->capturesVariableByCopy())
38  CapturedVars.push_back(
39  EmitLoadOfLValue(EmitLValue(*I), SourceLocation()).getScalarVal());
40  else {
41  assert(CurCap->capturesVariable() && "Expected capture by reference.");
42  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
43  }
44  }
45 }
46 
48  StringRef Name, LValue AddrLV,
49  bool isReferenceType = false) {
50  ASTContext &Ctx = CGF.getContext();
51 
52  auto *CastedPtr = CGF.EmitScalarConversion(
53  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
54  Ctx.getPointerType(DstType), SourceLocation());
55  auto TmpAddr =
56  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
57  .getAddress();
58 
59  // If we are dealing with references we need to return the address of the
60  // reference instead of the reference of the value.
61  if (isReferenceType) {
62  QualType RefType = Ctx.getLValueReferenceType(DstType);
63  auto *RefVal = TmpAddr.getPointer();
64  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name) + ".ref");
65  auto TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
66  CGF.EmitScalarInit(RefVal, TmpLVal);
67  }
68 
69  return TmpAddr;
70 }
71 
72 llvm::Function *
74  assert(
76  "CapturedStmtInfo should be set when generating the captured function");
77  const CapturedDecl *CD = S.getCapturedDecl();
78  const RecordDecl *RD = S.getCapturedRecordDecl();
79  assert(CD->hasBody() && "missing CapturedDecl body");
80 
81  // Build the argument list.
82  ASTContext &Ctx = CGM.getContext();
83  FunctionArgList Args;
84  Args.append(CD->param_begin(),
85  std::next(CD->param_begin(), CD->getContextParamPosition()));
86  auto I = S.captures().begin();
87  for (auto *FD : RD->fields()) {
88  QualType ArgType = FD->getType();
89  IdentifierInfo *II = nullptr;
90  VarDecl *CapVar = nullptr;
91 
92  // If this is a capture by copy and the type is not a pointer, the outlined
93  // function argument type should be uintptr and the value properly casted to
94  // uintptr. This is necessary given that the runtime library is only able to
95  // deal with pointers. We can pass in the same way the VLA type sizes to the
96  // outlined function.
97  if ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
98  I->capturesVariableArrayType())
99  ArgType = Ctx.getUIntPtrType();
100 
101  if (I->capturesVariable() || I->capturesVariableByCopy()) {
102  CapVar = I->getCapturedVar();
103  II = CapVar->getIdentifier();
104  } else if (I->capturesThis())
105  II = &getContext().Idents.get("this");
106  else {
107  assert(I->capturesVariableArrayType());
108  II = &getContext().Idents.get("vla");
109  }
110  if (ArgType->isVariablyModifiedType())
111  ArgType = getContext().getVariableArrayDecayedType(ArgType);
112  Args.push_back(ImplicitParamDecl::Create(getContext(), nullptr,
113  FD->getLocation(), II, ArgType));
114  ++I;
115  }
116  Args.append(
117  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
118  CD->param_end());
119 
120  // Create the function declaration.
122  const CGFunctionInfo &FuncInfo =
123  CGM.getTypes().arrangeFreeFunctionDeclaration(Ctx.VoidTy, Args, ExtInfo,
124  /*IsVariadic=*/false);
125  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
126 
127  llvm::Function *F = llvm::Function::Create(
130  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
131  if (CD->isNothrow())
132  F->addFnAttr(llvm::Attribute::NoUnwind);
133 
134  // Generate the function.
135  StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
136  CD->getBody()->getLocStart());
137  unsigned Cnt = CD->getContextParamPosition();
138  I = S.captures().begin();
139  for (auto *FD : RD->fields()) {
140  // If we are capturing a pointer by copy we don't need to do anything, just
141  // use the value that we get from the arguments.
142  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
143  setAddrOfLocalVar(I->getCapturedVar(), GetAddrOfLocalVar(Args[Cnt]));
144  ++Cnt, ++I;
145  continue;
146  }
147 
148  LValue ArgLVal =
149  MakeAddrLValue(GetAddrOfLocalVar(Args[Cnt]), Args[Cnt]->getType(),
151  if (FD->hasCapturedVLAType()) {
152  LValue CastedArgLVal =
153  MakeAddrLValue(castValueFromUintptr(*this, FD->getType(),
154  Args[Cnt]->getName(), ArgLVal),
156  auto *ExprArg =
157  EmitLoadOfLValue(CastedArgLVal, SourceLocation()).getScalarVal();
158  auto VAT = FD->getCapturedVLAType();
159  VLASizeMap[VAT->getSizeExpr()] = ExprArg;
160  } else if (I->capturesVariable()) {
161  auto *Var = I->getCapturedVar();
162  QualType VarTy = Var->getType();
163  Address ArgAddr = ArgLVal.getAddress();
164  if (!VarTy->isReferenceType()) {
165  ArgAddr = EmitLoadOfReference(
166  ArgAddr, ArgLVal.getType()->castAs<ReferenceType>());
167  }
168  setAddrOfLocalVar(
169  Var, Address(ArgAddr.getPointer(), getContext().getDeclAlign(Var)));
170  } else if (I->capturesVariableByCopy()) {
171  assert(!FD->getType()->isAnyPointerType() &&
172  "Not expecting a captured pointer.");
173  auto *Var = I->getCapturedVar();
174  QualType VarTy = Var->getType();
175  setAddrOfLocalVar(I->getCapturedVar(),
176  castValueFromUintptr(*this, FD->getType(),
177  Args[Cnt]->getName(), ArgLVal,
178  VarTy->isReferenceType()));
179  } else {
180  // If 'this' is captured, load it into CXXThisValue.
181  assert(I->capturesThis());
182  CXXThisValue =
183  EmitLoadOfLValue(ArgLVal, Args[Cnt]->getLocation()).getScalarVal();
184  }
185  ++Cnt, ++I;
186  }
187 
188  PGO.assignRegionCounters(GlobalDecl(CD), F);
189  CapturedStmtInfo->EmitBody(*this, CD->getBody());
191 
192  return F;
193 }
194 
195 //===----------------------------------------------------------------------===//
196 // OpenMP Directive Emission
197 //===----------------------------------------------------------------------===//
199  Address DestAddr, Address SrcAddr, QualType OriginalType,
200  const llvm::function_ref<void(Address, Address)> &CopyGen) {
201  // Perform element-by-element initialization.
202  QualType ElementTy;
203 
204  // Drill down to the base element type on both arrays.
205  auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
206  auto NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
207  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
208 
209  auto SrcBegin = SrcAddr.getPointer();
210  auto DestBegin = DestAddr.getPointer();
211  // Cast from pointer to array type to pointer to single element.
212  auto DestEnd = Builder.CreateGEP(DestBegin, NumElements);
213  // The basic structure here is a while-do loop.
214  auto BodyBB = createBasicBlock("omp.arraycpy.body");
215  auto DoneBB = createBasicBlock("omp.arraycpy.done");
216  auto IsEmpty =
217  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
218  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
219 
220  // Enter the loop body, making that address the current address.
221  auto EntryBB = Builder.GetInsertBlock();
222  EmitBlock(BodyBB);
223 
224  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
225 
226  llvm::PHINode *SrcElementPHI =
227  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
228  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
229  Address SrcElementCurrent =
230  Address(SrcElementPHI,
231  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
232 
233  llvm::PHINode *DestElementPHI =
234  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
235  DestElementPHI->addIncoming(DestBegin, EntryBB);
236  Address DestElementCurrent =
237  Address(DestElementPHI,
238  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
239 
240  // Emit copy.
241  CopyGen(DestElementCurrent, SrcElementCurrent);
242 
243  // Shift the address forward by one element.
244  auto DestElementNext = Builder.CreateConstGEP1_32(
245  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
246  auto SrcElementNext = Builder.CreateConstGEP1_32(
247  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
248  // Check whether we've reached the end.
249  auto Done =
250  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
251  Builder.CreateCondBr(Done, DoneBB, BodyBB);
252  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
253  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
254 
255  // Done.
256  EmitBlock(DoneBB, /*IsFinished=*/true);
257 }
258 
259 /// \brief Emit initialization of arrays of complex types.
260 /// \param DestAddr Address of the array.
261 /// \param Type Type of array.
262 /// \param Init Initial expression of array.
263 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
264  QualType Type, const Expr *Init) {
265  // Perform element-by-element initialization.
266  QualType ElementTy;
267 
268  // Drill down to the base element type on both arrays.
269  auto ArrayTy = Type->getAsArrayTypeUnsafe();
270  auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
271  DestAddr =
272  CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
273 
274  auto DestBegin = DestAddr.getPointer();
275  // Cast from pointer to array type to pointer to single element.
276  auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
277  // The basic structure here is a while-do loop.
278  auto BodyBB = CGF.createBasicBlock("omp.arrayinit.body");
279  auto DoneBB = CGF.createBasicBlock("omp.arrayinit.done");
280  auto IsEmpty =
281  CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty");
282  CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
283 
284  // Enter the loop body, making that address the current address.
285  auto EntryBB = CGF.Builder.GetInsertBlock();
286  CGF.EmitBlock(BodyBB);
287 
288  CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
289 
290  llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
291  DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
292  DestElementPHI->addIncoming(DestBegin, EntryBB);
293  Address DestElementCurrent =
294  Address(DestElementPHI,
295  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
296 
297  // Emit copy.
298  {
299  CodeGenFunction::RunCleanupsScope InitScope(CGF);
300  CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
301  /*IsInitializer=*/false);
302  }
303 
304  // Shift the address forward by one element.
305  auto DestElementNext = CGF.Builder.CreateConstGEP1_32(
306  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
307  // Check whether we've reached the end.
308  auto Done =
309  CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
310  CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB);
311  DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock());
312 
313  // Done.
314  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
315 }
316 
317 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
318  Address SrcAddr, const VarDecl *DestVD,
319  const VarDecl *SrcVD, const Expr *Copy) {
320  if (OriginalType->isArrayType()) {
321  auto *BO = dyn_cast<BinaryOperator>(Copy);
322  if (BO && BO->getOpcode() == BO_Assign) {
323  // Perform simple memcpy for simple copying.
324  EmitAggregateAssign(DestAddr, SrcAddr, OriginalType);
325  } else {
326  // For arrays with complex element types perform element by element
327  // copying.
329  DestAddr, SrcAddr, OriginalType,
330  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
331  // Working with the single array element, so have to remap
332  // destination and source variables to corresponding array
333  // elements.
335  Remap.addPrivate(DestVD, [DestElement]() -> Address {
336  return DestElement;
337  });
338  Remap.addPrivate(
339  SrcVD, [SrcElement]() -> Address { return SrcElement; });
340  (void)Remap.Privatize();
341  EmitIgnoredExpr(Copy);
342  });
343  }
344  } else {
345  // Remap pseudo source variable to private copy.
347  Remap.addPrivate(SrcVD, [SrcAddr]() -> Address { return SrcAddr; });
348  Remap.addPrivate(DestVD, [DestAddr]() -> Address { return DestAddr; });
349  (void)Remap.Privatize();
350  // Emit copying of the whole variable.
351  EmitIgnoredExpr(Copy);
352  }
353 }
354 
356  OMPPrivateScope &PrivateScope) {
357  if (!HaveInsertPoint())
358  return false;
359  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
360  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
361  auto IRef = C->varlist_begin();
362  auto InitsRef = C->inits().begin();
363  for (auto IInit : C->private_copies()) {
364  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
365  if (EmittedAsFirstprivate.count(OrigVD) == 0) {
366  EmittedAsFirstprivate.insert(OrigVD);
367  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
368  auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
369  bool IsRegistered;
370  DeclRefExpr DRE(
371  const_cast<VarDecl *>(OrigVD),
372  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
373  OrigVD) != nullptr,
374  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
375  Address OriginalAddr = EmitLValue(&DRE).getAddress();
376  QualType Type = OrigVD->getType();
377  if (Type->isArrayType()) {
378  // Emit VarDecl with copy init for arrays.
379  // Get the address of the original variable captured in current
380  // captured region.
381  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
382  auto Emission = EmitAutoVarAlloca(*VD);
383  auto *Init = VD->getInit();
384  if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
385  // Perform simple memcpy.
386  EmitAggregateAssign(Emission.getAllocatedAddress(), OriginalAddr,
387  Type);
388  } else {
390  Emission.getAllocatedAddress(), OriginalAddr, Type,
391  [this, VDInit, Init](Address DestElement,
392  Address SrcElement) {
393  // Clean up any temporaries needed by the initialization.
394  RunCleanupsScope InitScope(*this);
395  // Emit initialization for single element.
396  setAddrOfLocalVar(VDInit, SrcElement);
397  EmitAnyExprToMem(Init, DestElement,
398  Init->getType().getQualifiers(),
399  /*IsInitializer*/ false);
400  LocalDeclMap.erase(VDInit);
401  });
402  }
403  EmitAutoVarCleanups(Emission);
404  return Emission.getAllocatedAddress();
405  });
406  } else {
407  IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
408  // Emit private VarDecl with copy init.
409  // Remap temp VDInit variable to the address of the original
410  // variable
411  // (for proper handling of captured global variables).
412  setAddrOfLocalVar(VDInit, OriginalAddr);
413  EmitDecl(*VD);
414  LocalDeclMap.erase(VDInit);
415  return GetAddrOfLocalVar(VD);
416  });
417  }
418  assert(IsRegistered &&
419  "firstprivate var already registered as private");
420  // Silence the warning about unused variable.
421  (void)IsRegistered;
422  }
423  ++IRef, ++InitsRef;
424  }
425  }
426  return !EmittedAsFirstprivate.empty();
427 }
428 
430  const OMPExecutableDirective &D,
431  CodeGenFunction::OMPPrivateScope &PrivateScope) {
432  if (!HaveInsertPoint())
433  return;
434  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
435  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
436  auto IRef = C->varlist_begin();
437  for (auto IInit : C->private_copies()) {
438  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
439  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
440  auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
441  bool IsRegistered =
442  PrivateScope.addPrivate(OrigVD, [&]() -> Address {
443  // Emit private VarDecl with copy init.
444  EmitDecl(*VD);
445  return GetAddrOfLocalVar(VD);
446  });
447  assert(IsRegistered && "private var already registered as private");
448  // Silence the warning about unused variable.
449  (void)IsRegistered;
450  }
451  ++IRef;
452  }
453  }
454 }
455 
457  if (!HaveInsertPoint())
458  return false;
459  // threadprivate_var1 = master_threadprivate_var1;
460  // operator=(threadprivate_var2, master_threadprivate_var2);
461  // ...
462  // __kmpc_barrier(&loc, global_tid);
464  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
465  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
466  auto IRef = C->varlist_begin();
467  auto ISrcRef = C->source_exprs().begin();
468  auto IDestRef = C->destination_exprs().begin();
469  for (auto *AssignOp : C->assignment_ops()) {
470  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
471  QualType Type = VD->getType();
472  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
473 
474  // Get the address of the master variable. If we are emitting code with
475  // TLS support, the address is passed from the master as field in the
476  // captured declaration.
477  Address MasterAddr = Address::invalid();
478  if (getLangOpts().OpenMPUseTLS &&
479  getContext().getTargetInfo().isTLSSupported()) {
480  assert(CapturedStmtInfo->lookup(VD) &&
481  "Copyin threadprivates should have been captured!");
482  DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
483  VK_LValue, (*IRef)->getExprLoc());
484  MasterAddr = EmitLValue(&DRE).getAddress();
485  LocalDeclMap.erase(VD);
486  } else {
487  MasterAddr =
488  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
489  : CGM.GetAddrOfGlobal(VD),
490  getContext().getDeclAlign(VD));
491  }
492  // Get the address of the threadprivate variable.
493  Address PrivateAddr = EmitLValue(*IRef).getAddress();
494  if (CopiedVars.size() == 1) {
495  // At first check if current thread is a master thread. If it is, no
496  // need to copy data.
497  CopyBegin = createBasicBlock("copyin.not.master");
498  CopyEnd = createBasicBlock("copyin.not.master.end");
499  Builder.CreateCondBr(
500  Builder.CreateICmpNE(
501  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
502  Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy)),
503  CopyBegin, CopyEnd);
504  EmitBlock(CopyBegin);
505  }
506  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
507  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
508  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
509  }
510  ++IRef;
511  ++ISrcRef;
512  ++IDestRef;
513  }
514  }
515  if (CopyEnd) {
516  // Exit out of copying procedure for non-master thread.
517  EmitBlock(CopyEnd, /*IsFinished=*/true);
518  return true;
519  }
520  return false;
521 }
522 
524  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
525  if (!HaveInsertPoint())
526  return false;
527  bool HasAtLeastOneLastprivate = false;
528  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
529  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
530  HasAtLeastOneLastprivate = true;
531  auto IRef = C->varlist_begin();
532  auto IDestRef = C->destination_exprs().begin();
533  for (auto *IInit : C->private_copies()) {
534  // Keep the address of the original variable for future update at the end
535  // of the loop.
536  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
537  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
538  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
539  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() -> Address {
540  DeclRefExpr DRE(
541  const_cast<VarDecl *>(OrigVD),
542  /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
543  OrigVD) != nullptr,
544  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
545  return EmitLValue(&DRE).getAddress();
546  });
547  // Check if the variable is also a firstprivate: in this case IInit is
548  // not generated. Initialization of this variable will happen in codegen
549  // for 'firstprivate' clause.
550  if (IInit) {
551  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
552  bool IsRegistered =
553  PrivateScope.addPrivate(OrigVD, [&]() -> Address {
554  // Emit private VarDecl with copy init.
555  EmitDecl(*VD);
556  return GetAddrOfLocalVar(VD);
557  });
558  assert(IsRegistered &&
559  "lastprivate var already registered as private");
560  (void)IsRegistered;
561  }
562  }
563  ++IRef, ++IDestRef;
564  }
565  }
566  return HasAtLeastOneLastprivate;
567 }
568 
570  const OMPExecutableDirective &D, llvm::Value *IsLastIterCond) {
571  if (!HaveInsertPoint())
572  return;
573  // Emit following code:
574  // if (<IsLastIterCond>) {
575  // orig_var1 = private_orig_var1;
576  // ...
577  // orig_varn = private_orig_varn;
578  // }
579  llvm::BasicBlock *ThenBB = nullptr;
580  llvm::BasicBlock *DoneBB = nullptr;
581  if (IsLastIterCond) {
582  ThenBB = createBasicBlock(".omp.lastprivate.then");
583  DoneBB = createBasicBlock(".omp.lastprivate.done");
584  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
585  EmitBlock(ThenBB);
586  }
587  llvm::DenseMap<const Decl *, const Expr *> LoopCountersAndUpdates;
588  if (auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
589  auto IC = LoopDirective->counters().begin();
590  for (auto F : LoopDirective->finals()) {
591  auto *D = cast<DeclRefExpr>(*IC)->getDecl()->getCanonicalDecl();
592  LoopCountersAndUpdates[D] = F;
593  ++IC;
594  }
595  }
596  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
597  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
598  auto IRef = C->varlist_begin();
599  auto ISrcRef = C->source_exprs().begin();
600  auto IDestRef = C->destination_exprs().begin();
601  for (auto *AssignOp : C->assignment_ops()) {
602  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
603  QualType Type = PrivateVD->getType();
604  auto *CanonicalVD = PrivateVD->getCanonicalDecl();
605  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
606  // If lastprivate variable is a loop control variable for loop-based
607  // directive, update its value before copyin back to original
608  // variable.
609  if (auto *UpExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
610  EmitIgnoredExpr(UpExpr);
611  auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
612  auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
613  // Get the address of the original variable.
614  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
615  // Get the address of the private variable.
616  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
617  if (auto RefTy = PrivateVD->getType()->getAs<ReferenceType>())
618  PrivateAddr =
619  Address(Builder.CreateLoad(PrivateAddr),
620  getNaturalTypeAlignment(RefTy->getPointeeType()));
621  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
622  }
623  ++IRef;
624  ++ISrcRef;
625  ++IDestRef;
626  }
627  }
628  if (IsLastIterCond)
629  EmitBlock(DoneBB, /*IsFinished=*/true);
630 }
631 
633  const OMPExecutableDirective &D,
634  CodeGenFunction::OMPPrivateScope &PrivateScope) {
635  if (!HaveInsertPoint())
636  return;
637  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
638  auto ILHS = C->lhs_exprs().begin();
639  auto IRHS = C->rhs_exprs().begin();
640  auto IPriv = C->privates().begin();
641  for (auto IRef : C->varlists()) {
642  auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
643  auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
644  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
645  if (auto *OASE = dyn_cast<OMPArraySectionExpr>(IRef)) {
646  auto *Base = OASE->getBase()->IgnoreParenImpCasts();
647  while (auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
648  Base = TempOASE->getBase()->IgnoreParenImpCasts();
649  while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
650  Base = TempASE->getBase()->IgnoreParenImpCasts();
651  auto *DE = cast<DeclRefExpr>(Base);
652  auto *OrigVD = cast<VarDecl>(DE->getDecl());
653  auto OASELValueLB = EmitOMPArraySectionExpr(OASE);
654  auto OASELValueUB =
655  EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false);
656  auto OriginalBaseLValue = EmitLValue(DE);
657  auto BaseLValue = OriginalBaseLValue;
658  auto *Zero = Builder.getInt64(/*C=*/0);
660  Indexes.push_back(Zero);
661  auto *ItemTy =
662  OASELValueLB.getPointer()->getType()->getPointerElementType();
663  auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType();
664  while (Ty != ItemTy) {
665  Indexes.push_back(Zero);
666  Ty = Ty->getPointerElementType();
667  }
668  BaseLValue = MakeAddrLValue(
669  Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes),
670  OASELValueLB.getAlignment()),
671  OASELValueLB.getType(), OASELValueLB.getAlignmentSource());
672  // Store the address of the original variable associated with the LHS
673  // implicit variable.
674  PrivateScope.addPrivate(LHSVD, [this, OASELValueLB]() -> Address {
675  return OASELValueLB.getAddress();
676  });
677  // Emit reduction copy.
678  bool IsRegistered = PrivateScope.addPrivate(
679  OrigVD, [this, PrivateVD, BaseLValue, OASELValueLB, OASELValueUB,
680  OriginalBaseLValue]() -> Address {
681  // Emit VarDecl with copy init for arrays.
682  // Get the address of the original variable captured in current
683  // captured region.
684  auto *Size = Builder.CreatePtrDiff(OASELValueUB.getPointer(),
685  OASELValueLB.getPointer());
686  Size = Builder.CreateNUWAdd(
687  Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1));
689  *this, cast<OpaqueValueExpr>(
690  getContext()
691  .getAsVariableArrayType(PrivateVD->getType())
692  ->getSizeExpr()),
693  RValue::get(Size));
694  EmitVariablyModifiedType(PrivateVD->getType());
695  auto Emission = EmitAutoVarAlloca(*PrivateVD);
696  auto Addr = Emission.getAllocatedAddress();
697  auto *Init = PrivateVD->getInit();
698  EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init);
699  EmitAutoVarCleanups(Emission);
700  // Emit private VarDecl with reduction init.
701  auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
702  OASELValueLB.getPointer());
703  auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset);
705  Ptr, OriginalBaseLValue.getPointer()->getType());
706  return Address(Ptr, OriginalBaseLValue.getAlignment());
707  });
708  assert(IsRegistered && "private var already registered as private");
709  // Silence the warning about unused variable.
710  (void)IsRegistered;
711  PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
712  return GetAddrOfLocalVar(PrivateVD);
713  });
714  } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(IRef)) {
715  auto *Base = ASE->getBase()->IgnoreParenImpCasts();
716  while (auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
717  Base = TempASE->getBase()->IgnoreParenImpCasts();
718  auto *DE = cast<DeclRefExpr>(Base);
719  auto *OrigVD = cast<VarDecl>(DE->getDecl());
720  auto ASELValue = EmitLValue(ASE);
721  auto OriginalBaseLValue = EmitLValue(DE);
722  auto BaseLValue = OriginalBaseLValue;
723  auto *Zero = Builder.getInt64(/*C=*/0);
725  Indexes.push_back(Zero);
726  auto *ItemTy =
727  ASELValue.getPointer()->getType()->getPointerElementType();
728  auto *Ty = BaseLValue.getPointer()->getType()->getPointerElementType();
729  while (Ty != ItemTy) {
730  Indexes.push_back(Zero);
731  Ty = Ty->getPointerElementType();
732  }
733  BaseLValue = MakeAddrLValue(
734  Address(Builder.CreateInBoundsGEP(BaseLValue.getPointer(), Indexes),
735  ASELValue.getAlignment()),
736  ASELValue.getType(), ASELValue.getAlignmentSource());
737  // Store the address of the original variable associated with the LHS
738  // implicit variable.
739  PrivateScope.addPrivate(LHSVD, [this, ASELValue]() -> Address {
740  return ASELValue.getAddress();
741  });
742  // Emit reduction copy.
743  bool IsRegistered = PrivateScope.addPrivate(
744  OrigVD, [this, PrivateVD, BaseLValue, ASELValue,
745  OriginalBaseLValue]() -> Address {
746  // Emit private VarDecl with reduction init.
747  EmitDecl(*PrivateVD);
748  auto Addr = GetAddrOfLocalVar(PrivateVD);
749  auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
750  ASELValue.getPointer());
751  auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset);
753  Ptr, OriginalBaseLValue.getPointer()->getType());
754  return Address(Ptr, OriginalBaseLValue.getAlignment());
755  });
756  assert(IsRegistered && "private var already registered as private");
757  // Silence the warning about unused variable.
758  (void)IsRegistered;
759  PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
760  return GetAddrOfLocalVar(PrivateVD);
761  });
762  } else {
763  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
764  // Store the address of the original variable associated with the LHS
765  // implicit variable.
766  PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> Address {
767  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
768  CapturedStmtInfo->lookup(OrigVD) != nullptr,
769  IRef->getType(), VK_LValue, IRef->getExprLoc());
770  return EmitLValue(&DRE).getAddress();
771  });
772  // Emit reduction copy.
773  bool IsRegistered =
774  PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> Address {
775  // Emit private VarDecl with reduction init.
776  EmitDecl(*PrivateVD);
777  return GetAddrOfLocalVar(PrivateVD);
778  });
779  assert(IsRegistered && "private var already registered as private");
780  // Silence the warning about unused variable.
781  (void)IsRegistered;
782  PrivateScope.addPrivate(RHSVD, [this, PrivateVD]() -> Address {
783  return GetAddrOfLocalVar(PrivateVD);
784  });
785  }
786  ++ILHS, ++IRHS, ++IPriv;
787  }
788  }
789 }
790 
792  const OMPExecutableDirective &D) {
793  if (!HaveInsertPoint())
794  return;
799  bool HasAtLeastOneReduction = false;
800  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
801  HasAtLeastOneReduction = true;
802  Privates.append(C->privates().begin(), C->privates().end());
803  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
804  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
805  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
806  }
807  if (HasAtLeastOneReduction) {
808  // Emit nowait reduction if nowait clause is present or directive is a
809  // parallel directive (it always has implicit barrier).
810  CGM.getOpenMPRuntime().emitReduction(
811  *this, D.getLocEnd(), Privates, LHSExprs, RHSExprs, ReductionOps,
814  D.getDirectiveKind() == OMPD_simd,
815  D.getDirectiveKind() == OMPD_simd);
816  }
817 }
818 
820  const OMPExecutableDirective &S,
821  OpenMPDirectiveKind InnermostKind,
822  const RegionCodeGenTy &CodeGen) {
823  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
825  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
826  auto OutlinedFn = CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
827  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
828  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
829  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
830  auto NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
831  /*IgnoreResultAssign*/ true);
832  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
833  CGF, NumThreads, NumThreadsClause->getLocStart());
834  }
835  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
836  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
837  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
838  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
839  }
840  const Expr *IfCond = nullptr;
841  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
842  if (C->getNameModifier() == OMPD_unknown ||
843  C->getNameModifier() == OMPD_parallel) {
844  IfCond = C->getCondition();
845  break;
846  }
847  }
848  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getLocStart(), OutlinedFn,
849  CapturedVars, IfCond);
850 }
851 
853  LexicalScope Scope(*this, S.getSourceRange());
854  // Emit parallel region as a standalone region.
855  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
856  OMPPrivateScope PrivateScope(CGF);
857  bool Copyins = CGF.EmitOMPCopyinClause(S);
858  bool Firstprivates = CGF.EmitOMPFirstprivateClause(S, PrivateScope);
859  if (Copyins || Firstprivates) {
860  // Emit implicit barrier to synchronize threads and avoid data races on
861  // initialization of firstprivate variables or propagation master's thread
862  // values of threadprivate variables to local instances of that variables
863  // of all other implicit threads.
864  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
865  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
866  /*ForceSimpleCall=*/true);
867  }
868  CGF.EmitOMPPrivateClause(S, PrivateScope);
869  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
870  (void)PrivateScope.Privatize();
871  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
872  CGF.EmitOMPReductionClauseFinal(S);
873  };
874  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen);
875 }
876 
877 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
878  JumpDest LoopExit) {
879  RunCleanupsScope BodyScope(*this);
880  // Update counters values on current iteration.
881  for (auto I : D.updates()) {
883  }
884  // Update the linear variables.
885  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
886  for (auto U : C->updates()) {
887  EmitIgnoredExpr(U);
888  }
889  }
890 
891  // On a continue in the body, jump to the end.
892  auto Continue = getJumpDestInCurrentScope("omp.body.continue");
893  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
894  // Emit loop body.
895  EmitStmt(D.getBody());
896  // The end (updates/cleanups).
897  EmitBlock(Continue.getBlock());
898  BreakContinueStack.pop_back();
899 }
900 
902  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
903  const Expr *IncExpr,
904  const llvm::function_ref<void(CodeGenFunction &)> &BodyGen,
905  const llvm::function_ref<void(CodeGenFunction &)> &PostIncGen) {
906  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
907 
908  // Start the loop with a block that tests the condition.
909  auto CondBlock = createBasicBlock("omp.inner.for.cond");
910  EmitBlock(CondBlock);
911  LoopStack.push(CondBlock);
912 
913  // If there are any cleanups between here and the loop-exit scope,
914  // create a block to stage a loop exit along.
915  auto ExitBlock = LoopExit.getBlock();
916  if (RequiresCleanup)
917  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
918 
919  auto LoopBody = createBasicBlock("omp.inner.for.body");
920 
921  // Emit condition.
922  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
923  if (ExitBlock != LoopExit.getBlock()) {
924  EmitBlock(ExitBlock);
925  EmitBranchThroughCleanup(LoopExit);
926  }
927 
928  EmitBlock(LoopBody);
930 
931  // Create a block for the increment.
932  auto Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
933  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
934 
935  BodyGen(*this);
936 
937  // Emit "IV = IV + 1" and a back-edge to the condition block.
938  EmitBlock(Continue.getBlock());
939  EmitIgnoredExpr(IncExpr);
940  PostIncGen(*this);
941  BreakContinueStack.pop_back();
942  EmitBranch(CondBlock);
943  LoopStack.pop();
944  // Emit the fall-through block.
945  EmitBlock(LoopExit.getBlock());
946 }
947 
949  if (!HaveInsertPoint())
950  return;
951  // Emit inits for the linear variables.
952  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
953  for (auto Init : C->inits()) {
954  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
955  auto *OrigVD = cast<VarDecl>(
956  cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())->getDecl());
957  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
958  CapturedStmtInfo->lookup(OrigVD) != nullptr,
959  VD->getInit()->getType(), VK_LValue,
960  VD->getInit()->getExprLoc());
961  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
962  EmitExprAsInit(&DRE, VD,
963  MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
964  /*capturedByInit=*/false);
965  EmitAutoVarCleanups(Emission);
966  }
967  // Emit the linear steps for the linear clauses.
968  // If a step is not constant, it is pre-calculated before the loop.
969  if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
970  if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
971  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
972  // Emit calculation of the linear step.
973  EmitIgnoredExpr(CS);
974  }
975  }
976 }
977 
979  const OMPLoopDirective &D) {
980  if (!CGF.HaveInsertPoint())
981  return;
982  // Emit the final values of the linear variables.
983  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
984  auto IC = C->varlist_begin();
985  for (auto F : C->finals()) {
986  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
987  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
988  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
989  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
990  Address OrigAddr = CGF.EmitLValue(&DRE).getAddress();
991  CodeGenFunction::OMPPrivateScope VarScope(CGF);
992  VarScope.addPrivate(OrigVD,
993  [OrigAddr]() -> Address { return OrigAddr; });
994  (void)VarScope.Privatize();
995  CGF.EmitIgnoredExpr(F);
996  ++IC;
997  }
998  }
999 }
1000 
1002  const OMPExecutableDirective &D) {
1003  if (!CGF.HaveInsertPoint())
1004  return;
1005  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1006  unsigned ClauseAlignment = 0;
1007  if (auto AlignmentExpr = Clause->getAlignment()) {
1008  auto AlignmentCI =
1009  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1010  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1011  }
1012  for (auto E : Clause->varlists()) {
1013  unsigned Alignment = ClauseAlignment;
1014  if (Alignment == 0) {
1015  // OpenMP [2.8.1, Description]
1016  // If no optional parameter is specified, implementation-defined default
1017  // alignments for SIMD instructions on the target platforms are assumed.
1018  Alignment =
1019  CGF.getContext()
1021  E->getType()->getPointeeType()))
1022  .getQuantity();
1023  }
1024  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1025  "alignment is not power of 2");
1026  if (Alignment != 0) {
1027  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1028  CGF.EmitAlignmentAssumption(PtrValue, Alignment);
1029  }
1030  }
1031  }
1032 }
1033 
1036  ArrayRef<Expr *> Counters,
1037  ArrayRef<Expr *> PrivateCounters) {
1038  if (!CGF.HaveInsertPoint())
1039  return;
1040  auto I = PrivateCounters.begin();
1041  for (auto *E : Counters) {
1042  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1043  auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1044  Address Addr = Address::invalid();
1045  (void)LoopScope.addPrivate(PrivateVD, [&]() -> Address {
1046  // Emit var without initialization.
1047  auto VarEmission = CGF.EmitAutoVarAlloca(*PrivateVD);
1048  CGF.EmitAutoVarCleanups(VarEmission);
1049  Addr = VarEmission.getAllocatedAddress();
1050  return Addr;
1051  });
1052  (void)LoopScope.addPrivate(VD, [&]() -> Address { return Addr; });
1053  ++I;
1054  }
1055 }
1056 
1058  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1059  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1060  if (!CGF.HaveInsertPoint())
1061  return;
1062  {
1063  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1064  emitPrivateLoopCounters(CGF, PreCondScope, S.counters(),
1065  S.private_counters());
1066  (void)PreCondScope.Privatize();
1067  // Get initial values of real counters.
1068  for (auto I : S.inits()) {
1069  CGF.EmitIgnoredExpr(I);
1070  }
1071  }
1072  // Check that loop is executed at least one time.
1073  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1074 }
1075 
1076 static void
1078  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1079  if (!CGF.HaveInsertPoint())
1080  return;
1081  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1082  auto CurPrivate = C->privates().begin();
1083  for (auto *E : C->varlists()) {
1084  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1085  auto *PrivateVD =
1086  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1087  bool IsRegistered = PrivateScope.addPrivate(VD, [&]() -> Address {
1088  // Emit private VarDecl with copy init.
1089  CGF.EmitVarDecl(*PrivateVD);
1090  return CGF.GetAddrOfLocalVar(PrivateVD);
1091  });
1092  assert(IsRegistered && "linear var already registered as private");
1093  // Silence the warning about unused variable.
1094  (void)IsRegistered;
1095  ++CurPrivate;
1096  }
1097  }
1098 }
1099 
1101  const OMPExecutableDirective &D,
1102  bool IsMonotonic) {
1103  if (!CGF.HaveInsertPoint())
1104  return;
1105  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1106  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1107  /*ignoreResult=*/true);
1108  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1109  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1110  // In presence of finite 'safelen', it may be unsafe to mark all
1111  // the memory instructions parallel, because loop-carried
1112  // dependences of 'safelen' iterations are possible.
1113  if (!IsMonotonic)
1115  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1116  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1117  /*ignoreResult=*/true);
1118  llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1119  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1120  // In presence of finite 'safelen', it may be unsafe to mark all
1121  // the memory instructions parallel, because loop-carried
1122  // dependences of 'safelen' iterations are possible.
1123  CGF.LoopStack.setParallel(false);
1124  }
1125 }
1126 
1127 void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
1128  bool IsMonotonic) {
1129  // Walk clauses and process safelen/lastprivate.
1130  LoopStack.setParallel(!IsMonotonic);
1132  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1133 }
1134 
1135 void CodeGenFunction::EmitOMPSimdFinal(const OMPLoopDirective &D) {
1136  if (!HaveInsertPoint())
1137  return;
1138  auto IC = D.counters().begin();
1139  for (auto F : D.finals()) {
1140  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1141  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD)) {
1142  DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
1143  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1144  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1145  Address OrigAddr = EmitLValue(&DRE).getAddress();
1146  OMPPrivateScope VarScope(*this);
1147  VarScope.addPrivate(OrigVD,
1148  [OrigAddr]() -> Address { return OrigAddr; });
1149  (void)VarScope.Privatize();
1150  EmitIgnoredExpr(F);
1151  }
1152  ++IC;
1153  }
1154  emitLinearClauseFinal(*this, D);
1155 }
1156 
1158  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1159  // if (PreCond) {
1160  // for (IV in 0..LastIteration) BODY;
1161  // <Final counter/linear vars updates>;
1162  // }
1163  //
1164 
1165  // Emit: if (PreCond) - begin.
1166  // If the condition constant folds and can be elided, avoid emitting the
1167  // whole loop.
1168  bool CondConstant;
1169  llvm::BasicBlock *ContBlock = nullptr;
1170  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1171  if (!CondConstant)
1172  return;
1173  } else {
1174  auto *ThenBlock = CGF.createBasicBlock("simd.if.then");
1175  ContBlock = CGF.createBasicBlock("simd.if.end");
1176  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1177  CGF.getProfileCount(&S));
1178  CGF.EmitBlock(ThenBlock);
1179  CGF.incrementProfileCounter(&S);
1180  }
1181 
1182  // Emit the loop iteration variable.
1183  const Expr *IVExpr = S.getIterationVariable();
1184  const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1185  CGF.EmitVarDecl(*IVDecl);
1186  CGF.EmitIgnoredExpr(S.getInit());
1187 
1188  // Emit the iterations count variable.
1189  // If it is not a variable, Sema decided to calculate iterations count on
1190  // each iteration (e.g., it is foldable into a constant).
1191  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1192  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1193  // Emit calculation of the iterations count.
1194  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
1195  }
1196 
1197  CGF.EmitOMPSimdInit(S);
1198 
1199  emitAlignedClause(CGF, S);
1200  CGF.EmitOMPLinearClauseInit(S);
1201  bool HasLastprivateClause;
1202  {
1203  OMPPrivateScope LoopScope(CGF);
1204  emitPrivateLoopCounters(CGF, LoopScope, S.counters(),
1205  S.private_counters());
1206  emitPrivateLinearVars(CGF, S, LoopScope);
1207  CGF.EmitOMPPrivateClause(S, LoopScope);
1208  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1209  HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1210  (void)LoopScope.Privatize();
1211  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1212  S.getInc(),
1213  [&S](CodeGenFunction &CGF) {
1214  CGF.EmitOMPLoopBody(S, JumpDest());
1215  CGF.EmitStopPoint(&S);
1216  },
1217  [](CodeGenFunction &) {});
1218  // Emit final copy of the lastprivate variables at the end of loops.
1219  if (HasLastprivateClause) {
1220  CGF.EmitOMPLastprivateClauseFinal(S);
1221  }
1222  CGF.EmitOMPReductionClauseFinal(S);
1223  }
1224  CGF.EmitOMPSimdFinal(S);
1225  // Emit: if (PreCond) - end.
1226  if (ContBlock) {
1227  CGF.EmitBranch(ContBlock);
1228  CGF.EmitBlock(ContBlock, true);
1229  }
1230  };
1231  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1232 }
1233 
1234 void CodeGenFunction::EmitOMPForOuterLoop(
1235  OpenMPScheduleClauseKind ScheduleKind, bool IsMonotonic,
1236  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1237  Address LB, Address UB, Address ST, Address IL, llvm::Value *Chunk) {
1238  auto &RT = CGM.getOpenMPRuntime();
1239 
1240  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1241  const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind);
1242 
1243  assert((Ordered ||
1244  !RT.isStaticNonchunked(ScheduleKind, /*Chunked=*/Chunk != nullptr)) &&
1245  "static non-chunked schedule does not need outer loop");
1246 
1247  // Emit outer loop.
1248  //
1249  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1250  // When schedule(dynamic,chunk_size) is specified, the iterations are
1251  // distributed to threads in the team in chunks as the threads request them.
1252  // Each thread executes a chunk of iterations, then requests another chunk,
1253  // until no chunks remain to be distributed. Each chunk contains chunk_size
1254  // iterations, except for the last chunk to be distributed, which may have
1255  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1256  //
1257  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1258  // to threads in the team in chunks as the executing threads request them.
1259  // Each thread executes a chunk of iterations, then requests another chunk,
1260  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1261  // each chunk is proportional to the number of unassigned iterations divided
1262  // by the number of threads in the team, decreasing to 1. For a chunk_size
1263  // with value k (greater than 1), the size of each chunk is determined in the
1264  // same way, with the restriction that the chunks do not contain fewer than k
1265  // iterations (except for the last chunk to be assigned, which may have fewer
1266  // than k iterations).
1267  //
1268  // When schedule(auto) is specified, the decision regarding scheduling is
1269  // delegated to the compiler and/or runtime system. The programmer gives the
1270  // implementation the freedom to choose any possible mapping of iterations to
1271  // threads in the team.
1272  //
1273  // When schedule(runtime) is specified, the decision regarding scheduling is
1274  // deferred until run time, and the schedule and chunk size are taken from the
1275  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1276  // implementation defined
1277  //
1278  // while(__kmpc_dispatch_next(&LB, &UB)) {
1279  // idx = LB;
1280  // while (idx <= UB) { BODY; ++idx;
1281  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1282  // } // inner loop
1283  // }
1284  //
1285  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1286  // When schedule(static, chunk_size) is specified, iterations are divided into
1287  // chunks of size chunk_size, and the chunks are assigned to the threads in
1288  // the team in a round-robin fashion in the order of the thread number.
1289  //
1290  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1291  // while (idx <= UB) { BODY; ++idx; } // inner loop
1292  // LB = LB + ST;
1293  // UB = UB + ST;
1294  // }
1295  //
1296 
1297  const Expr *IVExpr = S.getIterationVariable();
1298  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1299  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1300 
1301  if (DynamicOrOrdered) {
1303  RT.emitForDispatchInit(*this, S.getLocStart(), ScheduleKind,
1304  IVSize, IVSigned, Ordered, UBVal, Chunk);
1305  } else {
1306  RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind,
1307  IVSize, IVSigned, Ordered, IL, LB, UB, ST, Chunk);
1308  }
1309 
1310  auto LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1311 
1312  // Start the loop with a block that tests the condition.
1313  auto CondBlock = createBasicBlock("omp.dispatch.cond");
1314  EmitBlock(CondBlock);
1315  LoopStack.push(CondBlock);
1316 
1317  llvm::Value *BoolCondVal = nullptr;
1318  if (!DynamicOrOrdered) {
1319  // UB = min(UB, GlobalUB)
1321  // IV = LB
1322  EmitIgnoredExpr(S.getInit());
1323  // IV < UB
1324  BoolCondVal = EvaluateExprAsBool(S.getCond());
1325  } else {
1326  BoolCondVal = RT.emitForNext(*this, S.getLocStart(), IVSize, IVSigned,
1327  IL, LB, UB, ST);
1328  }
1329 
1330  // If there are any cleanups between here and the loop-exit scope,
1331  // create a block to stage a loop exit along.
1332  auto ExitBlock = LoopExit.getBlock();
1333  if (LoopScope.requiresCleanups())
1334  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1335 
1336  auto LoopBody = createBasicBlock("omp.dispatch.body");
1337  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1338  if (ExitBlock != LoopExit.getBlock()) {
1339  EmitBlock(ExitBlock);
1340  EmitBranchThroughCleanup(LoopExit);
1341  }
1342  EmitBlock(LoopBody);
1343 
1344  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1345  // LB for loop condition and emitted it above).
1346  if (DynamicOrOrdered)
1347  EmitIgnoredExpr(S.getInit());
1348 
1349  // Create a block for the increment.
1350  auto Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1351  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1352 
1353  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1354  // with dynamic/guided scheduling and without ordered clause.
1356  LoopStack.setParallel(!IsMonotonic);
1357  else
1358  EmitOMPSimdInit(S, IsMonotonic);
1359 
1360  SourceLocation Loc = S.getLocStart();
1361  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
1362  [&S, LoopExit](CodeGenFunction &CGF) {
1363  CGF.EmitOMPLoopBody(S, LoopExit);
1364  CGF.EmitStopPoint(&S);
1365  },
1366  [Ordered, IVSize, IVSigned, Loc](CodeGenFunction &CGF) {
1367  if (Ordered) {
1368  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(
1369  CGF, Loc, IVSize, IVSigned);
1370  }
1371  });
1372 
1373  EmitBlock(Continue.getBlock());
1374  BreakContinueStack.pop_back();
1375  if (!DynamicOrOrdered) {
1376  // Emit "LB = LB + Stride", "UB = UB + Stride".
1379  }
1380 
1381  EmitBranch(CondBlock);
1382  LoopStack.pop();
1383  // Emit the fall-through block.
1384  EmitBlock(LoopExit.getBlock());
1385 
1386  // Tell the runtime we are done.
1387  if (!DynamicOrOrdered)
1388  RT.emitForStaticFinish(*this, S.getLocEnd());
1389 }
1390 
1391 /// \brief Emit a helper variable and return corresponding lvalue.
1393  const DeclRefExpr *Helper) {
1394  auto VDecl = cast<VarDecl>(Helper->getDecl());
1395  CGF.EmitVarDecl(*VDecl);
1396  return CGF.EmitLValue(Helper);
1397 }
1398 
1399 namespace {
1400  struct ScheduleKindModifiersTy {
1404  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
1407  : Kind(Kind), M1(M1), M2(M2) {}
1408  };
1409 } // namespace
1410 
1411 static std::pair<llvm::Value * /*Chunk*/, ScheduleKindModifiersTy>
1413  bool OuterRegion) {
1414  // Detect the loop schedule kind and chunk.
1415  auto ScheduleKind = OMPC_SCHEDULE_unknown;
1418  llvm::Value *Chunk = nullptr;
1419  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
1420  ScheduleKind = C->getScheduleKind();
1421  M1 = C->getFirstScheduleModifier();
1422  M2 = C->getSecondScheduleModifier();
1423  if (const auto *Ch = C->getChunkSize()) {
1424  if (auto *ImpRef = cast_or_null<DeclRefExpr>(C->getHelperChunkSize())) {
1425  if (OuterRegion) {
1426  const VarDecl *ImpVar = cast<VarDecl>(ImpRef->getDecl());
1427  CGF.EmitVarDecl(*ImpVar);
1429  CGF.EmitAnyExpr(Ch),
1430  CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(ImpVar),
1431  ImpVar->getType()));
1432  } else {
1433  Ch = ImpRef;
1434  }
1435  }
1436  if (!C->getHelperChunkSize() || !OuterRegion) {
1437  Chunk = CGF.EmitScalarExpr(Ch);
1438  Chunk = CGF.EmitScalarConversion(Chunk, Ch->getType(),
1440  S.getLocStart());
1441  }
1442  }
1443  }
1444  return std::make_pair(Chunk, ScheduleKindModifiersTy(ScheduleKind, M1, M2));
1445 }
1446 
1447 bool CodeGenFunction::EmitOMPWorksharingLoop(const OMPLoopDirective &S) {
1448  // Emit the loop iteration variable.
1449  auto IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
1450  auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
1451  EmitVarDecl(*IVDecl);
1452 
1453  // Emit the iterations count variable.
1454  // If it is not a variable, Sema decided to calculate iterations count on each
1455  // iteration (e.g., it is foldable into a constant).
1456  if (auto LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1457  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1458  // Emit calculation of the iterations count.
1460  }
1461 
1462  auto &RT = CGM.getOpenMPRuntime();
1463 
1464  bool HasLastprivateClause;
1465  // Check pre-condition.
1466  {
1467  // Skip the entire loop if we don't meet the precondition.
1468  // If the condition constant folds and can be elided, avoid emitting the
1469  // whole loop.
1470  bool CondConstant;
1471  llvm::BasicBlock *ContBlock = nullptr;
1472  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1473  if (!CondConstant)
1474  return false;
1475  } else {
1476  auto *ThenBlock = createBasicBlock("omp.precond.then");
1477  ContBlock = createBasicBlock("omp.precond.end");
1478  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
1479  getProfileCount(&S));
1480  EmitBlock(ThenBlock);
1482  }
1483 
1484  emitAlignedClause(*this, S);
1486  // Emit 'then' code.
1487  {
1488  // Emit helper vars inits.
1489  LValue LB =
1490  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1491  LValue UB =
1492  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1493  LValue ST =
1494  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
1495  LValue IL =
1496  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
1497 
1498  OMPPrivateScope LoopScope(*this);
1499  if (EmitOMPFirstprivateClause(S, LoopScope)) {
1500  // Emit implicit barrier to synchronize threads and avoid data races on
1501  // initialization of firstprivate variables.
1502  CGM.getOpenMPRuntime().emitBarrierCall(
1503  *this, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1504  /*ForceSimpleCall=*/true);
1505  }
1506  EmitOMPPrivateClause(S, LoopScope);
1507  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
1508  EmitOMPReductionClauseInit(S, LoopScope);
1509  emitPrivateLoopCounters(*this, LoopScope, S.counters(),
1510  S.private_counters());
1511  emitPrivateLinearVars(*this, S, LoopScope);
1512  (void)LoopScope.Privatize();
1513 
1514  // Detect the loop schedule kind and chunk.
1515  llvm::Value *Chunk;
1516  OpenMPScheduleClauseKind ScheduleKind;
1517  auto ScheduleInfo =
1518  emitScheduleClause(*this, S, /*OuterRegion=*/false);
1519  Chunk = ScheduleInfo.first;
1520  ScheduleKind = ScheduleInfo.second.Kind;
1521  const OpenMPScheduleClauseModifier M1 = ScheduleInfo.second.M1;
1522  const OpenMPScheduleClauseModifier M2 = ScheduleInfo.second.M2;
1523  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1524  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1525  const bool Ordered = S.getSingleClause<OMPOrderedClause>() != nullptr;
1526  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
1527  // If the static schedule kind is specified or if the ordered clause is
1528  // specified, and if no monotonic modifier is specified, the effect will
1529  // be as if the monotonic modifier was specified.
1530  if (RT.isStaticNonchunked(ScheduleKind,
1531  /* Chunked */ Chunk != nullptr) &&
1532  !Ordered) {
1534  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
1535  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1536  // When no chunk_size is specified, the iteration space is divided into
1537  // chunks that are approximately equal in size, and at most one chunk is
1538  // distributed to each thread. Note that the size of the chunks is
1539  // unspecified in this case.
1540  RT.emitForStaticInit(*this, S.getLocStart(), ScheduleKind,
1541  IVSize, IVSigned, Ordered,
1542  IL.getAddress(), LB.getAddress(),
1543  UB.getAddress(), ST.getAddress());
1544  auto LoopExit =
1545  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
1546  // UB = min(UB, GlobalUB);
1548  // IV = LB;
1549  EmitIgnoredExpr(S.getInit());
1550  // while (idx <= UB) { BODY; ++idx; }
1551  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1552  S.getInc(),
1553  [&S, LoopExit](CodeGenFunction &CGF) {
1554  CGF.EmitOMPLoopBody(S, LoopExit);
1555  CGF.EmitStopPoint(&S);
1556  },
1557  [](CodeGenFunction &) {});
1558  EmitBlock(LoopExit.getBlock());
1559  // Tell the runtime we are done.
1560  RT.emitForStaticFinish(*this, S.getLocStart());
1561  } else {
1562  const bool IsMonotonic = Ordered ||
1563  ScheduleKind == OMPC_SCHEDULE_static ||
1564  ScheduleKind == OMPC_SCHEDULE_unknown ||
1565  M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
1566  M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
1567  // Emit the outer loop, which requests its work chunk [LB..UB] from
1568  // runtime and runs the inner loop to process it.
1569  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
1570  LB.getAddress(), UB.getAddress(), ST.getAddress(),
1571  IL.getAddress(), Chunk);
1572  }
1574  // Emit final copy of the lastprivate variables if IsLastIter != 0.
1575  if (HasLastprivateClause)
1577  S, Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getLocStart())));
1578  }
1580  EmitOMPSimdFinal(S);
1581  }
1582  // We're now done with the loop, so jump to the continuation block.
1583  if (ContBlock) {
1584  EmitBranch(ContBlock);
1585  EmitBlock(ContBlock, true);
1586  }
1587  }
1588  return HasLastprivateClause;
1589 }
1590 
1592  LexicalScope Scope(*this, S.getSourceRange());
1593  bool HasLastprivates = false;
1594  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) {
1595  HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
1596  };
1597  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
1598  S.hasCancel());
1599 
1600  // Emit an implicit barrier at the end.
1601  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
1602  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
1603  }
1604 }
1605 
1607  LexicalScope Scope(*this, S.getSourceRange());
1608  bool HasLastprivates = false;
1609  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF) {
1610  HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
1611  };
1612  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1613 
1614  // Emit an implicit barrier at the end.
1615  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates) {
1616  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_for);
1617  }
1618 }
1619 
1621  const Twine &Name,
1622  llvm::Value *Init = nullptr) {
1623  auto LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
1624  if (Init)
1625  CGF.EmitScalarInit(Init, LVal);
1626  return LVal;
1627 }
1628 
1630 CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
1631  auto *Stmt = cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt();
1632  auto *CS = dyn_cast<CompoundStmt>(Stmt);
1633  bool HasLastprivates = false;
1634  auto &&CodeGen = [&S, Stmt, CS, &HasLastprivates](CodeGenFunction &CGF) {
1635  auto &C = CGF.CGM.getContext();
1636  auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1637  // Emit helper vars inits.
1638  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
1639  CGF.Builder.getInt32(0));
1640  auto *GlobalUBVal = CS != nullptr ? CGF.Builder.getInt32(CS->size() - 1)
1641  : CGF.Builder.getInt32(0);
1642  LValue UB =
1643  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
1644  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
1645  CGF.Builder.getInt32(1));
1646  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
1647  CGF.Builder.getInt32(0));
1648  // Loop counter.
1649  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
1650  OpaqueValueExpr IVRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
1651  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
1652  OpaqueValueExpr UBRefExpr(S.getLocStart(), KmpInt32Ty, VK_LValue);
1653  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
1654  // Generate condition for loop.
1655  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
1656  OK_Ordinary, S.getLocStart(),
1657  /*fpContractable=*/false);
1658  // Increment for loop counter.
1659  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
1660  S.getLocStart());
1661  auto BodyGen = [Stmt, CS, &S, &IV](CodeGenFunction &CGF) {
1662  // Iterate through all sections and emit a switch construct:
1663  // switch (IV) {
1664  // case 0:
1665  // <SectionStmt[0]>;
1666  // break;
1667  // ...
1668  // case <NumSection> - 1:
1669  // <SectionStmt[<NumSection> - 1]>;
1670  // break;
1671  // }
1672  // .omp.sections.exit:
1673  auto *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
1674  auto *SwitchStmt = CGF.Builder.CreateSwitch(
1675  CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
1676  CS == nullptr ? 1 : CS->size());
1677  if (CS) {
1678  unsigned CaseNumber = 0;
1679  for (auto *SubStmt : CS->children()) {
1680  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
1681  CGF.EmitBlock(CaseBB);
1682  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
1683  CGF.EmitStmt(SubStmt);
1684  CGF.EmitBranch(ExitBB);
1685  ++CaseNumber;
1686  }
1687  } else {
1688  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
1689  CGF.EmitBlock(CaseBB);
1690  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
1691  CGF.EmitStmt(Stmt);
1692  CGF.EmitBranch(ExitBB);
1693  }
1694  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1695  };
1696 
1697  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1698  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
1699  // Emit implicit barrier to synchronize threads and avoid data races on
1700  // initialization of firstprivate variables.
1701  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1702  CGF, S.getLocStart(), OMPD_unknown, /*EmitChecks=*/false,
1703  /*ForceSimpleCall=*/true);
1704  }
1705  CGF.EmitOMPPrivateClause(S, LoopScope);
1706  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1707  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1708  (void)LoopScope.Privatize();
1709 
1710  // Emit static non-chunked loop.
1711  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
1712  CGF, S.getLocStart(), OMPC_SCHEDULE_static, /*IVSize=*/32,
1713  /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(), LB.getAddress(),
1714  UB.getAddress(), ST.getAddress());
1715  // UB = min(UB, GlobalUB);
1716  auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
1717  auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
1718  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
1719  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
1720  // IV = LB;
1721  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
1722  // while (idx <= UB) { BODY; ++idx; }
1723  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
1724  [](CodeGenFunction &) {});
1725  // Tell the runtime we are done.
1726  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart());
1727  CGF.EmitOMPReductionClauseFinal(S);
1728 
1729  // Emit final copy of the lastprivate variables if IsLastIter != 0.
1730  if (HasLastprivates)
1731  CGF.EmitOMPLastprivateClauseFinal(
1732  S, CGF.Builder.CreateIsNotNull(
1733  CGF.EmitLoadOfScalar(IL, S.getLocStart())));
1734  };
1735 
1736  bool HasCancel = false;
1737  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
1738  HasCancel = OSD->hasCancel();
1739  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
1740  HasCancel = OPSD->hasCancel();
1741  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
1742  HasCancel);
1743  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
1744  // clause. Otherwise the barrier will be generated by the codegen for the
1745  // directive.
1746  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
1747  // Emit implicit barrier to synchronize threads and avoid data races on
1748  // initialization of firstprivate variables.
1749  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(),
1750  OMPD_unknown);
1751  }
1752  return OMPD_sections;
1753 }
1754 
1756  LexicalScope Scope(*this, S.getSourceRange());
1757  OpenMPDirectiveKind EmittedAs = EmitSections(S);
1758  // Emit an implicit barrier at the end.
1759  if (!S.getSingleClause<OMPNowaitClause>()) {
1760  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), EmittedAs);
1761  }
1762 }
1763 
1765  LexicalScope Scope(*this, S.getSourceRange());
1766  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1767  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1768  };
1769  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
1770  S.hasCancel());
1771 }
1772 
1774  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
1777  llvm::SmallVector<const Expr *, 8> AssignmentOps;
1778  // Check if there are any 'copyprivate' clauses associated with this
1779  // 'single'
1780  // construct.
1781  // Build a list of copyprivate variables along with helper expressions
1782  // (<source>, <destination>, <destination>=<source> expressions)
1783  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
1784  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
1785  DestExprs.append(C->destination_exprs().begin(),
1786  C->destination_exprs().end());
1787  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
1788  AssignmentOps.append(C->assignment_ops().begin(),
1789  C->assignment_ops().end());
1790  }
1791  LexicalScope Scope(*this, S.getSourceRange());
1792  // Emit code for 'single' region along with 'copyprivate' clauses
1793  bool HasFirstprivates;
1794  auto &&CodeGen = [&S, &HasFirstprivates](CodeGenFunction &CGF) {
1795  CodeGenFunction::OMPPrivateScope SingleScope(CGF);
1796  HasFirstprivates = CGF.EmitOMPFirstprivateClause(S, SingleScope);
1797  CGF.EmitOMPPrivateClause(S, SingleScope);
1798  (void)SingleScope.Privatize();
1799 
1800  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1801  };
1802  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getLocStart(),
1803  CopyprivateVars, DestExprs, SrcExprs,
1804  AssignmentOps);
1805  // Emit an implicit barrier at the end (to avoid data race on firstprivate
1806  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
1807  if ((!S.getSingleClause<OMPNowaitClause>() || HasFirstprivates) &&
1808  CopyprivateVars.empty()) {
1809  CGM.getOpenMPRuntime().emitBarrierCall(
1810  *this, S.getLocStart(),
1811  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
1812  }
1813 }
1814 
1816  LexicalScope Scope(*this, S.getSourceRange());
1817  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1818  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1819  };
1820  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getLocStart());
1821 }
1822 
1824  LexicalScope Scope(*this, S.getSourceRange());
1825  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1826  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
1827  };
1828  Expr *Hint = nullptr;
1829  if (auto *HintClause = S.getSingleClause<OMPHintClause>())
1830  Hint = HintClause->getHint();
1831  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
1833  CodeGen, S.getLocStart(), Hint);
1834 }
1835 
1837  const OMPParallelForDirective &S) {
1838  // Emit directive as a combined directive that consists of two implicit
1839  // directives: 'parallel' with 'for' directive.
1840  LexicalScope Scope(*this, S.getSourceRange());
1841  (void)emitScheduleClause(*this, S, /*OuterRegion=*/true);
1842  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1843  CGF.EmitOMPWorksharingLoop(S);
1844  };
1845  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen);
1846 }
1847 
1849  const OMPParallelForSimdDirective &S) {
1850  // Emit directive as a combined directive that consists of two implicit
1851  // directives: 'parallel' with 'for' directive.
1852  LexicalScope Scope(*this, S.getSourceRange());
1853  (void)emitScheduleClause(*this, S, /*OuterRegion=*/true);
1854  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1855  CGF.EmitOMPWorksharingLoop(S);
1856  };
1857  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen);
1858 }
1859 
1861  const OMPParallelSectionsDirective &S) {
1862  // Emit directive as a combined directive that consists of two implicit
1863  // directives: 'parallel' with 'sections' directive.
1864  LexicalScope Scope(*this, S.getSourceRange());
1865  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
1866  (void)CGF.EmitSections(S);
1867  };
1868  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen);
1869 }
1870 
1872  // Emit outlined function for task construct.
1873  LexicalScope Scope(*this, S.getSourceRange());
1874  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
1875  auto CapturedStruct = GenerateCapturedStmtArgument(*CS);
1876  auto *I = CS->getCapturedDecl()->param_begin();
1877  auto *PartId = std::next(I);
1878  // The first function argument for tasks is a thread id, the second one is a
1879  // part id (0 for tied tasks, >=0 for untied task).
1880  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
1881  // Get list of private variables.
1883  llvm::SmallVector<const Expr *, 8> PrivateCopies;
1884  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
1885  auto IRef = C->varlist_begin();
1886  for (auto *IInit : C->private_copies()) {
1887  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1888  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1889  PrivateVars.push_back(*IRef);
1890  PrivateCopies.push_back(IInit);
1891  }
1892  ++IRef;
1893  }
1894  }
1895  EmittedAsPrivate.clear();
1896  // Get list of firstprivate variables.
1897  llvm::SmallVector<const Expr *, 8> FirstprivateVars;
1898  llvm::SmallVector<const Expr *, 8> FirstprivateCopies;
1899  llvm::SmallVector<const Expr *, 8> FirstprivateInits;
1900  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
1901  auto IRef = C->varlist_begin();
1902  auto IElemInitRef = C->inits().begin();
1903  for (auto *IInit : C->private_copies()) {
1904  auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1905  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
1906  FirstprivateVars.push_back(*IRef);
1907  FirstprivateCopies.push_back(IInit);
1908  FirstprivateInits.push_back(*IElemInitRef);
1909  }
1910  ++IRef, ++IElemInitRef;
1911  }
1912  }
1913  // Build list of dependences.
1915  Dependences;
1916  for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
1917  for (auto *IRef : C->varlists()) {
1918  Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
1919  }
1920  }
1921  auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars](
1922  CodeGenFunction &CGF) {
1923  // Set proper addresses for generated private copies.
1924  auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
1925  OMPPrivateScope Scope(CGF);
1926  if (!PrivateVars.empty() || !FirstprivateVars.empty()) {
1927  auto *CopyFn = CGF.Builder.CreateLoad(
1928  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
1929  auto *PrivatesPtr = CGF.Builder.CreateLoad(
1930  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
1931  // Map privates.
1933  PrivatePtrs;
1935  CallArgs.push_back(PrivatesPtr);
1936  for (auto *E : PrivateVars) {
1937  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1938  Address PrivatePtr =
1939  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
1940  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
1941  CallArgs.push_back(PrivatePtr.getPointer());
1942  }
1943  for (auto *E : FirstprivateVars) {
1944  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1945  Address PrivatePtr =
1946  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
1947  PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
1948  CallArgs.push_back(PrivatePtr.getPointer());
1949  }
1950  CGF.EmitRuntimeCall(CopyFn, CallArgs);
1951  for (auto &&Pair : PrivatePtrs) {
1952  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
1953  CGF.getContext().getDeclAlign(Pair.first));
1954  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
1955  }
1956  }
1957  (void)Scope.Privatize();
1958  if (*PartId) {
1959  // TODO: emit code for untied tasks.
1960  }
1961  CGF.EmitStmt(CS->getCapturedStmt());
1962  };
1963  auto OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
1964  S, *I, OMPD_task, CodeGen);
1965  // Check if we should emit tied or untied task.
1966  bool Tied = !S.getSingleClause<OMPUntiedClause>();
1967  // Check if the task is final
1968  llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
1969  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
1970  // If the condition constant folds and can be elided, try to avoid emitting
1971  // the condition and the dead arm of the if/else.
1972  auto *Cond = Clause->getCondition();
1973  bool CondConstant;
1974  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
1975  Final.setInt(CondConstant);
1976  else
1977  Final.setPointer(EvaluateExprAsBool(Cond));
1978  } else {
1979  // By default the task is not final.
1980  Final.setInt(/*IntVal=*/false);
1981  }
1982  auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
1983  const Expr *IfCond = nullptr;
1984  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1985  if (C->getNameModifier() == OMPD_unknown ||
1986  C->getNameModifier() == OMPD_task) {
1987  IfCond = C->getCondition();
1988  break;
1989  }
1990  }
1991  CGM.getOpenMPRuntime().emitTaskCall(
1992  *this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy,
1993  CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars,
1994  FirstprivateCopies, FirstprivateInits, Dependences);
1995 }
1996 
1998  const OMPTaskyieldDirective &S) {
1999  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getLocStart());
2000 }
2001 
2003  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getLocStart(), OMPD_barrier);
2004 }
2005 
2007  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getLocStart());
2008 }
2009 
2011  const OMPTaskgroupDirective &S) {
2012  LexicalScope Scope(*this, S.getSourceRange());
2013  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2014  CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2015  };
2016  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getLocStart());
2017 }
2018 
2020  CGM.getOpenMPRuntime().emitFlush(*this, [&]() -> ArrayRef<const Expr *> {
2021  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>()) {
2022  return llvm::makeArrayRef(FlushClause->varlist_begin(),
2023  FlushClause->varlist_end());
2024  }
2025  return llvm::None;
2026  }(), S.getLocStart());
2027 }
2028 
2030  const OMPDistributeDirective &S) {
2031  llvm_unreachable("CodeGen for 'omp distribute' is not supported yet.");
2032 }
2033 
2034 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
2035  const CapturedStmt *S) {
2036  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
2038  CGF.CapturedStmtInfo = &CapStmtInfo;
2039  auto *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
2040  Fn->addFnAttr(llvm::Attribute::NoInline);
2041  return Fn;
2042 }
2043 
2045  if (!S.getAssociatedStmt())
2046  return;
2047  LexicalScope Scope(*this, S.getSourceRange());
2048  auto *C = S.getSingleClause<OMPSIMDClause>();
2049  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF) {
2050  if (C) {
2051  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2053  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
2054  auto *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
2055  CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars);
2056  } else {
2057  CGF.EmitStmt(
2058  cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
2059  }
2060  };
2061  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getLocStart(), !C);
2062 }
2063 
2064 static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
2065  QualType SrcType, QualType DestType,
2066  SourceLocation Loc) {
2067  assert(CGF.hasScalarEvaluationKind(DestType) &&
2068  "DestType must have scalar evaluation kind.");
2069  assert(!Val.isAggregate() && "Must be a scalar or complex.");
2070  return Val.isScalar()
2071  ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType, DestType,
2072  Loc)
2073  : CGF.EmitComplexToScalarConversion(Val.getComplexVal(), SrcType,
2074  DestType, Loc);
2075 }
2076 
2079  QualType DestType, SourceLocation Loc) {
2080  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
2081  "DestType must have complex evaluation kind.");
2082  CodeGenFunction::ComplexPairTy ComplexVal;
2083  if (Val.isScalar()) {
2084  // Convert the input element to the element type of the complex.
2085  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
2086  auto ScalarVal = CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
2087  DestElementType, Loc);
2088  ComplexVal = CodeGenFunction::ComplexPairTy(
2089  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
2090  } else {
2091  assert(Val.isComplex() && "Must be a scalar or complex.");
2092  auto SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
2093  auto DestElementType = DestType->castAs<ComplexType>()->getElementType();
2094  ComplexVal.first = CGF.EmitScalarConversion(
2095  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
2096  ComplexVal.second = CGF.EmitScalarConversion(
2097  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
2098  }
2099  return ComplexVal;
2100 }
2101 
2102 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
2103  LValue LVal, RValue RVal) {
2104  if (LVal.isGlobalReg()) {
2105  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
2106  } else {
2107  CGF.EmitAtomicStore(RVal, LVal, IsSeqCst ? llvm::SequentiallyConsistent
2108  : llvm::Monotonic,
2109  LVal.isVolatile(), /*IsInit=*/false);
2110  }
2111 }
2112 
2114  QualType RValTy, SourceLocation Loc) {
2115  switch (getEvaluationKind(LVal.getType())) {
2116  case TEK_Scalar:
2118  *this, RVal, RValTy, LVal.getType(), Loc)),
2119  LVal);
2120  break;
2121  case TEK_Complex:
2123  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
2124  /*isInit=*/false);
2125  break;
2126  case TEK_Aggregate:
2127  llvm_unreachable("Must be a scalar or complex.");
2128  }
2129 }
2130 
2131 static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
2132  const Expr *X, const Expr *V,
2133  SourceLocation Loc) {
2134  // v = x;
2135  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
2136  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
2137  LValue XLValue = CGF.EmitLValue(X);
2138  LValue VLValue = CGF.EmitLValue(V);
2139  RValue Res = XLValue.isGlobalReg()
2140  ? CGF.EmitLoadOfLValue(XLValue, Loc)
2141  : CGF.EmitAtomicLoad(XLValue, Loc,
2142  IsSeqCst ? llvm::SequentiallyConsistent
2143  : llvm::Monotonic,
2144  XLValue.isVolatile());
2145  // OpenMP, 2.12.6, atomic Construct
2146  // Any atomic construct with a seq_cst clause forces the atomically
2147  // performed operation to include an implicit flush operation without a
2148  // list.
2149  if (IsSeqCst)
2150  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2151  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
2152 }
2153 
2154 static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
2155  const Expr *X, const Expr *E,
2156  SourceLocation Loc) {
2157  // x = expr;
2158  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
2159  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
2160  // OpenMP, 2.12.6, atomic Construct
2161  // Any atomic construct with a seq_cst clause forces the atomically
2162  // performed operation to include an implicit flush operation without a
2163  // list.
2164  if (IsSeqCst)
2165  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2166 }
2167 
2168 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
2169  RValue Update,
2170  BinaryOperatorKind BO,
2171  llvm::AtomicOrdering AO,
2172  bool IsXLHSInRHSPart) {
2173  auto &Context = CGF.CGM.getContext();
2174  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
2175  // expression is simple and atomic is allowed for the given type for the
2176  // target platform.
2177  if (BO == BO_Comma || !Update.isScalar() ||
2178  !Update.getScalarVal()->getType()->isIntegerTy() ||
2179  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
2180  (Update.getScalarVal()->getType() !=
2181  X.getAddress().getElementType())) ||
2182  !X.getAddress().getElementType()->isIntegerTy() ||
2185  return std::make_pair(false, RValue::get(nullptr));
2186 
2187  llvm::AtomicRMWInst::BinOp RMWOp;
2188  switch (BO) {
2189  case BO_Add:
2190  RMWOp = llvm::AtomicRMWInst::Add;
2191  break;
2192  case BO_Sub:
2193  if (!IsXLHSInRHSPart)
2194  return std::make_pair(false, RValue::get(nullptr));
2195  RMWOp = llvm::AtomicRMWInst::Sub;
2196  break;
2197  case BO_And:
2198  RMWOp = llvm::AtomicRMWInst::And;
2199  break;
2200  case BO_Or:
2201  RMWOp = llvm::AtomicRMWInst::Or;
2202  break;
2203  case BO_Xor:
2204  RMWOp = llvm::AtomicRMWInst::Xor;
2205  break;
2206  case BO_LT:
2208  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
2209  : llvm::AtomicRMWInst::Max)
2210  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
2211  : llvm::AtomicRMWInst::UMax);
2212  break;
2213  case BO_GT:
2215  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
2216  : llvm::AtomicRMWInst::Min)
2217  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
2218  : llvm::AtomicRMWInst::UMin);
2219  break;
2220  case BO_Assign:
2221  RMWOp = llvm::AtomicRMWInst::Xchg;
2222  break;
2223  case BO_Mul:
2224  case BO_Div:
2225  case BO_Rem:
2226  case BO_Shl:
2227  case BO_Shr:
2228  case BO_LAnd:
2229  case BO_LOr:
2230  return std::make_pair(false, RValue::get(nullptr));
2231  case BO_PtrMemD:
2232  case BO_PtrMemI:
2233  case BO_LE:
2234  case BO_GE:
2235  case BO_EQ:
2236  case BO_NE:
2237  case BO_AddAssign:
2238  case BO_SubAssign:
2239  case BO_AndAssign:
2240  case BO_OrAssign:
2241  case BO_XorAssign:
2242  case BO_MulAssign:
2243  case BO_DivAssign:
2244  case BO_RemAssign:
2245  case BO_ShlAssign:
2246  case BO_ShrAssign:
2247  case BO_Comma:
2248  llvm_unreachable("Unsupported atomic update operation");
2249  }
2250  auto *UpdateVal = Update.getScalarVal();
2251  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
2252  UpdateVal = CGF.Builder.CreateIntCast(
2253  IC, X.getAddress().getElementType(),
2255  }
2256  auto *Res = CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
2257  return std::make_pair(true, RValue::get(Res));
2258 }
2259 
2261  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
2262  llvm::AtomicOrdering AO, SourceLocation Loc,
2263  const llvm::function_ref<RValue(RValue)> &CommonGen) {
2264  // Update expressions are allowed to have the following forms:
2265  // x binop= expr; -> xrval + expr;
2266  // x++, ++x -> xrval + 1;
2267  // x--, --x -> xrval - 1;
2268  // x = x binop expr; -> xrval binop expr
2269  // x = expr Op x; - > expr binop xrval;
2270  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
2271  if (!Res.first) {
2272  if (X.isGlobalReg()) {
2273  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
2274  // 'xrval'.
2275  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
2276  } else {
2277  // Perform compare-and-swap procedure.
2278  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
2279  }
2280  }
2281  return Res;
2282 }
2283 
2284 static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
2285  const Expr *X, const Expr *E,
2286  const Expr *UE, bool IsXLHSInRHSPart,
2287  SourceLocation Loc) {
2288  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
2289  "Update expr in 'atomic update' must be a binary operator.");
2290  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
2291  // Update expressions are allowed to have the following forms:
2292  // x binop= expr; -> xrval + expr;
2293  // x++, ++x -> xrval + 1;
2294  // x--, --x -> xrval - 1;
2295  // x = x binop expr; -> xrval binop expr
2296  // x = expr Op x; - > expr binop xrval;
2297  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
2298  LValue XLValue = CGF.EmitLValue(X);
2299  RValue ExprRValue = CGF.EmitAnyExpr(E);
2300  auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic;
2301  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
2302  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
2303  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
2304  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
2305  auto Gen =
2306  [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) -> RValue {
2307  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
2308  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
2309  return CGF.EmitAnyExpr(UE);
2310  };
2312  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
2313  // OpenMP, 2.12.6, atomic Construct
2314  // Any atomic construct with a seq_cst clause forces the atomically
2315  // performed operation to include an implicit flush operation without a
2316  // list.
2317  if (IsSeqCst)
2318  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2319 }
2320 
2322  QualType SourceType, QualType ResType,
2323  SourceLocation Loc) {
2324  switch (CGF.getEvaluationKind(ResType)) {
2325  case TEK_Scalar:
2326  return RValue::get(
2327  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
2328  case TEK_Complex: {
2329  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
2330  return RValue::getComplex(Res.first, Res.second);
2331  }
2332  case TEK_Aggregate:
2333  break;
2334  }
2335  llvm_unreachable("Must be a scalar or complex.");
2336 }
2337 
2338 static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
2339  bool IsPostfixUpdate, const Expr *V,
2340  const Expr *X, const Expr *E,
2341  const Expr *UE, bool IsXLHSInRHSPart,
2342  SourceLocation Loc) {
2343  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
2344  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
2345  RValue NewVVal;
2346  LValue VLValue = CGF.EmitLValue(V);
2347  LValue XLValue = CGF.EmitLValue(X);
2348  RValue ExprRValue = CGF.EmitAnyExpr(E);
2349  auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic;
2350  QualType NewVValType;
2351  if (UE) {
2352  // 'x' is updated with some additional value.
2353  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
2354  "Update expr in 'atomic capture' must be a binary operator.");
2355  auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
2356  // Update expressions are allowed to have the following forms:
2357  // x binop= expr; -> xrval + expr;
2358  // x++, ++x -> xrval + 1;
2359  // x--, --x -> xrval - 1;
2360  // x = x binop expr; -> xrval binop expr
2361  // x = expr Op x; - > expr binop xrval;
2362  auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
2363  auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
2364  auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
2365  NewVValType = XRValExpr->getType();
2366  auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
2367  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
2368  IsSeqCst, IsPostfixUpdate](RValue XRValue) -> RValue {
2369  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
2370  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
2371  RValue Res = CGF.EmitAnyExpr(UE);
2372  NewVVal = IsPostfixUpdate ? XRValue : Res;
2373  return Res;
2374  };
2375  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
2376  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
2377  if (Res.first) {
2378  // 'atomicrmw' instruction was generated.
2379  if (IsPostfixUpdate) {
2380  // Use old value from 'atomicrmw'.
2381  NewVVal = Res.second;
2382  } else {
2383  // 'atomicrmw' does not provide new value, so evaluate it using old
2384  // value of 'x'.
2385  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
2386  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
2387  NewVVal = CGF.EmitAnyExpr(UE);
2388  }
2389  }
2390  } else {
2391  // 'x' is simply rewritten with some 'expr'.
2392  NewVValType = X->getType().getNonReferenceType();
2393  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
2394  X->getType().getNonReferenceType(), Loc);
2395  auto &&Gen = [&CGF, &NewVVal, ExprRValue](RValue XRValue) -> RValue {
2396  NewVVal = XRValue;
2397  return ExprRValue;
2398  };
2399  // Try to perform atomicrmw xchg, otherwise simple exchange.
2400  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
2401  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
2402  Loc, Gen);
2403  if (Res.first) {
2404  // 'atomicrmw' instruction was generated.
2405  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
2406  }
2407  }
2408  // Emit post-update store to 'v' of old/new 'x' value.
2409  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
2410  // OpenMP, 2.12.6, atomic Construct
2411  // Any atomic construct with a seq_cst clause forces the atomically
2412  // performed operation to include an implicit flush operation without a
2413  // list.
2414  if (IsSeqCst)
2415  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
2416 }
2417 
2419  bool IsSeqCst, bool IsPostfixUpdate,
2420  const Expr *X, const Expr *V, const Expr *E,
2421  const Expr *UE, bool IsXLHSInRHSPart,
2422  SourceLocation Loc) {
2423  switch (Kind) {
2424  case OMPC_read:
2425  EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
2426  break;
2427  case OMPC_write:
2428  EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
2429  break;
2430  case OMPC_unknown:
2431  case OMPC_update:
2432  EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
2433  break;
2434  case OMPC_capture:
2435  EmitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
2436  IsXLHSInRHSPart, Loc);
2437  break;
2438  case OMPC_if:
2439  case OMPC_final:
2440  case OMPC_num_threads:
2441  case OMPC_private:
2442  case OMPC_firstprivate:
2443  case OMPC_lastprivate:
2444  case OMPC_reduction:
2445  case OMPC_safelen:
2446  case OMPC_simdlen:
2447  case OMPC_collapse:
2448  case OMPC_default:
2449  case OMPC_seq_cst:
2450  case OMPC_shared:
2451  case OMPC_linear:
2452  case OMPC_aligned:
2453  case OMPC_copyin:
2454  case OMPC_copyprivate:
2455  case OMPC_flush:
2456  case OMPC_proc_bind:
2457  case OMPC_schedule:
2458  case OMPC_ordered:
2459  case OMPC_nowait:
2460  case OMPC_untied:
2461  case OMPC_threadprivate:
2462  case OMPC_depend:
2463  case OMPC_mergeable:
2464  case OMPC_device:
2465  case OMPC_threads:
2466  case OMPC_simd:
2467  case OMPC_map:
2468  case OMPC_num_teams:
2469  case OMPC_thread_limit:
2470  case OMPC_priority:
2471  case OMPC_grainsize:
2472  case OMPC_nogroup:
2473  case OMPC_num_tasks:
2474  case OMPC_hint:
2475  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
2476  }
2477 }
2478 
2480  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
2482  for (auto *C : S.clauses()) {
2483  // Find first clause (skip seq_cst clause, if it is first).
2484  if (C->getClauseKind() != OMPC_seq_cst) {
2485  Kind = C->getClauseKind();
2486  break;
2487  }
2488  }
2489 
2490  const auto *CS =
2491  S.getAssociatedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
2492  if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
2493  enterFullExpression(EWC);
2494  }
2495  // Processing for statements under 'atomic capture'.
2496  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
2497  for (const auto *C : Compound->body()) {
2498  if (const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
2499  enterFullExpression(EWC);
2500  }
2501  }
2502  }
2503 
2504  LexicalScope Scope(*this, S.getSourceRange());
2505  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF) {
2506  CGF.EmitStopPoint(CS);
2507  EmitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
2508  S.getV(), S.getExpr(), S.getUpdateExpr(),
2509  S.isXLHSInRHSPart(), S.getLocStart());
2510  };
2511  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
2512 }
2513 
2515  LexicalScope Scope(*this, S.getSourceRange());
2516  const CapturedStmt &CS = *cast<CapturedStmt>(S.getAssociatedStmt());
2517 
2519  GenerateOpenMPCapturedVars(CS, CapturedVars);
2520 
2521  llvm::Function *Fn = nullptr;
2522  llvm::Constant *FnID = nullptr;
2523 
2524  // Check if we have any if clause associated with the directive.
2525  const Expr *IfCond = nullptr;
2526 
2527  if (auto *C = S.getSingleClause<OMPIfClause>()) {
2528  IfCond = C->getCondition();
2529  }
2530 
2531  // Check if we have any device clause associated with the directive.
2532  const Expr *Device = nullptr;
2533  if (auto *C = S.getSingleClause<OMPDeviceClause>()) {
2534  Device = C->getDevice();
2535  }
2536 
2537  // Check if we have an if clause whose conditional always evaluates to false
2538  // or if we do not have any targets specified. If so the target region is not
2539  // an offload entry point.
2540  bool IsOffloadEntry = true;
2541  if (IfCond) {
2542  bool Val;
2543  if (ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
2544  IsOffloadEntry = false;
2545  }
2546  if (CGM.getLangOpts().OMPTargetTriples.empty())
2547  IsOffloadEntry = false;
2548 
2549  assert(CurFuncDecl && "No parent declaration for target region!");
2550  StringRef ParentName;
2551  // In case we have Ctors/Dtors we use the complete type variant to produce
2552  // the mangling of the device outlined kernel.
2553  if (auto *D = dyn_cast<CXXConstructorDecl>(CurFuncDecl))
2554  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
2555  else if (auto *D = dyn_cast<CXXDestructorDecl>(CurFuncDecl))
2556  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
2557  else
2558  ParentName =
2559  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CurFuncDecl)));
2560 
2561  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
2562  IsOffloadEntry);
2563 
2564  CGM.getOpenMPRuntime().emitTargetCall(*this, S, Fn, FnID, IfCond, Device,
2565  CapturedVars);
2566 }
2567 
2569  llvm_unreachable("CodeGen for 'omp teams' is not supported yet.");
2570 }
2571 
2573  const OMPCancellationPointDirective &S) {
2574  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getLocStart(),
2575  S.getCancelRegion());
2576 }
2577 
2579  const Expr *IfCond = nullptr;
2580  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
2581  if (C->getNameModifier() == OMPD_unknown ||
2582  C->getNameModifier() == OMPD_cancel) {
2583  IfCond = C->getCondition();
2584  break;
2585  }
2586  }
2587  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getLocStart(), IfCond,
2588  S.getCancelRegion());
2589 }
2590 
2593  if (Kind == OMPD_parallel || Kind == OMPD_task)
2594  return ReturnBlock;
2595  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
2596  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for);
2597  return BreakContinueStack.back().BreakBlock;
2598 }
2599 
2600 // Generate the instructions for '#pragma omp target data' directive.
2602  const OMPTargetDataDirective &S) {
2603  // emit the code inside the construct for now
2604  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2605  CGM.getOpenMPRuntime().emitInlinedDirective(
2606  *this, OMPD_target_data,
2607  [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
2608 }
2609 
2611  // emit the code inside the construct for now
2612  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2613  CGM.getOpenMPRuntime().emitInlinedDirective(
2614  *this, OMPD_taskloop,
2615  [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
2616 }
2617 
2619  const OMPTaskLoopSimdDirective &S) {
2620  // emit the code inside the construct for now
2621  auto CS = cast<CapturedStmt>(S.getAssociatedStmt());
2622  CGM.getOpenMPRuntime().emitInlinedDirective(
2623  *this, OMPD_taskloop_simd,
2624  [&CS](CodeGenFunction &CGF) { CGF.EmitStmt(CS->getCapturedStmt()); });
2625 }
2626 
This represents '#pragma omp master' directive.
Definition: StmtOpenMP.h:1100
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
Address EmitLoadOfReference(Address Ref, const ReferenceType *RefTy, AlignmentSource *Source=nullptr)
Definition: CGExpr.cpp:1934
static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
This represents '#pragma omp task' directive.
Definition: StmtOpenMP.h:1440
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
Definition: StmtOpenMP.h:1907
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
This represents clause 'copyin' in the '#pragma omp ...' directives.
Complete object ctor.
Definition: ABI.h:26
A (possibly-)qualified type.
Definition: Type.h:575
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:2157
llvm::Value * getPointer() const
Definition: CGValue.h:327
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D)
Emit final update of reduction values to original variables at the end of the directive.
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:216
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:131
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static std::pair< llvm::Value *, ScheduleKindModifiersTy > emitScheduleClause(CodeGenFunction &CGF, const OMPLoopDirective &S, bool OuterRegion)
llvm::Module & getModule() const
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
IdentifierInfo * getIdentifier() const
getIdentifier - Get the identifier that names this declaration, if there is one.
Definition: Decl.h:164
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:1796
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:65
This represents '#pragma omp for simd' directive.
Definition: StmtOpenMP.h:850
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:171
Address getAddress() const
Definition: CGValue.h:331
This represents 'if' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:152
static void emitLinearClauseFinal(CodeGenFunction &CGF, const OMPLoopDirective &D)
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
The base class of the type hierarchy.
Definition: Type.h:1249
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1568
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
Definition: CGStmt.cpp:2127
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
This represents '#pragma omp parallel for' directive.
Definition: StmtOpenMP.h:1221
const LangOptions & getLangOpts() const
const CGFunctionInfo & arrangeFreeFunctionDeclaration(QualType ResTy, const FunctionArgList &Args, const FunctionType::ExtInfo &Info, bool isVariadic)
Definition: CGCall.cpp:490
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1725
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
static void emitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
QualType getVariableArrayDecayedType(QualType Ty) const
Returns a vla type where known sizes are replaced with [*].
VarDecl - An instance of this class is created to represent a variable declaration or definition...
Definition: Decl.h:699
This represents clause 'private' in the '#pragma omp ...' directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
This represents 'num_threads' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:289
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
field_iterator field_begin() const
Definition: Decl.cpp:3746
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:1793
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
Expr * getInc() const
Definition: StmtOpenMP.h:593
CharUnits getNaturalTypeAlignment(QualType T, AlignmentSource *Source=nullptr, bool forPointeeType=false)
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
This represents 'safelen' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:347
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:202
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:2755
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
RecordDecl - Represents a struct/union/class.
Definition: Decl.h:3166
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents '#pragma omp parallel' directive.
Definition: StmtOpenMP.h:232
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
Definition: Type.h:4381
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:1766
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:91
This represents 'simd' clause in the '#pragma omp ...' directive.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Definition: CGDecl.cpp:1275
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool isReferenceType() const
Definition: Type.h:5314
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
bool isAnyPointerType() const
Definition: Type.h:5308
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
Definition: CGExpr.cpp:2830
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:675
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:1828
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
This represents '#pragma omp barrier' directive.
Definition: StmtOpenMP.h:1552
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
Definition: StmtOpenMP.h:294
This represents '#pragma omp critical' directive.
Definition: StmtOpenMP.h:1147
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2214
IdentifierTable & Idents
Definition: ASTContext.h:451
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:102
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:139
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4006
BinaryOperatorKind
static bool hasScalarEvaluationKind(QualType T)
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:681
CharUnits getAlignment() const
Definition: CGValue.h:316
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:580
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
Definition: CGBuilder.h:176
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
uint32_t Offset
Definition: CacheTokens.cpp:44
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:224
Expr * getX()
Get 'x' part of the associated expression/statement.
Definition: StmtOpenMP.h:1891
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
field_range fields() const
Definition: Decl.h:3295
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTargetDirective(const OMPTargetDirective &S)
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:2875
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:135
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
This represents '#pragma omp cancellation point' directive.
Definition: StmtOpenMP.h:2107
void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy)
EmitAggregateCopy - Emit an aggregate assignment.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:38
void incrementProfileCounter(const Stmt *S)
Increment the profiler's counter for the given statement.
This represents 'final' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:235
void EmitStmt(const Stmt *S)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:48
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents '#pragma omp teams' directive.
Definition: StmtOpenMP.h:2050
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:127
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
Definition: CodeGenPGO.cpp:608
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, bool IsForDefinition=false)
void push(llvm::BasicBlock *Header)
Begin a new structured loop.
This represents clause 'reduction' in the '#pragma omp ...' directives.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:834
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:2101
An ordinary object is located at an address in memory.
Definition: Specifiers.h:118
This represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:3560
detail::InMemoryDirectory::const_iterator I
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
QualType getType() const
Definition: Decl.h:530
Expr * getCond() const
Definition: StmtOpenMP.h:585
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:5692
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:663
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2151
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:539
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:942
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents '#pragma omp taskgroup' directive.
Definition: StmtOpenMP.h:1640
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:168
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:38
This represents clause 'aligned' in the '#pragma omp ...' directives.
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
ASTContext * Context
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
Definition: StmtOpenMP.h:1910
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:726
This represents '#pragma omp distribute' directive.
Definition: StmtOpenMP.h:2361
This represents implicit clause 'depend' for the '#pragma omp task' directive.
virtual StringRef getHelperName() const
Get the name of the capture helper.
static TypeEvaluationKind getEvaluationKind(QualType T)
hasAggregateLLVMType - Return true if the specified AST type will map into an aggregate LLVM type or ...
llvm::Value * getPointer() const
Definition: Address.h:38
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:581
Expr - This represents one expression.
Definition: Expr.h:104
static Address invalid()
Definition: Address.h:35
bool isAggregate() const
Definition: CGValue.h:53
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents 'simdlen' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:402
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:569
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
ASTContext & getContext() const
struct ExtInfo * ExtInfo
Definition: CGCleanup.h:264
void EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
unsigned getContextParamPosition() const
Definition: Decl.h:3620
This represents 'ordered' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:852
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This represents '#pragma omp for' directive.
Definition: StmtOpenMP.h:773
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:121
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:597
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep)
Creates clause with a list of variables VL and a linear step Step.
UnaryOperator - This represents the unary-expression's (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1654
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:639
This represents '#pragma omp cancel' directive.
Definition: StmtOpenMP.h:2165
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
ValueDecl * getDecl()
Definition: Expr.h:1007
static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1304
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:28
This represents '#pragma omp flush' directive.
Definition: StmtOpenMP.h:1691
This represents '#pragma omp parallel for simd' directive.
Definition: StmtOpenMP.h:1301
bool isVolatile() const
Definition: CGValue.h:295
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:934
SourceLocation getLocStart() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:169
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
void EmitOMPFlushDirective(const OMPFlushDirective &S)
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, const Expr *Init)
Emit initialization of arrays of complex types.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:840
Kind
This captures a statement into a function.
Definition: Stmt.h:1984
bool isSimple() const
Definition: CGValue.h:246
ASTContext & getContext() const
This represents '#pragma omp single' directive.
Definition: StmtOpenMP.h:1045
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents 'hint' clause in the '#pragma omp ...' directive.
void EmitOMPForDirective(const OMPForDirective &S)
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
Definition: CGExpr.cpp:109
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CGExpr.cpp:1236
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:604
This represents 'schedule' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:653
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
Definition: StmtOpenMP.h:1917
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:3626
This represents '#pragma omp taskwait' directive.
Definition: StmtOpenMP.h:1596
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
Definition: CGDecl.cpp:1153
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:190
void EmitOMPSingleDirective(const OMPSingleDirective &S)
An aligned address.
Definition: Address.h:25
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
const LangOptions & getLangOpts() const
llvm::function_ref< void(CodeGenFunction &)> RegionCodeGenTy
This represents '#pragma omp target' directive.
Definition: StmtOpenMP.h:1935
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:5706
bool isGlobalReg() const
Definition: CGValue.h:250
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:611
static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Expr * getV()
Get 'v' part of the associated expression/statement.
Definition: StmtOpenMP.h:1912
Complete object dtor.
Definition: ABI.h:36
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents '#pragma omp ordered' directive.
Definition: StmtOpenMP.h:1746
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:149
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:5159
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
void enterFullExpression(const ExprWithCleanups *E)
void EmitDecl(const Decl &D)
EmitDecl - Emit a declaration.
Definition: CGDecl.cpp:36
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
Expr * getPreCond() const
Definition: StmtOpenMP.h:581
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:146
QualType getType() const
Definition: Expr.h:125
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
CGFunctionInfo - Class to encapsulate the information about a function definition.
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Definition: OpenMPKinds.h:66
static const Type * getElementType(const Expr *BaseExpr)
This represents 'device' clause in the '#pragma omp ...' directive.
bool isScalar() const
Definition: CGValue.h:51
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:92
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
Definition: CGExpr.cpp:97
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1030
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:58
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
This represents '#pragma omp section' directive.
Definition: StmtOpenMP.h:983
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:117
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
const Stmt * getBody() const
Definition: StmtOpenMP.h:646
This represents '#pragma omp simd' directive.
Definition: StmtOpenMP.h:708
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:78
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:246
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
This represents clause 'linear' in the '#pragma omp ...' directives.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1205
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:625
detail::InMemoryDirectory::const_iterator E
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
Definition: StmtOpenMP.h:1898
static void emitPrivateLoopCounters(CodeGenFunction &CGF, CodeGenFunction::OMPPrivateScope &LoopScope, ArrayRef< Expr * > Counters, ArrayRef< Expr * > PrivateCounters)
This represents '#pragma omp atomic' directive.
Definition: StmtOpenMP.h:1801
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:577
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:5255
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:657
JumpDest ReturnBlock
ReturnBlock - Unified return block.
SwitchStmt - This represents a 'switch' stmt.
Definition: Stmt.h:938
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1370
API for captured statement code generation.
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
Complex values, per C99 6.2.5p11.
Definition: Type.h:2087
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T)
Definition: Decl.cpp:3948
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:669
bool isNothrow() const
Definition: Decl.cpp:4009
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition: OpenMPKinds.h:58
void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
Definition: CGDecl.cpp:658
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2287
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:33
bool isComplex() const
Definition: CGValue.h:52
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:367
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:11761
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:3628
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1074
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:632
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
Definition: StmtOpenMP.h:131
This represents 'nowait' clause in the '#pragma omp ...' directive.
Definition: OpenMPClause.h:903
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:387
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:310
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:944
bool isArrayType() const
Definition: Type.h:5344
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1415
This represents '#pragma omp taskloop simd' directive.
Definition: StmtOpenMP.h:2295
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
QualType getType() const
Definition: CGValue.h:258
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
This represents '#pragma omp sections' directive.
Definition: StmtOpenMP.h:915
This represents '#pragma omp target data' directive.
Definition: StmtOpenMP.h:1993
capture_range captures()
Definition: Stmt.h:2118
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> &BodyGen, const llvm::function_ref< void(CodeGenFunction &)> &PostIncGen)
Emit inner loop of the worksharing/simd construct.
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:922
static RValue get(llvm::Value *V)
Definition: CGValue.h:85
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
Expr * getLastIteration() const
Definition: StmtOpenMP.h:573
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
Definition: CGLoopInfo.h:124
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
Definition: CGCleanup.cpp:980
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:106
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
SourceLocation getLocation() const
Definition: DeclBase.h:384
LValue - This represents an lvalue references.
Definition: CGValue.h:152
This represents '#pragma omp taskyield' directive.
Definition: StmtOpenMP.h:1508
This represents '#pragma omp parallel sections' directive.
Definition: StmtOpenMP.h:1369
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1730
SourceLocation getLocEnd() const
Returns ending location of directive.
Definition: StmtOpenMP.h:171
Expr * getInit() const
Definition: StmtOpenMP.h:589
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:183
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:2144
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:197
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:2167
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:866
void EmitOMPTaskDirective(const OMPTaskDirective &S)
A class which abstracts out some details necessary for making a call.
Definition: Type.h:2872
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:618
bool Privatize()
Privatizes local variables previously registered as private.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:5116
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1293
This represents '#pragma omp taskloop' directive.
Definition: StmtOpenMP.h:2230