clang  3.7.0
CGOpenMPRuntime.cpp
Go to the documentation of this file.
1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a class for OpenMP runtime code generation.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "CGCleanup.h"
17 #include "clang/AST/Decl.h"
18 #include "clang/AST/StmtOpenMP.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/IR/CallSite.h"
21 #include "llvm/IR/DerivedTypes.h"
22 #include "llvm/IR/GlobalValue.h"
23 #include "llvm/IR/Value.h"
24 #include "llvm/Support/raw_ostream.h"
25 #include <cassert>
26 
27 using namespace clang;
28 using namespace CodeGen;
29 
30 namespace {
31 /// \brief Base class for handling code generation inside OpenMP regions.
32 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
33 public:
34  /// \brief Kinds of OpenMP regions used in codegen.
35  enum CGOpenMPRegionKind {
36  /// \brief Region with outlined function for standalone 'parallel'
37  /// directive.
38  ParallelOutlinedRegion,
39  /// \brief Region with outlined function for standalone 'task' directive.
40  TaskOutlinedRegion,
41  /// \brief Region for constructs that do not require function outlining,
42  /// like 'for', 'sections', 'atomic' etc. directives.
43  InlinedRegion,
44  };
45 
46  CGOpenMPRegionInfo(const CapturedStmt &CS,
47  const CGOpenMPRegionKind RegionKind,
49  : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
50  CodeGen(CodeGen), Kind(Kind) {}
51 
52  CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
54  : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
55  Kind(Kind) {}
56 
57  /// \brief Get a variable or parameter for storing global thread id
58  /// inside OpenMP construct.
59  virtual const VarDecl *getThreadIDVariable() const = 0;
60 
61  /// \brief Emit the captured statement body.
62  virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
63 
64  /// \brief Get an LValue for the current ThreadID variable.
65  /// \return LValue for thread id variable. This LValue always has type int32*.
66  virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
67 
68  CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
69 
70  OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
71 
72  static bool classof(const CGCapturedStmtInfo *Info) {
73  return Info->getKind() == CR_OpenMP;
74  }
75 
76 protected:
77  CGOpenMPRegionKind RegionKind;
78  const RegionCodeGenTy &CodeGen;
80 };
81 
82 /// \brief API for captured statement code generation in OpenMP constructs.
83 class CGOpenMPOutlinedRegionInfo : public CGOpenMPRegionInfo {
84 public:
85  CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
86  const RegionCodeGenTy &CodeGen,
88  : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind),
89  ThreadIDVar(ThreadIDVar) {
90  assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
91  }
92  /// \brief Get a variable or parameter for storing global thread id
93  /// inside OpenMP construct.
94  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
95 
96  /// \brief Get the name of the capture helper.
97  StringRef getHelperName() const override { return ".omp_outlined."; }
98 
99  static bool classof(const CGCapturedStmtInfo *Info) {
100  return CGOpenMPRegionInfo::classof(Info) &&
101  cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
102  ParallelOutlinedRegion;
103  }
104 
105 private:
106  /// \brief A variable or parameter storing global thread id for OpenMP
107  /// constructs.
108  const VarDecl *ThreadIDVar;
109 };
110 
111 /// \brief API for captured statement code generation in OpenMP constructs.
112 class CGOpenMPTaskOutlinedRegionInfo : public CGOpenMPRegionInfo {
113 public:
114  CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
115  const VarDecl *ThreadIDVar,
116  const RegionCodeGenTy &CodeGen,
118  : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind),
119  ThreadIDVar(ThreadIDVar) {
120  assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
121  }
122  /// \brief Get a variable or parameter for storing global thread id
123  /// inside OpenMP construct.
124  const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
125 
126  /// \brief Get an LValue for the current ThreadID variable.
127  LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
128 
129  /// \brief Get the name of the capture helper.
130  StringRef getHelperName() const override { return ".omp_outlined."; }
131 
132  static bool classof(const CGCapturedStmtInfo *Info) {
133  return CGOpenMPRegionInfo::classof(Info) &&
134  cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
135  TaskOutlinedRegion;
136  }
137 
138 private:
139  /// \brief A variable or parameter storing global thread id for OpenMP
140  /// constructs.
141  const VarDecl *ThreadIDVar;
142 };
143 
144 /// \brief API for inlined captured statement code generation in OpenMP
145 /// constructs.
146 class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
147 public:
148  CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
149  const RegionCodeGenTy &CodeGen,
151  : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind), OldCSI(OldCSI),
152  OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
153  // \brief Retrieve the value of the context parameter.
154  llvm::Value *getContextValue() const override {
155  if (OuterRegionInfo)
156  return OuterRegionInfo->getContextValue();
157  llvm_unreachable("No context value for inlined OpenMP region");
158  }
159  virtual void setContextValue(llvm::Value *V) override {
160  if (OuterRegionInfo) {
161  OuterRegionInfo->setContextValue(V);
162  return;
163  }
164  llvm_unreachable("No context value for inlined OpenMP region");
165  }
166  /// \brief Lookup the captured field decl for a variable.
167  const FieldDecl *lookup(const VarDecl *VD) const override {
168  if (OuterRegionInfo)
169  return OuterRegionInfo->lookup(VD);
170  // If there is no outer outlined region,no need to lookup in a list of
171  // captured variables, we can use the original one.
172  return nullptr;
173  }
174  FieldDecl *getThisFieldDecl() const override {
175  if (OuterRegionInfo)
176  return OuterRegionInfo->getThisFieldDecl();
177  return nullptr;
178  }
179  /// \brief Get a variable or parameter for storing global thread id
180  /// inside OpenMP construct.
181  const VarDecl *getThreadIDVariable() const override {
182  if (OuterRegionInfo)
183  return OuterRegionInfo->getThreadIDVariable();
184  return nullptr;
185  }
186 
187  /// \brief Get the name of the capture helper.
188  StringRef getHelperName() const override {
189  if (auto *OuterRegionInfo = getOldCSI())
190  return OuterRegionInfo->getHelperName();
191  llvm_unreachable("No helper name for inlined OpenMP construct");
192  }
193 
194  CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
195 
196  static bool classof(const CGCapturedStmtInfo *Info) {
197  return CGOpenMPRegionInfo::classof(Info) &&
198  cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
199  }
200 
201 private:
202  /// \brief CodeGen info about outer OpenMP region.
204  CGOpenMPRegionInfo *OuterRegionInfo;
205 };
206 
207 /// \brief RAII for emitting code of OpenMP constructs.
208 class InlinedOpenMPRegionRAII {
209  CodeGenFunction &CGF;
210 
211 public:
212  /// \brief Constructs region for combined constructs.
213  /// \param CodeGen Code generation sequence for combined directives. Includes
214  /// a list of functions used for code generation of implicitly inlined
215  /// regions.
216  InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
218  : CGF(CGF) {
219  // Start emission for the construct.
220  CGF.CapturedStmtInfo =
221  new CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, CodeGen, Kind);
222  }
223  ~InlinedOpenMPRegionRAII() {
224  // Restore original CapturedStmtInfo only if we're done with code emission.
225  auto *OldCSI =
226  cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
227  delete CGF.CapturedStmtInfo;
228  CGF.CapturedStmtInfo = OldCSI;
229  }
230 };
231 
232 } // namespace
233 
234 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
235  return CGF.MakeNaturalAlignAddrLValue(
236  CGF.Builder.CreateAlignedLoad(
237  CGF.GetAddrOfLocalVar(getThreadIDVariable()),
238  CGF.PointerAlignInBytes),
239  getThreadIDVariable()
240  ->getType()
241  ->castAs<PointerType>()
242  ->getPointeeType());
243 }
244 
245 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) {
246  // 1.2.2 OpenMP Language Terminology
247  // Structured block - An executable statement with a single entry at the
248  // top and a single exit at the bottom.
249  // The point of exit cannot be a branch out of the structured block.
250  // longjmp() and throw() must not violate the entry/exit criteria.
251  CGF.EHStack.pushTerminate();
252  {
254  CodeGen(CGF);
255  }
256  CGF.EHStack.popTerminate();
257 }
258 
259 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
260  CodeGenFunction &CGF) {
261  return CGF.MakeNaturalAlignAddrLValue(
262  CGF.GetAddrOfLocalVar(getThreadIDVariable()),
263  getThreadIDVariable()->getType());
264 }
265 
267  : CGM(CGM), DefaultOpenMPPSource(nullptr), KmpRoutineEntryPtrTy(nullptr) {
269  "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
270  CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
271  CGM.Int8PtrTy /* psource */, nullptr);
272  // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
273  llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
274  llvm::PointerType::getUnqual(CGM.Int32Ty)};
275  Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
276  KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
277 }
278 
279 void CGOpenMPRuntime::clear() {
280  InternalVars.clear();
281 }
282 
284  const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
285  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
286  assert(ThreadIDVar->getType()->isPointerType() &&
287  "thread id variable must be of type kmp_int32 *");
288  const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
289  CodeGenFunction CGF(CGM, true);
290  CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind);
291  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
292  return CGF.GenerateCapturedStmtFunction(*CS);
293 }
294 
296  const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
297  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
298  assert(!ThreadIDVar->getType()->isPointerType() &&
299  "thread id variable must be of type kmp_int32 for tasks");
300  auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
301  CodeGenFunction CGF(CGM, true);
302  CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
303  InnermostKind);
304  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
305  return CGF.GenerateCapturedStmtFunction(*CS);
306 }
307 
308 llvm::Value *
309 CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
310  llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
311  if (!Entry) {
312  if (!DefaultOpenMPPSource) {
313  // Initialize default location for psource field of ident_t structure of
314  // all ident_t objects. Format is ";file;function;line;column;;".
315  // Taken from
316  // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
317  DefaultOpenMPPSource =
318  CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;");
319  DefaultOpenMPPSource =
320  llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
321  }
322  auto DefaultOpenMPLocation = new llvm::GlobalVariable(
323  CGM.getModule(), IdentTy, /*isConstant*/ true,
324  llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr);
325  DefaultOpenMPLocation->setUnnamedAddr(true);
326 
327  llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true);
328  llvm::Constant *Values[] = {Zero,
329  llvm::ConstantInt::get(CGM.Int32Ty, Flags),
330  Zero, Zero, DefaultOpenMPPSource};
331  llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
332  DefaultOpenMPLocation->setInitializer(Init);
333  OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation;
334  return DefaultOpenMPLocation;
335  }
336  return Entry;
337 }
338 
340  SourceLocation Loc,
341  OpenMPLocationFlags Flags) {
342  // If no debug info is generated - return global default location.
343  if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo ||
344  Loc.isInvalid())
345  return getOrCreateDefaultLocation(Flags);
346 
347  assert(CGF.CurFn && "No function in current CodeGenFunction.");
348 
349  llvm::Value *LocValue = nullptr;
350  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
351  if (I != OpenMPLocThreadIDMap.end())
352  LocValue = I->second.DebugLoc;
353  // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if
354  // GetOpenMPThreadID was called before this routine.
355  if (LocValue == nullptr) {
356  // Generate "ident_t .kmpc_loc.addr;"
357  llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr");
358  AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy));
359  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
360  Elem.second.DebugLoc = AI;
361  LocValue = AI;
362 
363  CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
364  CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
365  CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags),
366  llvm::ConstantExpr::getSizeOf(IdentTy),
367  CGM.PointerAlignInBytes);
368  }
369 
370  // char **psource = &.kmpc_loc_<flags>.addr.psource;
371  auto *PSource = CGF.Builder.CreateConstInBoundsGEP2_32(IdentTy, LocValue, 0,
373 
374  auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
375  if (OMPDebugLoc == nullptr) {
376  SmallString<128> Buffer2;
377  llvm::raw_svector_ostream OS2(Buffer2);
378  // Build debug location
380  OS2 << ";" << PLoc.getFilename() << ";";
381  if (const FunctionDecl *FD =
382  dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) {
383  OS2 << FD->getQualifiedNameAsString();
384  }
385  OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
386  OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
387  OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
388  }
389  // *psource = ";<File>;<Function>;<Line>;<Column>;;";
390  CGF.Builder.CreateStore(OMPDebugLoc, PSource);
391 
392  return LocValue;
393 }
394 
396  SourceLocation Loc) {
397  assert(CGF.CurFn && "No function in current CodeGenFunction.");
398 
399  llvm::Value *ThreadID = nullptr;
400  // Check whether we've already cached a load of the thread id in this
401  // function.
402  auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
403  if (I != OpenMPLocThreadIDMap.end()) {
404  ThreadID = I->second.ThreadID;
405  if (ThreadID != nullptr)
406  return ThreadID;
407  }
408  if (auto OMPRegionInfo =
409  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
410  if (OMPRegionInfo->getThreadIDVariable()) {
411  // Check if this an outlined function with thread id passed as argument.
412  auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
413  ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
414  // If value loaded in entry block, cache it and use it everywhere in
415  // function.
416  if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
417  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
418  Elem.second.ThreadID = ThreadID;
419  }
420  return ThreadID;
421  }
422  }
423 
424  // This is not an outlined function region - need to call __kmpc_int32
425  // kmpc_global_thread_num(ident_t *loc).
426  // Generate thread id value and cache this value for use across the
427  // function.
428  CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
429  CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
430  ThreadID =
431  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
432  emitUpdateLocation(CGF, Loc));
433  auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
434  Elem.second.ThreadID = ThreadID;
435  return ThreadID;
436 }
437 
439  assert(CGF.CurFn && "No function in current CodeGenFunction.");
440  if (OpenMPLocThreadIDMap.count(CGF.CurFn))
441  OpenMPLocThreadIDMap.erase(CGF.CurFn);
442 }
443 
445  return llvm::PointerType::getUnqual(IdentTy);
446 }
447 
449  return llvm::PointerType::getUnqual(Kmpc_MicroTy);
450 }
451 
452 llvm::Constant *
453 CGOpenMPRuntime::createRuntimeFunction(OpenMPRTLFunction Function) {
454  llvm::Constant *RTLFn = nullptr;
455  switch (Function) {
456  case OMPRTL__kmpc_fork_call: {
457  // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
458  // microtask, ...);
459  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
461  llvm::FunctionType *FnTy =
462  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
463  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
464  break;
465  }
466  case OMPRTL__kmpc_global_thread_num: {
467  // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
468  llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
469  llvm::FunctionType *FnTy =
470  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
471  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
472  break;
473  }
474  case OMPRTL__kmpc_threadprivate_cached: {
475  // Build void *__kmpc_threadprivate_cached(ident_t *loc,
476  // kmp_int32 global_tid, void *data, size_t size, void ***cache);
477  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
478  CGM.VoidPtrTy, CGM.SizeTy,
479  CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
480  llvm::FunctionType *FnTy =
481  llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
482  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
483  break;
484  }
485  case OMPRTL__kmpc_critical: {
486  // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
487  // kmp_critical_name *crit);
488  llvm::Type *TypeParams[] = {
490  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
491  llvm::FunctionType *FnTy =
492  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
493  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
494  break;
495  }
496  case OMPRTL__kmpc_threadprivate_register: {
497  // Build void __kmpc_threadprivate_register(ident_t *, void *data,
498  // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
499  // typedef void *(*kmpc_ctor)(void *);
500  auto KmpcCtorTy =
501  llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
502  /*isVarArg*/ false)->getPointerTo();
503  // typedef void *(*kmpc_cctor)(void *, void *);
504  llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
505  auto KmpcCopyCtorTy =
506  llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
507  /*isVarArg*/ false)->getPointerTo();
508  // typedef void (*kmpc_dtor)(void *);
509  auto KmpcDtorTy =
510  llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
511  ->getPointerTo();
512  llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
513  KmpcCopyCtorTy, KmpcDtorTy};
514  auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
515  /*isVarArg*/ false);
516  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
517  break;
518  }
519  case OMPRTL__kmpc_end_critical: {
520  // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
521  // kmp_critical_name *crit);
522  llvm::Type *TypeParams[] = {
524  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
525  llvm::FunctionType *FnTy =
526  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
527  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
528  break;
529  }
530  case OMPRTL__kmpc_cancel_barrier: {
531  // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
532  // global_tid);
533  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
534  llvm::FunctionType *FnTy =
535  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
536  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
537  break;
538  }
539  case OMPRTL__kmpc_barrier: {
540  // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
541  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
542  llvm::FunctionType *FnTy =
543  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
544  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
545  break;
546  }
547  case OMPRTL__kmpc_for_static_fini: {
548  // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);
549  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
550  llvm::FunctionType *FnTy =
551  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
552  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini");
553  break;
554  }
555  case OMPRTL__kmpc_push_num_threads: {
556  // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
557  // kmp_int32 num_threads)
558  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
559  CGM.Int32Ty};
560  llvm::FunctionType *FnTy =
561  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
562  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
563  break;
564  }
565  case OMPRTL__kmpc_serialized_parallel: {
566  // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
567  // global_tid);
568  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
569  llvm::FunctionType *FnTy =
570  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
571  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
572  break;
573  }
574  case OMPRTL__kmpc_end_serialized_parallel: {
575  // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
576  // global_tid);
577  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
578  llvm::FunctionType *FnTy =
579  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
580  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
581  break;
582  }
583  case OMPRTL__kmpc_flush: {
584  // Build void __kmpc_flush(ident_t *loc);
585  llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
586  llvm::FunctionType *FnTy =
587  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
588  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
589  break;
590  }
591  case OMPRTL__kmpc_master: {
592  // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
593  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
594  llvm::FunctionType *FnTy =
595  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
596  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
597  break;
598  }
599  case OMPRTL__kmpc_end_master: {
600  // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
601  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
602  llvm::FunctionType *FnTy =
603  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
604  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
605  break;
606  }
607  case OMPRTL__kmpc_omp_taskyield: {
608  // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid,
609  // int end_part);
610  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
611  llvm::FunctionType *FnTy =
612  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
613  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield");
614  break;
615  }
616  case OMPRTL__kmpc_single: {
617  // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid);
618  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
619  llvm::FunctionType *FnTy =
620  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
621  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single");
622  break;
623  }
624  case OMPRTL__kmpc_end_single: {
625  // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid);
626  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
627  llvm::FunctionType *FnTy =
628  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
629  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single");
630  break;
631  }
632  case OMPRTL__kmpc_omp_task_alloc: {
633  // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
634  // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
635  // kmp_routine_entry_t *task_entry);
636  assert(KmpRoutineEntryPtrTy != nullptr &&
637  "Type kmp_routine_entry_t must be created.");
638  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty,
640  // Return void * and then cast to particular kmp_task_t type.
641  llvm::FunctionType *FnTy =
642  llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false);
643  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc");
644  break;
645  }
646  case OMPRTL__kmpc_omp_task: {
647  // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
648  // *new_task);
649  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
650  CGM.VoidPtrTy};
651  llvm::FunctionType *FnTy =
652  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
653  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task");
654  break;
655  }
656  case OMPRTL__kmpc_copyprivate: {
657  // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,
658  // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *),
659  // kmp_int32 didit);
660  llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
661  auto *CpyFnTy =
662  llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false);
663  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy,
664  CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
665  CGM.Int32Ty};
666  llvm::FunctionType *FnTy =
667  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
668  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate");
669  break;
670  }
671  case OMPRTL__kmpc_reduce: {
672  // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid,
673  // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void
674  // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck);
675  llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
676  auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
677  /*isVarArg=*/false);
678  llvm::Type *TypeParams[] = {
679  getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
680  CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
681  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
682  llvm::FunctionType *FnTy =
683  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
684  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce");
685  break;
686  }
687  case OMPRTL__kmpc_reduce_nowait: {
688  // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32
689  // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
690  // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name
691  // *lck);
692  llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
693  auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
694  /*isVarArg=*/false);
695  llvm::Type *TypeParams[] = {
696  getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy,
697  CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
698  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
699  llvm::FunctionType *FnTy =
700  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
701  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait");
702  break;
703  }
704  case OMPRTL__kmpc_end_reduce: {
705  // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
706  // kmp_critical_name *lck);
707  llvm::Type *TypeParams[] = {
709  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
710  llvm::FunctionType *FnTy =
711  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
712  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce");
713  break;
714  }
715  case OMPRTL__kmpc_end_reduce_nowait: {
716  // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
717  // kmp_critical_name *lck);
718  llvm::Type *TypeParams[] = {
720  llvm::PointerType::getUnqual(KmpCriticalNameTy)};
721  llvm::FunctionType *FnTy =
722  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
723  RTLFn =
724  CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait");
725  break;
726  }
727  case OMPRTL__kmpc_omp_task_begin_if0: {
728  // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
729  // *new_task);
730  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
731  CGM.VoidPtrTy};
732  llvm::FunctionType *FnTy =
733  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
734  RTLFn =
735  CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0");
736  break;
737  }
738  case OMPRTL__kmpc_omp_task_complete_if0: {
739  // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
740  // *new_task);
741  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
742  CGM.VoidPtrTy};
743  llvm::FunctionType *FnTy =
744  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
745  RTLFn = CGM.CreateRuntimeFunction(FnTy,
746  /*Name=*/"__kmpc_omp_task_complete_if0");
747  break;
748  }
749  case OMPRTL__kmpc_ordered: {
750  // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid);
751  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
752  llvm::FunctionType *FnTy =
753  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
754  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered");
755  break;
756  }
757  case OMPRTL__kmpc_end_ordered: {
758  // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid);
759  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
760  llvm::FunctionType *FnTy =
761  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
762  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered");
763  break;
764  }
765  case OMPRTL__kmpc_omp_taskwait: {
766  // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid);
767  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
768  llvm::FunctionType *FnTy =
769  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
770  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait");
771  break;
772  }
773  case OMPRTL__kmpc_taskgroup: {
774  // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid);
775  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
776  llvm::FunctionType *FnTy =
777  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
778  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup");
779  break;
780  }
781  case OMPRTL__kmpc_end_taskgroup: {
782  // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid);
783  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
784  llvm::FunctionType *FnTy =
785  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
786  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup");
787  break;
788  }
789  case OMPRTL__kmpc_push_proc_bind: {
790  // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
791  // int proc_bind)
792  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
793  llvm::FunctionType *FnTy =
794  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
795  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind");
796  break;
797  }
798  case OMPRTL__kmpc_omp_task_with_deps: {
799  // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
800  // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
801  // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list);
802  llvm::Type *TypeParams[] = {
803  getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty,
804  CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
805  llvm::FunctionType *FnTy =
806  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
807  RTLFn =
808  CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps");
809  break;
810  }
811  case OMPRTL__kmpc_omp_wait_deps: {
812  // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
813  // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,
814  // kmp_depend_info_t *noalias_dep_list);
815  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
816  CGM.Int32Ty, CGM.VoidPtrTy,
817  CGM.Int32Ty, CGM.VoidPtrTy};
818  llvm::FunctionType *FnTy =
819  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
820  RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps");
821  break;
822  }
823  case OMPRTL__kmpc_cancellationpoint: {
824  // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
825  // global_tid, kmp_int32 cncl_kind)
826  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
827  llvm::FunctionType *FnTy =
828  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
829  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint");
830  break;
831  }
832  case OMPRTL__kmpc_cancel: {
833  // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
834  // kmp_int32 cncl_kind)
835  llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy};
836  llvm::FunctionType *FnTy =
837  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
838  RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel");
839  break;
840  }
841  }
842  return RTLFn;
843 }
844 
845 llvm::Constant *CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize,
846  bool IVSigned) {
847  assert((IVSize == 32 || IVSize == 64) &&
848  "IV size is not compatible with the omp runtime");
849  auto Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4"
850  : "__kmpc_for_static_init_4u")
851  : (IVSigned ? "__kmpc_for_static_init_8"
852  : "__kmpc_for_static_init_8u");
853  auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
854  auto PtrTy = llvm::PointerType::getUnqual(ITy);
855  llvm::Type *TypeParams[] = {
856  getIdentTyPointerTy(), // loc
857  CGM.Int32Ty, // tid
858  CGM.Int32Ty, // schedtype
859  llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
860  PtrTy, // p_lower
861  PtrTy, // p_upper
862  PtrTy, // p_stride
863  ITy, // incr
864  ITy // chunk
865  };
866  llvm::FunctionType *FnTy =
867  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
868  return CGM.CreateRuntimeFunction(FnTy, Name);
869 }
870 
871 llvm::Constant *CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize,
872  bool IVSigned) {
873  assert((IVSize == 32 || IVSize == 64) &&
874  "IV size is not compatible with the omp runtime");
875  auto Name =
876  IVSize == 32
877  ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u")
878  : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u");
879  auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
880  llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc
881  CGM.Int32Ty, // tid
882  CGM.Int32Ty, // schedtype
883  ITy, // lower
884  ITy, // upper
885  ITy, // stride
886  ITy // chunk
887  };
888  llvm::FunctionType *FnTy =
889  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
890  return CGM.CreateRuntimeFunction(FnTy, Name);
891 }
892 
893 llvm::Constant *CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize,
894  bool IVSigned) {
895  assert((IVSize == 32 || IVSize == 64) &&
896  "IV size is not compatible with the omp runtime");
897  auto Name =
898  IVSize == 32
899  ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u")
900  : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u");
901  llvm::Type *TypeParams[] = {
902  getIdentTyPointerTy(), // loc
903  CGM.Int32Ty, // tid
904  };
905  llvm::FunctionType *FnTy =
906  llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
907  return CGM.CreateRuntimeFunction(FnTy, Name);
908 }
909 
910 llvm::Constant *CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize,
911  bool IVSigned) {
912  assert((IVSize == 32 || IVSize == 64) &&
913  "IV size is not compatible with the omp runtime");
914  auto Name =
915  IVSize == 32
916  ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u")
917  : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u");
918  auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
919  auto PtrTy = llvm::PointerType::getUnqual(ITy);
920  llvm::Type *TypeParams[] = {
921  getIdentTyPointerTy(), // loc
922  CGM.Int32Ty, // tid
923  llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter
924  PtrTy, // p_lower
925  PtrTy, // p_upper
926  PtrTy // p_stride
927  };
928  llvm::FunctionType *FnTy =
929  llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
930  return CGM.CreateRuntimeFunction(FnTy, Name);
931 }
932 
933 llvm::Constant *
935  assert(!CGM.getLangOpts().OpenMPUseTLS ||
937  // Lookup the entry, lazily creating it if necessary.
939  Twine(CGM.getMangledName(VD)) + ".cache.");
940 }
941 
943  const VarDecl *VD,
944  llvm::Value *VDAddr,
945  SourceLocation Loc) {
946  if (CGM.getLangOpts().OpenMPUseTLS &&
948  return VDAddr;
949 
950  auto VarTy = VDAddr->getType()->getPointerElementType();
951  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
952  CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy),
953  CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
955  return CGF.EmitRuntimeCall(
956  createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
957 }
958 
960  CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor,
961  llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
962  // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
963  // library.
964  auto OMPLoc = emitUpdateLocation(CGF, Loc);
965  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num),
966  OMPLoc);
967  // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
968  // to register constructor/destructor for variable.
969  llvm::Value *Args[] = {OMPLoc,
970  CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy),
971  Ctor, CopyCtor, Dtor};
972  CGF.EmitRuntimeCall(
973  createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
974 }
975 
977  const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc,
978  bool PerformInit, CodeGenFunction *CGF) {
979  if (CGM.getLangOpts().OpenMPUseTLS &&
981  return nullptr;
982 
983  VD = VD->getDefinition(CGM.getContext());
984  if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
985  ThreadPrivateWithDefinition.insert(VD);
986  QualType ASTTy = VD->getType();
987 
988  llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
989  auto Init = VD->getAnyInitializer();
990  if (CGM.getLangOpts().CPlusPlus && PerformInit) {
991  // Generate function that re-emits the declaration's initializer into the
992  // threadprivate copy of the variable VD
993  CodeGenFunction CtorCGF(CGM);
994  FunctionArgList Args;
995  ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
996  /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
997  Args.push_back(&Dst);
998 
999  auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
1001  /*isVariadic=*/false);
1002  auto FTy = CGM.getTypes().GetFunctionType(FI);
1003  auto Fn = CGM.CreateGlobalInitOrDestructFunction(
1004  FTy, ".__kmpc_global_ctor_.", Loc);
1005  CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
1006  Args, SourceLocation());
1007  auto ArgVal = CtorCGF.EmitLoadOfScalar(
1008  CtorCGF.GetAddrOfLocalVar(&Dst),
1009  /*Volatile=*/false, CGM.PointerAlignInBytes,
1010  CGM.getContext().VoidPtrTy, Dst.getLocation());
1011  auto Arg = CtorCGF.Builder.CreatePointerCast(
1012  ArgVal,
1013  CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy)));
1014  CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
1015  /*IsInitializer=*/true);
1016  ArgVal = CtorCGF.EmitLoadOfScalar(
1017  CtorCGF.GetAddrOfLocalVar(&Dst),
1018  /*Volatile=*/false, CGM.PointerAlignInBytes,
1019  CGM.getContext().VoidPtrTy, Dst.getLocation());
1020  CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1021  CtorCGF.FinishFunction();
1022  Ctor = Fn;
1023  }
1024  if (VD->getType().isDestructedType() != QualType::DK_none) {
1025  // Generate function that emits destructor call for the threadprivate copy
1026  // of the variable VD
1027  CodeGenFunction DtorCGF(CGM);
1028  FunctionArgList Args;
1029  ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
1030  /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
1031  Args.push_back(&Dst);
1032 
1033  auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
1034  CGM.getContext().VoidTy, Args, FunctionType::ExtInfo(),
1035  /*isVariadic=*/false);
1036  auto FTy = CGM.getTypes().GetFunctionType(FI);
1037  auto Fn = CGM.CreateGlobalInitOrDestructFunction(
1038  FTy, ".__kmpc_global_dtor_.", Loc);
1039  DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
1040  SourceLocation());
1041  auto ArgVal = DtorCGF.EmitLoadOfScalar(
1042  DtorCGF.GetAddrOfLocalVar(&Dst),
1043  /*Volatile=*/false, CGM.PointerAlignInBytes,
1044  CGM.getContext().VoidPtrTy, Dst.getLocation());
1045  DtorCGF.emitDestroy(ArgVal, ASTTy,
1046  DtorCGF.getDestroyer(ASTTy.isDestructedType()),
1047  DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
1048  DtorCGF.FinishFunction();
1049  Dtor = Fn;
1050  }
1051  // Do not emit init function if it is not required.
1052  if (!Ctor && !Dtor)
1053  return nullptr;
1054 
1055  llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1056  auto CopyCtorTy =
1057  llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1058  /*isVarArg=*/false)->getPointerTo();
1059  // Copying constructor for the threadprivate variable.
1060  // Must be NULL - reserved by runtime, but currently it requires that this
1061  // parameter is always NULL. Otherwise it fires assertion.
1062  CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1063  if (Ctor == nullptr) {
1064  auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1065  /*isVarArg=*/false)->getPointerTo();
1066  Ctor = llvm::Constant::getNullValue(CtorTy);
1067  }
1068  if (Dtor == nullptr) {
1069  auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1070  /*isVarArg=*/false)->getPointerTo();
1071  Dtor = llvm::Constant::getNullValue(DtorTy);
1072  }
1073  if (!CGF) {
1074  auto InitFunctionTy =
1075  llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
1076  auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
1077  InitFunctionTy, ".__omp_threadprivate_init_.");
1078  CodeGenFunction InitCGF(CGM);
1079  FunctionArgList ArgList;
1080  InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
1081  CGM.getTypes().arrangeNullaryFunction(), ArgList,
1082  Loc);
1083  emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1084  InitCGF.FinishFunction();
1085  return InitFunction;
1086  }
1087  emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1088  }
1089  return nullptr;
1090 }
1091 
1092 /// \brief Emits code for OpenMP 'if' clause using specified \a CodeGen
1093 /// function. Here is the logic:
1094 /// if (Cond) {
1095 /// ThenGen();
1096 /// } else {
1097 /// ElseGen();
1098 /// }
1099 static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond,
1100  const RegionCodeGenTy &ThenGen,
1101  const RegionCodeGenTy &ElseGen) {
1102  CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1103 
1104  // If the condition constant folds and can be elided, try to avoid emitting
1105  // the condition and the dead arm of the if/else.
1106  bool CondConstant;
1107  if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) {
1109  if (CondConstant) {
1110  ThenGen(CGF);
1111  } else {
1112  ElseGen(CGF);
1113  }
1114  return;
1115  }
1116 
1117  // Otherwise, the condition did not fold, or we couldn't elide it. Just
1118  // emit the conditional branch.
1119  auto ThenBlock = CGF.createBasicBlock("omp_if.then");
1120  auto ElseBlock = CGF.createBasicBlock("omp_if.else");
1121  auto ContBlock = CGF.createBasicBlock("omp_if.end");
1122  CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0);
1123 
1124  // Emit the 'then' code.
1125  CGF.EmitBlock(ThenBlock);
1126  {
1127  CodeGenFunction::RunCleanupsScope ThenScope(CGF);
1128  ThenGen(CGF);
1129  }
1130  CGF.EmitBranch(ContBlock);
1131  // Emit the 'else' code if present.
1132  {
1133  // There is no need to emit line number for unconditional branch.
1134  auto NL = ApplyDebugLocation::CreateEmpty(CGF);
1135  CGF.EmitBlock(ElseBlock);
1136  }
1137  {
1138  CodeGenFunction::RunCleanupsScope ThenScope(CGF);
1139  ElseGen(CGF);
1140  }
1141  {
1142  // There is no need to emit line number for unconditional branch.
1143  auto NL = ApplyDebugLocation::CreateEmpty(CGF);
1144  CGF.EmitBranch(ContBlock);
1145  }
1146  // Emit the continuation block for code after the if.
1147  CGF.EmitBlock(ContBlock, /*IsFinished=*/true);
1148 }
1149 
1151  llvm::Value *OutlinedFn,
1152  llvm::Value *CapturedStruct,
1153  const Expr *IfCond) {
1154  auto *RTLoc = emitUpdateLocation(CGF, Loc);
1155  auto &&ThenGen =
1156  [this, OutlinedFn, CapturedStruct, RTLoc](CodeGenFunction &CGF) {
1157  // Build call __kmpc_fork_call(loc, 1, microtask,
1158  // captured_struct/*context*/)
1159  llvm::Value *Args[] = {
1160  RTLoc,
1161  CGF.Builder.getInt32(
1162  1), // Number of arguments after 'microtask' argument
1163  // (there is only one additional argument - 'context')
1164  CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
1165  CGF.EmitCastToVoidPtr(CapturedStruct)};
1166  auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call);
1167  CGF.EmitRuntimeCall(RTLFn, Args);
1168  };
1169  auto &&ElseGen = [this, OutlinedFn, CapturedStruct, RTLoc, Loc](
1170  CodeGenFunction &CGF) {
1171  auto ThreadID = getThreadID(CGF, Loc);
1172  // Build calls:
1173  // __kmpc_serialized_parallel(&Loc, GTid);
1174  llvm::Value *Args[] = {RTLoc, ThreadID};
1175  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_serialized_parallel),
1176  Args);
1177 
1178  // OutlinedFn(&GTid, &zero, CapturedStruct);
1179  auto ThreadIDAddr = emitThreadIDAddress(CGF, Loc);
1180  auto Int32Ty = CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32,
1181  /*Signed*/ true);
1182  auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
1183  CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
1184  llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
1185  CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
1186 
1187  // __kmpc_end_serialized_parallel(&Loc, GTid);
1188  llvm::Value *EndArgs[] = {emitUpdateLocation(CGF, Loc), ThreadID};
1189  CGF.EmitRuntimeCall(
1190  createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs);
1191  };
1192  if (IfCond) {
1193  emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen);
1194  } else {
1196  ThenGen(CGF);
1197  }
1198 }
1199 
1200 // If we're inside an (outlined) parallel region, use the region info's
1201 // thread-ID variable (it is passed in a first argument of the outlined function
1202 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
1203 // regular serial code region, get thread ID by calling kmp_int32
1204 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
1205 // return the address of that temp.
1207  SourceLocation Loc) {
1208  if (auto OMPRegionInfo =
1209  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
1210  if (OMPRegionInfo->getThreadIDVariable())
1211  return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1212 
1213  auto ThreadID = getThreadID(CGF, Loc);
1214  auto Int32Ty =
1215  CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
1216  auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
1217  CGF.EmitStoreOfScalar(ThreadID,
1218  CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty));
1219 
1220  return ThreadIDTemp;
1221 }
1222 
1223 llvm::Constant *
1225  const llvm::Twine &Name) {
1226  SmallString<256> Buffer;
1227  llvm::raw_svector_ostream Out(Buffer);
1228  Out << Name;
1229  auto RuntimeName = Out.str();
1230  auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName, nullptr)).first;
1231  if (Elem.second) {
1232  assert(Elem.second->getType()->getPointerElementType() == Ty &&
1233  "OMP internal variable has different type than requested");
1234  return &*Elem.second;
1235  }
1236 
1237  return Elem.second = new llvm::GlobalVariable(
1238  CGM.getModule(), Ty, /*IsConstant*/ false,
1239  llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
1240  Elem.first());
1241 }
1242 
1244  llvm::Twine Name(".gomp_critical_user_", CriticalName);
1245  return getOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
1246 }
1247 
1248 namespace {
1249 template <size_t N> class CallEndCleanup : public EHScopeStack::Cleanup {
1250  llvm::Value *Callee;
1251  llvm::Value *Args[N];
1252 
1253 public:
1254  CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs)
1255  : Callee(Callee) {
1256  assert(CleanupArgs.size() == N);
1257  std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args));
1258  }
1259  void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
1260  CGF.EmitRuntimeCall(Callee, Args);
1261  }
1262 };
1263 } // namespace
1264 
1266  StringRef CriticalName,
1267  const RegionCodeGenTy &CriticalOpGen,
1268  SourceLocation Loc) {
1269  // __kmpc_critical(ident_t *, gtid, Lock);
1270  // CriticalOpGen();
1271  // __kmpc_end_critical(ident_t *, gtid, Lock);
1272  // Prepare arguments and build a call to __kmpc_critical
1273  {
1275  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1276  getCriticalRegionLock(CriticalName)};
1277  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args);
1278  // Build a call to __kmpc_end_critical
1279  CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1280  NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical),
1281  llvm::makeArrayRef(Args));
1282  emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen);
1283  }
1284 }
1285 
1286 static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
1288  const RegionCodeGenTy &BodyOpGen) {
1289  llvm::Value *CallBool = CGF.EmitScalarConversion(
1290  IfCond,
1291  CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
1292  CGF.getContext().BoolTy);
1293 
1294  auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
1295  auto *ContBlock = CGF.createBasicBlock("omp_if.end");
1296  // Generate the branch (If-stmt)
1297  CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1298  CGF.EmitBlock(ThenBlock);
1299  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, Kind, BodyOpGen);
1300  // Emit the rest of bblocks/branches
1301  CGF.EmitBranch(ContBlock);
1302  CGF.EmitBlock(ContBlock, true);
1303 }
1304 
1306  const RegionCodeGenTy &MasterOpGen,
1307  SourceLocation Loc) {
1308  // if(__kmpc_master(ident_t *, gtid)) {
1309  // MasterOpGen();
1310  // __kmpc_end_master(ident_t *, gtid);
1311  // }
1312  // Prepare arguments and build a call to __kmpc_master
1313  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
1314  auto *IsMaster =
1315  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args);
1316  typedef CallEndCleanup<std::extent<decltype(Args)>::value>
1317  MasterCallEndCleanup;
1318  emitIfStmt(CGF, IsMaster, OMPD_master, [&](CodeGenFunction &CGF) -> void {
1320  CGF.EHStack.pushCleanup<MasterCallEndCleanup>(
1321  NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master),
1322  llvm::makeArrayRef(Args));
1323  MasterOpGen(CGF);
1324  });
1325 }
1326 
1328  SourceLocation Loc) {
1329  // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
1330  llvm::Value *Args[] = {
1331  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1332  llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)};
1333  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args);
1334 }
1335 
1337  const RegionCodeGenTy &TaskgroupOpGen,
1338  SourceLocation Loc) {
1339  // __kmpc_taskgroup(ident_t *, gtid);
1340  // TaskgroupOpGen();
1341  // __kmpc_end_taskgroup(ident_t *, gtid);
1342  // Prepare arguments and build a call to __kmpc_taskgroup
1343  {
1345  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
1346  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args);
1347  // Build a call to __kmpc_end_taskgroup
1348  CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1349  NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup),
1350  llvm::makeArrayRef(Args));
1351  emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen);
1352  }
1353 }
1354 
1356  CodeGenModule &CGM, llvm::Type *ArgsType,
1357  ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
1358  ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
1359  auto &C = CGM.getContext();
1360  // void copy_func(void *LHSArg, void *RHSArg);
1361  FunctionArgList Args;
1362  ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
1363  C.VoidPtrTy);
1364  ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
1365  C.VoidPtrTy);
1366  Args.push_back(&LHSArg);
1367  Args.push_back(&RHSArg);
1369  auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration(
1370  C.VoidTy, Args, EI, /*isVariadic=*/false);
1371  auto *Fn = llvm::Function::Create(
1373  ".omp.copyprivate.copy_func", &CGM.getModule());
1374  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
1375  CodeGenFunction CGF(CGM);
1376  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
1377  // Dest = (void*[n])(LHSArg);
1378  // Src = (void*[n])(RHSArg);
1379  auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1380  CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
1381  CGF.PointerAlignInBytes),
1382  ArgsType);
1383  auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1384  CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
1385  CGF.PointerAlignInBytes),
1386  ArgsType);
1387  // *(Type0*)Dst[0] = *(Type0*)Src[0];
1388  // *(Type1*)Dst[1] = *(Type1*)Src[1];
1389  // ...
1390  // *(Typen*)Dst[n] = *(Typen*)Src[n];
1391  for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
1392  auto *DestAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1393  CGF.Builder.CreateAlignedLoad(
1394  CGF.Builder.CreateStructGEP(nullptr, LHS, I),
1395  CGM.PointerAlignInBytes),
1396  CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
1397  auto *SrcAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1398  CGF.Builder.CreateAlignedLoad(
1399  CGF.Builder.CreateStructGEP(nullptr, RHS, I),
1400  CGM.PointerAlignInBytes),
1401  CGF.ConvertTypeForMem(C.getPointerType(SrcExprs[I]->getType())));
1402  auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();
1403  QualType Type = VD->getType();
1404  CGF.EmitOMPCopy(CGF, Type, DestAddr, SrcAddr,
1405  cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()),
1406  cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()),
1407  AssignmentOps[I]);
1408  }
1409  CGF.FinishFunction();
1410  return Fn;
1411 }
1412 
1414  const RegionCodeGenTy &SingleOpGen,
1415  SourceLocation Loc,
1416  ArrayRef<const Expr *> CopyprivateVars,
1417  ArrayRef<const Expr *> SrcExprs,
1418  ArrayRef<const Expr *> DstExprs,
1419  ArrayRef<const Expr *> AssignmentOps) {
1420  assert(CopyprivateVars.size() == SrcExprs.size() &&
1421  CopyprivateVars.size() == DstExprs.size() &&
1422  CopyprivateVars.size() == AssignmentOps.size());
1423  auto &C = CGM.getContext();
1424  // int32 did_it = 0;
1425  // if(__kmpc_single(ident_t *, gtid)) {
1426  // SingleOpGen();
1427  // __kmpc_end_single(ident_t *, gtid);
1428  // did_it = 1;
1429  // }
1430  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
1431  // <copy_func>, did_it);
1432 
1433  llvm::AllocaInst *DidIt = nullptr;
1434  if (!CopyprivateVars.empty()) {
1435  // int32 did_it = 0;
1436  auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
1437  DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it");
1438  CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(0), DidIt,
1439  DidIt->getAlignment());
1440  }
1441  // Prepare arguments and build a call to __kmpc_single
1442  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
1443  auto *IsSingle =
1444  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args);
1445  typedef CallEndCleanup<std::extent<decltype(Args)>::value>
1446  SingleCallEndCleanup;
1447  emitIfStmt(CGF, IsSingle, OMPD_single, [&](CodeGenFunction &CGF) -> void {
1449  CGF.EHStack.pushCleanup<SingleCallEndCleanup>(
1450  NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single),
1451  llvm::makeArrayRef(Args));
1452  SingleOpGen(CGF);
1453  if (DidIt) {
1454  // did_it = 1;
1455  CGF.Builder.CreateAlignedStore(CGF.Builder.getInt32(1), DidIt,
1456  DidIt->getAlignment());
1457  }
1458  });
1459  // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
1460  // <copy_func>, did_it);
1461  if (DidIt) {
1462  llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
1463  auto CopyprivateArrayTy =
1464  C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
1465  /*IndexTypeQuals=*/0);
1466  // Create a list of all private variables for copyprivate.
1467  auto *CopyprivateList =
1468  CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list");
1469  for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
1470  auto *Elem = CGF.Builder.CreateStructGEP(
1471  CopyprivateList->getAllocatedType(), CopyprivateList, I);
1472  CGF.Builder.CreateAlignedStore(
1473  CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1474  CGF.EmitLValue(CopyprivateVars[I]).getAddress(), CGF.VoidPtrTy),
1475  Elem, CGM.PointerAlignInBytes);
1476  }
1477  // Build function that copies private values from single region to all other
1478  // threads in the corresponding parallel region.
1479  auto *CpyFn = emitCopyprivateCopyFunction(
1480  CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(),
1481  CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
1482  auto *BufSize = llvm::ConstantInt::get(
1483  CGM.SizeTy, C.getTypeSizeInChars(CopyprivateArrayTy).getQuantity());
1484  auto *CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList,
1485  CGF.VoidPtrTy);
1486  auto *DidItVal =
1487  CGF.Builder.CreateAlignedLoad(DidIt, CGF.PointerAlignInBytes);
1488  llvm::Value *Args[] = {
1489  emitUpdateLocation(CGF, Loc), // ident_t *<loc>
1490  getThreadID(CGF, Loc), // i32 <gtid>
1491  BufSize, // size_t <buf_size>
1492  CL, // void *<copyprivate list>
1493  CpyFn, // void (*) (void *, void *) <copy_func>
1494  DidItVal // i32 did_it
1495  };
1496  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args);
1497  }
1498 }
1499 
1501  const RegionCodeGenTy &OrderedOpGen,
1502  SourceLocation Loc) {
1503  // __kmpc_ordered(ident_t *, gtid);
1504  // OrderedOpGen();
1505  // __kmpc_end_ordered(ident_t *, gtid);
1506  // Prepare arguments and build a call to __kmpc_ordered
1507  {
1509  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
1510  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args);
1511  // Build a call to __kmpc_end_ordered
1512  CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1513  NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered),
1514  llvm::makeArrayRef(Args));
1515  emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen);
1516  }
1517 }
1518 
1521  bool CheckForCancel) {
1522  // Build call __kmpc_cancel_barrier(loc, thread_id);
1523  // Build call __kmpc_barrier(loc, thread_id);
1524  OpenMPLocationFlags Flags = OMP_IDENT_KMPC;
1525  if (Kind == OMPD_for) {
1526  Flags =
1527  static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_FOR);
1528  } else if (Kind == OMPD_sections) {
1529  Flags = static_cast<OpenMPLocationFlags>(Flags |
1530  OMP_IDENT_BARRIER_IMPL_SECTIONS);
1531  } else if (Kind == OMPD_single) {
1532  Flags =
1533  static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL_SINGLE);
1534  } else if (Kind == OMPD_barrier) {
1535  Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_EXPL);
1536  } else {
1537  Flags = static_cast<OpenMPLocationFlags>(Flags | OMP_IDENT_BARRIER_IMPL);
1538  }
1539  // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
1540  // thread_id);
1541  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
1542  getThreadID(CGF, Loc)};
1543  if (auto *OMPRegionInfo =
1544  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
1545  auto CancelDestination =
1546  CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
1547  if (CancelDestination.isValid()) {
1548  auto *Result = CGF.EmitRuntimeCall(
1549  createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args);
1550  if (CheckForCancel) {
1551  // if (__kmpc_cancel_barrier()) {
1552  // exit from construct;
1553  // }
1554  auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
1555  auto *ContBB = CGF.createBasicBlock(".cancel.continue");
1556  auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
1557  CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
1558  CGF.EmitBlock(ExitBB);
1559  // exit from construct;
1560  CGF.EmitBranchThroughCleanup(CancelDestination);
1561  CGF.EmitBlock(ContBB, /*IsFinished=*/true);
1562  }
1563  return;
1564  }
1565  }
1566  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args);
1567 }
1568 
1569 /// \brief Schedule types for 'omp for' loops (these enumerators are taken from
1570 /// the enum sched_type in kmp.h).
1572  /// \brief Lower bound for default (unordered) versions.
1580  /// \brief Lower bound for 'ordered' versions.
1589 };
1590 
1591 /// \brief Map the OpenMP loop schedule to the runtime enumeration.
1593  bool Chunked, bool Ordered) {
1594  switch (ScheduleKind) {
1595  case OMPC_SCHEDULE_static:
1596  return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
1597  : (Ordered ? OMP_ord_static : OMP_sch_static);
1598  case OMPC_SCHEDULE_dynamic:
1600  case OMPC_SCHEDULE_guided:
1602  case OMPC_SCHEDULE_runtime:
1603  return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
1604  case OMPC_SCHEDULE_auto:
1605  return Ordered ? OMP_ord_auto : OMP_sch_auto;
1606  case OMPC_SCHEDULE_unknown:
1607  assert(!Chunked && "chunk was specified but schedule kind not known");
1608  return Ordered ? OMP_ord_static : OMP_sch_static;
1609  }
1610  llvm_unreachable("Unexpected runtime schedule");
1611 }
1612 
1614  bool Chunked) const {
1615  auto Schedule = getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
1616  return Schedule == OMP_sch_static;
1617 }
1618 
1620  auto Schedule =
1621  getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
1622  assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
1623  return Schedule != OMP_sch_static;
1624 }
1625 
1627  OpenMPScheduleClauseKind ScheduleKind,
1628  unsigned IVSize, bool IVSigned, bool Ordered,
1629  llvm::Value *IL, llvm::Value *LB,
1630  llvm::Value *UB, llvm::Value *ST,
1631  llvm::Value *Chunk) {
1632  OpenMPSchedType Schedule =
1633  getRuntimeSchedule(ScheduleKind, Chunk != nullptr, Ordered);
1634  if (Ordered ||
1635  (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
1636  Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked)) {
1637  // Call __kmpc_dispatch_init(
1638  // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
1639  // kmp_int[32|64] lower, kmp_int[32|64] upper,
1640  // kmp_int[32|64] stride, kmp_int[32|64] chunk);
1641 
1642  // If the Chunk was not specified in the clause - use default value 1.
1643  if (Chunk == nullptr)
1644  Chunk = CGF.Builder.getIntN(IVSize, 1);
1645  llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
1646  getThreadID(CGF, Loc),
1647  CGF.Builder.getInt32(Schedule), // Schedule type
1648  CGF.Builder.getIntN(IVSize, 0), // Lower
1649  UB, // Upper
1650  CGF.Builder.getIntN(IVSize, 1), // Stride
1651  Chunk // Chunk
1652  };
1653  CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args);
1654  } else {
1655  // Call __kmpc_for_static_init(
1656  // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
1657  // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
1658  // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
1659  // kmp_int[32|64] incr, kmp_int[32|64] chunk);
1660  if (Chunk == nullptr) {
1661  assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static) &&
1662  "expected static non-chunked schedule");
1663  // If the Chunk was not specified in the clause - use default value 1.
1664  Chunk = CGF.Builder.getIntN(IVSize, 1);
1665  } else
1666  assert((Schedule == OMP_sch_static_chunked ||
1667  Schedule == OMP_ord_static_chunked) &&
1668  "expected static chunked schedule");
1669  llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
1670  getThreadID(CGF, Loc),
1671  CGF.Builder.getInt32(Schedule), // Schedule type
1672  IL, // &isLastIter
1673  LB, // &LB
1674  UB, // &UB
1675  ST, // &Stride
1676  CGF.Builder.getIntN(IVSize, 1), // Incr
1677  Chunk // Chunk
1678  };
1679  CGF.EmitRuntimeCall(createForStaticInitFunction(IVSize, IVSigned), Args);
1680  }
1681 }
1682 
1684  SourceLocation Loc) {
1685  // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
1686  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
1687  getThreadID(CGF, Loc)};
1688  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini),
1689  Args);
1690 }
1691 
1693  SourceLocation Loc,
1694  unsigned IVSize,
1695  bool IVSigned) {
1696  // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
1697  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC),
1698  getThreadID(CGF, Loc)};
1699  CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args);
1700 }
1701 
1703  SourceLocation Loc, unsigned IVSize,
1704  bool IVSigned, llvm::Value *IL,
1705  llvm::Value *LB, llvm::Value *UB,
1706  llvm::Value *ST) {
1707  // Call __kmpc_dispatch_next(
1708  // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1709  // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1710  // kmp_int[32|64] *p_stride);
1711  llvm::Value *Args[] = {
1712  emitUpdateLocation(CGF, Loc, OMP_IDENT_KMPC), getThreadID(CGF, Loc),
1713  IL, // &isLastIter
1714  LB, // &Lower
1715  UB, // &Upper
1716  ST // &Stride
1717  };
1718  llvm::Value *Call =
1719  CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args);
1720  return CGF.EmitScalarConversion(
1721  Call, CGF.getContext().getIntTypeForBitwidth(32, /* Signed */ true),
1722  CGF.getContext().BoolTy);
1723 }
1724 
1726  llvm::Value *NumThreads,
1727  SourceLocation Loc) {
1728  // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
1729  llvm::Value *Args[] = {
1730  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1731  CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
1732  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads),
1733  Args);
1734 }
1735 
1737  OpenMPProcBindClauseKind ProcBind,
1738  SourceLocation Loc) {
1739  // Constants for proc bind value accepted by the runtime.
1740  enum ProcBindTy {
1741  ProcBindFalse = 0,
1742  ProcBindTrue,
1743  ProcBindMaster,
1744  ProcBindClose,
1745  ProcBindSpread,
1746  ProcBindIntel,
1747  ProcBindDefault
1748  } RuntimeProcBind;
1749  switch (ProcBind) {
1750  case OMPC_PROC_BIND_master:
1751  RuntimeProcBind = ProcBindMaster;
1752  break;
1753  case OMPC_PROC_BIND_close:
1754  RuntimeProcBind = ProcBindClose;
1755  break;
1756  case OMPC_PROC_BIND_spread:
1757  RuntimeProcBind = ProcBindSpread;
1758  break;
1760  llvm_unreachable("Unsupported proc_bind value.");
1761  }
1762  // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
1763  llvm::Value *Args[] = {
1764  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1765  llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)};
1766  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args);
1767 }
1768 
1770  SourceLocation Loc) {
1771  // Build call void __kmpc_flush(ident_t *loc)
1772  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush),
1773  emitUpdateLocation(CGF, Loc));
1774 }
1775 
1776 namespace {
1777 /// \brief Indexes of fields for type kmp_task_t.
1779  /// \brief List of shared variables.
1780  KmpTaskTShareds,
1781  /// \brief Task routine.
1782  KmpTaskTRoutine,
1783  /// \brief Partition id for the untied tasks.
1784  KmpTaskTPartId,
1785  /// \brief Function with call of destructors for private variables.
1786  KmpTaskTDestructors,
1787 };
1788 } // namespace
1789 
1791  if (!KmpRoutineEntryPtrTy) {
1792  // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
1793  auto &C = CGM.getContext();
1794  QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
1796  KmpRoutineEntryPtrQTy = C.getPointerType(
1797  C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
1799  }
1800 }
1801 
1803  QualType FieldTy) {
1804  auto *Field = FieldDecl::Create(
1805  C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy,
1807  /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1808  Field->setAccess(AS_public);
1809  DC->addDecl(Field);
1810 }
1811 
1812 namespace {
1813 struct PrivateHelpersTy {
1814  PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy,
1815  const VarDecl *PrivateElemInit)
1816  : Original(Original), PrivateCopy(PrivateCopy),
1817  PrivateElemInit(PrivateElemInit) {}
1818  const VarDecl *Original;
1819  const VarDecl *PrivateCopy;
1820  const VarDecl *PrivateElemInit;
1821 };
1822 typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
1823 } // namespace
1824 
1825 static RecordDecl *
1827  const ArrayRef<PrivateDataTy> Privates) {
1828  if (!Privates.empty()) {
1829  auto &C = CGM.getContext();
1830  // Build struct .kmp_privates_t. {
1831  // /* private vars */
1832  // };
1833  auto *RD = C.buildImplicitRecord(".kmp_privates.t");
1834  RD->startDefinition();
1835  for (auto &&Pair : Privates) {
1836  auto Type = Pair.second.Original->getType();
1837  Type = Type.getNonReferenceType();
1838  addFieldToRecordDecl(C, RD, Type);
1839  }
1840  RD->completeDefinition();
1841  return RD;
1842  }
1843  return nullptr;
1844 }
1845 
1846 static RecordDecl *
1848  QualType KmpRoutineEntryPointerQTy) {
1849  auto &C = CGM.getContext();
1850  // Build struct kmp_task_t {
1851  // void * shareds;
1852  // kmp_routine_entry_t routine;
1853  // kmp_int32 part_id;
1854  // kmp_routine_entry_t destructors;
1855  // };
1856  auto *RD = C.buildImplicitRecord("kmp_task_t");
1857  RD->startDefinition();
1858  addFieldToRecordDecl(C, RD, C.VoidPtrTy);
1859  addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
1860  addFieldToRecordDecl(C, RD, KmpInt32Ty);
1861  addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
1862  RD->completeDefinition();
1863  return RD;
1864 }
1865 
1866 static RecordDecl *
1868  const ArrayRef<PrivateDataTy> Privates) {
1869  auto &C = CGM.getContext();
1870  // Build struct kmp_task_t_with_privates {
1871  // kmp_task_t task_data;
1872  // .kmp_privates_t. privates;
1873  // };
1874  auto *RD = C.buildImplicitRecord("kmp_task_t_with_privates");
1875  RD->startDefinition();
1876  addFieldToRecordDecl(C, RD, KmpTaskTQTy);
1877  if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) {
1878  addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
1879  }
1880  RD->completeDefinition();
1881  return RD;
1882 }
1883 
1884 /// \brief Emit a proxy function which accepts kmp_task_t as the second
1885 /// argument.
1886 /// \code
1887 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1888 /// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map,
1889 /// tt->shareds);
1890 /// return 0;
1891 /// }
1892 /// \endcode
1893 static llvm::Value *
1895  QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy,
1896  QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
1897  QualType SharedsPtrTy, llvm::Value *TaskFunction,
1898  llvm::Value *TaskPrivatesMap) {
1899  auto &C = CGM.getContext();
1900  FunctionArgList Args;
1901  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
1902  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
1903  /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
1904  Args.push_back(&GtidArg);
1905  Args.push_back(&TaskTypeArg);
1906  FunctionType::ExtInfo Info;
1907  auto &TaskEntryFnInfo =
1908  CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info,
1909  /*isVariadic=*/false);
1910  auto *TaskEntryTy = CGM.getTypes().GetFunctionType(TaskEntryFnInfo);
1911  auto *TaskEntry =
1913  ".omp_task_entry.", &CGM.getModule());
1914  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskEntryFnInfo, TaskEntry);
1915  CodeGenFunction CGF(CGM);
1916  CGF.disableDebugInfo();
1917  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
1918 
1919  // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
1920  // tt->task_data.shareds);
1921  auto *GtidParam = CGF.EmitLoadOfScalar(
1922  CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false,
1923  C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc);
1924  auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
1925  CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
1926  LValue TDBase =
1927  CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
1928  auto *KmpTaskTWithPrivatesQTyRD =
1929  cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
1930  LValue Base =
1931  CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
1932  auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
1933  auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
1934  auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
1935  auto *PartidParam = CGF.EmitLoadOfLValue(PartIdLVal, Loc).getScalarVal();
1936 
1937  auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
1938  auto SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI);
1939  auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1940  CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(),
1941  CGF.ConvertTypeForMem(SharedsPtrTy));
1942 
1943  auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
1944  llvm::Value *PrivatesParam;
1945  if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
1946  auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
1947  PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1948  PrivatesLVal.getAddress(), CGF.VoidPtrTy);
1949  } else {
1950  PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
1951  }
1952 
1953  llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam,
1954  TaskPrivatesMap, SharedsParam};
1955  CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
1957  RValue::get(CGF.Builder.getInt32(/*C=*/0)),
1958  CGF.MakeNaturalAlignAddrLValue(CGF.ReturnValue, KmpInt32Ty));
1959  CGF.FinishFunction();
1960  return TaskEntry;
1961 }
1962 
1964  SourceLocation Loc,
1965  QualType KmpInt32Ty,
1966  QualType KmpTaskTWithPrivatesPtrQTy,
1967  QualType KmpTaskTWithPrivatesQTy) {
1968  auto &C = CGM.getContext();
1969  FunctionArgList Args;
1970  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
1971  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
1972  /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy);
1973  Args.push_back(&GtidArg);
1974  Args.push_back(&TaskTypeArg);
1975  FunctionType::ExtInfo Info;
1976  auto &DestructorFnInfo =
1977  CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info,
1978  /*isVariadic=*/false);
1979  auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo);
1980  auto *DestructorFn =
1982  ".omp_task_destructor.", &CGM.getModule());
1983  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn);
1984  CodeGenFunction CGF(CGM);
1985  CGF.disableDebugInfo();
1986  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
1987  Args);
1988 
1989  auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
1990  CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
1991  LValue Base =
1992  CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
1993  auto *KmpTaskTWithPrivatesQTyRD =
1994  cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
1995  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
1996  Base = CGF.EmitLValueForField(Base, *FI);
1997  for (auto *Field :
1998  cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
1999  if (auto DtorKind = Field->getType().isDestructedType()) {
2000  auto FieldLValue = CGF.EmitLValueForField(Base, Field);
2001  CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
2002  }
2003  }
2004  CGF.FinishFunction();
2005  return DestructorFn;
2006 }
2007 
2008 /// \brief Emit a privates mapping function for correct handling of private and
2009 /// firstprivate variables.
2010 /// \code
2011 /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
2012 /// **noalias priv1,..., <tyn> **noalias privn) {
2013 /// *priv1 = &.privates.priv1;
2014 /// ...;
2015 /// *privn = &.privates.privn;
2016 /// }
2017 /// \endcode
2018 static llvm::Value *
2020  const ArrayRef<const Expr *> PrivateVars,
2021  const ArrayRef<const Expr *> FirstprivateVars,
2022  QualType PrivatesQTy,
2023  const ArrayRef<PrivateDataTy> Privates) {
2024  auto &C = CGM.getContext();
2025  FunctionArgList Args;
2026  ImplicitParamDecl TaskPrivatesArg(
2027  C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
2028  C.getPointerType(PrivatesQTy).withConst().withRestrict());
2029  Args.push_back(&TaskPrivatesArg);
2030  llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
2031  unsigned Counter = 1;
2032  for (auto *E: PrivateVars) {
2033  Args.push_back(ImplicitParamDecl::Create(
2034  C, /*DC=*/nullptr, Loc,
2035  /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
2036  .withConst()
2037  .withRestrict()));
2038  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2039  PrivateVarsPos[VD] = Counter;
2040  ++Counter;
2041  }
2042  for (auto *E : FirstprivateVars) {
2043  Args.push_back(ImplicitParamDecl::Create(
2044  C, /*DC=*/nullptr, Loc,
2045  /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
2046  .withConst()
2047  .withRestrict()));
2048  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2049  PrivateVarsPos[VD] = Counter;
2050  ++Counter;
2051  }
2052  FunctionType::ExtInfo Info;
2053  auto &TaskPrivatesMapFnInfo =
2054  CGM.getTypes().arrangeFreeFunctionDeclaration(C.VoidTy, Args, Info,
2055  /*isVariadic=*/false);
2056  auto *TaskPrivatesMapTy =
2057  CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
2058  auto *TaskPrivatesMap = llvm::Function::Create(
2059  TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage,
2060  ".omp_task_privates_map.", &CGM.getModule());
2061  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo,
2062  TaskPrivatesMap);
2063  TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
2064  CodeGenFunction CGF(CGM);
2065  CGF.disableDebugInfo();
2066  CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
2067  TaskPrivatesMapFnInfo, Args);
2068 
2069  // *privi = &.privates.privi;
2070  auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad(
2071  CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes);
2072  LValue Base =
2073  CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy);
2074  auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
2075  Counter = 0;
2076  for (auto *Field : PrivatesQTyRD->fields()) {
2077  auto FieldLVal = CGF.EmitLValueForField(Base, Field);
2078  auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
2079  auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD),
2080  VD->getType());
2081  auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc);
2082  CGF.EmitStoreOfScalar(
2083  FieldLVal.getAddress(),
2084  CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(),
2085  RefLVal.getType()->getPointeeType()));
2086  ++Counter;
2087  }
2088  CGF.FinishFunction();
2089  return TaskPrivatesMap;
2090 }
2091 
2092 static int array_pod_sort_comparator(const PrivateDataTy *P1,
2093  const PrivateDataTy *P2) {
2094  return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
2095 }
2096 
2099  bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
2100  llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds,
2101  const Expr *IfCond, ArrayRef<const Expr *> PrivateVars,
2102  ArrayRef<const Expr *> PrivateCopies,
2103  ArrayRef<const Expr *> FirstprivateVars,
2104  ArrayRef<const Expr *> FirstprivateCopies,
2105  ArrayRef<const Expr *> FirstprivateInits,
2106  ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences) {
2107  auto &C = CGM.getContext();
2109  // Aggregate privates and sort them by the alignment.
2110  auto I = PrivateCopies.begin();
2111  for (auto *E : PrivateVars) {
2112  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2113  Privates.push_back(std::make_pair(
2114  C.getTypeAlignInChars(VD->getType()),
2115  PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
2116  /*PrivateElemInit=*/nullptr)));
2117  ++I;
2118  }
2119  I = FirstprivateCopies.begin();
2120  auto IElemInitRef = FirstprivateInits.begin();
2121  for (auto *E : FirstprivateVars) {
2122  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2123  Privates.push_back(std::make_pair(
2124  C.getTypeAlignInChars(VD->getType()),
2125  PrivateHelpersTy(
2126  VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
2127  cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
2128  ++I, ++IElemInitRef;
2129  }
2130  llvm::array_pod_sort(Privates.begin(), Privates.end(),
2132  auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2133  // Build type kmp_routine_entry_t (if not built yet).
2134  emitKmpRoutineEntryT(KmpInt32Ty);
2135  // Build type kmp_task_t (if not built yet).
2136  if (KmpTaskTQTy.isNull()) {
2137  KmpTaskTQTy = C.getRecordType(
2139  }
2140  auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
2141  // Build particular struct kmp_task_t for the given task.
2142  auto *KmpTaskTWithPrivatesQTyRD =
2144  auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
2145  QualType KmpTaskTWithPrivatesPtrQTy =
2146  C.getPointerType(KmpTaskTWithPrivatesQTy);
2147  auto *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy);
2148  auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
2149  auto KmpTaskTWithPrivatesTySize =
2150  CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy));
2151  QualType SharedsPtrTy = C.getPointerType(SharedsTy);
2152 
2153  // Emit initial values for private copies (if any).
2154  llvm::Value *TaskPrivatesMap = nullptr;
2155  auto *TaskPrivatesMapTy =
2156  std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().begin(),
2157  3)
2158  ->getType();
2159  if (!Privates.empty()) {
2160  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
2161  TaskPrivatesMap = emitTaskPrivateMappingFunction(
2162  CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(), Privates);
2163  TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2164  TaskPrivatesMap, TaskPrivatesMapTy);
2165  } else {
2166  TaskPrivatesMap = llvm::ConstantPointerNull::get(
2167  cast<llvm::PointerType>(TaskPrivatesMapTy));
2168  }
2169  // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
2170  // kmp_task_t *tt);
2171  auto *TaskEntry = emitProxyTaskFunction(
2172  CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy,
2173  KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap);
2174 
2175  // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
2176  // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2177  // kmp_routine_entry_t *task_entry);
2178  // Task flags. Format is taken from
2179  // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp.h,
2180  // description of kmp_tasking_flags struct.
2181  const unsigned TiedFlag = 0x1;
2182  const unsigned FinalFlag = 0x2;
2183  unsigned Flags = Tied ? TiedFlag : 0;
2184  auto *TaskFlags =
2185  Final.getPointer()
2186  ? CGF.Builder.CreateSelect(Final.getPointer(),
2187  CGF.Builder.getInt32(FinalFlag),
2188  CGF.Builder.getInt32(/*C=*/0))
2189  : CGF.Builder.getInt32(Final.getInt() ? FinalFlag : 0);
2190  TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags));
2191  auto SharedsSize = C.getTypeSizeInChars(SharedsTy);
2192  llvm::Value *AllocArgs[] = {
2193  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), TaskFlags,
2194  KmpTaskTWithPrivatesTySize, CGM.getSize(SharedsSize),
2195  CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskEntry,
2197  auto *NewTask = CGF.EmitRuntimeCall(
2198  createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs);
2199  auto *NewTaskNewTaskTTy = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2200  NewTask, KmpTaskTWithPrivatesPtrTy);
2201  LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy,
2202  KmpTaskTWithPrivatesQTy);
2203  LValue TDBase =
2204  CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
2205  // Fill the data in the resulting kmp_task_t record.
2206  // Copy shareds if there are any.
2207  llvm::Value *KmpTaskSharedsPtr = nullptr;
2208  if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
2209  KmpTaskSharedsPtr = CGF.EmitLoadOfScalar(
2210  CGF.EmitLValueForField(
2211  TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),
2212  Loc);
2213  CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
2214  }
2215  // Emit initial values for private copies (if any).
2216  bool NeedsCleanup = false;
2217  if (!Privates.empty()) {
2218  auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
2219  auto PrivatesBase = CGF.EmitLValueForField(Base, *FI);
2220  FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
2221  LValue SharedsBase;
2222  if (!FirstprivateVars.empty()) {
2223  SharedsBase = CGF.MakeNaturalAlignAddrLValue(
2224  CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2225  KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
2226  SharedsTy);
2227  }
2229  cast<CapturedStmt>(*D.getAssociatedStmt()));
2230  for (auto &&Pair : Privates) {
2231  auto *VD = Pair.second.PrivateCopy;
2232  auto *Init = VD->getAnyInitializer();
2233  LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
2234  if (Init) {
2235  if (auto *Elem = Pair.second.PrivateElemInit) {
2236  auto *OriginalVD = Pair.second.Original;
2237  auto *SharedField = CapturesInfo.lookup(OriginalVD);
2238  auto SharedRefLValue =
2239  CGF.EmitLValueForField(SharedsBase, SharedField);
2240  QualType Type = OriginalVD->getType();
2241  if (Type->isArrayType()) {
2242  // Initialize firstprivate array.
2243  if (!isa<CXXConstructExpr>(Init) ||
2244  CGF.isTrivialInitializer(Init)) {
2245  // Perform simple memcpy.
2246  CGF.EmitAggregateAssign(PrivateLValue.getAddress(),
2247  SharedRefLValue.getAddress(), Type);
2248  } else {
2249  // Initialize firstprivate array using element-by-element
2250  // intialization.
2252  PrivateLValue.getAddress(), SharedRefLValue.getAddress(),
2253  Type, [&CGF, Elem, Init, &CapturesInfo](
2254  llvm::Value *DestElement, llvm::Value *SrcElement) {
2255  // Clean up any temporaries needed by the initialization.
2256  CodeGenFunction::OMPPrivateScope InitScope(CGF);
2257  InitScope.addPrivate(Elem, [SrcElement]() -> llvm::Value *{
2258  return SrcElement;
2259  });
2260  (void)InitScope.Privatize();
2261  // Emit initialization for single element.
2263  CGF, &CapturesInfo);
2264  CGF.EmitAnyExprToMem(Init, DestElement,
2265  Init->getType().getQualifiers(),
2266  /*IsInitializer=*/false);
2267  });
2268  }
2269  } else {
2270  CodeGenFunction::OMPPrivateScope InitScope(CGF);
2271  InitScope.addPrivate(Elem, [SharedRefLValue]() -> llvm::Value *{
2272  return SharedRefLValue.getAddress();
2273  });
2274  (void)InitScope.Privatize();
2275  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
2276  CGF.EmitExprAsInit(Init, VD, PrivateLValue,
2277  /*capturedByInit=*/false);
2278  }
2279  } else {
2280  CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
2281  }
2282  }
2283  NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType();
2284  ++FI;
2285  }
2286  }
2287  // Provide pointer to function with destructors for privates.
2288  llvm::Value *DestructorFn =
2289  NeedsCleanup ? emitDestructorsFunction(CGM, Loc, KmpInt32Ty,
2290  KmpTaskTWithPrivatesPtrQTy,
2291  KmpTaskTWithPrivatesQTy)
2292  : llvm::ConstantPointerNull::get(
2293  cast<llvm::PointerType>(KmpRoutineEntryPtrTy));
2294  LValue Destructor = CGF.EmitLValueForField(
2295  TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTDestructors));
2296  CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2297  DestructorFn, KmpRoutineEntryPtrTy),
2298  Destructor);
2299 
2300  // Process list of dependences.
2301  llvm::Value *DependInfo = nullptr;
2302  unsigned DependencesNumber = Dependences.size();
2303  if (!Dependences.empty()) {
2304  // Dependence kind for RTL.
2305  enum RTLDependenceKindTy { DepIn = 1, DepOut = 2, DepInOut = 3 };
2306  enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
2307  RecordDecl *KmpDependInfoRD;
2308  QualType FlagsTy = C.getIntTypeForBitwidth(
2309  C.toBits(C.getTypeSizeInChars(C.BoolTy)), /*Signed=*/false);
2310  llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
2311  if (KmpDependInfoTy.isNull()) {
2312  KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info");
2313  KmpDependInfoRD->startDefinition();
2314  addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType());
2315  addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
2316  addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
2317  KmpDependInfoRD->completeDefinition();
2318  KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
2319  } else {
2320  KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl());
2321  }
2322  // Define type kmp_depend_info[<Dependences.size()>];
2323  QualType KmpDependInfoArrayTy = C.getConstantArrayType(
2324  KmpDependInfoTy, llvm::APInt(/*numBits=*/64, Dependences.size()),
2325  ArrayType::Normal, /*IndexTypeQuals=*/0);
2326  // kmp_depend_info[<Dependences.size()>] deps;
2327  DependInfo = CGF.CreateMemTemp(KmpDependInfoArrayTy);
2328  for (unsigned i = 0; i < DependencesNumber; ++i) {
2329  auto Addr = CGF.EmitLValue(Dependences[i].second);
2330  auto *Size = llvm::ConstantInt::get(
2331  CGF.SizeTy,
2332  C.getTypeSizeInChars(Dependences[i].second->getType()).getQuantity());
2333  auto Base = CGF.MakeNaturalAlignAddrLValue(
2334  CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, i),
2335  KmpDependInfoTy);
2336  // deps[i].base_addr = &<Dependences[i].second>;
2337  auto BaseAddrLVal = CGF.EmitLValueForField(
2338  Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr));
2339  CGF.EmitStoreOfScalar(
2340  CGF.Builder.CreatePtrToInt(Addr.getAddress(), CGF.IntPtrTy),
2341  BaseAddrLVal);
2342  // deps[i].len = sizeof(<Dependences[i].second>);
2343  auto LenLVal = CGF.EmitLValueForField(
2344  Base, *std::next(KmpDependInfoRD->field_begin(), Len));
2345  CGF.EmitStoreOfScalar(Size, LenLVal);
2346  // deps[i].flags = <Dependences[i].first>;
2347  RTLDependenceKindTy DepKind;
2348  switch (Dependences[i].first) {
2349  case OMPC_DEPEND_in:
2350  DepKind = DepIn;
2351  break;
2352  case OMPC_DEPEND_out:
2353  DepKind = DepOut;
2354  break;
2355  case OMPC_DEPEND_inout:
2356  DepKind = DepInOut;
2357  break;
2358  case OMPC_DEPEND_unknown:
2359  llvm_unreachable("Unknown task dependence type");
2360  }
2361  auto FlagsLVal = CGF.EmitLValueForField(
2362  Base, *std::next(KmpDependInfoRD->field_begin(), Flags));
2363  CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
2364  FlagsLVal);
2365  }
2366  DependInfo = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2367  CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, DependInfo, 0),
2368  CGF.VoidPtrTy);
2369  }
2370 
2371  // NOTE: routine and part_id fields are intialized by __kmpc_omp_task_alloc()
2372  // libcall.
2373  // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t
2374  // *new_task);
2375  // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
2376  // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
2377  // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
2378  // list is not empty
2379  auto *ThreadID = getThreadID(CGF, Loc);
2380  auto *UpLoc = emitUpdateLocation(CGF, Loc);
2381  llvm::Value *TaskArgs[] = {UpLoc, ThreadID, NewTask};
2382  llvm::Value *DepTaskArgs[] = {
2383  UpLoc,
2384  ThreadID,
2385  NewTask,
2386  DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr,
2387  DependInfo,
2388  DependInfo ? CGF.Builder.getInt32(0) : nullptr,
2389  DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr};
2390  auto &&ThenCodeGen = [this, DependInfo, &TaskArgs,
2391  &DepTaskArgs](CodeGenFunction &CGF) {
2392  // TODO: add check for untied tasks.
2393  CGF.EmitRuntimeCall(
2394  createRuntimeFunction(DependInfo ? OMPRTL__kmpc_omp_task_with_deps
2395  : OMPRTL__kmpc_omp_task),
2396  DependInfo ? makeArrayRef(DepTaskArgs) : makeArrayRef(TaskArgs));
2397  };
2398  typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value>
2399  IfCallEndCleanup;
2400  llvm::Value *DepWaitTaskArgs[] = {
2401  UpLoc,
2402  ThreadID,
2403  DependInfo ? CGF.Builder.getInt32(DependencesNumber) : nullptr,
2404  DependInfo,
2405  DependInfo ? CGF.Builder.getInt32(0) : nullptr,
2406  DependInfo ? llvm::ConstantPointerNull::get(CGF.VoidPtrTy) : nullptr};
2407  auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
2408  DependInfo, &DepWaitTaskArgs](CodeGenFunction &CGF) {
2409  CodeGenFunction::RunCleanupsScope LocalScope(CGF);
2410  // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
2411  // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
2412  // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
2413  // is specified.
2414  if (DependInfo)
2415  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps),
2416  DepWaitTaskArgs);
2417  // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
2418  // kmp_task_t *new_task);
2419  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0),
2420  TaskArgs);
2421  // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
2422  // kmp_task_t *new_task);
2423  CGF.EHStack.pushCleanup<IfCallEndCleanup>(
2425  createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0),
2426  llvm::makeArrayRef(TaskArgs));
2427 
2428  // Call proxy_task_entry(gtid, new_task);
2429  llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
2430  CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
2431  };
2432  if (IfCond) {
2433  emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
2434  } else {
2436  ThenCodeGen(CGF);
2437  }
2438 }
2439 
2441  llvm::Type *ArgsType,
2442  ArrayRef<const Expr *> LHSExprs,
2443  ArrayRef<const Expr *> RHSExprs,
2444  ArrayRef<const Expr *> ReductionOps) {
2445  auto &C = CGM.getContext();
2446 
2447  // void reduction_func(void *LHSArg, void *RHSArg);
2448  FunctionArgList Args;
2449  ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
2450  C.VoidPtrTy);
2451  ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, SourceLocation(), /*Id=*/nullptr,
2452  C.VoidPtrTy);
2453  Args.push_back(&LHSArg);
2454  Args.push_back(&RHSArg);
2456  auto &CGFI = CGM.getTypes().arrangeFreeFunctionDeclaration(
2457  C.VoidTy, Args, EI, /*isVariadic=*/false);
2458  auto *Fn = llvm::Function::Create(
2460  ".omp.reduction.reduction_func", &CGM.getModule());
2461  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, CGFI, Fn);
2462  CodeGenFunction CGF(CGM);
2463  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args);
2464 
2465  // Dst = (void*[n])(LHSArg);
2466  // Src = (void*[n])(RHSArg);
2467  auto *LHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2468  CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&LHSArg),
2469  CGF.PointerAlignInBytes),
2470  ArgsType);
2471  auto *RHS = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2472  CGF.Builder.CreateAlignedLoad(CGF.GetAddrOfLocalVar(&RHSArg),
2473  CGF.PointerAlignInBytes),
2474  ArgsType);
2475 
2476  // ...
2477  // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2478  // ...
2480  for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I) {
2481  Scope.addPrivate(
2482  cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()),
2483  [&]() -> llvm::Value *{
2484  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2485  CGF.Builder.CreateAlignedLoad(
2486  CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, RHS, I),
2487  CGM.PointerAlignInBytes),
2488  CGF.ConvertTypeForMem(C.getPointerType(RHSExprs[I]->getType())));
2489  });
2490  Scope.addPrivate(
2491  cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()),
2492  [&]() -> llvm::Value *{
2493  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2494  CGF.Builder.CreateAlignedLoad(
2495  CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, LHS, I),
2496  CGM.PointerAlignInBytes),
2497  CGF.ConvertTypeForMem(C.getPointerType(LHSExprs[I]->getType())));
2498  });
2499  }
2500  Scope.Privatize();
2501  for (auto *E : ReductionOps) {
2502  CGF.EmitIgnoredExpr(E);
2503  }
2504  Scope.ForceCleanup();
2505  CGF.FinishFunction();
2506  return Fn;
2507 }
2508 
2510  ArrayRef<const Expr *> LHSExprs,
2511  ArrayRef<const Expr *> RHSExprs,
2512  ArrayRef<const Expr *> ReductionOps,
2513  bool WithNowait, bool SimpleReduction) {
2514  // Next code should be emitted for reduction:
2515  //
2516  // static kmp_critical_name lock = { 0 };
2517  //
2518  // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2519  // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
2520  // ...
2521  // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
2522  // *(Type<n>-1*)rhs[<n>-1]);
2523  // }
2524  //
2525  // ...
2526  // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2527  // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2528  // RedList, reduce_func, &<lock>)) {
2529  // case 1:
2530  // ...
2531  // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2532  // ...
2533  // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2534  // break;
2535  // case 2:
2536  // ...
2537  // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2538  // ...
2539  // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
2540  // break;
2541  // default:;
2542  // }
2543  //
2544  // if SimpleReduction is true, only the next code is generated:
2545  // ...
2546  // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2547  // ...
2548 
2549  auto &C = CGM.getContext();
2550 
2551  if (SimpleReduction) {
2553  for (auto *E : ReductionOps) {
2554  CGF.EmitIgnoredExpr(E);
2555  }
2556  return;
2557  }
2558 
2559  // 1. Build a list of reduction variables.
2560  // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
2561  llvm::APInt ArraySize(/*unsigned int numBits=*/32, RHSExprs.size());
2562  QualType ReductionArrayTy =
2563  C.getConstantArrayType(C.VoidPtrTy, ArraySize, ArrayType::Normal,
2564  /*IndexTypeQuals=*/0);
2565  auto *ReductionList =
2566  CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list");
2567  for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I) {
2568  auto *Elem = CGF.Builder.CreateStructGEP(/*Ty=*/nullptr, ReductionList, I);
2569  CGF.Builder.CreateAlignedStore(
2570  CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2571  CGF.EmitLValue(RHSExprs[I]).getAddress(), CGF.VoidPtrTy),
2572  Elem, CGM.PointerAlignInBytes);
2573  }
2574 
2575  // 2. Emit reduce_func().
2576  auto *ReductionFn = emitReductionFunction(
2577  CGM, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), LHSExprs,
2578  RHSExprs, ReductionOps);
2579 
2580  // 3. Create static kmp_critical_name lock = { 0 };
2581  auto *Lock = getCriticalRegionLock(".reduction");
2582 
2583  // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2584  // RedList, reduce_func, &<lock>);
2585  auto *IdentTLoc = emitUpdateLocation(
2586  CGF, Loc,
2587  static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE));
2588  auto *ThreadId = getThreadID(CGF, Loc);
2589  auto *ReductionArrayTySize = llvm::ConstantInt::get(
2590  CGM.SizeTy, C.getTypeSizeInChars(ReductionArrayTy).getQuantity());
2591  auto *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(ReductionList,
2592  CGF.VoidPtrTy);
2593  llvm::Value *Args[] = {
2594  IdentTLoc, // ident_t *<loc>
2595  ThreadId, // i32 <gtid>
2596  CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
2597  ReductionArrayTySize, // size_type sizeof(RedList)
2598  RL, // void *RedList
2599  ReductionFn, // void (*) (void *, void *) <reduce_func>
2600  Lock // kmp_critical_name *&<lock>
2601  };
2602  auto Res = CGF.EmitRuntimeCall(
2603  createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait
2604  : OMPRTL__kmpc_reduce),
2605  Args);
2606 
2607  // 5. Build switch(res)
2608  auto *DefaultBB = CGF.createBasicBlock(".omp.reduction.default");
2609  auto *SwInst = CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2);
2610 
2611  // 6. Build case 1:
2612  // ...
2613  // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2614  // ...
2615  // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2616  // break;
2617  auto *Case1BB = CGF.createBasicBlock(".omp.reduction.case1");
2618  SwInst->addCase(CGF.Builder.getInt32(1), Case1BB);
2619  CGF.EmitBlock(Case1BB);
2620 
2621  {
2623  // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2624  llvm::Value *EndArgs[] = {
2625  IdentTLoc, // ident_t *<loc>
2626  ThreadId, // i32 <gtid>
2627  Lock // kmp_critical_name *&<lock>
2628  };
2629  CGF.EHStack
2630  .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
2632  createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait
2633  : OMPRTL__kmpc_end_reduce),
2634  llvm::makeArrayRef(EndArgs));
2635  for (auto *E : ReductionOps) {
2636  CGF.EmitIgnoredExpr(E);
2637  }
2638  }
2639 
2640  CGF.EmitBranch(DefaultBB);
2641 
2642  // 7. Build case 2:
2643  // ...
2644  // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2645  // ...
2646  // break;
2647  auto *Case2BB = CGF.createBasicBlock(".omp.reduction.case2");
2648  SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
2649  CGF.EmitBlock(Case2BB);
2650 
2651  {
2653  if (!WithNowait) {
2654  // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
2655  llvm::Value *EndArgs[] = {
2656  IdentTLoc, // ident_t *<loc>
2657  ThreadId, // i32 <gtid>
2658  Lock // kmp_critical_name *&<lock>
2659  };
2660  CGF.EHStack
2661  .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
2663  createRuntimeFunction(OMPRTL__kmpc_end_reduce),
2664  llvm::makeArrayRef(EndArgs));
2665  }
2666  auto I = LHSExprs.begin();
2667  for (auto *E : ReductionOps) {
2668  const Expr *XExpr = nullptr;
2669  const Expr *EExpr = nullptr;
2670  const Expr *UpExpr = nullptr;
2672  if (auto *BO = dyn_cast<BinaryOperator>(E)) {
2673  if (BO->getOpcode() == BO_Assign) {
2674  XExpr = BO->getLHS();
2675  UpExpr = BO->getRHS();
2676  }
2677  }
2678  // Try to emit update expression as a simple atomic.
2679  auto *RHSExpr = UpExpr;
2680  if (RHSExpr) {
2681  // Analyze RHS part of the whole expression.
2682  if (auto *ACO = dyn_cast<AbstractConditionalOperator>(
2683  RHSExpr->IgnoreParenImpCasts())) {
2684  // If this is a conditional operator, analyze its condition for
2685  // min/max reduction operator.
2686  RHSExpr = ACO->getCond();
2687  }
2688  if (auto *BORHS =
2689  dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
2690  EExpr = BORHS->getRHS();
2691  BO = BORHS->getOpcode();
2692  }
2693  }
2694  if (XExpr) {
2695  auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
2696  LValue X = CGF.EmitLValue(XExpr);
2697  RValue E;
2698  if (EExpr)
2699  E = CGF.EmitAnyExpr(EExpr);
2701  X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc,
2702  [&CGF, UpExpr, VD](RValue XRValue) {
2703  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
2704  PrivateScope.addPrivate(
2705  VD, [&CGF, VD, XRValue]() -> llvm::Value *{
2706  auto *LHSTemp = CGF.CreateMemTemp(VD->getType());
2708  XRValue,
2709  CGF.MakeNaturalAlignAddrLValue(LHSTemp, VD->getType()));
2710  return LHSTemp;
2711  });
2712  (void)PrivateScope.Privatize();
2713  return CGF.EmitAnyExpr(UpExpr);
2714  });
2715  } else {
2716  // Emit as a critical region.
2717  emitCriticalRegion(CGF, ".atomic_reduction", [E](CodeGenFunction &CGF) {
2718  CGF.EmitIgnoredExpr(E);
2719  }, Loc);
2720  }
2721  ++I;
2722  }
2723  }
2724 
2725  CGF.EmitBranch(DefaultBB);
2726  CGF.EmitBlock(DefaultBB, /*IsFinished=*/true);
2727 }
2728 
2730  SourceLocation Loc) {
2731  // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
2732  // global_tid);
2733  llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2734  // Ignore return result until untied tasks are supported.
2735  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args);
2736 }
2737 
2739  OpenMPDirectiveKind InnerKind,
2740  const RegionCodeGenTy &CodeGen) {
2741  InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind);
2742  CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
2743 }
2744 
2745 namespace {
2747  CancelNoreq = 0,
2748  CancelParallel = 1,
2749  CancelLoop = 2,
2750  CancelSections = 3,
2751  CancelTaskgroup = 4
2752 };
2753 }
2754 
2756  RTCancelKind CancelKind = CancelNoreq;
2757  if (CancelRegion == OMPD_parallel)
2758  CancelKind = CancelParallel;
2759  else if (CancelRegion == OMPD_for)
2760  CancelKind = CancelLoop;
2761  else if (CancelRegion == OMPD_sections)
2762  CancelKind = CancelSections;
2763  else {
2764  assert(CancelRegion == OMPD_taskgroup);
2765  CancelKind = CancelTaskgroup;
2766  }
2767  return CancelKind;
2768 }
2769 
2771  CodeGenFunction &CGF, SourceLocation Loc,
2772  OpenMPDirectiveKind CancelRegion) {
2773  // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
2774  // global_tid, kmp_int32 cncl_kind);
2775  if (auto *OMPRegionInfo =
2776  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
2777  auto CancelDest =
2778  CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2779  if (CancelDest.isValid()) {
2780  llvm::Value *Args[] = {
2781  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2782  CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
2783  // Ignore return result until untied tasks are supported.
2784  auto *Result = CGF.EmitRuntimeCall(
2785  createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args);
2786  // if (__kmpc_cancellationpoint()) {
2787  // __kmpc_cancel_barrier();
2788  // exit from construct;
2789  // }
2790  auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
2791  auto *ContBB = CGF.createBasicBlock(".cancel.continue");
2792  auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
2793  CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2794  CGF.EmitBlock(ExitBB);
2795  // __kmpc_cancel_barrier();
2796  emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false);
2797  // exit from construct;
2798  CGF.EmitBranchThroughCleanup(CancelDest);
2799  CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2800  }
2801  }
2802 }
2803 
2805  OpenMPDirectiveKind CancelRegion) {
2806  // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
2807  // kmp_int32 cncl_kind);
2808  if (auto *OMPRegionInfo =
2809  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
2810  auto CancelDest =
2811  CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
2812  if (CancelDest.isValid()) {
2813  llvm::Value *Args[] = {
2814  emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2815  CGF.Builder.getInt32(getCancellationKind(CancelRegion))};
2816  // Ignore return result until untied tasks are supported.
2817  auto *Result =
2818  CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_cancel), Args);
2819  // if (__kmpc_cancel()) {
2820  // __kmpc_cancel_barrier();
2821  // exit from construct;
2822  // }
2823  auto *ExitBB = CGF.createBasicBlock(".cancel.exit");
2824  auto *ContBB = CGF.createBasicBlock(".cancel.continue");
2825  auto *Cmp = CGF.Builder.CreateIsNotNull(Result);
2826  CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2827  CGF.EmitBlock(ExitBB);
2828  // __kmpc_cancel_barrier();
2829  emitBarrierCall(CGF, Loc, OMPD_unknown, /*CheckForCancel=*/false);
2830  // exit from construct;
2831  CGF.EmitBranchThroughCleanup(CancelDest);
2832  CGF.EmitBlock(ContBB, /*IsFinished=*/true);
2833  }
2834  }
2835 }
2836 
llvm::PointerType * Int8PtrPtrTy
void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void pushTerminate()
Push a terminate handler on the stack.
Definition: CGCleanup.cpp:240
virtual llvm::Value * emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool CheckForCancel=true)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
llvm::IntegerType * IntTy
int
CanQualType VoidPtrTy
Definition: ASTContext.h:831
unsigned getColumn() const
Return the presumed column number of this location.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
DestructionKind isDestructedType() const
Definition: Type.h:999
llvm::Module & getModule() const
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
Definition: ASTContext.cpp:901
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
virtual llvm::Value * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D. This outlined function has typ...
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
Definition: CGExpr.cpp:57
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
void EmitOMPCopy(CodeGenFunction &CGF, QualType OriginalType, llvm::Value *DestAddr, llvm::Value *SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
Definition: Decl.cpp:1987
const llvm::DataLayout & getDataLayout() const
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
Definition: CGExpr.cpp:1469
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
virtual void completeDefinition()
Definition: Decl.cpp:3638
bool addPrivate(const VarDecl *LocalVD, const std::function< llvm::Value *()> &PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
const CGFunctionInfo & arrangeFreeFunctionDeclaration(QualType ResTy, const FunctionArgList &Args, const FunctionType::ExtInfo &Info, bool isVariadic)
Definition: CGCall.cpp:460
void EmitOMPAggregateAssign(llvm::Value *DestAddr, llvm::Value *SrcAddr, QualType OriginalType, const llvm::function_ref< void(llvm::Value *, llvm::Value *)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
llvm::Value * getAddress() const
Definition: CGValue.h:265
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
ExtProtoInfo - Extra information about a function prototype.
Definition: Type.h:3042
field_iterator field_begin() const
Definition: Decl.cpp:3629
static llvm::Value * emitReductionFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
llvm::StringMap< llvm::AssertingVH< llvm::Constant >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names. It stores variables with the follow...
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
*QualType KmpTaskTQTy
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, llvm::Value *CapturedStruct, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
Definition: Type.cpp:1509
String describing the source location. The string is composed of semi-colon separated fields which de...
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:89
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
llvm::CallInst * EmitRuntimeCall(llvm::Value *callee, const Twine &name="")
void startDefinition()
Starts the definition of this tag declaration.
Definition: Decl.cpp:3419
llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, ArrayRef< llvm::Value * > Args, const Twine &Name="")
Definition: CGCall.cpp:3069
static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function. Here is the logic: if (Cond) { Th...
QualType KmpRoutineEntryPtrQTy
void EmitAggregateAssign(llvm::Value *DestPtr, llvm::Value *SrcPtr, QualType EltTy)
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy)
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc)
Emits a critical region.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
Definition: CGExpr.cpp:119
CGOpenMPRuntime(CodeGenModule &CGM)
BinaryOperatorKind
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:518
RAII for correct setting/restoring of CapturedStmtInfo.
llvm::GlobalVariable * GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr, unsigned Alignment=0)
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call(). Original representation is: typedef...
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
RecordDecl * getDecl() const
Definition: Type.h:3527
void pushDestroy(QualType::DestructionKind dtorKind, llvm::Value *addr, QualType type)
Definition: CGDecl.cpp:1381
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep)
Creates clause with a list of variables VL and a linear step Step.
virtual void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:107
llvm::Constant * createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
unsigned getLine() const
Return the presumed line number of this location.
QualType getType() const
Definition: Decl.h:538
bool isInvalid() const
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
Definition: CGExpr.cpp:2680
virtual void emitForInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPScheduleClauseKind SchedKind, unsigned IVSize, bool IVSigned, bool Ordered, llvm::Value *IL, llvm::Value *LB, llvm::Value *UB, llvm::Value *ST, llvm::Value *Chunk=nullptr)
Call the appropriate runtime routine to initialize it before start of loop.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags=OMP_IDENT_KMPC)
Emits object of ident_t type with info for source location.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, bool WithNowait, bool SimpleReduction)
Emit a code for reduction clause. Next code should be emitted for reduction:
KmpTaskTFields
Indexes of fields for type kmp_task_t.
void SetLLVMFunctionAttributes(const Decl *D, const CGFunctionInfo &Info, llvm::Function *F)
Set the LLVM function attributes (sext, zext, etc).
void InitTempAlloca(llvm::AllocaInst *Alloca, llvm::Value *Value)
InitTempAlloca - Provide an initial value for the given alloca.
Definition: CGExpr.cpp:64
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
llvm::Constant * createRuntimeFunction(OpenMPRTLFunction Function)
Returns specified OpenMP runtime function.
llvm::Value * GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
const CGFunctionInfo & arrangeNullaryFunction()
Definition: CGCall.cpp:475
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked. This kind of worksharing directive is emit...
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
ASTContext & getContext() const
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
Definition: CGStmt.cpp:2200
virtual llvm::Value * getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Represents an unpacked "presumed" location which can be presented to the user.
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:43
The result type of a method or function.
OpenMPProcBindClauseKind
OpenMP attributes for 'proc_bind' clause.
Definition: OpenMPKinds.h:50
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, const ArrayRef< PrivateDataTy > Privates)
const Expr * getAnyInitializer() const
Definition: Decl.h:1056
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeSet ExtraAttrs=llvm::AttributeSet())
Create a new runtime function with the specified type and name.
Kind
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
This captures a statement into a function. For example, the following pragma annotated compound state...
Definition: Stmt.h:1989
const char * getFilename() const
Return the presumed filename of this location.
ASTContext & getContext() const
Encodes a location in the source. The SourceManager can decode this to get at the full include stack...
void EmitAggregateCopy(llvm::Value *DestPtr, llvm::Value *SrcPtr, QualType EltTy, bool isVolatile=false, CharUnits Alignment=CharUnits::Zero(), bool isAssignment=false)
Definition: CGExprAgg.cpp:1425
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
Lower bound for 'ordered' versions.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result)
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
llvm::Constant * createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
llvm::Constant * createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable. It emits a call to runtime library which ad...
virtual void clear()
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
Definition: CGDecl.cpp:1087
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param PrivateVars List of references to private variables for the task *directive *param PrivateCopies List of private copies for each private variable in *p PrivateVars *param FirstprivateVars List of references to private variables for the *task directive *param FirstprivateCopies List of private copies for each private variable *in p FirstprivateVars *param FirstprivateInits List of references to auto generated variables *used for initialization of a single array element Used if firstprivate *variable is of array type *param Dependences List of dependences for the task including *original expression and dependency type *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, bool Tied, llvm::PointerIntPair< llvm::Value *, 1, bool > Final, llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds, const Expr *IfCond, ArrayRef< const Expr * > PrivateVars, ArrayRef< const Expr * > PrivateCopies, ArrayRef< const Expr * > FirstprivateVars, ArrayRef< const Expr * > FirstprivateCopies, ArrayRef< const Expr * > FirstprivateInits, ArrayRef< std::pair< OpenMPDependClauseKind, const Expr * >> Dependences)
CanQualType VoidTy
Definition: ASTContext.h:817
const CodeGenOptions & getCodeGenOpts() const
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
const LangOptions & getLangOpts() const
llvm::AllocaInst * CreateMemTemp(QualType T, const Twine &Name="tmp")
Definition: CGExpr.cpp:80
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name. If the lock object does not...
llvm::function_ref< void(CodeGenFunction &)> RegionCodeGenTy
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
void FinishFunction(SourceLocation EndLoc=SourceLocation())
OpenMPDebugLocMapTy OpenMPDebugLocMap
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, llvm::Value *IL, llvm::Value *LB, llvm::Value *UB, llvm::Value *ST)
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:54
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
RTCancelKind
bool field_empty() const
Definition: Decl.h:3358
void EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr, bool Volatile, unsigned Alignment, QualType Ty, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0)
Definition: CGExpr.cpp:1244
const RecordType * getAsStructureType() const
Definition: Type.cpp:430
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emit code for the directive that does not require outlining.
static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
API for captured statement code generation.
static bool classof(const OMPClause *T)
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc)
Emit an ordered region.
This file defines OpenMP AST classes for executable directives and clauses.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T)
Definition: Decl.cpp:3850
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Definition: OpenMPKinds.h:58
virtual llvm::Value * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the OpenMP task directive D. This outlined function has type void(*)(kmp_...
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:33
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
Definition: CGStmt.cpp:348
void addDecl(Decl *D)
Add the declaration D into this context.
Definition: DeclBase.cpp:1228
SourceManager & getSourceManager()
Definition: ASTContext.h:494
bool isTLSSupported() const
Whether the target supports thread-local storage.
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::DenseSet< const VarDecl * > ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
X
Definition: SemaDecl.cpp:11429
Lower bound for default (unordered) versions.
void EmitBranch(llvm::BasicBlock *Block)
Definition: CGStmt.cpp:368
llvm::Type * ConvertType(QualType T)
static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, OpenMPDirectiveKind Kind, const RegionCodeGenTy &BodyOpGen)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
LValue EmitLValue(const Expr *E)
Definition: CGExpr.cpp:831
void popTerminate()
Pops a terminate handler off the stack.
Definition: CGCleanup.h:534
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
bool isArrayType() const
Definition: Type.h:5271
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
Definition: CGExpr.cpp:1316
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, SourceLocation Loc=SourceLocation(), bool TLS=false)
Definition: CGDeclCXX.cpp:251
static RValue get(llvm::Value *V)
Definition: CGValue.h:71
static llvm::Value * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Value *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
llvm::Value * EmitLoadOfScalar(llvm::Value *Addr, bool Volatile, unsigned Alignment, QualType Ty, SourceLocation Loc, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0)
Definition: CGExpr.cpp:1120
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void EmitBranchThroughCleanup(JumpDest Dest)
Definition: CGCleanup.cpp:940
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Definition: CGDebugInfo.h:542
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, const ArrayRef< PrivateDataTy > Privates)
static int array_pod_sort_comparator(const PrivateDataTy *P1, const PrivateDataTy *P2)
void emitThreadPrivateVarInit(CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic. This kind of worksharing directive is emitted without...
llvm::Constant * createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
void setAccess(AccessSpecifier AS)
Definition: DeclBase.h:411
CanQualType BoolTy
Definition: ASTContext.h:818
bool isNull() const
isNull - Return true if this QualType doesn't point to a type yet.
Definition: Type.h:633
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const ArrayRef< const Expr * > PrivateVars, const ArrayRef< const Expr * > FirstprivateVars, QualType PrivatesQTy, const ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Definition: Decl.cpp:3319
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:184
No in-class initializer.
Definition: Specifiers.h:198
llvm::StructType * IdentTy
bool Privatize()
Privatizes local variables previously registered as private. Registration is separate from the actual...
bool isPointerType() const
Definition: Type.h:5232
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1253
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.