20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/Bitcode/ReaderWriter.h"
22 #include "llvm/IR/CallSite.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/IR/GlobalValue.h"
25 #include "llvm/IR/Value.h"
26 #include "llvm/Support/Format.h"
27 #include "llvm/Support/raw_ostream.h"
30 using namespace clang;
31 using namespace CodeGen;
38 enum CGOpenMPRegionKind {
41 ParallelOutlinedRegion,
52 const CGOpenMPRegionKind RegionKind,
55 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
56 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
61 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
62 Kind(Kind), HasCancel(HasCancel) {}
66 virtual const VarDecl *getThreadIDVariable()
const = 0;
75 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
79 bool hasCancel()
const {
return HasCancel; }
81 static bool classof(
const CGCapturedStmtInfo *Info) {
86 CGOpenMPRegionKind RegionKind;
93 class CGOpenMPOutlinedRegionInfo :
public CGOpenMPRegionInfo {
98 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
100 ThreadIDVar(ThreadIDVar) {
101 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
105 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
108 StringRef getHelperName()
const override {
return ".omp_outlined."; }
110 static bool classof(
const CGCapturedStmtInfo *Info) {
112 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
113 ParallelOutlinedRegion;
123 class CGOpenMPTaskOutlinedRegionInfo :
public CGOpenMPRegionInfo {
129 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
130 ThreadIDVar(ThreadIDVar) {
131 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
135 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
141 StringRef getHelperName()
const override {
return ".omp_outlined."; }
143 static bool classof(
const CGCapturedStmtInfo *Info) {
145 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
157 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
162 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
164 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
168 return OuterRegionInfo->getContextValue();
169 llvm_unreachable(
"No context value for inlined OpenMP region");
172 if (OuterRegionInfo) {
173 OuterRegionInfo->setContextValue(V);
176 llvm_unreachable(
"No context value for inlined OpenMP region");
181 return OuterRegionInfo->lookup(VD);
186 FieldDecl *getThisFieldDecl()
const override {
188 return OuterRegionInfo->getThisFieldDecl();
193 const VarDecl *getThreadIDVariable()
const override {
195 return OuterRegionInfo->getThreadIDVariable();
200 StringRef getHelperName()
const override {
201 if (
auto *OuterRegionInfo = getOldCSI())
202 return OuterRegionInfo->getHelperName();
203 llvm_unreachable(
"No helper name for inlined OpenMP construct");
208 static bool classof(
const CGCapturedStmtInfo *Info) {
210 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
216 CGOpenMPRegionInfo *OuterRegionInfo;
224 class CGOpenMPTargetRegionInfo :
public CGOpenMPRegionInfo {
228 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
230 HelperName(HelperName) {}
234 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
237 StringRef getHelperName()
const override {
return HelperName; }
239 static bool classof(
const CGCapturedStmtInfo *Info) {
241 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
245 StringRef HelperName;
249 class InlinedOpenMPRegionRAII {
264 ~InlinedOpenMPRegionRAII() {
267 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI();
268 delete CGF.CapturedStmtInfo;
269 CGF.CapturedStmtInfo = OldCSI;
286 getThreadIDVariable()->
getType());
305 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
308 getThreadIDVariable()->
getType(),
320 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.
Int32Ty),
321 llvm::PointerType::getUnqual(CGM.
Int32Ty)};
346 const llvm::Twine &
Name =
"") {
355 "thread id variable must be of type kmp_int32 *");
358 bool HasCancel =
false;
359 if (
auto *OPD = dyn_cast<OMPParallelDirective>(&D))
360 HasCancel = OPD->hasCancel();
361 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
362 HasCancel = OPSD->hasCancel();
363 else if (
auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
364 HasCancel = OPFD->hasCancel();
365 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
375 "thread id variable must be of type kmp_int32 for tasks");
378 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
380 cast<OMPTaskDirective>(D).hasCancel());
385 Address CGOpenMPRuntime::getOrCreateDefaultLocation(OpenMPLocationFlags Flags) {
387 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
389 if (!DefaultOpenMPPSource) {
394 DefaultOpenMPPSource =
396 DefaultOpenMPPSource =
397 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.
Int8PtrTy);
399 auto DefaultOpenMPLocation =
new llvm::GlobalVariable(
401 llvm::GlobalValue::PrivateLinkage,
nullptr);
402 DefaultOpenMPLocation->setUnnamedAddr(
true);
403 DefaultOpenMPLocation->setAlignment(Align.
getQuantity());
405 llvm::Constant *Zero = llvm::ConstantInt::get(CGM.
Int32Ty, 0,
true);
406 llvm::Constant *Values[] = {Zero,
407 llvm::ConstantInt::get(CGM.
Int32Ty, Flags),
408 Zero, Zero, DefaultOpenMPPSource};
409 llvm::Constant *Init = llvm::ConstantStruct::get(
IdentTy, Values);
410 DefaultOpenMPLocation->setInitializer(Init);
411 OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
418 OpenMPLocationFlags Flags) {
422 return getOrCreateDefaultLocation(Flags).
getPointer();
424 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
441 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
451 if (OMPDebugLoc ==
nullptr) {
453 llvm::raw_svector_ostream OS2(Buffer2);
459 OS2 << FD->getQualifiedNameAsString();
462 OMPDebugLoc = CGF.
Builder.CreateGlobalStringPtr(OS2.str());
475 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
482 ThreadID =
I->second.ThreadID;
483 if (ThreadID !=
nullptr)
486 if (
auto *OMPRegionInfo =
488 if (OMPRegionInfo->getThreadIDVariable()) {
490 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
496 Elem.second.ThreadID = ThreadID;
506 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
512 Elem.second.ThreadID = ThreadID;
517 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
523 return llvm::PointerType::getUnqual(
IdentTy);
532 llvm::Constant *RTLFn =
nullptr;
534 case OMPRTL__kmpc_fork_call: {
539 llvm::FunctionType *FnTy =
540 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
true);
544 case OMPRTL__kmpc_global_thread_num: {
547 llvm::FunctionType *FnTy =
548 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
552 case OMPRTL__kmpc_threadprivate_cached: {
557 CGM.
VoidPtrTy->getPointerTo()->getPointerTo()};
558 llvm::FunctionType *FnTy =
559 llvm::FunctionType::get(CGM.
VoidPtrTy, TypeParams,
false);
563 case OMPRTL__kmpc_critical: {
569 llvm::FunctionType *FnTy =
570 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
574 case OMPRTL__kmpc_critical_with_hint: {
580 llvm::FunctionType *FnTy =
581 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
585 case OMPRTL__kmpc_threadprivate_register: {
591 false)->getPointerTo();
594 auto KmpcCopyCtorTy =
595 llvm::FunctionType::get(CGM.
VoidPtrTy, KmpcCopyCtorTyArgs,
596 false)->getPointerTo();
602 KmpcCopyCtorTy, KmpcDtorTy};
603 auto FnTy = llvm::FunctionType::get(CGM.
VoidTy, FnTyArgs,
608 case OMPRTL__kmpc_end_critical: {
614 llvm::FunctionType *FnTy =
615 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
619 case OMPRTL__kmpc_cancel_barrier: {
623 llvm::FunctionType *FnTy =
624 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
628 case OMPRTL__kmpc_barrier: {
631 llvm::FunctionType *FnTy =
632 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
636 case OMPRTL__kmpc_for_static_fini: {
639 llvm::FunctionType *FnTy =
640 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
644 case OMPRTL__kmpc_push_num_threads: {
649 llvm::FunctionType *FnTy =
650 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
654 case OMPRTL__kmpc_serialized_parallel: {
658 llvm::FunctionType *FnTy =
659 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
663 case OMPRTL__kmpc_end_serialized_parallel: {
667 llvm::FunctionType *FnTy =
668 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
672 case OMPRTL__kmpc_flush: {
675 llvm::FunctionType *FnTy =
676 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
680 case OMPRTL__kmpc_master: {
683 llvm::FunctionType *FnTy =
684 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
688 case OMPRTL__kmpc_end_master: {
691 llvm::FunctionType *FnTy =
692 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
696 case OMPRTL__kmpc_omp_taskyield: {
700 llvm::FunctionType *FnTy =
701 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
705 case OMPRTL__kmpc_single: {
708 llvm::FunctionType *FnTy =
709 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
713 case OMPRTL__kmpc_end_single: {
716 llvm::FunctionType *FnTy =
717 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
721 case OMPRTL__kmpc_omp_task_alloc: {
726 "Type kmp_routine_entry_t must be created.");
730 llvm::FunctionType *FnTy =
731 llvm::FunctionType::get(CGM.
VoidPtrTy, TypeParams,
false);
735 case OMPRTL__kmpc_omp_task: {
740 llvm::FunctionType *FnTy =
741 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
745 case OMPRTL__kmpc_copyprivate: {
751 llvm::FunctionType::get(CGM.
VoidTy, CpyTypeParams,
false);
755 llvm::FunctionType *FnTy =
756 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
760 case OMPRTL__kmpc_reduce: {
765 auto *ReduceFnTy = llvm::FunctionType::get(CGM.
VoidTy, ReduceTypeParams,
769 CGM.
VoidPtrTy, ReduceFnTy->getPointerTo(),
771 llvm::FunctionType *FnTy =
772 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
776 case OMPRTL__kmpc_reduce_nowait: {
782 auto *ReduceFnTy = llvm::FunctionType::get(CGM.
VoidTy, ReduceTypeParams,
786 CGM.
VoidPtrTy, ReduceFnTy->getPointerTo(),
788 llvm::FunctionType *FnTy =
789 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
793 case OMPRTL__kmpc_end_reduce: {
799 llvm::FunctionType *FnTy =
800 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
804 case OMPRTL__kmpc_end_reduce_nowait: {
810 llvm::FunctionType *FnTy =
811 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
816 case OMPRTL__kmpc_omp_task_begin_if0: {
821 llvm::FunctionType *FnTy =
822 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
827 case OMPRTL__kmpc_omp_task_complete_if0: {
832 llvm::FunctionType *FnTy =
833 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
835 "__kmpc_omp_task_complete_if0");
838 case OMPRTL__kmpc_ordered: {
841 llvm::FunctionType *FnTy =
842 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
846 case OMPRTL__kmpc_end_ordered: {
849 llvm::FunctionType *FnTy =
850 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
854 case OMPRTL__kmpc_omp_taskwait: {
857 llvm::FunctionType *FnTy =
858 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
862 case OMPRTL__kmpc_taskgroup: {
865 llvm::FunctionType *FnTy =
866 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
870 case OMPRTL__kmpc_end_taskgroup: {
873 llvm::FunctionType *FnTy =
874 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
878 case OMPRTL__kmpc_push_proc_bind: {
882 llvm::FunctionType *FnTy =
883 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
887 case OMPRTL__kmpc_omp_task_with_deps: {
894 llvm::FunctionType *FnTy =
895 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
900 case OMPRTL__kmpc_omp_wait_deps: {
907 llvm::FunctionType *FnTy =
908 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
912 case OMPRTL__kmpc_cancellationpoint: {
916 llvm::FunctionType *FnTy =
917 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
921 case OMPRTL__kmpc_cancel: {
925 llvm::FunctionType *FnTy =
926 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
930 case OMPRTL__tgt_target: {
939 CGM.
SizeTy->getPointerTo(),
941 llvm::FunctionType *FnTy =
942 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
946 case OMPRTL__tgt_register_lib: {
951 llvm::FunctionType *FnTy =
952 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
956 case OMPRTL__tgt_unregister_lib: {
961 llvm::FunctionType *FnTy =
962 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
973 auto SizeInChars =
C.getTypeSizeInChars(Ty);
974 if (SizeInChars.isZero()) {
976 while (
auto *VAT =
C.getAsVariableArrayType(Ty)) {
978 std::tie(ArraySize, Ty) = CGF.
getVLASize(VAT);
979 Size = Size ? CGF.
Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
981 SizeInChars =
C.getTypeSizeInChars(Ty);
982 assert(!SizeInChars.isZero());
983 Size = CGF.
Builder.CreateNUWMul(
984 Size, llvm::ConstantInt::get(CGF.
SizeTy, SizeInChars.getQuantity()));
986 Size = llvm::ConstantInt::get(CGF.
SizeTy, SizeInChars.getQuantity());
992 assert((IVSize == 32 || IVSize == 64) &&
993 "IV size is not compatible with the omp runtime");
994 auto Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4"
995 :
"__kmpc_for_static_init_4u")
996 : (IVSigned ?
"__kmpc_for_static_init_8"
997 :
"__kmpc_for_static_init_8u");
999 auto PtrTy = llvm::PointerType::getUnqual(ITy);
1004 llvm::PointerType::getUnqual(CGM.
Int32Ty),
1011 llvm::FunctionType *FnTy =
1012 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
1018 assert((IVSize == 32 || IVSize == 64) &&
1019 "IV size is not compatible with the omp runtime");
1022 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
1023 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
1033 llvm::FunctionType *FnTy =
1034 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
1040 assert((IVSize == 32 || IVSize == 64) &&
1041 "IV size is not compatible with the omp runtime");
1044 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
1045 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
1050 llvm::FunctionType *FnTy =
1051 llvm::FunctionType::get(CGM.
VoidTy, TypeParams,
false);
1057 assert((IVSize == 32 || IVSize == 64) &&
1058 "IV size is not compatible with the omp runtime");
1061 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
1062 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
1064 auto PtrTy = llvm::PointerType::getUnqual(ITy);
1068 llvm::PointerType::getUnqual(CGM.
Int32Ty),
1073 llvm::FunctionType *FnTy =
1074 llvm::FunctionType::get(CGM.
Int32Ty, TypeParams,
false);
1119 Ctor, CopyCtor, Dtor};
1136 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1145 Args.push_back(&Dst);
1152 FTy,
".__kmpc_global_ctor_.", FI, Loc);
1155 auto ArgVal = CtorCGF.EmitLoadOfScalar(
1156 CtorCGF.GetAddrOfLocalVar(&Dst),
false,
1159 Arg = CtorCGF.Builder.CreateElementBitCast(Arg,
1160 CtorCGF.ConvertTypeForMem(ASTTy));
1161 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->
getType().getQualifiers(),
1163 ArgVal = CtorCGF.EmitLoadOfScalar(
1164 CtorCGF.GetAddrOfLocalVar(&Dst),
false,
1166 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
1167 CtorCGF.FinishFunction();
1177 Args.push_back(&Dst);
1184 FTy,
".__kmpc_global_dtor_.", FI, Loc);
1187 auto ArgVal = DtorCGF.EmitLoadOfScalar(
1188 DtorCGF.GetAddrOfLocalVar(&Dst),
1193 DtorCGF.FinishFunction();
1202 llvm::FunctionType::get(CGM.
VoidPtrTy, CopyCtorTyArgs,
1203 false)->getPointerTo();
1207 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1208 if (Ctor ==
nullptr) {
1210 false)->getPointerTo();
1211 Ctor = llvm::Constant::getNullValue(CtorTy);
1213 if (Dtor ==
nullptr) {
1215 false)->getPointerTo();
1216 Dtor = llvm::Constant::getNullValue(DtorTy);
1219 auto InitFunctionTy =
1220 llvm::FunctionType::get(CGM.
VoidTy,
false);
1222 InitFunctionTy,
".__omp_threadprivate_init_.",
1230 InitCGF.FinishFunction();
1231 return InitFunction;
1299 const Expr *IfCond) {
1303 auto &&ThenGen = [
this, OutlinedFn, CapturedVars,
1308 CGF.
Builder.getInt32(CapturedVars.size()),
1312 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1317 auto &&ElseGen = [
this, OutlinedFn, CapturedVars, RTLoc,
1333 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
1334 OutlinedFnArgs.push_back(ZeroAddr.
getPointer());
1335 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1359 if (
auto *OMPRegionInfo =
1361 if (OMPRegionInfo->getThreadIDVariable())
1362 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
1367 auto ThreadIDTemp = CGF.
CreateMemTemp(Int32Ty,
".threadid_temp.");
1371 return ThreadIDTemp;
1376 const llvm::Twine &
Name) {
1378 llvm::raw_svector_ostream Out(Buffer);
1380 auto RuntimeName = Out.str();
1381 auto &Elem = *
InternalVars.insert(std::make_pair(RuntimeName,
nullptr)).first;
1383 assert(Elem.second->getType()->getPointerElementType() == Ty &&
1384 "OMP internal variable has different type than requested");
1385 return &*Elem.second;
1388 return Elem.second =
new llvm::GlobalVariable(
1390 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
1395 llvm::Twine
Name(
".gomp_critical_user_", CriticalName);
1407 assert(CleanupArgs.size() == N);
1408 std::copy(CleanupArgs.begin(), CleanupArgs.end(),
std::begin(Args));
1419 StringRef CriticalName,
1435 ArgsWithHint.push_back(
1442 CGF.
EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1444 llvm::makeArrayRef(Args));
1459 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
1480 typedef CallEndCleanup<std::extent<decltype(Args)>::value>
1481 MasterCallEndCleanup;
1485 CGF.
EHStack.pushCleanup<MasterCallEndCleanup>(
1487 llvm::makeArrayRef(Args));
1499 llvm::ConstantInt::get(CGM.
IntTy, 0,
true)};
1517 CGF.
EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1519 llvm::makeArrayRef(Args));
1527 unsigned Index,
const VarDecl *Var) {
1550 Args.push_back(&LHSArg);
1551 Args.push_back(&RHSArg);
1554 C.VoidTy, Args, EI,
false);
1557 ".omp.copyprivate.copy_func", &CGM.
getModule());
1573 for (
unsigned I = 0,
E = AssignmentOps.size();
I <
E; ++
I) {
1574 auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[
I])->getDecl());
1577 auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[
I])->getDecl());
1580 auto *VD = cast<DeclRefExpr>(CopyprivateVars[
I])->getDecl();
1582 CGF.
EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[
I]);
1597 assert(CopyprivateVars.size() == SrcExprs.size() &&
1598 CopyprivateVars.size() == DstExprs.size() &&
1599 CopyprivateVars.size() == AssignmentOps.size());
1611 if (!CopyprivateVars.empty()) {
1613 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
1614 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
1621 typedef CallEndCleanup<std::extent<decltype(Args)>::value>
1622 SingleCallEndCleanup;
1626 CGF.
EHStack.pushCleanup<SingleCallEndCleanup>(
1628 llvm::makeArrayRef(Args));
1638 llvm::APInt ArraySize(32, CopyprivateVars.size());
1639 auto CopyprivateArrayTy =
1644 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
1645 for (
unsigned I = 0,
E = CopyprivateVars.size();
I <
E; ++
I) {
1657 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
1658 auto *BufSize =
getTypeSize(CGF, CopyprivateArrayTy);
1689 CGF.
EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>(
1691 llvm::makeArrayRef(Args));
1698 bool ForceSimpleCall) {
1703 OpenMPLocationFlags Flags = OMP_IDENT_KMPC;
1704 if (Kind == OMPD_for) {
1706 static_cast<OpenMPLocationFlags
>(Flags | OMP_IDENT_BARRIER_IMPL_FOR);
1707 }
else if (Kind == OMPD_sections) {
1708 Flags =
static_cast<OpenMPLocationFlags
>(Flags |
1709 OMP_IDENT_BARRIER_IMPL_SECTIONS);
1710 }
else if (Kind == OMPD_single) {
1712 static_cast<OpenMPLocationFlags
>(Flags | OMP_IDENT_BARRIER_IMPL_SINGLE);
1713 }
else if (Kind == OMPD_barrier) {
1714 Flags =
static_cast<OpenMPLocationFlags
>(Flags | OMP_IDENT_BARRIER_EXPL);
1716 Flags =
static_cast<OpenMPLocationFlags
>(Flags | OMP_IDENT_BARRIER_IMPL);
1722 if (
auto *OMPRegionInfo =
1724 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
1733 auto *Cmp = CGF.
Builder.CreateIsNotNull(Result);
1734 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
1737 auto CancelDestination =
1772 bool Chunked,
bool Ordered) {
1773 switch (ScheduleKind) {
1774 case OMPC_SCHEDULE_static:
1777 case OMPC_SCHEDULE_dynamic:
1779 case OMPC_SCHEDULE_guided:
1781 case OMPC_SCHEDULE_runtime:
1783 case OMPC_SCHEDULE_auto:
1786 assert(!Chunked &&
"chunk was specified but schedule kind not known");
1789 llvm_unreachable(
"Unexpected runtime schedule");
1793 bool Chunked)
const {
1808 unsigned IVSize,
bool IVSigned,
1824 if (Chunk ==
nullptr)
1825 Chunk = CGF.
Builder.getIntN(IVSize, 1);
1829 CGF.
Builder.getInt32(Schedule),
1830 CGF.
Builder.getIntN(IVSize, 0),
1832 CGF.
Builder.getIntN(IVSize, 1),
1841 unsigned IVSize,
bool IVSigned,
1858 if (Chunk ==
nullptr) {
1860 "expected static non-chunked schedule");
1862 Chunk = CGF.
Builder.getIntN(IVSize, 1);
1866 "expected static chunked schedule");
1871 CGF.
Builder.getInt32(Schedule),
1876 CGF.
Builder.getIntN(IVSize, 1),
1957 case OMPC_PROC_BIND_master:
1958 RuntimeProcBind = ProcBindMaster;
1960 case OMPC_PROC_BIND_close:
1961 RuntimeProcBind = ProcBindClose;
1963 case OMPC_PROC_BIND_spread:
1964 RuntimeProcBind = ProcBindSpread;
1967 llvm_unreachable(
"Unsupported proc_bind value.");
1972 llvm::ConstantInt::get(CGM.
IntTy, RuntimeProcBind,
true)};
1995 KmpTaskTDestructors,
1999 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
2001 return OffloadEntriesTargetRegion.empty();
2005 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2006 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2007 StringRef ParentName,
unsigned LineNum,
2008 unsigned ColNum,
unsigned Order) {
2009 assert(CGM.
getLangOpts().OpenMPIsDevice &&
"Initialization of entries is "
2010 "only required for the device "
2011 "code generation.");
2012 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum][ColNum] =
2013 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr);
2014 ++OffloadingEntriesNum;
2017 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2018 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2019 StringRef ParentName,
unsigned LineNum,
2020 unsigned ColNum, llvm::Constant *Addr,
2021 llvm::Constant *
ID) {
2025 assert(hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum,
2027 "Entry must exist.");
2028 auto &Entry = OffloadEntriesTargetRegion[DeviceID][FileID][ParentName]
2030 assert(Entry.isValid() &&
"Entry not initialized!");
2031 Entry.setAddress(Addr);
2035 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID);
2036 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum][ColNum] =
2041 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2042 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
unsigned LineNum,
2043 unsigned ColNum)
const {
2044 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2045 if (PerDevice == OffloadEntriesTargetRegion.end())
2047 auto PerFile = PerDevice->second.find(FileID);
2048 if (PerFile == PerDevice->second.end())
2050 auto PerParentName = PerFile->second.find(ParentName);
2051 if (PerParentName == PerFile->second.end())
2053 auto PerLine = PerParentName->second.find(LineNum);
2054 if (PerLine == PerParentName->second.end())
2056 auto PerColumn = PerLine->second.find(ColNum);
2057 if (PerColumn == PerLine->second.end())
2060 if (PerColumn->second.getAddress() || PerColumn->second.getID())
2065 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
2066 const OffloadTargetRegionEntryInfoActTy &
Action) {
2068 for (
auto &D : OffloadEntriesTargetRegion)
2069 for (
auto &F : D.second)
2070 for (
auto &
P : F.second)
2071 for (
auto &L :
P.second)
2072 for (
auto &C : L.second)
2073 Action(D.first, F.first,
P.first(), L.first, C.first, C.second);
2079 static llvm::Function *
2085 nullptr, C.VoidPtrTy);
2086 Args.push_back(&DummyPtr);
2118 assert(!Devices.empty() &&
"No OpenMP offloading devices??");
2122 auto *OffloadEntryTy =
2124 llvm::GlobalVariable *HostEntriesBegin =
new llvm::GlobalVariable(
2125 M, OffloadEntryTy,
true,
2127 ".omp_offloading.entries_begin");
2128 llvm::GlobalVariable *HostEntriesEnd =
new llvm::GlobalVariable(
2129 M, OffloadEntryTy,
true,
2131 ".omp_offloading.entries_end");
2135 auto *DeviceImageTy = cast<llvm::StructType>(
2138 for (
unsigned i = 0; i < Devices.size(); ++i) {
2139 StringRef T = Devices[i].getTriple();
2140 auto *ImgBegin =
new llvm::GlobalVariable(
2142 0, Twine(
".omp_offloading.img_start.") + Twine(T));
2143 auto *ImgEnd =
new llvm::GlobalVariable(
2145 0, Twine(
".omp_offloading.img_end.") + Twine(T));
2147 llvm::Constant *Dev =
2148 llvm::ConstantStruct::get(DeviceImageTy, ImgBegin, ImgEnd,
2149 HostEntriesBegin, HostEntriesEnd,
nullptr);
2150 DeviceImagesEntires.push_back(Dev);
2154 llvm::ArrayType *DeviceImagesInitTy =
2155 llvm::ArrayType::get(DeviceImageTy, DeviceImagesEntires.size());
2156 llvm::Constant *DeviceImagesInit =
2157 llvm::ConstantArray::get(DeviceImagesInitTy, DeviceImagesEntires);
2159 llvm::GlobalVariable *DeviceImages =
new llvm::GlobalVariable(
2160 M, DeviceImagesInitTy,
true,
2162 ".omp_offloading.device_images");
2163 DeviceImages->setUnnamedAddr(
true);
2166 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.
Int32Ty),
2167 llvm::Constant::getNullValue(CGM.
Int32Ty)};
2170 auto *BinaryDescriptorTy = cast<llvm::StructType>(
2172 llvm::Constant *TargetRegionsDescriptorInit = llvm::ConstantStruct::get(
2173 BinaryDescriptorTy, llvm::ConstantInt::get(CGM.
Int32Ty, Devices.size()),
2174 llvm::ConstantExpr::getGetElementPtr(DeviceImagesInitTy, DeviceImages,
2176 HostEntriesBegin, HostEntriesEnd,
nullptr);
2178 auto *Desc =
new llvm::GlobalVariable(
2179 M, BinaryDescriptorTy,
true,
2181 ".omp_offloading.descriptor");
2188 auto *IdentInfo = &C.Idents.get(
".omp_offloading.reg_unreg_var");
2190 IdentInfo, C.CharTy);
2208 auto *TgtOffloadEntryType = cast<llvm::StructType>(
2210 llvm::LLVMContext &C = CGM.
getModule().getContext();
2214 llvm::Constant *AddrPtr = llvm::ConstantExpr::getBitCast(Addr, CGM.
VoidPtrTy);
2217 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
2219 llvm::GlobalVariable *Str =
2220 new llvm::GlobalVariable(M, StrPtrInit->getType(),
true,
2222 ".omp_offloading.entry_name");
2223 Str->setUnnamedAddr(
true);
2224 llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.
Int8PtrTy);
2227 llvm::Constant *EntryInit = llvm::ConstantStruct::get(
2228 TgtOffloadEntryType, AddrPtr, StrPtr,
2229 llvm::ConstantInt::get(CGM.
SizeTy, Size),
nullptr);
2230 llvm::GlobalVariable *Entry =
new llvm::GlobalVariable(
2232 EntryInit,
".omp_offloading.entry");
2235 Entry->setSection(
".omp_offloading.entries");
2238 Entry->setAlignment(1);
2257 llvm::LLVMContext &C = M.getContext();
2262 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
2265 auto getMDInt = [&](
unsigned v) {
2266 return llvm::ConstantAsMetadata::get(
2267 llvm::ConstantInt::get(llvm::Type::getInt32Ty(C),
v));
2270 auto getMDString = [&](StringRef
v) {
return llvm::MDString::get(C,
v); };
2273 auto &&TargetRegionMetadataEmitter = [&](
2274 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
unsigned Line,
2288 Ops.push_back(getMDInt(E.getKind()));
2289 Ops.push_back(getMDInt(DeviceID));
2290 Ops.push_back(getMDInt(FileID));
2291 Ops.push_back(getMDString(ParentName));
2292 Ops.push_back(getMDInt(Line));
2293 Ops.push_back(getMDInt(Column));
2294 Ops.push_back(getMDInt(E.getOrder()));
2297 OrderedEntries[E.getOrder()] = &
E;
2300 MD->addOperand(llvm::MDNode::get(C, Ops));
2304 TargetRegionMetadataEmitter);
2306 for (
auto *E : OrderedEntries) {
2307 assert(E &&
"All ordered entries must exist!");
2309 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
2311 assert(CE->getID() && CE->getAddress() &&
2312 "Entry ID and Addr are invalid!");
2315 llvm_unreachable(
"Unsupported entry kind.");
2335 llvm::LLVMContext
C;
2336 auto ME = llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(),
C);
2341 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
2345 for (
auto I : MD->operands()) {
2346 llvm::MDNode *MN = cast<llvm::MDNode>(
I);
2348 auto getMDInt = [&](
unsigned Idx) {
2349 llvm::ConstantAsMetadata *V =
2350 cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
2351 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
2354 auto getMDString = [&](
unsigned Idx) {
2355 llvm::MDString *V = cast<llvm::MDString>(MN->getOperand(Idx));
2356 return V->getString();
2359 switch (getMDInt(0)) {
2361 llvm_unreachable(
"Unexpected metadata!");
2366 getMDInt(1), getMDInt(2),
2367 getMDString(3), getMDInt(4),
2368 getMDInt(5), getMDInt(6));
2378 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
2381 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
2414 RD->completeDefinition();
2440 RD->completeDefinition();
2465 RD->completeDefinition();
2472 struct PrivateHelpersTy {
2473 PrivateHelpersTy(
const VarDecl *Original,
const VarDecl *PrivateCopy,
2474 const VarDecl *PrivateElemInit)
2475 : Original(Original), PrivateCopy(PrivateCopy),
2476 PrivateElemInit(PrivateElemInit) {}
2479 const VarDecl *PrivateElemInit;
2481 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
2486 if (!Privates.empty()) {
2493 for (
auto &&Pair : Privates) {
2494 auto *VD = Pair.second.Original;
2505 RD->completeDefinition();
2513 QualType KmpRoutineEntryPointerQTy) {
2527 RD->completeDefinition();
2545 RD->completeDefinition();
2570 Args.push_back(&GtidArg);
2571 Args.push_back(&TaskTypeArg);
2573 auto &TaskEntryFnInfo =
2591 auto *KmpTaskTWithPrivatesQTyRD =
2592 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
2595 auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
2596 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
2600 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
2606 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
2608 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
2611 PrivatesLVal.getPointer(), CGF.
VoidPtrTy);
2613 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
2616 llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam,
2617 TaskPrivatesMap, SharedsParam};
2629 QualType KmpTaskTWithPrivatesPtrQTy,
2630 QualType KmpTaskTWithPrivatesQTy) {
2637 Args.push_back(&GtidArg);
2638 Args.push_back(&TaskTypeArg);
2640 auto &DestructorFnInfo =
2644 auto *DestructorFn =
2646 ".omp_task_destructor.", &CGM.
getModule());
2656 auto *KmpTaskTWithPrivatesQTyRD =
2657 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
2658 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
2661 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
2662 if (
auto DtorKind = Field->getType().isDestructedType()) {
2664 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
2668 return DestructorFn;
2690 C,
nullptr, Loc,
nullptr,
2691 C.getPointerType(PrivatesQTy).withConst().withRestrict());
2692 Args.push_back(&TaskPrivatesArg);
2693 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
2694 unsigned Counter = 1;
2695 for (
auto *E: PrivateVars) {
2698 nullptr, C.getPointerType(C.getPointerType(E->getType()))
2701 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2702 PrivateVarsPos[VD] = Counter;
2705 for (
auto *E : FirstprivateVars) {
2708 nullptr, C.getPointerType(C.getPointerType(E->getType()))
2711 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2712 PrivateVarsPos[VD] = Counter;
2716 auto &TaskPrivatesMapFnInfo =
2719 auto *TaskPrivatesMapTy =
2723 ".omp_task_privates_map.", &CGM.
getModule());
2725 TaskPrivatesMapFnInfo);
2726 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
2730 TaskPrivatesMapFnInfo, Args);
2735 auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
2737 for (
auto *Field : PrivatesQTyRD->fields()) {
2739 auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
2747 return TaskPrivatesMap;
2751 const PrivateDataTy *P2) {
2752 return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
2757 bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
2764 ArrayRef<std::pair<OpenMPDependClauseKind, const Expr *>> Dependences) {
2770 auto I = PrivateCopies.begin();
2771 for (
auto *E : PrivateVars) {
2772 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2773 Privates.push_back(std::make_pair(
2775 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
2779 I = FirstprivateCopies.begin();
2780 auto IElemInitRef = FirstprivateInits.begin();
2781 for (
auto *E : FirstprivateVars) {
2782 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2783 Privates.push_back(std::make_pair(
2786 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
2787 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
2788 ++
I, ++IElemInitRef;
2790 llvm::array_pod_sort(Privates.begin(), Privates.end(),
2792 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
2802 auto *KmpTaskTWithPrivatesQTyRD =
2804 auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
2805 QualType KmpTaskTWithPrivatesPtrQTy =
2806 C.getPointerType(KmpTaskTWithPrivatesQTy);
2807 auto *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
2808 auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
2809 auto *KmpTaskTWithPrivatesTySize =
getTypeSize(CGF, KmpTaskTWithPrivatesQTy);
2810 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
2814 auto *TaskPrivatesMapTy =
2815 std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().
begin(),
2818 if (!Privates.empty()) {
2819 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
2821 CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(),
Privates);
2823 TaskPrivatesMap, TaskPrivatesMapTy);
2825 TaskPrivatesMap = llvm::ConstantPointerNull::get(
2826 cast<llvm::PointerType>(TaskPrivatesMapTy));
2831 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy,
2832 KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap);
2840 const unsigned TiedFlag = 0x1;
2841 const unsigned FinalFlag = 0x2;
2842 unsigned Flags = Tied ? TiedFlag : 0;
2845 ? CGF.
Builder.CreateSelect(Final.getPointer(),
2846 CGF.
Builder.getInt32(FinalFlag),
2848 : CGF.
Builder.getInt32(Final.getInt() ? FinalFlag : 0);
2849 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
2850 auto *SharedsSize = CGM.
getSize(C.getTypeSizeInChars(SharedsTy));
2853 KmpTaskTWithPrivatesTySize, SharedsSize,
2859 NewTask, KmpTaskTWithPrivatesPtrTy);
2861 KmpTaskTWithPrivatesQTy);
2871 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
2878 bool NeedsCleanup =
false;
2879 if (!Privates.empty()) {
2880 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
2882 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
2884 if (!FirstprivateVars.empty()) {
2892 for (
auto &&Pair : Privates) {
2893 auto *VD = Pair.second.PrivateCopy;
2897 if (
auto *Elem = Pair.second.PrivateElemInit) {
2898 auto *OriginalVD = Pair.second.Original;
2899 auto *SharedField = CapturesInfo.lookup(OriginalVD);
2900 auto SharedRefLValue =
2903 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
2908 if (!isa<CXXConstructExpr>(Init) ||
2912 SharedRefLValue.getAddress(),
Type);
2917 PrivateLValue.
getAddress(), SharedRefLValue.getAddress(),
2918 Type, [&CGF, Elem, Init, &CapturesInfo](
2922 InitScope.addPrivate(Elem, [SrcElement]() ->
Address {
2925 (void)InitScope.Privatize();
2928 CGF, &CapturesInfo);
2929 CGF.EmitAnyExprToMem(Init, DestElement,
2930 Init->getType().getQualifiers(),
2936 InitScope.addPrivate(Elem, [SharedRefLValue]() ->
Address {
2937 return SharedRefLValue.getAddress();
2939 (void)InitScope.Privatize();
2941 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
2945 CGF.EmitExprAsInit(Init, VD, PrivateLValue,
false);
2948 NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType();
2955 KmpTaskTWithPrivatesPtrQTy,
2956 KmpTaskTWithPrivatesQTy)
2957 : llvm::ConstantPointerNull::get(
2959 LValue Destructor = CGF.EmitLValueForField(
2960 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTDestructors));
2961 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2967 unsigned NumDependencies = Dependences.size();
2968 if (NumDependencies) {
2970 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
2971 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
2974 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy),
false);
2975 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy);
2977 KmpDependInfoRD = C.buildImplicitRecord(
"kmp_depend_info");
2989 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
2993 DependenciesArray = CGF.CreateMemTemp(KmpDependInfoArrayTy);
2994 for (
unsigned i = 0; i < NumDependencies; ++i) {
2995 const Expr *E = Dependences[i].second;
2996 auto Addr = CGF.EmitLValue(E);
3001 CGF.EmitOMPArraySectionExpr(ASE,
false);
3003 CGF.Builder.CreateConstGEP1_32(UpAddrLVal.
getPointer(), 1);
3005 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGM.
SizeTy);
3007 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3010 auto Base = CGF.MakeAddrLValue(
3011 CGF.Builder.CreateConstArrayGEP(DependenciesArray, i, DependencySize),
3014 auto BaseAddrLVal = CGF.EmitLValueForField(
3015 Base, *std::next(KmpDependInfoRD->
field_begin(), BaseAddr));
3016 CGF.EmitStoreOfScalar(
3017 CGF.Builder.CreatePtrToInt(Addr.getPointer(), CGF.IntPtrTy),
3020 auto LenLVal = CGF.EmitLValueForField(
3021 Base, *std::next(KmpDependInfoRD->
field_begin(), Len));
3022 CGF.EmitStoreOfScalar(Size, LenLVal);
3024 RTLDependenceKindTy DepKind;
3025 switch (Dependences[i].first) {
3026 case OMPC_DEPEND_in:
3030 case OMPC_DEPEND_out:
3031 case OMPC_DEPEND_inout:
3034 case OMPC_DEPEND_source:
3035 case OMPC_DEPEND_sink:
3037 llvm_unreachable(
"Unknown task dependence type");
3039 auto FlagsLVal = CGF.EmitLValueForField(
3040 Base, *std::next(KmpDependInfoRD->
field_begin(), Flags));
3041 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind),
3044 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3059 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
3061 if (NumDependencies) {
3062 DepTaskArgs[0] = UpLoc;
3063 DepTaskArgs[1] = ThreadID;
3064 DepTaskArgs[2] = NewTask;
3065 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies);
3066 DepTaskArgs[4] = DependenciesArray.
getPointer();
3067 DepTaskArgs[5] = CGF.Builder.getInt32(0);
3068 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3070 auto &&ThenCodeGen = [
this, NumDependencies,
3073 if (NumDependencies) {
3081 typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value>
3085 if (NumDependencies) {
3086 DepWaitTaskArgs[0] = UpLoc;
3087 DepWaitTaskArgs[1] = ThreadID;
3088 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
3089 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
3090 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
3091 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
3093 auto &&ElseCodeGen = [
this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
3100 if (NumDependencies)
3109 CGF.EHStack.pushCleanup<IfCallEndCleanup>(
3112 llvm::makeArrayRef(TaskArgs));
3115 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
3116 CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
3140 const Expr *,
const Expr *)> &RedOpGen,
3141 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
3142 const Expr *UpExpr =
nullptr) {
3155 auto LHSEnd = CGF.
Builder.CreateGEP(LHSBegin, NumElements);
3160 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
3161 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
3164 auto EntryBB = CGF.
Builder.GetInsertBlock();
3169 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
3170 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
3171 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
3176 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
3177 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
3178 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
3185 Scope.addPrivate(LHSVar, [=]() ->
Address {
return LHSElementCurrent; });
3186 Scope.addPrivate(RHSVar, [=]() ->
Address {
return RHSElementCurrent; });
3188 RedOpGen(CGF, XExpr, EExpr, UpExpr);
3189 Scope.ForceCleanup();
3192 auto LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
3193 LHSElementPHI, 1,
"omp.arraycpy.dest.element");
3194 auto RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
3195 RHSElementPHI, 1,
"omp.arraycpy.src.element");
3198 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
3199 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
3200 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
3201 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
3221 Args.push_back(&LHSArg);
3222 Args.push_back(&RHSArg);
3225 C.VoidTy, Args, EI,
false);
3228 ".omp.reduction.reduction_func", &CGM.
getModule());
3246 auto IPriv = Privates.begin();
3248 for (
unsigned I = 0, E = ReductionOps.size(); I <
E; ++
I, ++IPriv, ++Idx) {
3249 auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[
I])->getDecl());
3250 Scope.addPrivate(RHSVar, [&]() ->
Address {
3253 auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[
I])->getDecl());
3254 Scope.addPrivate(LHSVar, [&]() ->
Address {
3257 QualType PrivTy = (*IPriv)->getType();
3266 cast<OpaqueValueExpr>(
3273 IPriv = Privates.begin();
3274 auto ILHS = LHSExprs.begin();
3275 auto IRHS = RHSExprs.begin();
3276 for (
auto *E : ReductionOps) {
3277 if ((*IPriv)->getType()->isArrayType()) {
3279 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3280 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3284 const Expr *) { CGF.EmitIgnoredExpr(E); });
3288 ++IPriv, ++ILHS, ++IRHS;
3290 Scope.ForceCleanup();
3300 bool WithNowait,
bool SimpleReduction) {
3340 if (SimpleReduction) {
3342 auto IPriv = Privates.begin();
3343 auto ILHS = LHSExprs.begin();
3344 auto IRHS = RHSExprs.begin();
3345 for (
auto *E : ReductionOps) {
3346 if ((*IPriv)->getType()->isArrayType()) {
3347 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3348 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3350 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
3352 const Expr *) { CGF.EmitIgnoredExpr(E); });
3354 CGF.EmitIgnoredExpr(E);
3355 ++IPriv, ++ILHS, ++IRHS;
3362 auto Size = RHSExprs.size();
3363 for (
auto *E : Privates) {
3368 llvm::APInt ArraySize(32, Size);
3373 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
3374 auto IPriv = Privates.begin();
3376 for (
unsigned I = 0, E = RHSExprs.size(); I <
E; ++
I, ++IPriv, ++Idx) {
3383 if ((*IPriv)->getType()->isArrayType()) {
3392 (*IPriv)->getType()))
3403 LHSExprs, RHSExprs, ReductionOps);
3412 static_cast<OpenMPLocationFlags>(OMP_IDENT_KMPC | OMP_ATOMIC_REDUCE));
3414 auto *ReductionArrayTySize =
getTypeSize(CGF, ReductionArrayTy);
3421 CGF.
Builder.getInt32(RHSExprs.size()),
3422 ReductionArrayTySize,
3429 : OMPRTL__kmpc_reduce),
3434 auto *SwInst = CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
3443 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
3455 .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
3458 : OMPRTL__kmpc_end_reduce),
3459 llvm::makeArrayRef(EndArgs));
3460 auto IPriv = Privates.begin();
3461 auto ILHS = LHSExprs.begin();
3462 auto IRHS = RHSExprs.begin();
3463 for (
auto *E : ReductionOps) {
3464 if ((*IPriv)->getType()->isArrayType()) {
3466 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3467 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3469 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
3471 const Expr *) { CGF.EmitIgnoredExpr(E); });
3474 CGF.EmitIgnoredExpr(E);
3475 ++IPriv, ++ILHS, ++IRHS;
3487 SwInst->addCase(CGF.
Builder.getInt32(2), Case2BB);
3500 .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>(
3503 llvm::makeArrayRef(EndArgs));
3505 auto ILHS = LHSExprs.begin();
3506 auto IRHS = RHSExprs.begin();
3507 auto IPriv = Privates.begin();
3508 for (
auto *E : ReductionOps) {
3509 const Expr *XExpr =
nullptr;
3510 const Expr *EExpr =
nullptr;
3511 const Expr *UpExpr =
nullptr;
3513 if (
auto *BO = dyn_cast<BinaryOperator>(E)) {
3515 XExpr = BO->getLHS();
3516 UpExpr = BO->getRHS();
3520 auto *RHSExpr = UpExpr;
3523 if (
auto *ACO = dyn_cast<AbstractConditionalOperator>(
3524 RHSExpr->IgnoreParenImpCasts())) {
3527 RHSExpr = ACO->getCond();
3530 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
3531 EExpr = BORHS->getRHS();
3532 BO = BORHS->getOpcode();
3536 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3537 auto &&AtomicRedGen = [
this, BO, VD, IPriv,
3539 const Expr *EExpr,
const Expr *UpExpr) {
3540 LValue X = CGF.EmitLValue(XExpr);
3543 E = CGF.EmitAnyExpr(EExpr);
3544 CGF.EmitOMPAtomicSimpleUpdateExpr(
3545 X, E, BO,
true, llvm::Monotonic, Loc,
3546 [&CGF, UpExpr, VD, IPriv, Loc](
RValue XRValue) {
3548 PrivateScope.addPrivate(
3549 VD, [&CGF, VD, XRValue, Loc]() ->
Address {
3550 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
3551 CGF.emitOMPSimpleStore(
3552 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
3553 VD->getType().getNonReferenceType(), Loc);
3556 (void)PrivateScope.Privatize();
3557 return CGF.EmitAnyExpr(UpExpr);
3560 if ((*IPriv)->getType()->isArrayType()) {
3562 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3564 AtomicRedGen, XExpr, EExpr, UpExpr);
3567 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
3571 const Expr *,
const Expr *) {
3573 CGF,
".atomic_reduction",
3576 if ((*IPriv)->getType()->isArrayType()) {
3577 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
3578 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
3582 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
3584 ++ILHS, ++IRHS, ++IPriv;
3609 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
3625 if (CancelRegion == OMPD_parallel)
3626 CancelKind = CancelParallel;
3627 else if (CancelRegion == OMPD_for)
3628 CancelKind = CancelLoop;
3629 else if (CancelRegion == OMPD_sections)
3630 CancelKind = CancelSections;
3632 assert(CancelRegion == OMPD_taskgroup);
3633 CancelKind = CancelTaskgroup;
3645 if (
auto *OMPRegionInfo =
3647 if (OMPRegionInfo->hasCancel()) {
3660 auto *Cmp = CGF.
Builder.CreateIsNotNull(Result);
3661 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3681 if (
auto *OMPRegionInfo =
3683 auto &&ThenGen = [
this, Loc, CancelRegion,
3697 auto *Cmp = CGF.
Builder.CreateIsNotNull(Result);
3698 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
3719 unsigned &DeviceID,
unsigned &FileID,
3720 unsigned &LineNum,
unsigned &ColumnNum) {
3727 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
3728 assert(Loc.
isFileID() &&
"Source location is expected to refer to a file.");
3731 assert(PLoc.isValid() &&
"Source location is expected to be always valid.");
3733 llvm::sys::fs::UniqueID
ID;
3734 if (llvm::sys::fs::getUniqueID(PLoc.getFilename(),
ID))
3735 llvm_unreachable(
"Source file with target region no longer exists!");
3737 DeviceID = ID.getDevice();
3738 FileID = ID.getFile();
3739 LineNum = PLoc.getLine();
3740 ColumnNum = PLoc.getColumn();
3746 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
3747 bool IsOffloadEntry) {
3749 assert(!ParentName.empty() &&
"Invalid target region parent name!");
3755 CGF.
EmitStmt(CS.getCapturedStmt());
3777 llvm::raw_svector_ostream OS(EntryFnName);
3778 OS <<
".omp_offloading" << llvm::format(
".%x", DeviceID)
3779 << llvm::format(
".%x.", FileID) << ParentName <<
".l" << Line <<
".c"
3784 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
3791 if (!IsOffloadEntry)
3806 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.
Int8PtrTy);
3809 OutlinedFnID =
new llvm::GlobalVariable(
3811 llvm::GlobalValue::PrivateLinkage,
3812 llvm::Constant::getNullValue(CGM.
Int8Ty),
".omp_offload.region_id");
3816 DeviceID, FileID, ParentName, Line, Column, OutlinedFn, OutlinedFnID);
3824 const Expr *IfCond,
const Expr *Device,
3830 enum OpenMPOffloadMappingFlags {
3834 OMP_MAP_FROM = 0x02,
3838 OMP_MAP_BYCOPY = 0x80,
3841 enum OpenMPOffloadingReservedDeviceIDs {
3844 OMP_DEVICEID_UNDEF = -1,
3847 assert(OutlinedFn &&
"Invalid outlined function!");
3857 bool hasVLACaptures =
false;
3862 auto CV = CapturedVars.begin();
3865 CI != CE; ++CI, ++RI, ++CV) {
3874 if (CI->capturesVariableArrayType()) {
3875 BasePointer = Pointer = *CV;
3878 MapType = OMP_MAP_BYCOPY;
3879 hasVLACaptures =
true;
3880 }
else if (CI->capturesThis()) {
3881 BasePointer = Pointer = *CV;
3882 const PointerType *PtrTy = cast<PointerType>(RI->getType().getTypePtr());
3885 MapType = OMP_MAP_TO | OMP_MAP_FROM;
3886 }
else if (CI->capturesVariableByCopy()) {
3887 MapType = OMP_MAP_BYCOPY;
3888 if (!RI->getType()->isAnyPointerType()) {
3892 Ctx.getUIntPtrType(),
3893 Twine(CI->getCapturedVar()->getName()) +
".casted");
3897 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
3906 BasePointer = Pointer =
3909 MapType |= OMP_MAP_PTR;
3910 BasePointer = Pointer = *CV;
3914 assert(CI->capturesVariable() &&
"Expected captured reference.");
3915 BasePointer = Pointer = *CV;
3918 cast<ReferenceType>(RI->getType().getTypePtr());
3924 MapType = ElementType->isAggregateType() ? (OMP_MAP_TO | OMP_MAP_FROM)
3926 if (ElementType->isAnyPointerType())
3927 MapType |= OMP_MAP_PTR;
3930 BasePointers.push_back(BasePointer);
3931 Pointers.push_back(Pointer);
3932 Sizes.push_back(Size);
3933 MapTypes.push_back(MapType);
3937 auto OffloadErrorQType =
3938 Ctx.getIntTypeForBitwidth(32,
true);
3946 auto &&ThenGen = [
this, &Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes,
3947 hasVLACaptures, Device, OutlinedFnID, OffloadError,
3949 unsigned PointerNumVal = BasePointers.size();
3956 if (PointerNumVal) {
3957 llvm::APInt PointerNumAP(32, PointerNumVal,
true);
3958 QualType PointerArrayType = Ctx.getConstantArrayType(
3970 if (hasVLACaptures) {
3971 QualType SizeArrayType = Ctx.getConstantArrayType(
3980 for (
auto S : Sizes)
3981 ConstSizes.push_back(cast<llvm::Constant>(
S));
3983 auto *SizesArrayInit = llvm::ConstantArray::get(
3984 llvm::ArrayType::get(CGM.
SizeTy, ConstSizes.size()), ConstSizes);
3985 auto *SizesArrayGbl =
new llvm::GlobalVariable(
3986 CGM.
getModule(), SizesArrayInit->getType(),
3987 true, llvm::GlobalValue::PrivateLinkage,
3988 SizesArrayInit,
".offload_sizes");
3989 SizesArrayGbl->setUnnamedAddr(
true);
3990 SizesArray = SizesArrayGbl;
3995 llvm::Constant *MapTypesArrayInit =
3996 llvm::ConstantDataArray::get(CGF.
Builder.getContext(), MapTypes);
3997 auto *MapTypesArrayGbl =
new llvm::GlobalVariable(
3998 CGM.
getModule(), MapTypesArrayInit->getType(),
3999 true, llvm::GlobalValue::PrivateLinkage,
4000 MapTypesArrayInit,
".offload_maptypes");
4001 MapTypesArrayGbl->setUnnamedAddr(
true);
4002 MapTypesArray = MapTypesArrayGbl;
4004 for (
unsigned i = 0; i < PointerNumVal; ++i) {
4007 if (BPVal->getType()->isPointerTy())
4010 assert(BPVal->getType()->isIntegerTy() &&
4011 "If not a pointer, the value type must be an integer.");
4015 llvm::ArrayType::get(CGM.
VoidPtrTy, PointerNumVal),
4016 BasePointersArray, 0, i);
4017 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
4021 if (PVal->getType()->isPointerTy())
4024 assert(PVal->getType()->isIntegerTy() &&
4025 "If not a pointer, the value type must be an integer.");
4029 llvm::ArrayType::get(CGM.
VoidPtrTy, PointerNumVal), PointersArray,
4031 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
4034 if (hasVLACaptures) {
4036 llvm::ArrayType::get(CGM.
SizeTy, PointerNumVal), SizesArray,
4039 Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
4041 Sizes[i], CGM.
SizeTy,
true),
4046 BasePointersArray = CGF.
Builder.CreateConstInBoundsGEP2_32(
4047 llvm::ArrayType::get(CGM.
VoidPtrTy, PointerNumVal), BasePointersArray,
4049 PointersArray = CGF.
Builder.CreateConstInBoundsGEP2_32(
4050 llvm::ArrayType::get(CGM.
VoidPtrTy, PointerNumVal), PointersArray,
4053 SizesArray = CGF.
Builder.CreateConstInBoundsGEP2_32(
4054 llvm::ArrayType::get(CGM.
SizeTy, PointerNumVal), SizesArray,
4056 MapTypesArray = CGF.
Builder.CreateConstInBoundsGEP2_32(
4057 llvm::ArrayType::get(CGM.
Int32Ty, PointerNumVal), MapTypesArray,
4062 BasePointersArray = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
4063 PointersArray = llvm::ConstantPointerNull::get(CGM.
VoidPtrPtrTy);
4064 SizesArray = llvm::ConstantPointerNull::get(CGM.
SizeTy->getPointerTo());
4066 llvm::ConstantPointerNull::get(CGM.
Int32Ty->getPointerTo());
4079 assert(OutlinedFnID &&
"Invalid outlined function ID!");
4087 DeviceID = CGF.
Builder.getInt32(OMP_DEVICEID_UNDEF);
4090 DeviceID, OutlinedFnID, PointerNum, BasePointersArray,
4091 PointersArray, SizesArray, MapTypesArray};
4099 auto &&ElseGen = [
this, OffloadError,
4125 auto Failed = CGF.
Builder.CreateIsNotNull(OffloadErrorVal);
4126 CGF.
Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
4129 CGF.
Builder.CreateCall(OutlinedFn, BasePointers);
4137 StringRef ParentName) {
4144 bool isTargetDirective = isa<OMPTargetDirective>(
S);
4146 if (isTargetDirective) {
4147 auto *E = cast<OMPExecutableDirective>(
S);
4158 DeviceID, FileID, ParentName, Line, Column))
4162 llvm::Constant *Addr;
4165 assert(Fn && Addr &&
"Target region emission failed.");
4170 if (!E->getAssociatedStmt())
4174 cast<CapturedStmt>(E->getAssociatedStmt())->getCapturedStmt(),
4180 if (
auto *L = dyn_cast<LambdaExpr>(S))
4184 for (
auto *II : S->children())
4191 auto &FD = *cast<FunctionDecl>(GD.
getDecl());
4216 for (
auto *Ctor : RD->ctors()) {
4217 StringRef ParentName =
4221 auto *Dtor = RD->getDestructor();
4223 StringRef ParentName =
4236 if (isa<FunctionDecl>(VD))
llvm::PointerType * Int8PtrPtrTy
void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
void pushTerminate()
Push a terminate handler on the stack.
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ArrayRef< const Expr * > PrivateVars, ArrayRef< const Expr * > FirstprivateVars, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Value *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device, ArrayRef< llvm::Value * > CapturedVars)
Emit the target offloading code associated with D.
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
llvm::IntegerType * IntTy
int
External linkage, which indicates that the entity can be referred to from other translation units...
PointerType - C99 6.7.5.1 - Pointer Declarators.
A (possibly-)qualified type.
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
llvm::Value * getPointer() const
CodeGenTypes & getTypes()
unsigned getColumn() const
Return the presumed column number of this location.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after...
llvm::Module & getModule() const
AlignmentSource
The source of the alignment of an l-value; an expression of confidence in the alignment actually matc...
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
virtual llvm::Value * emitParallelOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateTempAlloca - This creates a alloca and inserts it into the entry block.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
QualType getTgtDeviceImageQTy()
Returns __tgt_device_image type.
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
Address getAddress() const
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
QualType getRecordType(const RecordDecl *Decl) const
std::unique_ptr< llvm::MemoryBuffer > Buffer
virtual void completeDefinition()
completeDefinition - Notes that the definition of this type is now complete.
QualType TgtOffloadEntryQTy
Type struct __tgt_offload_entry{ void *addr; // Pointer to the offload entry info.
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
const CGFunctionInfo & arrangeFreeFunctionDeclaration(QualType ResTy, const FunctionArgList &Args, const FunctionType::ExtInfo &Info, bool isVariadic)
Target region entries related.
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS...
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
VarDecl - An instance of this class is created to represent a variable declaration or definition...
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Extra information about a function prototype.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
field_iterator field_begin() const
llvm::StringMap< llvm::AssertingVH< llvm::Constant >, llvm::BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const
Return the store size, in character units, of the given LLVM type.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
CharUnits getNaturalTypeAlignment(QualType T, AlignmentSource *Source=nullptr, bool forPointeeType=false)
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
llvm::Type * ConvertTypeForMem(QualType T)
ConvertTypeForMem - Convert type T into a llvm::Type.
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
static llvm::Value * getTypeSize(CodeGenFunction &CGF, QualType Ty)
llvm::Constant * getPointer() const
RecordDecl - Represents a struct/union/class.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
String describing the source location.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Expr * getSizeExpr() const
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr, CGOpenMPRuntime::IdentFieldIndex Field, const llvm::Twine &Name="")
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
llvm::IntegerType * Int64Ty
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
FieldDecl - An instance of this class is created by Sema::ActOnField to represent a member of a struc...
llvm::IntegerType * SizeTy
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
llvm::CallInst * EmitRuntimeCall(llvm::Value *callee, const Twine &name="")
void startDefinition()
Starts the definition of this tag declaration.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPScheduleClauseKind SchedKind, unsigned IVSize, bool IVSigned, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk=nullptr)
Call the appropriate runtime routine to initialize it before start of loop.
static LValue emitLoadOfPointerLValue(CodeGenFunction &CGF, Address PtrAddr, QualType Ty)
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable. ...
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, ArrayRef< llvm::Value * > Args, const Twine &Name="")
Emits a call or invoke instruction to the given function, depending on the current state of the EH st...
static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
QualType KmpRoutineEntryPtrQTy
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata...
void InitTempAlloca(Address Alloca, llvm::Value *Value)
InitTempAlloca - Provide an initial value for the given alloca which will be observable at all locati...
const Decl * getDecl() const
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and and emit all target regions found along the way.
llvm::ArrayType * KmpCriticalNameTy
Type kmp_critical_name, originally defined as typedef kmp_int32 kmp_critical_name[8];.
llvm::Function * createOffloadingBinaryDescriptorRegistration()
Creates and registers offloading binary descriptor for the current compilation unit.
CGOpenMPRuntime(CodeGenModule &CGM)
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
const TargetInfo & getTargetInfo() const
static CharUnits getIdentAlign(CodeGenModule &CGM)
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
QualType TgtDeviceImageQTy
struct __tgt_device_image{ void *ImageStart; // Pointer to the target code start. ...
RAII for correct setting/restoring of CapturedStmtInfo.
llvm::PointerType * VoidPtrTy
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned ColNum, llvm::Constant *Addr, llvm::Constant *ID)
Register target region entry.
llvm::FunctionType * Kmpc_MicroTy
The type for a microtask which gets passed to __kmpc_fork_call().
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
RecordDecl * getDecl() const
CharUnits getPointerSize() const
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy)
EmitAggregateCopy - Emit an aggregate assignment.
Scope - A scope is a transient data structure that is used while parsing the program.
llvm::PointerType * VoidPtrPtrTy
virtual void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void EmitStmt(const Stmt *S)
EmitStmt - Emit the code for the statement.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
llvm::Constant * createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPScheduleClauseKind SchedKind, unsigned IVSize, bool IVSigned, bool Ordered, llvm::Value *UB, llvm::Value *Chunk=nullptr)
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
unsigned getLine() const
Return the presumed line number of this location.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
detail::InMemoryDirectory::const_iterator I
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param PrivateVars List of references to private variables for the task *directive *param PrivateCopies List of private copies for each private variable in *p PrivateVars *param FirstprivateVars List of references to private variables for the *task directive *param FirstprivateCopies List of private copies for each private variable *in p FirstprivateVars *param FirstprivateInits List of references to auto generated variables *used for initialization of a single array element Used if firstprivate *variable is of array type *param Dependences List of dependences for the task including *original expression and dependency type *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, bool Tied, llvm::PointerIntPair< llvm::Value *, 1, bool > Final, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, ArrayRef< const Expr * > PrivateVars, ArrayRef< const Expr * > PrivateCopies, ArrayRef< const Expr * > FirstprivateVars, ArrayRef< const Expr * > FirstprivateCopies, ArrayRef< const Expr * > FirstprivateInits, ArrayRef< std::pair< OpenMPDependClauseKind, const Expr * >> Dependences)
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
Describes the capture of either a variable, or 'this', or variable-length array type.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
ID
Defines the set of possible language-specific address spaces.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags=OMP_IDENT_KMPC)
Emits object of ident_t type with info for source location.
KmpTaskTFields
Indexes of fields for type kmp_task_t.
llvm::Value * getPointer() const
Expr - This represents one expression.
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
QualType getTgtBinaryDescriptorQTy()
Returns __tgt_bin_desc type.
CGCXXABI & getCXXABI() const
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
llvm::Constant * createRuntimeFunction(OpenMPRTLFunction Function)
Returns specified OpenMP runtime function.
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
const CGFunctionInfo & arrangeNullaryFunction()
getNullaryFunctionInfo - Get the function info for a void() function with standard CC...
std::string OMPHostIRFile
Name of the IR file that contains the result of the OpenMP target host code generation.
static llvm::Function * createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name, const RegionCodeGenTy &Codegen)
Create a Ctor/Dtor-like function whose body is emitted through Codegen.
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, OpenMPDirectiveKind Kind, SourceLocation Loc, const RegionCodeGenTy &BodyOpGen)
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
ASTContext & getContext() const
CharUnits getNaturalPointeeTypeAlignment(QualType T, AlignmentSource *Source=nullptr)
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
virtual void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D, llvm::Constant *Dtor, llvm::Constant *Addr)=0
Emit code to force the execution of a destructor during global teardown.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
llvm::IntegerType * Int32Ty
static llvm::Value * emitReductionFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
QualType getTgtOffloadEntryQTy()
Returns __tgt_offload_entry type.
llvm::Function * GenerateCapturedStmtFunction(const CapturedStmt &S)
Creates the outlined function for a CapturedStmt.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Represents an unpacked "presumed" location which can be presented to the user.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep)
Creates clause with a list of variables VL and a linear step Step.
The result type of a method or function.
void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned ColNum, unsigned Order)
Initialize target region entry.
OpenMPProcBindClauseKind
OpenMP attributes for 'proc_bind' clause.
GlobalDecl - represents a global declaration.
const Expr * getAnyInitializer() const
getAnyInitializer - Get the initializer for this variable, no matter which declaration it is attached...
SourceLocation getLocStart() const
Returns starting location of directive kind.
The l-value was considered opaque, so the alignment was determined from a type.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeSet ExtraAttrs=llvm::AttributeSet())
Create a new runtime function with the specified type and name.
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
This captures a statement into a function.
const char * getFilename() const
Return the presumed filename of this location.
ASTContext & getContext() const
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup...
Encodes a location in the source.
CharUnits getPointerAlign() const
This is a basic class for representing single OpenMP executable directive.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
bool isValid() const
Return true if this is a valid SourceLocation object.
Lower bound for 'ordered' versions.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
OpenMPDirectiveKind
OpenMP directives.
llvm::Constant * createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
llvm::Constant * createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
static bool classof(const EHScope *Scope)
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
const CodeGenOptions & getCodeGenOpts() const
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
const LangOptions & getLangOpts() const
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
llvm::function_ref< void(CodeGenFunction &)> RegionCodeGenTy
bool empty() const
Return true if a there are no entries defined.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
OpenMPDebugLocMapTy OpenMPDebugLocMap
QualType getPointeeType() const
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned ColNum) const
Return true if a target region entry with the provided information exists.
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
llvm::Type * KmpRoutineEntryPtrTy
Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
U cast(CodeGen::Address addr)
detail::InMemoryDirectory::const_iterator E
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry)
Emit outilined function for 'target' directive.
const RecordType * getAsStructureType() const
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
const VariableArrayType * getAsVariableArrayType(QualType T) const
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
static CharUnits getIdentSize(CodeGenModule &CGM)
API for captured statement code generation.
llvm::PointerType * getType() const
Return the type of the pointer value.
static bool classof(const OMPClause *T)
This file defines OpenMP AST classes for executable directives and clauses.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T)
Address CreateConstArrayGEP(Address Addr, uint64_t Index, CharUnits EltSize, const llvm::Twine &Name="")
Given addr = [n x T]* ...
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
unsigned size() const
Return number of entries defined so far.
llvm::PointerType * Int8PtrTy
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
virtual llvm::Value * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the OpenMP task directive D.
Base for LValueReferenceType and RValueReferenceType.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
StringRef getMangledName(GlobalDecl GD)
QualType withRestrict() const
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void addDecl(Decl *D)
Add the declaration D into this context.
QualType getPointeeType() const
SourceManager & getSourceManager()
bool isTLSSupported() const
Whether the target supports thread-local storage.
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
llvm::DenseSet< const VarDecl * > ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Lower bound for default (unordered) versions.
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, unsigned &DeviceID, unsigned &FileID, unsigned &LineNum, unsigned &ColumnNum)
Obtain information that uniquely identifies a target entry.
void createOffloadEntry(llvm::Constant *Addr, StringRef Name, uint64_t Size)
Creates offloading entry for the provided address Addr, name Name and size Size.
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, bool WithNowait, bool SimpleReduction)
Emit a code for reduction clause.
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
void loadOffloadInfoMetadata()
Loads all the offload entries information from the host IR metadata.
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false)
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
llvm::Type * ConvertType(QualType T)
Privates[]
Gets the list of initial values for linear variables.
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
void popTerminate()
Pops a terminate handler off the stack.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
std::pair< llvm::Value *, QualType > getVLASize(const VariableArrayType *vla)
getVLASize - Returns an LLVM value that corresponds to the size, in non-variably-sized elements...
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
virtual llvm::Function * emitRegistrationFunction()
Creates the offloading descriptor in the event any target region was emitted in the current module an...
static RValue get(llvm::Value *V)
static llvm::Value * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Value *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
static int array_pod_sort_comparator(const PrivateDataTy *P1, const PrivateDataTy *P2)
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
llvm::Constant * createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
LValue - This represents an lvalue references.
Information for lazily generating a cleanup.
void setAccess(AccessSpecifier AS)
bool isNull() const
Return true if this QualType doesn't point to a type yet.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
static CharUnits getOffsetOfIdentField(CGOpenMPRuntime::IdentFieldIndex Field)
A class which abstracts out some details necessary for making a call.
llvm::StructType * IdentTy
QualType TgtBinaryDescriptorQTy
struct __tgt_bin_desc{ int32_t NumDevices; // Number of devices supported.
bool isPointerType() const
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.