clang  3.8.0
CGAtomic.cpp
Go to the documentation of this file.
1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the code for emitting atomic operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CodeGenFunction.h"
15 #include "CGCall.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenModule.h"
18 #include "clang/AST/ASTContext.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/Intrinsics.h"
23 #include "llvm/IR/Operator.h"
24 
25 using namespace clang;
26 using namespace CodeGen;
27 
28 namespace {
29  class AtomicInfo {
30  CodeGenFunction &CGF;
31  QualType AtomicTy;
32  QualType ValueTy;
33  uint64_t AtomicSizeInBits;
34  uint64_t ValueSizeInBits;
35  CharUnits AtomicAlign;
36  CharUnits ValueAlign;
37  CharUnits LValueAlign;
38  TypeEvaluationKind EvaluationKind;
39  bool UseLibcall;
40  LValue LVal;
41  CGBitFieldInfo BFI;
42  public:
43  AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44  : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45  EvaluationKind(TEK_Scalar), UseLibcall(true) {
46  assert(!lvalue.isGlobalReg());
47  ASTContext &C = CGF.getContext();
48  if (lvalue.isSimple()) {
49  AtomicTy = lvalue.getType();
50  if (auto *ATy = AtomicTy->getAs<AtomicType>())
51  ValueTy = ATy->getValueType();
52  else
53  ValueTy = AtomicTy;
54  EvaluationKind = CGF.getEvaluationKind(ValueTy);
55 
56  uint64_t ValueAlignInBits;
57  uint64_t AtomicAlignInBits;
58  TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59  ValueSizeInBits = ValueTI.Width;
60  ValueAlignInBits = ValueTI.Align;
61 
62  TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63  AtomicSizeInBits = AtomicTI.Width;
64  AtomicAlignInBits = AtomicTI.Align;
65 
66  assert(ValueSizeInBits <= AtomicSizeInBits);
67  assert(ValueAlignInBits <= AtomicAlignInBits);
68 
69  AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70  ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71  if (lvalue.getAlignment().isZero())
72  lvalue.setAlignment(AtomicAlign);
73 
74  LVal = lvalue;
75  } else if (lvalue.isBitField()) {
76  ValueTy = lvalue.getType();
77  ValueSizeInBits = C.getTypeSize(ValueTy);
78  auto &OrigBFI = lvalue.getBitFieldInfo();
79  auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80  AtomicSizeInBits = C.toBits(
81  C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82  .RoundUpToAlignment(lvalue.getAlignment()));
83  auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84  auto OffsetInChars =
85  (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86  lvalue.getAlignment();
87  VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88  VoidPtrAddr, OffsetInChars.getQuantity());
90  VoidPtrAddr,
91  CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92  "atomic_bitfield_base");
93  BFI = OrigBFI;
94  BFI.Offset = Offset;
95  BFI.StorageSize = AtomicSizeInBits;
96  BFI.StorageOffset += OffsetInChars;
97  LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98  BFI, lvalue.getType(),
99  lvalue.getAlignmentSource());
100  LVal.setTBAAInfo(lvalue.getTBAAInfo());
101  AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102  if (AtomicTy.isNull()) {
103  llvm::APInt Size(
104  /*numBits=*/32,
105  C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106  AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107  /*IndexTypeQuals=*/0);
108  }
109  AtomicAlign = ValueAlign = lvalue.getAlignment();
110  } else if (lvalue.isVectorElt()) {
111  ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112  ValueSizeInBits = C.getTypeSize(ValueTy);
113  AtomicTy = lvalue.getType();
114  AtomicSizeInBits = C.getTypeSize(AtomicTy);
115  AtomicAlign = ValueAlign = lvalue.getAlignment();
116  LVal = lvalue;
117  } else {
118  assert(lvalue.isExtVectorElt());
119  ValueTy = lvalue.getType();
120  ValueSizeInBits = C.getTypeSize(ValueTy);
121  AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122  lvalue.getType(), lvalue.getExtVectorAddress()
123  .getElementType()->getVectorNumElements());
124  AtomicSizeInBits = C.getTypeSize(AtomicTy);
125  AtomicAlign = ValueAlign = lvalue.getAlignment();
126  LVal = lvalue;
127  }
128  UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129  AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130  }
131 
132  QualType getAtomicType() const { return AtomicTy; }
133  QualType getValueType() const { return ValueTy; }
134  CharUnits getAtomicAlignment() const { return AtomicAlign; }
135  CharUnits getValueAlignment() const { return ValueAlign; }
136  uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137  uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138  TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139  bool shouldUseLibcall() const { return UseLibcall; }
140  const LValue &getAtomicLValue() const { return LVal; }
141  llvm::Value *getAtomicPointer() const {
142  if (LVal.isSimple())
143  return LVal.getPointer();
144  else if (LVal.isBitField())
145  return LVal.getBitFieldPointer();
146  else if (LVal.isVectorElt())
147  return LVal.getVectorPointer();
148  assert(LVal.isExtVectorElt());
149  return LVal.getExtVectorPointer();
150  }
151  Address getAtomicAddress() const {
152  return Address(getAtomicPointer(), getAtomicAlignment());
153  }
154 
155  Address getAtomicAddressAsAtomicIntPointer() const {
156  return emitCastToAtomicIntPointer(getAtomicAddress());
157  }
158 
159  /// Is the atomic size larger than the underlying value type?
160  ///
161  /// Note that the absence of padding does not mean that atomic
162  /// objects are completely interchangeable with non-atomic
163  /// objects: we might have promoted the alignment of a type
164  /// without making it bigger.
165  bool hasPadding() const {
166  return (ValueSizeInBits != AtomicSizeInBits);
167  }
168 
169  bool emitMemSetZeroIfNecessary() const;
170 
171  llvm::Value *getAtomicSizeValue() const {
172  CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173  return CGF.CGM.getSize(size);
174  }
175 
176  /// Cast the given pointer to an integer pointer suitable for atomic
177  /// operations if the source.
178  Address emitCastToAtomicIntPointer(Address Addr) const;
179 
180  /// If Addr is compatible with the iN that will be used for an atomic
181  /// operation, bitcast it. Otherwise, create a temporary that is suitable
182  /// and copy the value across.
183  Address convertToAtomicIntPointer(Address Addr) const;
184 
185  /// Turn an atomic-layout object into an r-value.
186  RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187  SourceLocation loc, bool AsValue) const;
188 
189  /// \brief Converts a rvalue to integer value.
190  llvm::Value *convertRValueToInt(RValue RVal) const;
191 
192  RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193  AggValueSlot ResultSlot,
194  SourceLocation Loc, bool AsValue) const;
195 
196  /// Copy an atomic r-value into atomic-layout memory.
197  void emitCopyIntoMemory(RValue rvalue) const;
198 
199  /// Project an l-value down to the value field.
200  LValue projectValue() const {
201  assert(LVal.isSimple());
202  Address addr = getAtomicAddress();
203  if (hasPadding())
204  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205 
206  return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207  LVal.getAlignmentSource(), LVal.getTBAAInfo());
208  }
209 
210  /// \brief Emits atomic load.
211  /// \returns Loaded value.
212  RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213  bool AsValue, llvm::AtomicOrdering AO,
214  bool IsVolatile);
215 
216  /// \brief Emits atomic compare-and-exchange sequence.
217  /// \param Expected Expected value.
218  /// \param Desired Desired value.
219  /// \param Success Atomic ordering for success operation.
220  /// \param Failure Atomic ordering for failed operation.
221  /// \param IsWeak true if atomic operation is weak, false otherwise.
222  /// \returns Pair of values: previous value from storage (value type) and
223  /// boolean flag (i1 type) with true if success and false otherwise.
224  std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
225  RValue Expected, RValue Desired,
226  llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
227  llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
228  bool IsWeak = false);
229 
230  /// \brief Emits atomic update.
231  /// \param AO Atomic ordering.
232  /// \param UpdateOp Update operation for the current lvalue.
233  void EmitAtomicUpdate(llvm::AtomicOrdering AO,
234  const llvm::function_ref<RValue(RValue)> &UpdateOp,
235  bool IsVolatile);
236  /// \brief Emits atomic update.
237  /// \param AO Atomic ordering.
238  void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
239  bool IsVolatile);
240 
241  /// Materialize an atomic r-value in atomic-layout memory.
242  Address materializeRValue(RValue rvalue) const;
243 
244  /// \brief Translates LLVM atomic ordering to GNU atomic ordering for
245  /// libcalls.
247  translateAtomicOrdering(const llvm::AtomicOrdering AO);
248 
249  /// \brief Creates temp alloca for intermediate operations on atomic value.
250  Address CreateTempAlloca() const;
251  private:
252  bool requiresMemSetZero(llvm::Type *type) const;
253 
254 
255  /// \brief Emits atomic load as a libcall.
256  void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
257  llvm::AtomicOrdering AO, bool IsVolatile);
258  /// \brief Emits atomic load as LLVM instruction.
259  llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
260  /// \brief Emits atomic compare-and-exchange op as a libcall.
261  llvm::Value *EmitAtomicCompareExchangeLibcall(
262  llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
263  llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
264  llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent);
265  /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
266  std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
267  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
268  llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
269  llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
270  bool IsWeak = false);
271  /// \brief Emit atomic update as libcalls.
272  void
273  EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
274  const llvm::function_ref<RValue(RValue)> &UpdateOp,
275  bool IsVolatile);
276  /// \brief Emit atomic update as LLVM instructions.
277  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
278  const llvm::function_ref<RValue(RValue)> &UpdateOp,
279  bool IsVolatile);
280  /// \brief Emit atomic update as libcalls.
281  void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
282  bool IsVolatile);
283  /// \brief Emit atomic update as LLVM instructions.
284  void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
285  bool IsVolatile);
286  };
287 }
288 
290 AtomicInfo::translateAtomicOrdering(const llvm::AtomicOrdering AO) {
291  switch (AO) {
292  case llvm::Unordered:
293  case llvm::NotAtomic:
294  case llvm::Monotonic:
296  case llvm::Acquire:
298  case llvm::Release:
300  case llvm::AcquireRelease:
302  case llvm::SequentiallyConsistent:
304  }
305  llvm_unreachable("Unhandled AtomicOrdering");
306 }
307 
308 Address AtomicInfo::CreateTempAlloca() const {
309  Address TempAlloca = CGF.CreateMemTemp(
310  (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
311  : AtomicTy,
312  getAtomicAlignment(),
313  "atomic-temp");
314  // Cast to pointer to value type for bitfields.
315  if (LVal.isBitField())
317  TempAlloca, getAtomicAddress().getType());
318  return TempAlloca;
319 }
320 
322  StringRef fnName,
323  QualType resultType,
324  CallArgList &args) {
325  const CGFunctionInfo &fnInfo =
326  CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
328  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
329  llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
330  return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
331 }
332 
333 /// Does a store of the given IR type modify the full expected width?
335  uint64_t expectedSize) {
336  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
337 }
338 
339 /// Does the atomic type require memsetting to zero before initialization?
340 ///
341 /// The IR type is provided as a way of making certain queries faster.
342 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343  // If the atomic type has size padding, we definitely need a memset.
344  if (hasPadding()) return true;
345 
346  // Otherwise, do some simple heuristics to try to avoid it:
347  switch (getEvaluationKind()) {
348  // For scalars and complexes, check whether the store size of the
349  // type uses the full size.
350  case TEK_Scalar:
351  return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
352  case TEK_Complex:
353  return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
354  AtomicSizeInBits / 2);
355 
356  // Padding in structs has an undefined bit pattern. User beware.
357  case TEK_Aggregate:
358  return false;
359  }
360  llvm_unreachable("bad evaluation kind");
361 }
362 
363 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364  assert(LVal.isSimple());
365  llvm::Value *addr = LVal.getPointer();
366  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
367  return false;
368 
369  CGF.Builder.CreateMemSet(
370  addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
371  CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
372  LVal.getAlignment().getQuantity());
373  return true;
374 }
375 
376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377  Address Dest, Address Ptr,
378  Address Val1, Address Val2,
379  uint64_t Size,
380  llvm::AtomicOrdering SuccessOrder,
381  llvm::AtomicOrdering FailureOrder) {
382  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
383  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
384  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
385 
386  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
387  Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
388  Pair->setVolatile(E->isVolatile());
389  Pair->setWeak(IsWeak);
390 
391  // Cmp holds the result of the compare-exchange operation: true on success,
392  // false on failure.
393  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395 
396  // This basic block is used to hold the store instruction if the operation
397  // failed.
398  llvm::BasicBlock *StoreExpectedBB =
399  CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400 
401  // This basic block is the exit point of the operation, we should end up
402  // here regardless of whether or not the operation succeeded.
403  llvm::BasicBlock *ContinueBB =
404  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405 
406  // Update Expected if Expected isn't equal to Old, otherwise branch to the
407  // exit point.
408  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409 
410  CGF.Builder.SetInsertPoint(StoreExpectedBB);
411  // Update the memory at Expected with Old's value.
412  CGF.Builder.CreateStore(Old, Val1);
413  // Finally, branch to the exit point.
414  CGF.Builder.CreateBr(ContinueBB);
415 
416  CGF.Builder.SetInsertPoint(ContinueBB);
417  // Update the memory at Dest with Cmp's value.
418  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419 }
420 
421 /// Given an ordering required on success, emit all possible cmpxchg
422 /// instructions to cope with the provided (but possibly only dynamically known)
423 /// FailureOrder.
425  bool IsWeak, Address Dest,
426  Address Ptr, Address Val1,
427  Address Val2,
428  llvm::Value *FailureOrderVal,
429  uint64_t Size,
430  llvm::AtomicOrdering SuccessOrder) {
431  llvm::AtomicOrdering FailureOrder;
432  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433  switch (FO->getSExtValue()) {
434  default:
435  FailureOrder = llvm::Monotonic;
436  break;
439  FailureOrder = llvm::Acquire;
440  break;
442  FailureOrder = llvm::SequentiallyConsistent;
443  break;
444  }
445  if (FailureOrder >= SuccessOrder) {
446  // Don't assert on undefined behaviour.
447  FailureOrder =
448  llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
449  }
450  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size,
451  SuccessOrder, FailureOrder);
452  return;
453  }
454 
455  // Create all the relevant BB's
456  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
457  *SeqCstBB = nullptr;
458  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
459  if (SuccessOrder != llvm::Monotonic && SuccessOrder != llvm::Release)
460  AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
461  if (SuccessOrder == llvm::SequentiallyConsistent)
462  SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
463 
464  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
465 
466  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
467 
468  // Emit all the different atomics
469 
470  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
471  // doesn't matter unless someone is crazy enough to use something that
472  // doesn't fold to a constant for the ordering.
473  CGF.Builder.SetInsertPoint(MonotonicBB);
474  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
475  Size, SuccessOrder, llvm::Monotonic);
476  CGF.Builder.CreateBr(ContBB);
477 
478  if (AcquireBB) {
479  CGF.Builder.SetInsertPoint(AcquireBB);
480  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
481  Size, SuccessOrder, llvm::Acquire);
482  CGF.Builder.CreateBr(ContBB);
483  SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
484  AcquireBB);
485  SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
486  AcquireBB);
487  }
488  if (SeqCstBB) {
489  CGF.Builder.SetInsertPoint(SeqCstBB);
490  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
491  Size, SuccessOrder, llvm::SequentiallyConsistent);
492  CGF.Builder.CreateBr(ContBB);
493  SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
494  SeqCstBB);
495  }
496 
497  CGF.Builder.SetInsertPoint(ContBB);
498 }
499 
501  Address Ptr, Address Val1, Address Val2,
502  llvm::Value *IsWeak, llvm::Value *FailureOrder,
503  uint64_t Size, llvm::AtomicOrdering Order) {
504  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
505  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
506 
507  switch (E->getOp()) {
508  case AtomicExpr::AO__c11_atomic_init:
509  llvm_unreachable("Already handled!");
510 
511  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
512  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
513  FailureOrder, Size, Order);
514  return;
515  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
516  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
517  FailureOrder, Size, Order);
518  return;
519  case AtomicExpr::AO__atomic_compare_exchange:
520  case AtomicExpr::AO__atomic_compare_exchange_n: {
521  if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
522  emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
523  Val1, Val2, FailureOrder, Size, Order);
524  } else {
525  // Create all the relevant BB's
526  llvm::BasicBlock *StrongBB =
527  CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
528  llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
529  llvm::BasicBlock *ContBB =
530  CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
531 
532  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
533  SI->addCase(CGF.Builder.getInt1(false), StrongBB);
534 
535  CGF.Builder.SetInsertPoint(StrongBB);
536  emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
537  FailureOrder, Size, Order);
538  CGF.Builder.CreateBr(ContBB);
539 
540  CGF.Builder.SetInsertPoint(WeakBB);
541  emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
542  FailureOrder, Size, Order);
543  CGF.Builder.CreateBr(ContBB);
544 
545  CGF.Builder.SetInsertPoint(ContBB);
546  }
547  return;
548  }
549  case AtomicExpr::AO__c11_atomic_load:
550  case AtomicExpr::AO__atomic_load_n:
551  case AtomicExpr::AO__atomic_load: {
552  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
553  Load->setAtomic(Order);
554  Load->setVolatile(E->isVolatile());
555  CGF.Builder.CreateStore(Load, Dest);
556  return;
557  }
558 
559  case AtomicExpr::AO__c11_atomic_store:
560  case AtomicExpr::AO__atomic_store:
561  case AtomicExpr::AO__atomic_store_n: {
562  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
563  llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
564  Store->setAtomic(Order);
565  Store->setVolatile(E->isVolatile());
566  return;
567  }
568 
569  case AtomicExpr::AO__c11_atomic_exchange:
570  case AtomicExpr::AO__atomic_exchange_n:
571  case AtomicExpr::AO__atomic_exchange:
572  Op = llvm::AtomicRMWInst::Xchg;
573  break;
574 
575  case AtomicExpr::AO__atomic_add_fetch:
576  PostOp = llvm::Instruction::Add;
577  // Fall through.
578  case AtomicExpr::AO__c11_atomic_fetch_add:
579  case AtomicExpr::AO__atomic_fetch_add:
580  Op = llvm::AtomicRMWInst::Add;
581  break;
582 
583  case AtomicExpr::AO__atomic_sub_fetch:
584  PostOp = llvm::Instruction::Sub;
585  // Fall through.
586  case AtomicExpr::AO__c11_atomic_fetch_sub:
587  case AtomicExpr::AO__atomic_fetch_sub:
588  Op = llvm::AtomicRMWInst::Sub;
589  break;
590 
591  case AtomicExpr::AO__atomic_and_fetch:
592  PostOp = llvm::Instruction::And;
593  // Fall through.
594  case AtomicExpr::AO__c11_atomic_fetch_and:
595  case AtomicExpr::AO__atomic_fetch_and:
597  break;
598 
599  case AtomicExpr::AO__atomic_or_fetch:
600  PostOp = llvm::Instruction::Or;
601  // Fall through.
602  case AtomicExpr::AO__c11_atomic_fetch_or:
603  case AtomicExpr::AO__atomic_fetch_or:
604  Op = llvm::AtomicRMWInst::Or;
605  break;
606 
607  case AtomicExpr::AO__atomic_xor_fetch:
608  PostOp = llvm::Instruction::Xor;
609  // Fall through.
610  case AtomicExpr::AO__c11_atomic_fetch_xor:
611  case AtomicExpr::AO__atomic_fetch_xor:
612  Op = llvm::AtomicRMWInst::Xor;
613  break;
614 
615  case AtomicExpr::AO__atomic_nand_fetch:
616  PostOp = llvm::Instruction::And; // the NOT is special cased below
617  // Fall through.
618  case AtomicExpr::AO__atomic_fetch_nand:
619  Op = llvm::AtomicRMWInst::Nand;
620  break;
621  }
622 
623  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
624  llvm::AtomicRMWInst *RMWI =
625  CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
626  RMWI->setVolatile(E->isVolatile());
627 
628  // For __atomic_*_fetch operations, perform the operation again to
629  // determine the value which was written.
630  llvm::Value *Result = RMWI;
631  if (PostOp)
632  Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
633  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
634  Result = CGF.Builder.CreateNot(Result);
635  CGF.Builder.CreateStore(Result, Dest);
636 }
637 
638 // This function emits any expression (scalar, complex, or aggregate)
639 // into a temporary alloca.
640 static Address
642  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
643  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
644  /*Init*/ true);
645  return DeclPtr;
646 }
647 
648 static void
650  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
651  SourceLocation Loc, CharUnits SizeInChars) {
652  if (UseOptimizedLibcall) {
653  // Load value and pass it to the function directly.
654  CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
655  int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
656  ValTy =
657  CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
658  llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
659  SizeInBits)->getPointerTo();
660  Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
661  Val = CGF.EmitLoadOfScalar(Ptr, false,
662  CGF.getContext().getPointerType(ValTy),
663  Loc);
664  // Coerce the value into an appropriately sized integer type.
665  Args.add(RValue::get(Val), ValTy);
666  } else {
667  // Non-optimized functions always take a reference.
668  Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
669  CGF.getContext().VoidPtrTy);
670  }
671 }
672 
674  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
675  QualType MemTy = AtomicTy;
676  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
677  MemTy = AT->getValueType();
678  CharUnits sizeChars, alignChars;
679  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
680  uint64_t Size = sizeChars.getQuantity();
681  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
682  bool UseLibcall = (sizeChars != alignChars ||
683  getContext().toBits(sizeChars) > MaxInlineWidthInBits);
684 
685  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
686 
687  Address Val1 = Address::invalid();
688  Address Val2 = Address::invalid();
689  Address Dest = Address::invalid();
690  Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
691 
692  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
693  LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
694  EmitAtomicInit(E->getVal1(), lvalue);
695  return RValue::get(nullptr);
696  }
697 
698  llvm::Value *Order = EmitScalarExpr(E->getOrder());
699 
700  switch (E->getOp()) {
701  case AtomicExpr::AO__c11_atomic_init:
702  llvm_unreachable("Already handled above with EmitAtomicInit!");
703 
704  case AtomicExpr::AO__c11_atomic_load:
705  case AtomicExpr::AO__atomic_load_n:
706  break;
707 
708  case AtomicExpr::AO__atomic_load:
709  Dest = EmitPointerWithAlignment(E->getVal1());
710  break;
711 
712  case AtomicExpr::AO__atomic_store:
713  Val1 = EmitPointerWithAlignment(E->getVal1());
714  break;
715 
716  case AtomicExpr::AO__atomic_exchange:
717  Val1 = EmitPointerWithAlignment(E->getVal1());
718  Dest = EmitPointerWithAlignment(E->getVal2());
719  break;
720 
721  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
722  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
723  case AtomicExpr::AO__atomic_compare_exchange_n:
724  case AtomicExpr::AO__atomic_compare_exchange:
725  Val1 = EmitPointerWithAlignment(E->getVal1());
726  if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
727  Val2 = EmitPointerWithAlignment(E->getVal2());
728  else
729  Val2 = EmitValToTemp(*this, E->getVal2());
730  OrderFail = EmitScalarExpr(E->getOrderFail());
731  if (E->getNumSubExprs() == 6)
732  IsWeak = EmitScalarExpr(E->getWeak());
733  break;
734 
735  case AtomicExpr::AO__c11_atomic_fetch_add:
736  case AtomicExpr::AO__c11_atomic_fetch_sub:
737  if (MemTy->isPointerType()) {
738  // For pointer arithmetic, we're required to do a bit of math:
739  // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
740  // ... but only for the C11 builtins. The GNU builtins expect the
741  // user to multiply by sizeof(T).
742  QualType Val1Ty = E->getVal1()->getType();
743  llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
744  CharUnits PointeeIncAmt =
746  Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
747  auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
748  Val1 = Temp;
749  EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
750  break;
751  }
752  // Fall through.
753  case AtomicExpr::AO__atomic_fetch_add:
754  case AtomicExpr::AO__atomic_fetch_sub:
755  case AtomicExpr::AO__atomic_add_fetch:
756  case AtomicExpr::AO__atomic_sub_fetch:
757  case AtomicExpr::AO__c11_atomic_store:
758  case AtomicExpr::AO__c11_atomic_exchange:
759  case AtomicExpr::AO__atomic_store_n:
760  case AtomicExpr::AO__atomic_exchange_n:
761  case AtomicExpr::AO__c11_atomic_fetch_and:
762  case AtomicExpr::AO__c11_atomic_fetch_or:
763  case AtomicExpr::AO__c11_atomic_fetch_xor:
764  case AtomicExpr::AO__atomic_fetch_and:
765  case AtomicExpr::AO__atomic_fetch_or:
766  case AtomicExpr::AO__atomic_fetch_xor:
767  case AtomicExpr::AO__atomic_fetch_nand:
768  case AtomicExpr::AO__atomic_and_fetch:
769  case AtomicExpr::AO__atomic_or_fetch:
770  case AtomicExpr::AO__atomic_xor_fetch:
771  case AtomicExpr::AO__atomic_nand_fetch:
772  Val1 = EmitValToTemp(*this, E->getVal1());
773  break;
774  }
775 
776  QualType RValTy = E->getType().getUnqualifiedType();
777 
778  // The inlined atomics only function on iN types, where N is a power of 2. We
779  // need to make sure (via temporaries if necessary) that all incoming values
780  // are compatible.
781  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
782  AtomicInfo Atomics(*this, AtomicVal);
783 
784  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
785  if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
786  if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
787  if (Dest.isValid())
788  Dest = Atomics.emitCastToAtomicIntPointer(Dest);
789  else if (E->isCmpXChg())
790  Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
791  else if (!RValTy->isVoidType())
792  Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
793 
794  // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
795  if (UseLibcall) {
796  bool UseOptimizedLibcall = false;
797  switch (E->getOp()) {
798  case AtomicExpr::AO__c11_atomic_init:
799  llvm_unreachable("Already handled above with EmitAtomicInit!");
800 
801  case AtomicExpr::AO__c11_atomic_fetch_add:
802  case AtomicExpr::AO__atomic_fetch_add:
803  case AtomicExpr::AO__c11_atomic_fetch_and:
804  case AtomicExpr::AO__atomic_fetch_and:
805  case AtomicExpr::AO__c11_atomic_fetch_or:
806  case AtomicExpr::AO__atomic_fetch_or:
807  case AtomicExpr::AO__atomic_fetch_nand:
808  case AtomicExpr::AO__c11_atomic_fetch_sub:
809  case AtomicExpr::AO__atomic_fetch_sub:
810  case AtomicExpr::AO__c11_atomic_fetch_xor:
811  case AtomicExpr::AO__atomic_fetch_xor:
812  case AtomicExpr::AO__atomic_add_fetch:
813  case AtomicExpr::AO__atomic_and_fetch:
814  case AtomicExpr::AO__atomic_nand_fetch:
815  case AtomicExpr::AO__atomic_or_fetch:
816  case AtomicExpr::AO__atomic_sub_fetch:
817  case AtomicExpr::AO__atomic_xor_fetch:
818  // For these, only library calls for certain sizes exist.
819  UseOptimizedLibcall = true;
820  break;
821 
822  case AtomicExpr::AO__c11_atomic_load:
823  case AtomicExpr::AO__c11_atomic_store:
824  case AtomicExpr::AO__c11_atomic_exchange:
825  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
826  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
827  case AtomicExpr::AO__atomic_load_n:
828  case AtomicExpr::AO__atomic_load:
829  case AtomicExpr::AO__atomic_store_n:
830  case AtomicExpr::AO__atomic_store:
831  case AtomicExpr::AO__atomic_exchange_n:
832  case AtomicExpr::AO__atomic_exchange:
833  case AtomicExpr::AO__atomic_compare_exchange_n:
834  case AtomicExpr::AO__atomic_compare_exchange:
835  // Only use optimized library calls for sizes for which they exist.
836  if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
837  UseOptimizedLibcall = true;
838  break;
839  }
840 
841  CallArgList Args;
842  if (!UseOptimizedLibcall) {
843  // For non-optimized library calls, the size is the first parameter
844  Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
845  getContext().getSizeType());
846  }
847  // Atomic address is the first or second parameter
850 
851  std::string LibCallName;
852  QualType LoweredMemTy =
853  MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
854  QualType RetTy;
855  bool HaveRetTy = false;
856  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
857  switch (E->getOp()) {
858  case AtomicExpr::AO__c11_atomic_init:
859  llvm_unreachable("Already handled!");
860 
861  // There is only one libcall for compare an exchange, because there is no
862  // optimisation benefit possible from a libcall version of a weak compare
863  // and exchange.
864  // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
865  // void *desired, int success, int failure)
866  // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
867  // int success, int failure)
868  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
869  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
870  case AtomicExpr::AO__atomic_compare_exchange:
871  case AtomicExpr::AO__atomic_compare_exchange_n:
872  LibCallName = "__atomic_compare_exchange";
873  RetTy = getContext().BoolTy;
874  HaveRetTy = true;
877  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
878  MemTy, E->getExprLoc(), sizeChars);
879  Args.add(RValue::get(Order), getContext().IntTy);
880  Order = OrderFail;
881  break;
882  // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
883  // int order)
884  // T __atomic_exchange_N(T *mem, T val, int order)
885  case AtomicExpr::AO__c11_atomic_exchange:
886  case AtomicExpr::AO__atomic_exchange_n:
887  case AtomicExpr::AO__atomic_exchange:
888  LibCallName = "__atomic_exchange";
889  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
890  MemTy, E->getExprLoc(), sizeChars);
891  break;
892  // void __atomic_store(size_t size, void *mem, void *val, int order)
893  // void __atomic_store_N(T *mem, T val, int order)
894  case AtomicExpr::AO__c11_atomic_store:
895  case AtomicExpr::AO__atomic_store:
896  case AtomicExpr::AO__atomic_store_n:
897  LibCallName = "__atomic_store";
898  RetTy = getContext().VoidTy;
899  HaveRetTy = true;
900  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
901  MemTy, E->getExprLoc(), sizeChars);
902  break;
903  // void __atomic_load(size_t size, void *mem, void *return, int order)
904  // T __atomic_load_N(T *mem, int order)
905  case AtomicExpr::AO__c11_atomic_load:
906  case AtomicExpr::AO__atomic_load:
907  case AtomicExpr::AO__atomic_load_n:
908  LibCallName = "__atomic_load";
909  break;
910  // T __atomic_add_fetch_N(T *mem, T val, int order)
911  // T __atomic_fetch_add_N(T *mem, T val, int order)
912  case AtomicExpr::AO__atomic_add_fetch:
913  PostOp = llvm::Instruction::Add;
914  // Fall through.
915  case AtomicExpr::AO__c11_atomic_fetch_add:
916  case AtomicExpr::AO__atomic_fetch_add:
917  LibCallName = "__atomic_fetch_add";
918  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
919  LoweredMemTy, E->getExprLoc(), sizeChars);
920  break;
921  // T __atomic_and_fetch_N(T *mem, T val, int order)
922  // T __atomic_fetch_and_N(T *mem, T val, int order)
923  case AtomicExpr::AO__atomic_and_fetch:
924  PostOp = llvm::Instruction::And;
925  // Fall through.
926  case AtomicExpr::AO__c11_atomic_fetch_and:
927  case AtomicExpr::AO__atomic_fetch_and:
928  LibCallName = "__atomic_fetch_and";
929  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
930  MemTy, E->getExprLoc(), sizeChars);
931  break;
932  // T __atomic_or_fetch_N(T *mem, T val, int order)
933  // T __atomic_fetch_or_N(T *mem, T val, int order)
934  case AtomicExpr::AO__atomic_or_fetch:
935  PostOp = llvm::Instruction::Or;
936  // Fall through.
937  case AtomicExpr::AO__c11_atomic_fetch_or:
938  case AtomicExpr::AO__atomic_fetch_or:
939  LibCallName = "__atomic_fetch_or";
940  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
941  MemTy, E->getExprLoc(), sizeChars);
942  break;
943  // T __atomic_sub_fetch_N(T *mem, T val, int order)
944  // T __atomic_fetch_sub_N(T *mem, T val, int order)
945  case AtomicExpr::AO__atomic_sub_fetch:
946  PostOp = llvm::Instruction::Sub;
947  // Fall through.
948  case AtomicExpr::AO__c11_atomic_fetch_sub:
949  case AtomicExpr::AO__atomic_fetch_sub:
950  LibCallName = "__atomic_fetch_sub";
951  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
952  LoweredMemTy, E->getExprLoc(), sizeChars);
953  break;
954  // T __atomic_xor_fetch_N(T *mem, T val, int order)
955  // T __atomic_fetch_xor_N(T *mem, T val, int order)
956  case AtomicExpr::AO__atomic_xor_fetch:
957  PostOp = llvm::Instruction::Xor;
958  // Fall through.
959  case AtomicExpr::AO__c11_atomic_fetch_xor:
960  case AtomicExpr::AO__atomic_fetch_xor:
961  LibCallName = "__atomic_fetch_xor";
962  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
963  MemTy, E->getExprLoc(), sizeChars);
964  break;
965  // T __atomic_nand_fetch_N(T *mem, T val, int order)
966  // T __atomic_fetch_nand_N(T *mem, T val, int order)
967  case AtomicExpr::AO__atomic_nand_fetch:
968  PostOp = llvm::Instruction::And; // the NOT is special cased below
969  // Fall through.
970  case AtomicExpr::AO__atomic_fetch_nand:
971  LibCallName = "__atomic_fetch_nand";
972  AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
973  MemTy, E->getExprLoc(), sizeChars);
974  break;
975  }
976 
977  // Optimized functions have the size in their name.
978  if (UseOptimizedLibcall)
979  LibCallName += "_" + llvm::utostr(Size);
980  // By default, assume we return a value of the atomic type.
981  if (!HaveRetTy) {
982  if (UseOptimizedLibcall) {
983  // Value is returned directly.
984  // The function returns an appropriately sized integer type.
986  getContext().toBits(sizeChars), /*Signed=*/false);
987  } else {
988  // Value is returned through parameter before the order.
989  RetTy = getContext().VoidTy;
992  }
993  }
994  // order is always the last parameter
995  Args.add(RValue::get(Order),
996  getContext().IntTy);
997 
998  // PostOp is only needed for the atomic_*_fetch operations, and
999  // thus is only needed for and implemented in the
1000  // UseOptimizedLibcall codepath.
1001  assert(UseOptimizedLibcall || !PostOp);
1002 
1003  RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1004  // The value is returned directly from the libcall.
1005  if (E->isCmpXChg())
1006  return Res;
1007 
1008  // The value is returned directly for optimized libcalls but the expr
1009  // provided an out-param.
1010  if (UseOptimizedLibcall && Res.getScalarVal()) {
1011  llvm::Value *ResVal = Res.getScalarVal();
1012  if (PostOp) {
1013  llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1014  ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1015  }
1016  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1017  ResVal = Builder.CreateNot(ResVal);
1018 
1020  ResVal,
1021  Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1022  }
1023 
1024  if (RValTy->isVoidType())
1025  return RValue::get(nullptr);
1026 
1027  return convertTempToRValue(
1028  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1029  RValTy, E->getExprLoc());
1030  }
1031 
1032  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1033  E->getOp() == AtomicExpr::AO__atomic_store ||
1034  E->getOp() == AtomicExpr::AO__atomic_store_n;
1035  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1036  E->getOp() == AtomicExpr::AO__atomic_load ||
1037  E->getOp() == AtomicExpr::AO__atomic_load_n;
1038 
1039  if (isa<llvm::ConstantInt>(Order)) {
1040  int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1041  switch (ord) {
1043  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1044  Size, llvm::Monotonic);
1045  break;
1048  if (IsStore)
1049  break; // Avoid crashing on code with undefined behavior
1050  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1051  Size, llvm::Acquire);
1052  break;
1054  if (IsLoad)
1055  break; // Avoid crashing on code with undefined behavior
1056  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1057  Size, llvm::Release);
1058  break;
1060  if (IsLoad || IsStore)
1061  break; // Avoid crashing on code with undefined behavior
1062  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1063  Size, llvm::AcquireRelease);
1064  break;
1066  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1067  Size, llvm::SequentiallyConsistent);
1068  break;
1069  default: // invalid order
1070  // We should not ever get here normally, but it's hard to
1071  // enforce that in general.
1072  break;
1073  }
1074  if (RValTy->isVoidType())
1075  return RValue::get(nullptr);
1076 
1077  return convertTempToRValue(
1078  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1079  RValTy, E->getExprLoc());
1080  }
1081 
1082  // Long case, when Order isn't obviously constant.
1083 
1084  // Create all the relevant BB's
1085  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1086  *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1087  *SeqCstBB = nullptr;
1088  MonotonicBB = createBasicBlock("monotonic", CurFn);
1089  if (!IsStore)
1090  AcquireBB = createBasicBlock("acquire", CurFn);
1091  if (!IsLoad)
1092  ReleaseBB = createBasicBlock("release", CurFn);
1093  if (!IsLoad && !IsStore)
1094  AcqRelBB = createBasicBlock("acqrel", CurFn);
1095  SeqCstBB = createBasicBlock("seqcst", CurFn);
1096  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1097 
1098  // Create the switch for the split
1099  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1100  // doesn't matter unless someone is crazy enough to use something that
1101  // doesn't fold to a constant for the ordering.
1102  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1103  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1104 
1105  // Emit all the different atomics
1106  Builder.SetInsertPoint(MonotonicBB);
1107  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1108  Size, llvm::Monotonic);
1109  Builder.CreateBr(ContBB);
1110  if (!IsStore) {
1111  Builder.SetInsertPoint(AcquireBB);
1112  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1113  Size, llvm::Acquire);
1114  Builder.CreateBr(ContBB);
1115  SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
1116  AcquireBB);
1117  SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
1118  AcquireBB);
1119  }
1120  if (!IsLoad) {
1121  Builder.SetInsertPoint(ReleaseBB);
1122  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1123  Size, llvm::Release);
1124  Builder.CreateBr(ContBB);
1125  SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_release),
1126  ReleaseBB);
1127  }
1128  if (!IsLoad && !IsStore) {
1129  Builder.SetInsertPoint(AcqRelBB);
1130  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1131  Size, llvm::AcquireRelease);
1132  Builder.CreateBr(ContBB);
1133  SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acq_rel),
1134  AcqRelBB);
1135  }
1136  Builder.SetInsertPoint(SeqCstBB);
1137  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1138  Size, llvm::SequentiallyConsistent);
1139  Builder.CreateBr(ContBB);
1140  SI->addCase(Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
1141  SeqCstBB);
1142 
1143  // Cleanup and return
1144  Builder.SetInsertPoint(ContBB);
1145  if (RValTy->isVoidType())
1146  return RValue::get(nullptr);
1147 
1148  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1149  return convertTempToRValue(
1150  Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1151  RValTy, E->getExprLoc());
1152 }
1153 
1154 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1155  unsigned addrspace =
1156  cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1157  llvm::IntegerType *ty =
1158  llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1159  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1160 }
1161 
1162 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1163  llvm::Type *Ty = Addr.getElementType();
1164  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1165  if (SourceSizeInBits != AtomicSizeInBits) {
1166  Address Tmp = CreateTempAlloca();
1167  CGF.Builder.CreateMemCpy(Tmp, Addr,
1168  std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1169  Addr = Tmp;
1170  }
1171 
1172  return emitCastToAtomicIntPointer(Addr);
1173 }
1174 
1175 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1176  AggValueSlot resultSlot,
1177  SourceLocation loc,
1178  bool asValue) const {
1179  if (LVal.isSimple()) {
1180  if (EvaluationKind == TEK_Aggregate)
1181  return resultSlot.asRValue();
1182 
1183  // Drill into the padding structure if we have one.
1184  if (hasPadding())
1185  addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1186 
1187  // Otherwise, just convert the temporary to an r-value using the
1188  // normal conversion routine.
1189  return CGF.convertTempToRValue(addr, getValueType(), loc);
1190  }
1191  if (!asValue)
1192  // Get RValue from temp memory as atomic for non-simple lvalues
1193  return RValue::get(CGF.Builder.CreateLoad(addr));
1194  if (LVal.isBitField())
1195  return CGF.EmitLoadOfBitfieldLValue(
1196  LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1197  LVal.getAlignmentSource()));
1198  if (LVal.isVectorElt())
1199  return CGF.EmitLoadOfLValue(
1200  LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1201  LVal.getAlignmentSource()), loc);
1202  assert(LVal.isExtVectorElt());
1204  addr, LVal.getExtVectorElts(), LVal.getType(),
1205  LVal.getAlignmentSource()));
1206 }
1207 
1208 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1209  AggValueSlot ResultSlot,
1210  SourceLocation Loc,
1211  bool AsValue) const {
1212  // Try not to in some easy cases.
1213  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1214  if (getEvaluationKind() == TEK_Scalar &&
1215  (((!LVal.isBitField() ||
1216  LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1217  !hasPadding()) ||
1218  !AsValue)) {
1219  auto *ValTy = AsValue
1220  ? CGF.ConvertTypeForMem(ValueTy)
1221  : getAtomicAddress().getType()->getPointerElementType();
1222  if (ValTy->isIntegerTy()) {
1223  assert(IntVal->getType() == ValTy && "Different integer types.");
1224  return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1225  } else if (ValTy->isPointerTy())
1226  return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1227  else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1228  return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1229  }
1230 
1231  // Create a temporary. This needs to be big enough to hold the
1232  // atomic integer.
1233  Address Temp = Address::invalid();
1234  bool TempIsVolatile = false;
1235  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1236  assert(!ResultSlot.isIgnored());
1237  Temp = ResultSlot.getAddress();
1238  TempIsVolatile = ResultSlot.isVolatile();
1239  } else {
1240  Temp = CreateTempAlloca();
1241  }
1242 
1243  // Slam the integer into the temporary.
1244  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1245  CGF.Builder.CreateStore(IntVal, CastTemp)
1246  ->setVolatile(TempIsVolatile);
1247 
1248  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1249 }
1250 
1251 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1252  llvm::AtomicOrdering AO, bool) {
1253  // void __atomic_load(size_t size, void *mem, void *return, int order);
1254  CallArgList Args;
1255  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1256  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1257  CGF.getContext().VoidPtrTy);
1258  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1259  CGF.getContext().VoidPtrTy);
1260  Args.add(RValue::get(
1261  llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1262  CGF.getContext().IntTy);
1263  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1264 }
1265 
1266 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1267  bool IsVolatile) {
1268  // Okay, we're doing this natively.
1269  Address Addr = getAtomicAddressAsAtomicIntPointer();
1270  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1271  Load->setAtomic(AO);
1272 
1273  // Other decoration.
1274  if (IsVolatile)
1275  Load->setVolatile(true);
1276  if (LVal.getTBAAInfo())
1277  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1278  return Load;
1279 }
1280 
1281 /// An LValue is a candidate for having its loads and stores be made atomic if
1282 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1283 /// performing such an operation can be performed without a libcall.
1285  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1286  AtomicInfo AI(*this, LV);
1287  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1288  // An atomic is inline if we don't need to use a libcall.
1289  bool AtomicIsInline = !AI.shouldUseLibcall();
1290  return IsVolatile && AtomicIsInline;
1291 }
1292 
1293 /// An type is a candidate for having its loads and stores be made atomic if
1294 /// we are operating under /volatile:ms *and* we know the access is volatile and
1295 /// performing such an operation can be performed without a libcall.
1297  bool IsVolatile) const {
1298  // An atomic is inline if we don't need to use a libcall (e.g. it is builtin).
1299  bool AtomicIsInline = getContext().getTargetInfo().hasBuiltinAtomic(
1300  getContext().getTypeSize(Ty), getContext().getTypeAlign(Ty));
1301  return CGM.getCodeGenOpts().MSVolatile && IsVolatile && AtomicIsInline;
1302 }
1303 
1305  AggValueSlot Slot) {
1306  llvm::AtomicOrdering AO;
1307  bool IsVolatile = LV.isVolatileQualified();
1308  if (LV.getType()->isAtomicType()) {
1309  AO = llvm::SequentiallyConsistent;
1310  } else {
1311  AO = llvm::Acquire;
1312  IsVolatile = true;
1313  }
1314  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1315 }
1316 
1317 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1318  bool AsValue, llvm::AtomicOrdering AO,
1319  bool IsVolatile) {
1320  // Check whether we should use a library call.
1321  if (shouldUseLibcall()) {
1322  Address TempAddr = Address::invalid();
1323  if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1324  assert(getEvaluationKind() == TEK_Aggregate);
1325  TempAddr = ResultSlot.getAddress();
1326  } else
1327  TempAddr = CreateTempAlloca();
1328 
1329  EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1330 
1331  // Okay, turn that back into the original value or whole atomic (for
1332  // non-simple lvalues) type.
1333  return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1334  }
1335 
1336  // Okay, we're doing this natively.
1337  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1338 
1339  // If we're ignoring an aggregate return, don't do anything.
1340  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1341  return RValue::getAggregate(Address::invalid(), false);
1342 
1343  // Okay, turn that back into the original value or atomic (for non-simple
1344  // lvalues) type.
1345  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1346 }
1347 
1348 /// Emit a load from an l-value of atomic type. Note that the r-value
1349 /// we produce is an r-value of the atomic *value* type.
1351  llvm::AtomicOrdering AO, bool IsVolatile,
1352  AggValueSlot resultSlot) {
1353  AtomicInfo Atomics(*this, src);
1354  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1355  IsVolatile);
1356 }
1357 
1358 /// Copy an r-value into memory as part of storing to an atomic type.
1359 /// This needs to create a bit-pattern suitable for atomic operations.
1360 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1361  assert(LVal.isSimple());
1362  // If we have an r-value, the rvalue should be of the atomic type,
1363  // which means that the caller is responsible for having zeroed
1364  // any padding. Just do an aggregate copy of that type.
1365  if (rvalue.isAggregate()) {
1366  CGF.EmitAggregateCopy(getAtomicAddress(),
1367  rvalue.getAggregateAddress(),
1368  getAtomicType(),
1369  (rvalue.isVolatileQualified()
1370  || LVal.isVolatileQualified()));
1371  return;
1372  }
1373 
1374  // Okay, otherwise we're copying stuff.
1375 
1376  // Zero out the buffer if necessary.
1377  emitMemSetZeroIfNecessary();
1378 
1379  // Drill past the padding if present.
1380  LValue TempLVal = projectValue();
1381 
1382  // Okay, store the rvalue in.
1383  if (rvalue.isScalar()) {
1384  CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1385  } else {
1386  CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1387  }
1388 }
1389 
1390 
1391 /// Materialize an r-value into memory for the purposes of storing it
1392 /// to an atomic type.
1393 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1394  // Aggregate r-values are already in memory, and EmitAtomicStore
1395  // requires them to be values of the atomic type.
1396  if (rvalue.isAggregate())
1397  return rvalue.getAggregateAddress();
1398 
1399  // Otherwise, make a temporary and materialize into it.
1400  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1401  AtomicInfo Atomics(CGF, TempLV);
1402  Atomics.emitCopyIntoMemory(rvalue);
1403  return TempLV.getAddress();
1404 }
1405 
1406 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1407  // If we've got a scalar value of the right size, try to avoid going
1408  // through memory.
1409  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1410  llvm::Value *Value = RVal.getScalarVal();
1411  if (isa<llvm::IntegerType>(Value->getType()))
1412  return CGF.EmitToMemory(Value, ValueTy);
1413  else {
1414  llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1415  CGF.getLLVMContext(),
1416  LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1417  if (isa<llvm::PointerType>(Value->getType()))
1418  return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1419  else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1420  return CGF.Builder.CreateBitCast(Value, InputIntTy);
1421  }
1422  }
1423  // Otherwise, we need to go through memory.
1424  // Put the r-value in memory.
1425  Address Addr = materializeRValue(RVal);
1426 
1427  // Cast the temporary to the atomic int type and pull a value out.
1428  Addr = emitCastToAtomicIntPointer(Addr);
1429  return CGF.Builder.CreateLoad(Addr);
1430 }
1431 
1432 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1433  llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1434  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1435  // Do the atomic store.
1436  Address Addr = getAtomicAddressAsAtomicIntPointer();
1437  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1438  ExpectedVal, DesiredVal,
1439  Success, Failure);
1440  // Other decoration.
1441  Inst->setVolatile(LVal.isVolatileQualified());
1442  Inst->setWeak(IsWeak);
1443 
1444  // Okay, turn that back into the original value type.
1445  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1446  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1447  return std::make_pair(PreviousVal, SuccessFailureVal);
1448 }
1449 
1450 llvm::Value *
1451 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1452  llvm::Value *DesiredAddr,
1453  llvm::AtomicOrdering Success,
1454  llvm::AtomicOrdering Failure) {
1455  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1456  // void *desired, int success, int failure);
1457  CallArgList Args;
1458  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1459  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1460  CGF.getContext().VoidPtrTy);
1461  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1462  CGF.getContext().VoidPtrTy);
1463  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1464  CGF.getContext().VoidPtrTy);
1465  Args.add(RValue::get(llvm::ConstantInt::get(
1466  CGF.IntTy, translateAtomicOrdering(Success))),
1467  CGF.getContext().IntTy);
1468  Args.add(RValue::get(llvm::ConstantInt::get(
1469  CGF.IntTy, translateAtomicOrdering(Failure))),
1470  CGF.getContext().IntTy);
1471  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1472  CGF.getContext().BoolTy, Args);
1473 
1474  return SuccessFailureRVal.getScalarVal();
1475 }
1476 
1477 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1478  RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1479  llvm::AtomicOrdering Failure, bool IsWeak) {
1480  if (Failure >= Success)
1481  // Don't assert on undefined behavior.
1482  Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1483 
1484  // Check whether we should use a library call.
1485  if (shouldUseLibcall()) {
1486  // Produce a source address.
1487  Address ExpectedAddr = materializeRValue(Expected);
1488  Address DesiredAddr = materializeRValue(Desired);
1489  auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1490  DesiredAddr.getPointer(),
1491  Success, Failure);
1492  return std::make_pair(
1493  convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1494  SourceLocation(), /*AsValue=*/false),
1495  Res);
1496  }
1497 
1498  // If we've got a scalar value of the right size, try to avoid going
1499  // through memory.
1500  auto *ExpectedVal = convertRValueToInt(Expected);
1501  auto *DesiredVal = convertRValueToInt(Desired);
1502  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1503  Failure, IsWeak);
1504  return std::make_pair(
1505  ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1506  SourceLocation(), /*AsValue=*/false),
1507  Res.second);
1508 }
1509 
1510 static void
1511 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1512  const llvm::function_ref<RValue(RValue)> &UpdateOp,
1513  Address DesiredAddr) {
1514  RValue UpRVal;
1515  LValue AtomicLVal = Atomics.getAtomicLValue();
1516  LValue DesiredLVal;
1517  if (AtomicLVal.isSimple()) {
1518  UpRVal = OldRVal;
1519  DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1520  } else {
1521  // Build new lvalue for temp address
1522  Address Ptr = Atomics.materializeRValue(OldRVal);
1523  LValue UpdateLVal;
1524  if (AtomicLVal.isBitField()) {
1525  UpdateLVal =
1526  LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1527  AtomicLVal.getType(),
1528  AtomicLVal.getAlignmentSource());
1529  DesiredLVal =
1530  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1531  AtomicLVal.getType(),
1532  AtomicLVal.getAlignmentSource());
1533  } else if (AtomicLVal.isVectorElt()) {
1534  UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1535  AtomicLVal.getType(),
1536  AtomicLVal.getAlignmentSource());
1537  DesiredLVal = LValue::MakeVectorElt(
1538  DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1539  AtomicLVal.getAlignmentSource());
1540  } else {
1541  assert(AtomicLVal.isExtVectorElt());
1542  UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1543  AtomicLVal.getType(),
1544  AtomicLVal.getAlignmentSource());
1545  DesiredLVal = LValue::MakeExtVectorElt(
1546  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1547  AtomicLVal.getAlignmentSource());
1548  }
1549  UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1550  DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1551  UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1552  }
1553  // Store new value in the corresponding memory area
1554  RValue NewRVal = UpdateOp(UpRVal);
1555  if (NewRVal.isScalar()) {
1556  CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1557  } else {
1558  assert(NewRVal.isComplex());
1559  CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1560  /*isInit=*/false);
1561  }
1562 }
1563 
1564 void AtomicInfo::EmitAtomicUpdateLibcall(
1565  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1566  bool IsVolatile) {
1567  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1568 
1569  Address ExpectedAddr = CreateTempAlloca();
1570 
1571  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1572  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1573  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1574  CGF.EmitBlock(ContBB);
1575  Address DesiredAddr = CreateTempAlloca();
1576  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1577  requiresMemSetZero(getAtomicAddress().getElementType())) {
1578  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1579  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1580  }
1581  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1583  SourceLocation(), /*AsValue=*/false);
1584  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1585  auto *Res =
1586  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1587  DesiredAddr.getPointer(),
1588  AO, Failure);
1589  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1590  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1591 }
1592 
1593 void AtomicInfo::EmitAtomicUpdateOp(
1594  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1595  bool IsVolatile) {
1596  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1597 
1598  // Do the atomic load.
1599  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1600  // For non-simple lvalues perform compare-and-swap procedure.
1601  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1602  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1603  auto *CurBB = CGF.Builder.GetInsertBlock();
1604  CGF.EmitBlock(ContBB);
1605  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1606  /*NumReservedValues=*/2);
1607  PHI->addIncoming(OldVal, CurBB);
1608  Address NewAtomicAddr = CreateTempAlloca();
1609  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1610  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1611  requiresMemSetZero(getAtomicAddress().getElementType())) {
1612  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1613  }
1614  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1615  SourceLocation(), /*AsValue=*/false);
1616  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1617  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1618  // Try to write new value using cmpxchg operation
1619  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1620  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1621  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1622  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1623 }
1624 
1625 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1626  RValue UpdateRVal, Address DesiredAddr) {
1627  LValue AtomicLVal = Atomics.getAtomicLValue();
1628  LValue DesiredLVal;
1629  // Build new lvalue for temp address
1630  if (AtomicLVal.isBitField()) {
1631  DesiredLVal =
1632  LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1633  AtomicLVal.getType(),
1634  AtomicLVal.getAlignmentSource());
1635  } else if (AtomicLVal.isVectorElt()) {
1636  DesiredLVal =
1637  LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1638  AtomicLVal.getType(),
1639  AtomicLVal.getAlignmentSource());
1640  } else {
1641  assert(AtomicLVal.isExtVectorElt());
1642  DesiredLVal = LValue::MakeExtVectorElt(
1643  DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1644  AtomicLVal.getAlignmentSource());
1645  }
1646  DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1647  // Store new value in the corresponding memory area
1648  assert(UpdateRVal.isScalar());
1649  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1650 }
1651 
1652 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1653  RValue UpdateRVal, bool IsVolatile) {
1654  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1655 
1656  Address ExpectedAddr = CreateTempAlloca();
1657 
1658  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1659  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1660  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1661  CGF.EmitBlock(ContBB);
1662  Address DesiredAddr = CreateTempAlloca();
1663  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1664  requiresMemSetZero(getAtomicAddress().getElementType())) {
1665  auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1666  CGF.Builder.CreateStore(OldVal, DesiredAddr);
1667  }
1668  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1669  auto *Res =
1670  EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1671  DesiredAddr.getPointer(),
1672  AO, Failure);
1673  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1674  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1675 }
1676 
1677 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1678  bool IsVolatile) {
1679  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1680 
1681  // Do the atomic load.
1682  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1683  // For non-simple lvalues perform compare-and-swap procedure.
1684  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1685  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1686  auto *CurBB = CGF.Builder.GetInsertBlock();
1687  CGF.EmitBlock(ContBB);
1688  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1689  /*NumReservedValues=*/2);
1690  PHI->addIncoming(OldVal, CurBB);
1691  Address NewAtomicAddr = CreateTempAlloca();
1692  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1693  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1694  requiresMemSetZero(getAtomicAddress().getElementType())) {
1695  CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1696  }
1697  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1698  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1699  // Try to write new value using cmpxchg operation
1700  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1701  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1702  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1703  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1704 }
1705 
1706 void AtomicInfo::EmitAtomicUpdate(
1707  llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1708  bool IsVolatile) {
1709  if (shouldUseLibcall()) {
1710  EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1711  } else {
1712  EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1713  }
1714 }
1715 
1716 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1717  bool IsVolatile) {
1718  if (shouldUseLibcall()) {
1719  EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1720  } else {
1721  EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1722  }
1723 }
1724 
1726  bool isInit) {
1727  bool IsVolatile = lvalue.isVolatileQualified();
1728  llvm::AtomicOrdering AO;
1729  if (lvalue.getType()->isAtomicType()) {
1730  AO = llvm::SequentiallyConsistent;
1731  } else {
1732  AO = llvm::Release;
1733  IsVolatile = true;
1734  }
1735  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1736 }
1737 
1738 /// Emit a store to an l-value of atomic type.
1739 ///
1740 /// Note that the r-value is expected to be an r-value *of the atomic
1741 /// type*; this means that for aggregate r-values, it should include
1742 /// storage for any padding that was necessary.
1744  llvm::AtomicOrdering AO, bool IsVolatile,
1745  bool isInit) {
1746  // If this is an aggregate r-value, it should agree in type except
1747  // maybe for address-space qualification.
1748  assert(!rvalue.isAggregate() ||
1750  == dest.getAddress().getElementType());
1751 
1752  AtomicInfo atomics(*this, dest);
1753  LValue LVal = atomics.getAtomicLValue();
1754 
1755  // If this is an initialization, just put the value there normally.
1756  if (LVal.isSimple()) {
1757  if (isInit) {
1758  atomics.emitCopyIntoMemory(rvalue);
1759  return;
1760  }
1761 
1762  // Check whether we should use a library call.
1763  if (atomics.shouldUseLibcall()) {
1764  // Produce a source address.
1765  Address srcAddr = atomics.materializeRValue(rvalue);
1766 
1767  // void __atomic_store(size_t size, void *mem, void *val, int order)
1768  CallArgList args;
1769  args.add(RValue::get(atomics.getAtomicSizeValue()),
1770  getContext().getSizeType());
1771  args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1772  getContext().VoidPtrTy);
1773  args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1774  getContext().VoidPtrTy);
1775  args.add(RValue::get(llvm::ConstantInt::get(
1776  IntTy, AtomicInfo::translateAtomicOrdering(AO))),
1777  getContext().IntTy);
1778  emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1779  return;
1780  }
1781 
1782  // Okay, we're doing this natively.
1783  llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1784 
1785  // Do the atomic store.
1786  Address addr =
1787  atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1788  intValue = Builder.CreateIntCast(
1789  intValue, addr.getElementType(), /*isSigned=*/false);
1790  llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1791 
1792  // Initializations don't need to be atomic.
1793  if (!isInit)
1794  store->setAtomic(AO);
1795 
1796  // Other decoration.
1797  if (IsVolatile)
1798  store->setVolatile(true);
1799  if (dest.getTBAAInfo())
1801  return;
1802  }
1803 
1804  // Emit simple atomic update operation.
1805  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1806 }
1807 
1808 /// Emit a compare-and-exchange op for atomic type.
1809 ///
1810 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1811  LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1812  llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1813  AggValueSlot Slot) {
1814  // If this is an aggregate r-value, it should agree in type except
1815  // maybe for address-space qualification.
1816  assert(!Expected.isAggregate() ||
1817  Expected.getAggregateAddress().getElementType() ==
1818  Obj.getAddress().getElementType());
1819  assert(!Desired.isAggregate() ||
1820  Desired.getAggregateAddress().getElementType() ==
1821  Obj.getAddress().getElementType());
1822  AtomicInfo Atomics(*this, Obj);
1823 
1824  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1825  IsWeak);
1826 }
1827 
1829  LValue LVal, llvm::AtomicOrdering AO,
1830  const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1831  AtomicInfo Atomics(*this, LVal);
1832  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1833 }
1834 
1836  AtomicInfo atomics(*this, dest);
1837 
1838  switch (atomics.getEvaluationKind()) {
1839  case TEK_Scalar: {
1840  llvm::Value *value = EmitScalarExpr(init);
1841  atomics.emitCopyIntoMemory(RValue::get(value));
1842  return;
1843  }
1844 
1845  case TEK_Complex: {
1846  ComplexPairTy value = EmitComplexExpr(init);
1847  atomics.emitCopyIntoMemory(RValue::getComplex(value));
1848  return;
1849  }
1850 
1851  case TEK_Aggregate: {
1852  // Fix up the destination if the initializer isn't an expression
1853  // of atomic type.
1854  bool Zeroed = false;
1855  if (!init->getType()->isAtomicType()) {
1856  Zeroed = atomics.emitMemSetZeroIfNecessary();
1857  dest = atomics.projectValue();
1858  }
1859 
1860  // Evaluate the expression directly into the destination.
1865  Zeroed ? AggValueSlot::IsZeroed :
1867 
1868  EmitAggExpr(init, slot);
1869  return;
1870  }
1871  }
1872  llvm_unreachable("bad evaluation kind");
1873 }
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Definition: CGCall.h:151
Defines the clang::ASTContext interface.
llvm::IntegerType * IntTy
int
CanQualType VoidPtrTy
Definition: ASTContext.h:895
A (possibly-)qualified type.
Definition: Type.h:575
llvm::Type * ConvertTypeForMem(QualType T)
static LValue MakeBitfield(Address Addr, const CGBitFieldInfo &Info, QualType type, AlignmentSource alignSource)
Create a new object to represent a bit-field access.
Definition: CGValue.h:414
static AggValueSlot forLValue(const LValue &LV, IsDestructed_t isDestructed, NeedsGCBarriers_t needsGC, IsAliased_t isAliased, IsZeroed_t isZeroed=IsNotZeroed)
Definition: CGValue.h:524
void setAlignment(CharUnits A)
Definition: CGValue.h:317
const TargetInfo & getTarget() const
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:65
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:171
Address getAddress() const
Definition: CGValue.h:331
void setTBAAInfo(llvm::MDNode *N)
Definition: CGValue.h:309
const llvm::DataLayout & getDataLayout() const
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
Definition: StoreRef.h:26
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1568
bool typeIsSuitableForInlineAtomic(QualType Ty, bool IsVolatile) const
An type is a candidate for having its loads and stores be made atomic if we are operating under /vola...
Definition: CGAtomic.cpp:1296
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
Expr * getVal1() const
Definition: Expr.h:4863
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1725
bool isVolatile() const
Definition: CGValue.h:542
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
bool isVolatile() const
Definition: Expr.h:4889
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
static llvm::Value * getTypeSize(CodeGenFunction &CGF, QualType Ty)
bool isVoidType() const
Definition: Type.h:5546
bool isVolatileQualified() const
Definition: CGValue.h:252
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:91
QualType getExtVectorType(QualType VectorType, unsigned NumElts) const
Return the unique reference to an extended vector type of the specified element type and size...
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
Definition: CGAtomic.cpp:1828
Expr * getOrder() const
Definition: Expr.h:4860
RValue EmitCall(const CGFunctionInfo &FnInfo, llvm::Value *Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, CGCalleeInfo CalleeInfo=CGCalleeInfo(), llvm::Instruction **callOrInvoke=nullptr)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
Definition: CGCall.cpp:3159
RValue EmitLoadOfExtVectorElementLValue(LValue V)
Definition: CGExpr.cpp:1492
Expr * getVal2() const
Definition: Expr.h:4873
CharUnits getAlignment() const
Definition: CGValue.h:316
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:580
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
uint32_t Offset
Definition: CacheTokens.cpp:44
QualType getIntPtrType() const
Return a type compatible with "intptr_t" (C99 7.18.1.4), as defined by the target.
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:272
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, const llvm::function_ref< RValue(RValue)> &UpdateOp, Address DesiredAddr)
Definition: CGAtomic.cpp:1511
unsigned Align
Definition: ASTContext.h:82
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitAtomicInit(Expr *E, LValue lvalue)
Definition: CGAtomic.cpp:1835
unsigned Offset
The offset within a contiguous run of bitfields that are represented as a single "field" within the L...
RValue EmitAtomicExpr(AtomicExpr *E)
Definition: CGAtomic.cpp:673
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
bool isExtVectorElt() const
Definition: CGValue.h:249
RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc)
Given the address of a temporary variable, produce an r-value of its type.
Definition: CGExpr.cpp:3940
bool isValid() const
Definition: Address.h:36
static LValue MakeExtVectorElt(Address vecAddress, llvm::Constant *Elts, QualType type, AlignmentSource alignSource)
Definition: CGValue.h:397
std::pair< CharUnits, CharUnits > getTypeInfoInChars(const Type *T) const
static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, uint64_t expectedSize)
Does a store of the given IR type modify the full expected width?
Definition: CGAtomic.cpp:334
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
Definition: CGExpr.cpp:168
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:38
bool isCmpXChg() const
Definition: Expr.h:4893
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:415
static unsigned getNumSubExprs(AtomicOp Op)
Determine the number of arguments the specified atomic builtin should have.
Definition: Expr.cpp:3967
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, llvm::MDNode *TBAAInfo, bool ConvertTypeToTag=true)
Decorate the instruction with a TBAA tag.
static TypeEvaluationKind getEvaluationKind(QualType T)
hasAggregateLLVMType - Return true if the specified AST type will map into an aggregate LLVM type or ...
llvm::Value * getPointer() const
Definition: Address.h:38
Expr - This represents one expression.
Definition: Expr.h:104
static Address invalid()
Definition: Address.h:35
bool isAggregate() const
Definition: CGValue.h:53
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
Definition: CGExpr.cpp:1348
bool isAtomicType() const
Definition: Type.h:5387
static LValue MakeVectorElt(Address vecAddress, llvm::Value *Idx, QualType type, AlignmentSource alignSource)
Definition: CGValue.h:386
RValue asRValue() const
Definition: CGValue.h:578
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
Definition: CGExpr.cpp:1323
bool isVectorElt() const
Definition: CGValue.h:247
static LValue MakeAddr(Address address, QualType type, ASTContext &Context, AlignmentSource alignSource, llvm::MDNode *TBAAInfo=nullptr)
Definition: CGValue.h:371
void add(RValue rvalue, QualType type, bool needscopy=false)
Definition: CGCall.h:81
llvm::LLVMContext & getLLVMContext()
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
Address EmitPointerWithAlignment(const Expr *Addr, AlignmentSource *Source=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
Definition: CGExpr.cpp:795
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
Definition: CGBuilder.h:294
Represents a GCC generic vector type.
Definition: Type.h:2724
llvm::Value * EmitCastToVoidPtr(llvm::Value *value)
Emit a cast to void* in the appropriate address space.
Definition: CGExpr.cpp:43
static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *IsWeak, llvm::Value *FailureOrder, uint64_t Size, llvm::AtomicOrdering Order)
Definition: CGAtomic.cpp:500
static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, llvm::Value *FailureOrderVal, uint64_t Size, llvm::AtomicOrdering SuccessOrder)
Given an ordering required on success, emit all possible cmpxchg instructions to cope with the provid...
Definition: CGAtomic.cpp:424
The result type of a method or function.
AtomicOp getOp() const
Definition: Expr.h:4884
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1304
bool isVolatile() const
Definition: CGValue.h:295
The l-value was considered opaque, so the alignment was determined from a type.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:168
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeSet ExtraAttrs=llvm::AttributeSet())
Create a new runtime function with the specified type and name.
bool isSimple() const
Definition: CGValue.h:246
llvm::Value * getBitFieldPointer() const
Definition: CGValue.h:362
ASTContext & getContext() const
Encodes a location in the source.
bool LValueIsSuitableForInlineAtomic(LValue Src)
An LValue is a candidate for having its loads and stores be made atomic if we are operating under /vo...
Definition: CGAtomic.cpp:1284
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
Emit a compare-and-exchange op for atomic type.
Definition: CGAtomic.cpp:1810
Expr * getPtr() const
Definition: Expr.h:4857
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
Definition: CGExpr.cpp:1337
const CGBitFieldInfo & getBitFieldInfo() const
Definition: CGValue.h:363
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Definition: CGExpr.cpp:1236
llvm::MDNode * getTBAAInfo() const
Definition: CGValue.h:308
An aggregate value slot.
Definition: CGValue.h:441
AtomicExpr - Variadic atomic builtins: __atomic_exchange, __atomic_fetch_*, __atomic_load, __atomic_store, and __atomic_compare_exchange_*, for the similarly-named C++11 instructions, and __c11 variants for <stdatomic.h>.
Definition: Expr.h:4817
CanQualType VoidTy
Definition: ASTContext.h:881
const CodeGenOptions & getCodeGenOpts() const
An aligned address.
Definition: Address.h:25
bool isGlobalReg() const
Definition: CGValue.h:250
Address getExtVectorAddress() const
Definition: CGValue.h:346
bool isBitField() const
Definition: CGValue.h:248
AlignmentSource getAlignmentSource() const
Definition: CGValue.h:319
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:193
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
QualType getType() const
Definition: Expr.h:125
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
static const Type * getElementType(const Expr *BaseExpr)
bool isScalar() const
Definition: CGValue.h:51
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:92
bool isZero() const
isZero - Test whether the quantity equals zero.
Definition: CharUnits.h:116
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
Definition: CGExpr.cpp:97
const internal::VariadicAllOfMatcher< Type > type
Matches Types in the clang AST.
Definition: ASTMatchers.h:1723
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:58
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:487
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
Definition: CGBuilder.h:191
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:78
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
detail::InMemoryDirectory::const_iterator E
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
Definition: CGExprAgg.cpp:1423
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Definition: CGBuilder.h:121
unsigned getMaxAtomicInlineWidth() const
Return the maximum width lock-free atomic operation which can be inlined given the supported features...
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type. ...
Definition: CGExprAgg.cpp:1401
llvm::Constant * getExtVectorElts() const
Definition: CGValue.h:353
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static void AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, SourceLocation Loc, CharUnits SizeInChars)
Definition: CGAtomic.cpp:649
const T * getAs() const
Member-template getAs<specific type>'.
Definition: Type.h:5675
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
Address getAddress() const
Definition: CGValue.h:562
bool isComplex() const
Definition: CGValue.h:52
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:367
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type, returning the result.
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
llvm::Value * getVectorIdx() const
Definition: CGValue.h:343
Expr * getWeak() const
Definition: Expr.h:4879
QualType getUnqualifiedType() const
Retrieve the unqualified variant of the given type, removing as little sugar as possible.
Definition: Type.h:5169
Address getAggregateAddress() const
getAggregateAddr() - Return the Value* of the address of the aggregate.
Definition: CGValue.h:70
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1415
QualType getType() const
Definition: CGValue.h:258
uint64_t Width
Definition: ASTContext.h:81
CanQualType IntTy
Definition: ASTContext.h:889
RValue EmitLoadOfBitfieldLValue(LValue LV)
Definition: CGExpr.cpp:1461
bool hasVolatileMember(QualType T)
hasVolatileMember - returns true if aggregate type has a volatile member.
static RValue get(llvm::Value *V)
Definition: CGValue.h:85
static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr, Address Val1, Address Val2, uint64_t Size, llvm::AtomicOrdering SuccessOrder, llvm::AtomicOrdering FailureOrder)
Definition: CGAtomic.cpp:376
bool isVolatileQualified() const
Definition: CGValue.h:55
const CGFunctionInfo & arrangeFreeFunctionCall(const CallArgList &Args, const FunctionType *Ty, bool ChainCall)
Figure out the rules for calling a function with the given formal type using the given arguments...
Definition: CGCall.cpp:444
static RValue emitAtomicLibcall(CodeGenFunction &CGF, StringRef fnName, QualType resultType, CallArgList &args)
Definition: CGAtomic.cpp:321
#define true
Definition: stdbool.h:32
static RValue getAggregate(Address addr, bool isVolatile=false)
Definition: CGValue.h:106
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
LValue - This represents an lvalue references.
Definition: CGValue.h:152
CanQualType BoolTy
Definition: ASTContext.h:882
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Definition: CGBuilder.h:183
CallArgList - Type for representing both the value and type of arguments in a call.
Definition: CGCall.h:56
static Address EmitValToTemp(CodeGenFunction &CGF, Expr *E)
Definition: CGAtomic.cpp:641
Expr * getOrderFail() const
Definition: Expr.h:4869
bool isIgnored() const
Definition: CGValue.h:566
A class which abstracts out some details necessary for making a call.
Definition: Type.h:2872
Structure with information about how a bitfield should be accessed.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:5116
bool isPointerType() const
Definition: Type.h:5305
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1293