| 4068 | } |
| 4069 | |
| 4070 | void compileAtomicRMWOp(unsigned MemoryIndex, uint64_t MemoryOffset, |
| 4071 | [[maybe_unused]] unsigned Alignment, |
| 4072 | LLVMAtomicRMWBinOp BinOp, LLVM::Type IntType, |
| 4073 | LLVM::Type TargetType, bool Signed = false) noexcept { |
| 4074 | auto Value = Builder.createSExtOrTrunc(stackPop(), TargetType); |
| 4075 | auto Offset = Builder.createZExt(Stack.back(), Context.Int64Ty); |
| 4076 | if (MemoryOffset != 0) { |
| 4077 | Offset = Builder.createAdd(Offset, LLContext.getInt64(MemoryOffset)); |
| 4078 | } |
| 4079 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
| 4080 | auto VPtr = Builder.createInBoundsGEP1( |
| 4081 | Context.Int8Ty, Context.getMemory(Builder, ExecCtx, MemoryIndex), |
| 4082 | Offset); |
| 4083 | auto Ptr = Builder.createBitCast(VPtr, TargetType.getPointerTo()); |
| 4084 | |
| 4085 | LLVM::Value Ret; |
| 4086 | if constexpr (Endian::native == Endian::big) { |
| 4087 | if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpAdd || |
| 4088 | BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpSub) { |
| 4089 | auto AtomicBB = LLVM::BasicBlock::create(LLContext, F.Fn, "atomic.rmw"); |
| 4090 | auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "atomic.rmw.ok"); |
| 4091 | Builder.createBr(AtomicBB); |
| 4092 | Builder.positionAtEnd(AtomicBB); |
| 4093 | |
| 4094 | auto Load = Builder.createLoad(TargetType, Ptr, true); |
| 4095 | Load.setOrdering(LLVMAtomicOrderingMonotonic); |
| 4096 | Load.setAlignment(1 << Alignment); |
| 4097 | |
| 4098 | LLVM::Value New; |
| 4099 | if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpAdd) |
| 4100 | New = Builder.createAdd(switchEndian(Load), Value); |
| 4101 | else if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpSub) { |
| 4102 | New = Builder.createSub(switchEndian(Load), Value); |
| 4103 | } else { |
| 4104 | assumingUnreachable(); |
| 4105 | } |
| 4106 | New = switchEndian(New); |
| 4107 | |
| 4108 | auto Exchange = Builder.createAtomicCmpXchg( |
| 4109 | Ptr, Load, New, LLVMAtomicOrderingSequentiallyConsistent, |
| 4110 | LLVMAtomicOrderingSequentiallyConsistent); |
| 4111 | |
| 4112 | Ret = Builder.createExtractValue(Exchange, 0); |
| 4113 | auto Success = Builder.createExtractValue(Exchange, 1); |
| 4114 | Builder.createCondBr(Success, OkBB, AtomicBB); |
| 4115 | Builder.positionAtEnd(OkBB); |
| 4116 | } else { |
| 4117 | Ret = Builder.createAtomicRMW(BinOp, Ptr, switchEndian(Value), |
| 4118 | LLVMAtomicOrderingSequentiallyConsistent); |
| 4119 | } |
| 4120 | } else { |
| 4121 | Ret = Builder.createAtomicRMW(BinOp, Ptr, switchEndian(Value), |
| 4122 | LLVMAtomicOrderingSequentiallyConsistent); |
| 4123 | } |
| 4124 | Ret = switchEndian(Ret); |
| 4125 | #if LLVM_VERSION_MAJOR >= 13 |
| 4126 | Ret.setAlignment(1 << Alignment); |
| 4127 | #endif |
nothing calls this directly
no test coverage detected