Add support atomic instructions as used with storage images
- OpAtomicIAdd
- OpAtomicSMin
- OpAtomicSMax
- OpAtomicUMin
- OpAtomicUMax
- OpAtomicAnd
- OpAtomicOr
- OpAtomicXor
- OpAtomicExchange
Includes required plumbing down through Reactor. Subzero support is just
stubbed for now.
Bug: b/130768731
Bug: b/127472316
Test: dEQP-VK.image.*
Change-Id: Iecf9e9ed602c4fde674f54491658048c96fd02c6
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29453
Tested-by: Chris Forbes <chrisforbes@google.com>
Presubmit-Ready: Chris Forbes <chrisforbes@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index 6c30207..9bfc908 100644
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -1451,10 +1451,46 @@
T(llvm::PointerType::get(T(type), 0)));
}
- Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
+ Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
- return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), llvm::AtomicOrdering::SequentiallyConsistent));
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::And, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Or, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xor, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Min, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Max, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createTrunc(Value *v, Type *destType)
diff --git a/src/Reactor/Nucleus.hpp b/src/Reactor/Nucleus.hpp
index 1adeaf3..593f7f1 100644
--- a/src/Reactor/Nucleus.hpp
+++ b/src/Reactor/Nucleus.hpp
@@ -101,7 +101,13 @@
static Value *createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex);
// Atomic instructions
- static Value *createAtomicAdd(Value *ptr, Value *value);
+ static Value *createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
// Cast/Conversion Operators
static Value *createTrunc(Value *V, Type *destType);
diff --git a/src/Reactor/Reactor.cpp b/src/Reactor/Reactor.cpp
index 96a8d25..5952254 100644
--- a/src/Reactor/Reactor.cpp
+++ b/src/Reactor/Reactor.cpp
@@ -2507,6 +2507,51 @@
return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
}
+ RValue<UInt> AddAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicAdd(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> AndAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicAnd(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> OrAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicOr(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> XorAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicXor(x.value, y.value, memoryOrder));
+ }
+
+ RValue<Int> MinAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
+ {
+ return RValue<Int>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> MinAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
+ }
+
+ RValue<Int> MaxAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
+ {
+ return RValue<Int>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> MaxAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder));
+ }
+
UInt::UInt(Argument<UInt> argument)
{
materialize(); // FIXME(b/129757459)
diff --git a/src/Reactor/Reactor.hpp b/src/Reactor/Reactor.hpp
index 4e95806..0b14131 100644
--- a/src/Reactor/Reactor.hpp
+++ b/src/Reactor/Reactor.hpp
@@ -1267,6 +1267,17 @@
RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max);
+
+ RValue<UInt> AddAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> AndAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> OrAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> XorAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<Int> MinAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
+ RValue<Int> MaxAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
+ RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+
// RValue<UInt> RoundUInt(RValue<Float> cast);
class Int2 : public LValue<Int2>
diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp
index 3afed7b..e4ba796 100644
--- a/src/Reactor/SubzeroReactor.cpp
+++ b/src/Reactor/SubzeroReactor.cpp
@@ -1033,12 +1033,48 @@
return createAdd(ptr, index);
}
- Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
+ Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicAdd");
return nullptr;
}
+ Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicAnd");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicOr");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicXor");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicMin");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicMax");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicExchange");
+ return nullptr;
+ }
+
static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
{
if(v->getType() == T(destType))
