Add support atomic instructions as used with storage images
- OpAtomicIAdd
- OpAtomicSMin
- OpAtomicSMax
- OpAtomicUMin
- OpAtomicUMax
- OpAtomicAnd
- OpAtomicOr
- OpAtomicXor
- OpAtomicExchange
Includes required plumbing down through Reactor. Subzero support is just
stubbed for now.
Bug: b/130768731
Bug: b/127472316
Test: dEQP-VK.image.*
Change-Id: Iecf9e9ed602c4fde674f54491658048c96fd02c6
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29453
Tested-by: Chris Forbes <chrisforbes@google.com>
Presubmit-Ready: Chris Forbes <chrisforbes@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/src/Pipeline/SpirvShader.cpp b/src/Pipeline/SpirvShader.cpp
index 53974fe..2525911 100644
--- a/src/Pipeline/SpirvShader.cpp
+++ b/src/Pipeline/SpirvShader.cpp
@@ -678,6 +678,15 @@
case spv::OpDPdyFine:
case spv::OpFwidthFine:
case spv::OpAtomicLoad:
+ case spv::OpAtomicIAdd:
+ case spv::OpAtomicSMin:
+ case spv::OpAtomicSMax:
+ case spv::OpAtomicUMin:
+ case spv::OpAtomicUMax:
+ case spv::OpAtomicAnd:
+ case spv::OpAtomicOr:
+ case spv::OpAtomicXor:
+ case spv::OpAtomicExchange:
case spv::OpPhi:
case spv::OpImageSampleImplicitLod:
case spv::OpImageQuerySize:
@@ -1978,6 +1987,17 @@
case spv::OpAtomicStore:
return EmitStore(insn, state);
+ case spv::OpAtomicIAdd:
+ case spv::OpAtomicSMin:
+ case spv::OpAtomicSMax:
+ case spv::OpAtomicUMin:
+ case spv::OpAtomicUMax:
+ case spv::OpAtomicAnd:
+ case spv::OpAtomicOr:
+ case spv::OpAtomicXor:
+ case spv::OpAtomicExchange:
+ return EmitAtomicOp(insn, state);
+
case spv::OpAccessChain:
case spv::OpInBoundsAccessChain:
return EmitAccessChain(insn, state);
@@ -4695,13 +4715,75 @@
state->routine->createPointer(resultId, imageBase);
- SIMD::Int texelOffset = GetTexelOffset(coordinate, imageType, binding, sizeof(uint32_t));
+ // TODO: texelOffset is in bytes. get rid of shift once Ben's changes for DivergentPointer land
+ SIMD::Int texelOffset = GetTexelOffset(coordinate, imageType, binding, sizeof(uint32_t)) >> 2;
auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
dst.move(0, texelOffset);
return EmitResult::Continue;
}
+ SpirvShader::EmitResult SpirvShader::EmitAtomicOp(InsnIterator insn, EmitState *state) const
+ {
+ auto &resultType = getType(Type::ID(insn.word(1)));
+ Object::ID resultId = insn.word(2);
+ Object::ID semanticsId = insn.word(5);
+ auto memorySemantics = static_cast<spv::MemorySemanticsMask>(getObject(semanticsId).constantValue[0]);
+ auto memoryOrder = MemoryOrder(memorySemantics);
+ auto value = GenericValue(this, state->routine, insn.word(6));
+ auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
+ auto ptr = Pointer<UInt>(state->routine->getPointer(insn.word(3)));
+ auto offsets = state->routine->getIntermediate(insn.word(3)).UInt(0);
+
+ SIMD::UInt x;
+ for (int j = 0; j < SIMD::Width; j++)
+ {
+ If(Extract(state->activeLaneMask(), j) != 0)
+ {
+ auto offset = Extract(offsets, j);
+ auto laneValue = Extract(value.UInt(0), j);
+ UInt v;
+ switch (insn.opcode())
+ {
+ case spv::OpAtomicIAdd:
+ v = AddAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicAnd:
+ v = AndAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicOr:
+ v = OrAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicXor:
+ v = XorAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicSMin:
+ v = As<UInt>(MinAtomic(As<Pointer<Int>>(&ptr[offset]), As<Int>(laneValue), memoryOrder));
+ break;
+ case spv::OpAtomicSMax:
+ v = As<UInt>(MaxAtomic(As<Pointer<Int>>(&ptr[offset]), As<Int>(laneValue), memoryOrder));
+ break;
+ case spv::OpAtomicUMin:
+ v = MinAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicUMax:
+ v = MaxAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ case spv::OpAtomicExchange:
+ v = ExchangeAtomic(&ptr[offset], laneValue, memoryOrder);
+ break;
+ default:
+ UNIMPLEMENTED("Atomic op", OpcodeName(insn.opcode()).c_str());
+ break;
+ }
+ x = Insert(x, v, j);
+ }
+ }
+
+ dst.move(0, x);
+ return EmitResult::Continue;
+ }
+
void SpirvShader::emitEpilog(SpirvRoutine *routine) const
{
for (auto insn : *this)
diff --git a/src/Pipeline/SpirvShader.hpp b/src/Pipeline/SpirvShader.hpp
index 76474b4..2eb3cd8 100644
--- a/src/Pipeline/SpirvShader.hpp
+++ b/src/Pipeline/SpirvShader.hpp
@@ -729,6 +729,7 @@
EmitResult EmitImageRead(InsnIterator insn, EmitState *state) const;
EmitResult EmitImageWrite(InsnIterator insn, EmitState *state) const;
EmitResult EmitImageTexelPointer(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAtomicOp(InsnIterator insn, EmitState *state) const;
SIMD::Int GetTexelOffset(GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize) const;
diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index 6c30207..9bfc908 100644
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -1451,10 +1451,46 @@
T(llvm::PointerType::get(T(type), 0)));
}
- Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
+ Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
- return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), llvm::AtomicOrdering::SequentiallyConsistent));
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::And, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Or, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xor, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Min, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Max, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
+ }
+
+ Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createTrunc(Value *v, Type *destType)
diff --git a/src/Reactor/Nucleus.hpp b/src/Reactor/Nucleus.hpp
index 1adeaf3..593f7f1 100644
--- a/src/Reactor/Nucleus.hpp
+++ b/src/Reactor/Nucleus.hpp
@@ -101,7 +101,13 @@
static Value *createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex);
// Atomic instructions
- static Value *createAtomicAdd(Value *ptr, Value *value);
+ static Value *createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
// Cast/Conversion Operators
static Value *createTrunc(Value *V, Type *destType);
diff --git a/src/Reactor/Reactor.cpp b/src/Reactor/Reactor.cpp
index 96a8d25..5952254 100644
--- a/src/Reactor/Reactor.cpp
+++ b/src/Reactor/Reactor.cpp
@@ -2507,6 +2507,51 @@
return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
}
+ RValue<UInt> AddAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicAdd(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> AndAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicAnd(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> OrAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicOr(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> XorAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicXor(x.value, y.value, memoryOrder));
+ }
+
+ RValue<Int> MinAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
+ {
+ return RValue<Int>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> MinAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
+ }
+
+ RValue<Int> MaxAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
+ {
+ return RValue<Int>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> MaxAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
+ }
+
+ RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
+ {
+ return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder));
+ }
+
UInt::UInt(Argument<UInt> argument)
{
materialize(); // FIXME(b/129757459)
diff --git a/src/Reactor/Reactor.hpp b/src/Reactor/Reactor.hpp
index 4e95806..0b14131 100644
--- a/src/Reactor/Reactor.hpp
+++ b/src/Reactor/Reactor.hpp
@@ -1267,6 +1267,17 @@
RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max);
+
+ RValue<UInt> AddAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> AndAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> OrAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> XorAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<Int> MinAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
+ RValue<Int> MaxAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
+ RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+
// RValue<UInt> RoundUInt(RValue<Float> cast);
class Int2 : public LValue<Int2>
diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp
index 3afed7b..e4ba796 100644
--- a/src/Reactor/SubzeroReactor.cpp
+++ b/src/Reactor/SubzeroReactor.cpp
@@ -1033,12 +1033,48 @@
return createAdd(ptr, index);
}
- Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
+ Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicAdd");
return nullptr;
}
+ Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicAnd");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicOr");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicXor");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicMin");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicMax");
+ return nullptr;
+ }
+
+ Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
+ {
+ UNIMPLEMENTED("createAtomicExchange");
+ return nullptr;
+ }
+
static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
{
if(v->getType() == T(destType))
