Add support for OpAtomicCompareExchange
Includes associated Reactor plumbing.
Bug: b/130768731
Bug: b/127472316
Test: dEQP-VK.image.*
Change-Id: I700eb29f976410b59778e4ed162066b84a55a974
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29456
Tested-by: Chris Forbes <chrisforbes@google.com>
Presubmit-Ready: Chris Forbes <chrisforbes@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/src/Pipeline/SpirvShader.cpp b/src/Pipeline/SpirvShader.cpp
index 2525911..32d8bd7 100644
--- a/src/Pipeline/SpirvShader.cpp
+++ b/src/Pipeline/SpirvShader.cpp
@@ -687,6 +687,7 @@
case spv::OpAtomicOr:
case spv::OpAtomicXor:
case spv::OpAtomicExchange:
+ case spv::OpAtomicCompareExchange:
case spv::OpPhi:
case spv::OpImageSampleImplicitLod:
case spv::OpImageQuerySize:
@@ -1998,6 +1999,9 @@
case spv::OpAtomicExchange:
return EmitAtomicOp(insn, state);
+ case spv::OpAtomicCompareExchange:
+ return EmitAtomicCompareExchange(insn, state);
+
case spv::OpAccessChain:
case spv::OpInBoundsAccessChain:
return EmitAccessChain(insn, state);
@@ -4784,6 +4788,40 @@
return EmitResult::Continue;
}
+ SpirvShader::EmitResult SpirvShader::EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const
+ {
+ // Separate from EmitAtomicOp due to different instruction encoding
+ auto &resultType = getType(Type::ID(insn.word(1)));
+ Object::ID resultId = insn.word(2);
+
+ auto memorySemanticsEqual = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(5)).constantValue[0]);
+ auto memoryOrderEqual = MemoryOrder(memorySemanticsEqual);
+ auto memorySemanticsUnequal = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(6)).constantValue[0]);
+ auto memoryOrderUnequal = MemoryOrder(memorySemanticsUnequal);
+
+ auto value = GenericValue(this, state->routine, insn.word(7));
+ auto comparator = GenericValue(this, state->routine, insn.word(8));
+ auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
+ auto ptr = Pointer<UInt>(state->routine->getPointer(insn.word(3)));
+ auto offsets = state->routine->getIntermediate(insn.word(3)).UInt(0);
+
+ SIMD::UInt x;
+ for (int j = 0; j < SIMD::Width; j++)
+ {
+ If(Extract(state->activeLaneMask(), j) != 0)
+ {
+ auto offset = Extract(offsets, j);
+ auto laneValue = Extract(value.UInt(0), j);
+ auto laneComparator = Extract(comparator.UInt(0), j);
+ UInt v = CompareExchangeAtomic(&ptr[offset], laneValue, laneComparator, memoryOrderEqual, memoryOrderUnequal);
+ x = Insert(x, v, j);
+ }
+ }
+
+ dst.move(0, x);
+ return EmitResult::Continue;
+ }
+
void SpirvShader::emitEpilog(SpirvRoutine *routine) const
{
for (auto insn : *this)
diff --git a/src/Pipeline/SpirvShader.hpp b/src/Pipeline/SpirvShader.hpp
index 2eb3cd8..4b0879c 100644
--- a/src/Pipeline/SpirvShader.hpp
+++ b/src/Pipeline/SpirvShader.hpp
@@ -730,6 +730,7 @@
EmitResult EmitImageWrite(InsnIterator insn, EmitState *state) const;
EmitResult EmitImageTexelPointer(InsnIterator insn, EmitState *state) const;
EmitResult EmitAtomicOp(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const;
SIMD::Int GetTexelOffset(GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize) const;
diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp
index 9bfc908..c196880 100644
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -1493,6 +1493,15 @@
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
+ Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+ {
+ RR_DEBUG_INFO_UPDATE_LOC();
+ // Note: AtomicCmpXchgInstruction returns a 1-member struct containing the result, not the result directly.
+ return V(::builder->CreateExtractValue(
+ ::builder->CreateAtomicCmpXchg(V(ptr), V(compare), V(value), atomicOrdering(true, memoryOrderEqual), atomicOrdering(true, memoryOrderUnequal)),
+ llvm::ArrayRef<unsigned>(0u)));
+ }
+
Value *Nucleus::createTrunc(Value *v, Type *destType)
{
RR_DEBUG_INFO_UPDATE_LOC();
diff --git a/src/Reactor/Nucleus.hpp b/src/Reactor/Nucleus.hpp
index 593f7f1..82ccafa 100644
--- a/src/Reactor/Nucleus.hpp
+++ b/src/Reactor/Nucleus.hpp
@@ -108,6 +108,7 @@
static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+ static Value *createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal);
// Cast/Conversion Operators
static Value *createTrunc(Value *V, Type *destType);
diff --git a/src/Reactor/Reactor.cpp b/src/Reactor/Reactor.cpp
index 5952254..43bc1d6 100644
--- a/src/Reactor/Reactor.cpp
+++ b/src/Reactor/Reactor.cpp
@@ -2552,6 +2552,11 @@
return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder));
}
+ RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+ {
+ return RValue<UInt>(Nucleus::createAtomicCompareExchange(x.value, y.value, compare.value, memoryOrderEqual, memoryOrderUnequal));
+ }
+
UInt::UInt(Argument<UInt> argument)
{
materialize(); // FIXME(b/129757459)
diff --git a/src/Reactor/Reactor.hpp b/src/Reactor/Reactor.hpp
index 0b14131..f28da09 100644
--- a/src/Reactor/Reactor.hpp
+++ b/src/Reactor/Reactor.hpp
@@ -1277,6 +1277,7 @@
RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+ RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal);
// RValue<UInt> RoundUInt(RValue<Float> cast);
diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp
index e4ba796..b8a91db 100644
--- a/src/Reactor/SubzeroReactor.cpp
+++ b/src/Reactor/SubzeroReactor.cpp
@@ -1075,6 +1075,12 @@
return nullptr;
}
+ Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+ {
+ UNIMPLEMENTED("createAtomicCompareExchange");
+ return nullptr;
+ }
+
static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
{
if(v->getType() == T(destType))
