Add support for OpAtomicCompareExchange Includes associated Reactor plumbing. Bug: b/130768731 Bug: b/127472316 Test: dEQP-VK.image.* Change-Id: I700eb29f976410b59778e4ed162066b84a55a974 Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29456 Tested-by: Chris Forbes <chrisforbes@google.com> Presubmit-Ready: Chris Forbes <chrisforbes@google.com> Kokoro-Presubmit: kokoro <noreply+kokoro@google.com> Reviewed-by: Ben Clayton <bclayton@google.com> Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/src/Pipeline/SpirvShader.cpp b/src/Pipeline/SpirvShader.cpp index 2525911..32d8bd7 100644 --- a/src/Pipeline/SpirvShader.cpp +++ b/src/Pipeline/SpirvShader.cpp
@@ -687,6 +687,7 @@ case spv::OpAtomicOr: case spv::OpAtomicXor: case spv::OpAtomicExchange: + case spv::OpAtomicCompareExchange: case spv::OpPhi: case spv::OpImageSampleImplicitLod: case spv::OpImageQuerySize: @@ -1998,6 +1999,9 @@ case spv::OpAtomicExchange: return EmitAtomicOp(insn, state); + case spv::OpAtomicCompareExchange: + return EmitAtomicCompareExchange(insn, state); + case spv::OpAccessChain: case spv::OpInBoundsAccessChain: return EmitAccessChain(insn, state); @@ -4784,6 +4788,40 @@ return EmitResult::Continue; } + SpirvShader::EmitResult SpirvShader::EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const + { + // Separate from EmitAtomicOp due to different instruction encoding + auto &resultType = getType(Type::ID(insn.word(1))); + Object::ID resultId = insn.word(2); + + auto memorySemanticsEqual = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(5)).constantValue[0]); + auto memoryOrderEqual = MemoryOrder(memorySemanticsEqual); + auto memorySemanticsUnequal = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(6)).constantValue[0]); + auto memoryOrderUnequal = MemoryOrder(memorySemanticsUnequal); + + auto value = GenericValue(this, state->routine, insn.word(7)); + auto comparator = GenericValue(this, state->routine, insn.word(8)); + auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents); + auto ptr = Pointer<UInt>(state->routine->getPointer(insn.word(3))); + auto offsets = state->routine->getIntermediate(insn.word(3)).UInt(0); + + SIMD::UInt x; + for (int j = 0; j < SIMD::Width; j++) + { + If(Extract(state->activeLaneMask(), j) != 0) + { + auto offset = Extract(offsets, j); + auto laneValue = Extract(value.UInt(0), j); + auto laneComparator = Extract(comparator.UInt(0), j); + UInt v = CompareExchangeAtomic(&ptr[offset], laneValue, laneComparator, memoryOrderEqual, memoryOrderUnequal); + x = Insert(x, v, j); + } + } + + dst.move(0, x); + return EmitResult::Continue; + } + void SpirvShader::emitEpilog(SpirvRoutine *routine) const { for (auto insn : *this)
diff --git a/src/Pipeline/SpirvShader.hpp b/src/Pipeline/SpirvShader.hpp index 2eb3cd8..4b0879c 100644 --- a/src/Pipeline/SpirvShader.hpp +++ b/src/Pipeline/SpirvShader.hpp
@@ -730,6 +730,7 @@ EmitResult EmitImageWrite(InsnIterator insn, EmitState *state) const; EmitResult EmitImageTexelPointer(InsnIterator insn, EmitState *state) const; EmitResult EmitAtomicOp(InsnIterator insn, EmitState *state) const; + EmitResult EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const; SIMD::Int GetTexelOffset(GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize) const;
diff --git a/src/Reactor/LLVMReactor.cpp b/src/Reactor/LLVMReactor.cpp index 9bfc908..c196880 100644 --- a/src/Reactor/LLVMReactor.cpp +++ b/src/Reactor/LLVMReactor.cpp
@@ -1493,6 +1493,15 @@ return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder))); } + Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal) + { + RR_DEBUG_INFO_UPDATE_LOC(); + // Note: AtomicCmpXchgInstruction returns a 1-member struct containing the result, not the result directly. + return V(::builder->CreateExtractValue( + ::builder->CreateAtomicCmpXchg(V(ptr), V(compare), V(value), atomicOrdering(true, memoryOrderEqual), atomicOrdering(true, memoryOrderUnequal)), + llvm::ArrayRef<unsigned>(0u))); + } + Value *Nucleus::createTrunc(Value *v, Type *destType) { RR_DEBUG_INFO_UPDATE_LOC();
diff --git a/src/Reactor/Nucleus.hpp b/src/Reactor/Nucleus.hpp index 593f7f1..82ccafa 100644 --- a/src/Reactor/Nucleus.hpp +++ b/src/Reactor/Nucleus.hpp
@@ -108,6 +108,7 @@ static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed); static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed); static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed); + static Value *createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal); // Cast/Conversion Operators static Value *createTrunc(Value *V, Type *destType);
diff --git a/src/Reactor/Reactor.cpp b/src/Reactor/Reactor.cpp index 5952254..43bc1d6 100644 --- a/src/Reactor/Reactor.cpp +++ b/src/Reactor/Reactor.cpp
@@ -2552,6 +2552,11 @@ return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder)); } + RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal) + { + return RValue<UInt>(Nucleus::createAtomicCompareExchange(x.value, y.value, compare.value, memoryOrderEqual, memoryOrderUnequal)); + } + UInt::UInt(Argument<UInt> argument) { materialize(); // FIXME(b/129757459)
diff --git a/src/Reactor/Reactor.hpp b/src/Reactor/Reactor.hpp index 0b14131..f28da09 100644 --- a/src/Reactor/Reactor.hpp +++ b/src/Reactor/Reactor.hpp
@@ -1277,6 +1277,7 @@ RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder); RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder); RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder); + RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal); // RValue<UInt> RoundUInt(RValue<Float> cast);
diff --git a/src/Reactor/SubzeroReactor.cpp b/src/Reactor/SubzeroReactor.cpp index e4ba796..b8a91db 100644 --- a/src/Reactor/SubzeroReactor.cpp +++ b/src/Reactor/SubzeroReactor.cpp
@@ -1075,6 +1075,12 @@ return nullptr; } + Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal) + { + UNIMPLEMENTED("createAtomicCompareExchange"); + return nullptr; + } + static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType) { if(v->getType() == T(destType))