SpirvShader: Implement OpControlBarrier.
Use the new coroutines to yield when hitting a ControlBarrier.
A barrier pushes the shader subgroup to the end of the compute invocation queue, which forces all subgroups to be brought to the fence before continuing.
Tests: dEQP-VK.spirv_assembly.instruction.compute.workgroup_memory.*
Tests: dEQP-VK.subgroups.basic.compute.*
Tests: dEQP-VK.compute.basic.*
Bug: b/131672705
Bug: b/132232716
Change-Id: Id78be9ce9d9455cb2cb7254482568985845b8b6a
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/30851
Presubmit-Ready: Ben Clayton <bclayton@google.com>
Tested-by: Chris Forbes <chrisforbes@google.com>
Reviewed-by: Chris Forbes <chrisforbes@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
diff --git a/src/Pipeline/ComputeProgram.cpp b/src/Pipeline/ComputeProgram.cpp
index 361cbb1..081831c 100644
--- a/src/Pipeline/ComputeProgram.cpp
+++ b/src/Pipeline/ComputeProgram.cpp
@@ -18,6 +18,8 @@
#include "Vulkan/VkDebug.hpp"
#include "Vulkan/VkPipelineLayout.hpp"
+#include <queue>
+
namespace
{
enum { X, Y, Z };
@@ -154,17 +156,18 @@
void ComputeProgram::emit()
{
+ Int workgroupX = Arg<1>();
+ Int workgroupY = Arg<2>();
+ Int workgroupZ = Arg<3>();
+ Pointer<Byte> workgroupMemory = Arg<4>();
+ Int firstSubgroup = Arg<5>();
+ Int subgroupCount = Arg<6>();
+
routine.descriptorSets = data + OFFSET(Data, descriptorSets);
routine.descriptorDynamicOffsets = data + OFFSET(Data, descriptorDynamicOffsets);
routine.pushConstants = data + OFFSET(Data, pushConstants);
routine.constants = *Pointer<Pointer<Byte>>(data + OFFSET(Data, constants));
- routine.workgroupMemory = *Pointer<Pointer<Byte>>(data + OFFSET(Data, workgroupMemory));
-
- Int workgroupX = Arg<1>();
- Int workgroupY = Arg<2>();
- Int workgroupZ = Arg<3>();
- Int firstSubgroup = Arg<4>();
- Int subgroupCount = Arg<5>();
+ routine.workgroupMemory = workgroupMemory;
Int invocationsPerWorkgroup = *Pointer<Int>(data + OFFSET(Data, invocationsPerWorkgroup));
@@ -210,8 +213,8 @@
auto subgroupsPerWorkgroup = (invocationsPerWorkgroup + invocationsPerSubgroup - 1) / invocationsPerSubgroup;
// We're sharing a buffer here across all workgroups.
- // We can only do this because we know workgroups are executed
- // serially.
+ // We can only do this because we know a single workgroup is in flight
+ // at any time.
std::vector<uint8_t> workgroupMemory(shader->workgroupMemory.size());
Data data;
@@ -230,19 +233,51 @@
data.subgroupsPerWorkgroup = subgroupsPerWorkgroup;
data.pushConstants = pushConstants;
data.constants = &sw::constants;
- data.workgroupMemory = workgroupMemory.data();
- // TODO(bclayton): Split work across threads.
for (uint32_t groupZ = 0; groupZ < groupCountZ; groupZ++)
{
for (uint32_t groupY = 0; groupY < groupCountY; groupY++)
{
for (uint32_t groupX = 0; groupX < groupCountX; groupX++)
{
- (*this)(&data, groupX, groupY, groupZ, 0, subgroupsPerWorkgroup);
- }
- }
- }
+
+ // TODO(bclayton): Split work across threads.
+ using Coroutine = std::unique_ptr<rr::Stream<SpirvShader::YieldResult>>;
+ std::queue<Coroutine> coroutines;
+
+ if (shader->getModes().ContainsControlBarriers)
+ {
+ // Make a function call per subgroup so each subgroup
+ // can yield, bringing all subgroups to the barrier
+ // together.
+ for(int subgroupIndex = 0; subgroupIndex < subgroupsPerWorkgroup; subgroupIndex++)
+ {
+ auto coroutine = (*this)(&data, groupX, groupY, groupZ, workgroupMemory.data(), subgroupIndex, 1);
+ coroutines.push(std::move(coroutine));
+ }
+ }
+ else
+ {
+ auto coroutine = (*this)(&data, groupX, groupY, groupZ, workgroupMemory.data(), 0, subgroupsPerWorkgroup);
+ coroutines.push(std::move(coroutine));
+ }
+
+ while (coroutines.size() > 0)
+ {
+ auto coroutine = std::move(coroutines.front());
+ coroutines.pop();
+
+ SpirvShader::YieldResult result;
+ if (coroutine->await(result))
+ {
+ // TODO: Consider result (when the enum is more than 1 entry).
+ coroutines.push(std::move(coroutine));
+ }
+ }
+
+ } // groupX
+ } // groupY
+ } // groupZ
}
} // namespace sw
diff --git a/src/Pipeline/ComputeProgram.hpp b/src/Pipeline/ComputeProgram.hpp
index b2a3785..762a44b 100644
--- a/src/Pipeline/ComputeProgram.hpp
+++ b/src/Pipeline/ComputeProgram.hpp
@@ -37,11 +37,12 @@
struct Constants;
// ComputeProgram builds a SPIR-V compute shader.
- class ComputeProgram : public Coroutine<int(
+ class ComputeProgram : public Coroutine<SpirvShader::YieldResult(
void* data,
int32_t workgroupX,
int32_t workgroupY,
int32_t workgroupZ,
+ void* workgroupMemory,
int32_t firstSubgroup,
int32_t subgroupCount)>
{
@@ -80,7 +81,6 @@
uint32_t invocationsPerWorkgroup; // Total number of invocations per workgroup.
PushConstantStorage pushConstants;
const Constants *constants;
- uint8_t* workgroupMemory;
};
SpirvRoutine routine;
diff --git a/src/Pipeline/SpirvShader.cpp b/src/Pipeline/SpirvShader.cpp
index 020309e..ae720a2 100644
--- a/src/Pipeline/SpirvShader.cpp
+++ b/src/Pipeline/SpirvShader.cpp
@@ -13,8 +13,9 @@
// limitations under the License.
#include "SpirvShader.hpp"
-
#include "SamplerCore.hpp"
+
+#include "Reactor/Coroutine.hpp"
#include "System/Math.hpp"
#include "Vulkan/VkBuffer.hpp"
#include "Vulkan/VkBufferView.hpp"
@@ -884,6 +885,10 @@
// Don't need to do anything during analysis pass
break;
+ case spv::OpControlBarrier:
+ modes.ContainsControlBarriers = true;
+ break;
+
case spv::OpExtension:
{
auto ext = reinterpret_cast<char const *>(insn.wordPointer(1));
@@ -2462,6 +2467,9 @@
case spv::OpCopyMemory:
return EmitCopyMemory(insn, state);
+ case spv::OpControlBarrier:
+ return EmitControlBarrier(insn, state);
+
case spv::OpMemoryBarrier:
return EmitMemoryBarrier(insn, state);
@@ -4889,6 +4897,11 @@
return ptr;
}
+ void SpirvShader::Yield(YieldResult res) const
+ {
+ rr::Yield(RValue<Int>(int(res)));
+ }
+
SpirvShader::EmitResult SpirvShader::EmitImageRead(InsnIterator insn, EmitState *state) const
{
auto &resultType = getType(Type::ID(insn.word(1)));
@@ -5468,6 +5481,29 @@
return EmitResult::Continue;
}
+ SpirvShader::EmitResult SpirvShader::EmitControlBarrier(InsnIterator insn, EmitState *state) const
+ {
+ auto executionScope = spv::Scope(GetConstScalarInt(insn.word(1)));
+ auto semantics = spv::MemorySemanticsMask(GetConstScalarInt(insn.word(3)));
+ // TODO: We probably want to consider the memory scope here. For now,
+ // just always emit the full fence.
+ Fence(semantics);
+
+ switch (executionScope)
+ {
+ case spv::ScopeWorkgroup:
+ case spv::ScopeSubgroup:
+ Yield(YieldResult::ControlBarrier);
+ break;
+ default:
+ // See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module.
+ UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup");
+ break;
+ }
+
+ return EmitResult::Continue;
+ }
+
SpirvShader::EmitResult SpirvShader::EmitMemoryBarrier(InsnIterator insn, EmitState *state) const
{
auto semantics = spv::MemorySemanticsMask(GetConstScalarInt(insn.word(2)));
diff --git a/src/Pipeline/SpirvShader.hpp b/src/Pipeline/SpirvShader.hpp
index cb37cce..1eaa6f8 100644
--- a/src/Pipeline/SpirvShader.hpp
+++ b/src/Pipeline/SpirvShader.hpp
@@ -262,6 +262,11 @@
using ImageSampler = void(void* texture, void *sampler, void* uvsIn, void* texelOut, void* constants);
using GetImageSampler = ImageSampler*(const vk::ImageView *imageView, const vk::Sampler *sampler);
+ enum class YieldResult
+ {
+ ControlBarrier,
+ };
+
/* Pseudo-iterator over SPIRV instructions, designed to support range-based-for. */
class InsnIterator
{
@@ -543,6 +548,7 @@
bool DepthLess : 1;
bool DepthUnchanged : 1;
bool ContainsKill : 1;
+ bool ContainsControlBarriers : 1;
bool NeedsCentroid : 1;
// Compute workgroup dimensions
@@ -934,6 +940,7 @@
EmitResult EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const;
EmitResult EmitSampledImageCombineOrSplit(InsnIterator insn, EmitState *state) const;
EmitResult EmitCopyMemory(InsnIterator insn, EmitState *state) const;
+ EmitResult EmitControlBarrier(InsnIterator insn, EmitState *state) const;
EmitResult EmitMemoryBarrier(InsnIterator insn, EmitState *state) const;
EmitResult EmitGroupNonUniform(InsnIterator insn, EmitState *state) const;
@@ -944,6 +951,9 @@
// Emits a rr::Fence for the given MemorySemanticsMask.
void Fence(spv::MemorySemanticsMask semantics) const;
+ // Helper for calling rr::Yield with res cast to an rr::Int.
+ void Yield(YieldResult res) const;
+
// OpcodeName() returns the name of the opcode op.
// If NDEBUG is defined, then OpcodeName() will only return the numerical code.
static std::string OpcodeName(spv::Op op);
