src/Pipeline/SpirvShaderGroup.cpp - SwiftShader - Git at Google

 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 #include "SpirvShader.hpp"

 #include <spirv/unified1/spirv.hpp>

 namespace sw {

 // Template function to perform a binary group operation.
 // |TYPE| should be the type of the binary operation (as a SIMD::<ScalarType>).
 // |I| should be a type suitable to initialize the identity value.
 // |APPLY| should be a callable object that takes two RValue<TYPE> parameters
 // and returns a new RValue<TYPE> corresponding to the operation's result.
 template<typename TYPE, typename I, typename APPLY>
 static RValue<TYPE> BinaryOperation(
     spv::GroupOperation operation,
     RValue<SIMD::UInt> value,
     RValue<SIMD::UInt> mask,
     const I identityValue,
     APPLY &&apply)
 {
 	auto identity = TYPE(identityValue);
 	SIMD::UInt v_uint = (value & mask) | (As<SIMD::UInt>(identity) & ~mask);
 	TYPE v = As<TYPE>(v_uint);

 	switch(operation)
 	{
 	case spv::GroupOperationReduce:
 		{
 			// NOTE: floating-point add and multiply are not really commutative so
 			//       ensure that all values in the final lanes are identical
 			TYPE v2 = apply(v.xxzz, v.yyww);  // [xy]   [xy]   [zw]   [zw]
 			return apply(v2.xxxx, v2.zzzz);   // [xyzw] [xyzw] [xyzw] [xyzw]
 		}
 		break;
 	case spv::GroupOperationInclusiveScan:
 		{
 			TYPE v2 = apply(v, Shuffle(v, identity, 0x4012) /* [id, v.y, v.z, v.w] */);   // [x] [xy] [yz]  [zw]
 			return apply(v2, Shuffle(v2, identity, 0x4401) /* [id,  id, v2.x, v2.y] */);  // [x] [xy] [xyz] [xyzw]
 		}
 		break;
 	case spv::GroupOperationExclusiveScan:
 		{
 			TYPE v2 = apply(v, Shuffle(v, identity, 0x4012) /* [id, v.y, v.z, v.w] */);      // [x] [xy] [yz]  [zw]
 			TYPE v3 = apply(v2, Shuffle(v2, identity, 0x4401) /* [id,  id, v2.x, v2.y] */);  // [x] [xy] [xyz] [xyzw]
 			return Shuffle(v3, identity, 0x4012 /* [id, v3.x, v3.y, v3.z] */);               // [i] [x]  [xy]  [xyz]
 		}
 		break;
 	default:
 		UNSUPPORTED("Group operation: %d", operation);
 		return identity;
 	}
 }

 void SpirvEmitter::EmitGroupNonUniform(InsnIterator insn)
 {
 	ASSERT(SIMD::Width == 4);  // EmitGroupNonUniform makes many assumptions that the SIMD vector width is 4

 	auto &type = shader.getType(Type::ID(insn.word(1)));
 	Object::ID resultId = insn.word(2);
 	auto scope = spv::Scope(shader.GetConstScalarInt(insn.word(3)));
 	ASSERT_MSG(scope == spv::ScopeSubgroup, "Scope for Non Uniform Group Operations must be Subgroup for Vulkan 1.1");

 	auto &dst = createIntermediate(resultId, type.componentCount);

 	switch(insn.opcode())
 	{
 	case spv::OpGroupNonUniformElect:
 		{
 			// Result is true only in the active invocation with the lowest id
 			// in the group, otherwise result is false.
 			SIMD::Int active = activeLaneMask();  // Considers helper invocations active. See b/151137030
 			// TODO: Would be nice if we could write this as:
 			//   elect = active & ~(active.Oxyz | active.OOxy | active.OOOx)
 			auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 			auto elect = active & ~(v0111 & (active.xxyz | active.xxxy | active.xxxx));
 			dst.move(0, elect);
 		}
 		break;

 	case spv::OpGroupNonUniformAll:
 		{
 			Operand predicate(shader, *this, insn.word(4));
 			dst.move(0, AndAll(predicate.UInt(0) | ~As<SIMD::UInt>(activeLaneMask())));  // Considers helper invocations active. See b/151137030
 		}
 		break;

 	case spv::OpGroupNonUniformAny:
 		{
 			Operand predicate(shader, *this, insn.word(4));
 			dst.move(0, OrAll(predicate.UInt(0) & As<SIMD::UInt>(activeLaneMask())));  // Considers helper invocations active. See b/151137030
 		}
 		break;

 	case spv::OpGroupNonUniformAllEqual:
 		{
 			Operand value(shader, *this, insn.word(4));
 			auto res = SIMD::UInt(0xffffffff);
 			SIMD::UInt active = As<SIMD::UInt>(activeLaneMask());  // Considers helper invocations active. See b/151137030
 			SIMD::UInt inactive = ~active;
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::UInt v = value.UInt(i) & active;
 				SIMD::UInt filled = v;
 				for(int j = 0; j < SIMD::Width - 1; j++)
 				{
 					filled |= filled.yzwx & inactive;  // Populate inactive 'holes' with a live value
 				}
 				res &= AndAll(CmpEQ(filled.xyzw, filled.yzwx));
 			}
 			dst.move(0, res);
 		}
 		break;

 	case spv::OpGroupNonUniformBroadcast:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			auto idId = Object::ID(insn.word(5));
 			Operand value(shader, *this, valueId);

 			// Decide between the fast path for constants and the slow path for
 			// intermediates.
 			if(shader.getObject(idId).kind == Object::Kind::Constant)
 			{
 				auto id = SIMD::Int(shader.GetConstScalarInt(insn.word(5)));
 				auto mask = CmpEQ(id, SIMD::Int(0, 1, 2, 3));
 				for(auto i = 0u; i < type.componentCount; i++)
 				{
 					dst.move(i, OrAll(value.Int(i) & mask));
 				}
 			}
 			else
 			{
 				Operand id(shader, *this, idId);

 				SIMD::UInt active = As<SIMD::UInt>(activeLaneMask());  // Considers helper invocations active. See b/151137030
 				SIMD::UInt inactive = ~active;
 				SIMD::UInt filled = id.UInt(0) & active;

 				for(int j = 0; j < SIMD::Width - 1; j++)
 				{
 					filled |= filled.yzwx & inactive;  // Populate inactive 'holes' with a live value
 				}

 				auto mask = CmpEQ(filled, SIMD::UInt(0, 1, 2, 3));

 				for(uint32_t i = 0u; i < type.componentCount; i++)
 				{
 					dst.move(i, OrAll(value.UInt(i) & mask));
 				}
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformBroadcastFirst:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			Operand value(shader, *this, valueId);
 			// Result is true only in the active invocation with the lowest id
 			// in the group, otherwise result is false.
 			SIMD::Int active = activeLaneMask();  // Considers helper invocations active. See b/151137030
 			// TODO: Would be nice if we could write this as:
 			//   elect = active & ~(active.Oxyz | active.OOxy | active.OOOx)
 			auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 			auto elect = active & ~(v0111 & (active.xxyz | active.xxxy | active.xxxx));
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				dst.move(i, OrAll(value.Int(i) & elect));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformQuadBroadcast:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			Operand value(shader, *this, valueId);

 			ASSERT(shader.getType(shader.getObject(insn.word(5))).componentCount == 1);
 			auto indexId = Object::ID(insn.word(5));
 			SIMD::Int index = Operand(shader, *this, indexId).Int(0);

 			SIMD::Int active = activeLaneMask();
 			// Populate all lanes in index with the same value. Index is required to be
 			// uniform per the SPIR-V spec, so all active lanes should be identical.
 			index = OrAll(active & index);
 			SIMD::Int mask = CmpEQ(index, SIMD::Int(0, 1, 2, 3));

 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				dst.move(i, OrAll(value.Int(i) & mask));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformQuadSwap:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			// SPIR-V spec: Drection must be a scalar of integer type and come from a constant instruction
 			int direction = shader.GetConstScalarInt(insn.word(5));

 			Operand value(shader, *this, valueId);
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::Int v = value.Int(i);
 				switch(direction)
 				{
 				case 0:  // Horizontal
 					dst.move(i, v.yxwz);
 					break;
 				case 1:  // Vertical
 					dst.move(i, v.zwxy);
 					break;
 				case 2:  // Diagonal
 					dst.move(i, v.wzyx);
 					break;
 				default:
 					// The SPIR-V spec doesn't define what happens in this case,
 					// so the result in undefined.
 					UNSUPPORTED("SPIR-V does not define a OpGroupNonUniformQuadSwap result for a direction of %d", direction);
 					break;
 				}
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformBallot:
 		{
 			ASSERT(type.componentCount == 4);
 			Operand predicate(shader, *this, insn.word(4));
 			dst.move(0, SIMD::Int(SignMask(activeLaneMask() & predicate.Int(0))));  // Considers helper invocations active. See b/151137030
 			dst.move(1, SIMD::Int(0));
 			dst.move(2, SIMD::Int(0));
 			dst.move(3, SIMD::Int(0));
 		}
 		break;

 	case spv::OpGroupNonUniformInverseBallot:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			ASSERT(type.componentCount == 1);
 			ASSERT(shader.getObjectType(valueId).componentCount == 4);
 			Operand value(shader, *this, valueId);
 			auto bit = (value.Int(0) >> SIMD::Int(0, 1, 2, 3)) & SIMD::Int(1);
 			dst.move(0, -bit);
 		}
 		break;

 	case spv::OpGroupNonUniformBallotBitExtract:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			auto indexId = Object::ID(insn.word(5));
 			ASSERT(type.componentCount == 1);
 			ASSERT(shader.getObjectType(valueId).componentCount == 4);
 			ASSERT(shader.getObjectType(indexId).componentCount == 1);
 			Operand value(shader, *this, valueId);
 			Operand index(shader, *this, indexId);
 			auto vecIdx = index.Int(0) / SIMD::Int(32);
 			auto bitIdx = index.Int(0) & SIMD::Int(31);
 			auto bits = (value.Int(0) & CmpEQ(vecIdx, SIMD::Int(0))) |
 			            (value.Int(1) & CmpEQ(vecIdx, SIMD::Int(1))) |
 			            (value.Int(2) & CmpEQ(vecIdx, SIMD::Int(2))) |
 			            (value.Int(3) & CmpEQ(vecIdx, SIMD::Int(3)));
 			dst.move(0, -((bits >> bitIdx) & SIMD::Int(1)));
 		}
 		break;

 	case spv::OpGroupNonUniformBallotBitCount:
 		{
 			auto operation = spv::GroupOperation(insn.word(4));
 			auto valueId = Object::ID(insn.word(5));
 			ASSERT(type.componentCount == 1);
 			ASSERT(shader.getObjectType(valueId).componentCount == 4);
 			Operand value(shader, *this, valueId);
 			switch(operation)
 			{
 			case spv::GroupOperationReduce:
 				dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(15)));
 				break;
 			case spv::GroupOperationInclusiveScan:
 				dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(1, 3, 7, 15)));
 				break;
 			case spv::GroupOperationExclusiveScan:
 				dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(0, 1, 3, 7)));
 				break;
 			default:
 				UNSUPPORTED("GroupOperation %d", int(operation));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformBallotFindLSB:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			ASSERT(type.componentCount == 1);
 			ASSERT(shader.getObjectType(valueId).componentCount == 4);
 			Operand value(shader, *this, valueId);
 			dst.move(0, Cttz(value.UInt(0) & SIMD::UInt(15), true));
 		}
 		break;

 	case spv::OpGroupNonUniformBallotFindMSB:
 		{
 			auto valueId = Object::ID(insn.word(4));
 			ASSERT(type.componentCount == 1);
 			ASSERT(shader.getObjectType(valueId).componentCount == 4);
 			Operand value(shader, *this, valueId);
 			dst.move(0, SIMD::UInt(31) - Ctlz(value.UInt(0) & SIMD::UInt(15), false));
 		}
 		break;

 	case spv::OpGroupNonUniformShuffle:
 		{
 			Operand value(shader, *this, insn.word(4));
 			Operand id(shader, *this, insn.word(5));
 			auto x = CmpEQ(SIMD::Int(0), id.Int(0));
 			auto y = CmpEQ(SIMD::Int(1), id.Int(0));
 			auto z = CmpEQ(SIMD::Int(2), id.Int(0));
 			auto w = CmpEQ(SIMD::Int(3), id.Int(0));
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::Int v = value.Int(i);
 				dst.move(i, (x & v.xxxx) | (y & v.yyyy) | (z & v.zzzz) | (w & v.wwww));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformShuffleXor:
 		{
 			Operand value(shader, *this, insn.word(4));
 			Operand mask(shader, *this, insn.word(5));
 			auto x = CmpEQ(SIMD::Int(0), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
 			auto y = CmpEQ(SIMD::Int(1), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
 			auto z = CmpEQ(SIMD::Int(2), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
 			auto w = CmpEQ(SIMD::Int(3), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::Int v = value.Int(i);
 				dst.move(i, (x & v.xxxx) | (y & v.yyyy) | (z & v.zzzz) | (w & v.wwww));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformShuffleUp:
 		{
 			Operand value(shader, *this, insn.word(4));
 			Operand delta(shader, *this, insn.word(5));
 			auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0));
 			auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0));
 			auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0));
 			auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0));
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::Int v = value.Int(i);
 				dst.move(i, (d0 & v.xyzw) | (d1 & v.xxyz) | (d2 & v.xxxy) | (d3 & v.xxxx));
 			}
 		}
 		break;

 	case spv::OpGroupNonUniformShuffleDown:
 		{
 			Operand value(shader, *this, insn.word(4));
 			Operand delta(shader, *this, insn.word(5));
 			auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0));
 			auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0));
 			auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0));
 			auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0));
 			for(auto i = 0u; i < type.componentCount; i++)
 			{
 				SIMD::Int v = value.Int(i);
 				dst.move(i, (d0 & v.xyzw) | (d1 & v.yzww) | (d2 & v.zwww) | (d3 & v.wwww));
 			}
 		}
 		break;

 	// The remaining instructions are GroupNonUniformArithmetic operations
 	default:
 		auto &type = shader.getType(Type::ID(insn.word(1)));
 		auto operation = static_cast<spv::GroupOperation>(insn.word(4));
 		Operand value(shader, *this, insn.word(5));
 		auto mask = As<SIMD::UInt>(activeLaneMask());  // Considers helper invocations active. See b/151137030

 		for(uint32_t i = 0; i < type.componentCount; i++)
 		{
 			switch(insn.opcode())
 			{
 			case spv::OpGroupNonUniformIAdd:
 				dst.move(i, BinaryOperation<SIMD::Int>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) { return a + b; }));
 				break;
 			case spv::OpGroupNonUniformFAdd:
 				dst.move(i, BinaryOperation<SIMD::Float>(
 				                operation, value.UInt(i), mask, 0.0f,
 				                [](auto a, auto b) { return a + b; }));
 				break;

 			case spv::OpGroupNonUniformIMul:
 				dst.move(i, BinaryOperation<SIMD::Int>(
 				                operation, value.UInt(i), mask, 1,
 				                [](auto a, auto b) { return a * b; }));
 				break;

 			case spv::OpGroupNonUniformFMul:
 				dst.move(i, BinaryOperation<SIMD::Float>(
 				                operation, value.UInt(i), mask, 1.0f,
 				                [](auto a, auto b) { return a * b; }));
 				break;

 			case spv::OpGroupNonUniformBitwiseAnd:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, ~0u,
 				                [](auto a, auto b) { return a & b; }));
 				break;

 			case spv::OpGroupNonUniformBitwiseOr:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) { return a | b; }));
 				break;

 			case spv::OpGroupNonUniformBitwiseXor:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) { return a ^ b; }));
 				break;

 			case spv::OpGroupNonUniformSMin:
 				dst.move(i, BinaryOperation<SIMD::Int>(
 				                operation, value.UInt(i), mask, INT32_MAX,
 				                [](auto a, auto b) { return Min(a, b); }));
 				break;

 			case spv::OpGroupNonUniformUMin:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, ~0u,
 				                [](auto a, auto b) { return Min(a, b); }));
 				break;

 			case spv::OpGroupNonUniformFMin:
 				dst.move(i, BinaryOperation<SIMD::Float>(
 				                operation, value.UInt(i), mask, SIMD::Float::infinity(),
 				                [](auto a, auto b) { return NMin(a, b); }));
 				break;

 			case spv::OpGroupNonUniformSMax:
 				dst.move(i, BinaryOperation<SIMD::Int>(
 				                operation, value.UInt(i), mask, INT32_MIN,
 				                [](auto a, auto b) { return Max(a, b); }));
 				break;

 			case spv::OpGroupNonUniformUMax:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) { return Max(a, b); }));
 				break;

 			case spv::OpGroupNonUniformFMax:
 				dst.move(i, BinaryOperation<SIMD::Float>(
 				                operation, value.UInt(i), mask, -SIMD::Float::infinity(),
 				                [](auto a, auto b) { return NMax(a, b); }));
 				break;

 			case spv::OpGroupNonUniformLogicalAnd:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, ~0u,
 				                [](auto a, auto b) {
 					                SIMD::UInt zero = SIMD::UInt(0);
 					                return CmpNEQ(a, zero) & CmpNEQ(b, zero);
 				                }));
 				break;

 			case spv::OpGroupNonUniformLogicalOr:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) {
 					                SIMD::UInt zero = SIMD::UInt(0);
 					                return CmpNEQ(a, zero) | CmpNEQ(b, zero);
 				                }));
 				break;

 			case spv::OpGroupNonUniformLogicalXor:
 				dst.move(i, BinaryOperation<SIMD::UInt>(
 				                operation, value.UInt(i), mask, 0,
 				                [](auto a, auto b) {
 					                SIMD::UInt zero = SIMD::UInt(0);
 					                return CmpNEQ(a, zero) ^ CmpNEQ(b, zero);
 				                }));
 				break;

 			default:
 				UNSUPPORTED("EmitGroupNonUniform op: %s", shader.OpcodeName(type.opcode()));
 			}
 		}
 		break;
 	}
 }

 }  // namespace sw
	// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	#include "SpirvShader.hpp"

	#include <spirv/unified1/spirv.hpp>

	namespace sw {

	// Template function to perform a binary group operation.
	// \|TYPE\| should be the type of the binary operation (as a SIMD::<ScalarType>).
	// \|I\| should be a type suitable to initialize the identity value.
	// \|APPLY\| should be a callable object that takes two RValue<TYPE> parameters
	// and returns a new RValue<TYPE> corresponding to the operation's result.
	template<typename TYPE, typename I, typename APPLY>
	static RValue<TYPE> BinaryOperation(
	spv::GroupOperation operation,
	RValue<SIMD::UInt> value,
	RValue<SIMD::UInt> mask,
	const I identityValue,
	APPLY &&apply)
	{
	auto identity = TYPE(identityValue);
	SIMD::UInt v_uint = (value & mask) \| (As<SIMD::UInt>(identity) & ~mask);
	TYPE v = As<TYPE>(v_uint);

	switch(operation)
	{
	case spv::GroupOperationReduce:
	{
	// NOTE: floating-point add and multiply are not really commutative so
	// ensure that all values in the final lanes are identical
	TYPE v2 = apply(v.xxzz, v.yyww); // [xy] [xy] [zw] [zw]
	return apply(v2.xxxx, v2.zzzz); // [xyzw] [xyzw] [xyzw] [xyzw]
	}
	break;
	case spv::GroupOperationInclusiveScan:
	{
	TYPE v2 = apply(v, Shuffle(v, identity, 0x4012) /* [id, v.y, v.z, v.w] */); // [x] [xy] [yz] [zw]
	return apply(v2, Shuffle(v2, identity, 0x4401) /* [id, id, v2.x, v2.y] */); // [x] [xy] [xyz] [xyzw]
	}
	break;
	case spv::GroupOperationExclusiveScan:
	{
	TYPE v2 = apply(v, Shuffle(v, identity, 0x4012) /* [id, v.y, v.z, v.w] */); // [x] [xy] [yz] [zw]
	TYPE v3 = apply(v2, Shuffle(v2, identity, 0x4401) /* [id, id, v2.x, v2.y] */); // [x] [xy] [xyz] [xyzw]
	return Shuffle(v3, identity, 0x4012 /* [id, v3.x, v3.y, v3.z] */); // [i] [x] [xy] [xyz]
	}
	break;
	default:
	UNSUPPORTED("Group operation: %d", operation);
	return identity;
	}
	}

	void SpirvEmitter::EmitGroupNonUniform(InsnIterator insn)
	{
	ASSERT(SIMD::Width == 4); // EmitGroupNonUniform makes many assumptions that the SIMD vector width is 4

	auto &type = shader.getType(Type::ID(insn.word(1)));
	Object::ID resultId = insn.word(2);
	auto scope = spv::Scope(shader.GetConstScalarInt(insn.word(3)));
	ASSERT_MSG(scope == spv::ScopeSubgroup, "Scope for Non Uniform Group Operations must be Subgroup for Vulkan 1.1");

	auto &dst = createIntermediate(resultId, type.componentCount);

	switch(insn.opcode())
	{
	case spv::OpGroupNonUniformElect:
	{
	// Result is true only in the active invocation with the lowest id
	// in the group, otherwise result is false.
	SIMD::Int active = activeLaneMask(); // Considers helper invocations active. See b/151137030
	// TODO: Would be nice if we could write this as:
	// elect = active & ~(active.Oxyz \| active.OOxy \| active.OOOx)
	auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
	auto elect = active & ~(v0111 & (active.xxyz \| active.xxxy \| active.xxxx));
	dst.move(0, elect);
	}
	break;

	case spv::OpGroupNonUniformAll:
	{
	Operand predicate(shader, *this, insn.word(4));
	dst.move(0, AndAll(predicate.UInt(0) \| ~As<SIMD::UInt>(activeLaneMask()))); // Considers helper invocations active. See b/151137030
	}
	break;

	case spv::OpGroupNonUniformAny:
	{
	Operand predicate(shader, *this, insn.word(4));
	dst.move(0, OrAll(predicate.UInt(0) & As<SIMD::UInt>(activeLaneMask()))); // Considers helper invocations active. See b/151137030
	}
	break;

	case spv::OpGroupNonUniformAllEqual:
	{
	Operand value(shader, *this, insn.word(4));
	auto res = SIMD::UInt(0xffffffff);
	SIMD::UInt active = As<SIMD::UInt>(activeLaneMask()); // Considers helper invocations active. See b/151137030
	SIMD::UInt inactive = ~active;
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::UInt v = value.UInt(i) & active;
	SIMD::UInt filled = v;
	for(int j = 0; j < SIMD::Width - 1; j++)
	{
	filled \|= filled.yzwx & inactive; // Populate inactive 'holes' with a live value
	}
	res &= AndAll(CmpEQ(filled.xyzw, filled.yzwx));
	}
	dst.move(0, res);
	}
	break;

	case spv::OpGroupNonUniformBroadcast:
	{
	auto valueId = Object::ID(insn.word(4));
	auto idId = Object::ID(insn.word(5));
	Operand value(shader, *this, valueId);

	// Decide between the fast path for constants and the slow path for
	// intermediates.
	if(shader.getObject(idId).kind == Object::Kind::Constant)
	{
	auto id = SIMD::Int(shader.GetConstScalarInt(insn.word(5)));
	auto mask = CmpEQ(id, SIMD::Int(0, 1, 2, 3));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	dst.move(i, OrAll(value.Int(i) & mask));
	}
	}
	else
	{
	Operand id(shader, *this, idId);

	SIMD::UInt active = As<SIMD::UInt>(activeLaneMask()); // Considers helper invocations active. See b/151137030
	SIMD::UInt inactive = ~active;
	SIMD::UInt filled = id.UInt(0) & active;

	for(int j = 0; j < SIMD::Width - 1; j++)
	{
	filled \|= filled.yzwx & inactive; // Populate inactive 'holes' with a live value
	}

	auto mask = CmpEQ(filled, SIMD::UInt(0, 1, 2, 3));

	for(uint32_t i = 0u; i < type.componentCount; i++)
	{
	dst.move(i, OrAll(value.UInt(i) & mask));
	}
	}
	}
	break;

	case spv::OpGroupNonUniformBroadcastFirst:
	{
	auto valueId = Object::ID(insn.word(4));
	Operand value(shader, *this, valueId);
	// Result is true only in the active invocation with the lowest id
	// in the group, otherwise result is false.
	SIMD::Int active = activeLaneMask(); // Considers helper invocations active. See b/151137030
	// TODO: Would be nice if we could write this as:
	// elect = active & ~(active.Oxyz \| active.OOxy \| active.OOOx)
	auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
	auto elect = active & ~(v0111 & (active.xxyz \| active.xxxy \| active.xxxx));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	dst.move(i, OrAll(value.Int(i) & elect));
	}
	}
	break;

	case spv::OpGroupNonUniformQuadBroadcast:
	{
	auto valueId = Object::ID(insn.word(4));
	Operand value(shader, *this, valueId);

	ASSERT(shader.getType(shader.getObject(insn.word(5))).componentCount == 1);
	auto indexId = Object::ID(insn.word(5));
	SIMD::Int index = Operand(shader, *this, indexId).Int(0);

	SIMD::Int active = activeLaneMask();
	// Populate all lanes in index with the same value. Index is required to be
	// uniform per the SPIR-V spec, so all active lanes should be identical.
	index = OrAll(active & index);
	SIMD::Int mask = CmpEQ(index, SIMD::Int(0, 1, 2, 3));

	for(auto i = 0u; i < type.componentCount; i++)
	{
	dst.move(i, OrAll(value.Int(i) & mask));
	}
	}
	break;

	case spv::OpGroupNonUniformQuadSwap:
	{
	auto valueId = Object::ID(insn.word(4));
	// SPIR-V spec: Drection must be a scalar of integer type and come from a constant instruction
	int direction = shader.GetConstScalarInt(insn.word(5));

	Operand value(shader, *this, valueId);
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::Int v = value.Int(i);
	switch(direction)
	{
	case 0: // Horizontal
	dst.move(i, v.yxwz);
	break;
	case 1: // Vertical
	dst.move(i, v.zwxy);
	break;
	case 2: // Diagonal
	dst.move(i, v.wzyx);
	break;
	default:
	// The SPIR-V spec doesn't define what happens in this case,
	// so the result in undefined.
	UNSUPPORTED("SPIR-V does not define a OpGroupNonUniformQuadSwap result for a direction of %d", direction);
	break;
	}
	}
	}
	break;

	case spv::OpGroupNonUniformBallot:
	{
	ASSERT(type.componentCount == 4);
	Operand predicate(shader, *this, insn.word(4));
	dst.move(0, SIMD::Int(SignMask(activeLaneMask() & predicate.Int(0)))); // Considers helper invocations active. See b/151137030
	dst.move(1, SIMD::Int(0));
	dst.move(2, SIMD::Int(0));
	dst.move(3, SIMD::Int(0));
	}
	break;

	case spv::OpGroupNonUniformInverseBallot:
	{
	auto valueId = Object::ID(insn.word(4));
	ASSERT(type.componentCount == 1);
	ASSERT(shader.getObjectType(valueId).componentCount == 4);
	Operand value(shader, *this, valueId);
	auto bit = (value.Int(0) >> SIMD::Int(0, 1, 2, 3)) & SIMD::Int(1);
	dst.move(0, -bit);
	}
	break;

	case spv::OpGroupNonUniformBallotBitExtract:
	{
	auto valueId = Object::ID(insn.word(4));
	auto indexId = Object::ID(insn.word(5));
	ASSERT(type.componentCount == 1);
	ASSERT(shader.getObjectType(valueId).componentCount == 4);
	ASSERT(shader.getObjectType(indexId).componentCount == 1);
	Operand value(shader, *this, valueId);
	Operand index(shader, *this, indexId);
	auto vecIdx = index.Int(0) / SIMD::Int(32);
	auto bitIdx = index.Int(0) & SIMD::Int(31);
	auto bits = (value.Int(0) & CmpEQ(vecIdx, SIMD::Int(0))) \|
	(value.Int(1) & CmpEQ(vecIdx, SIMD::Int(1))) \|
	(value.Int(2) & CmpEQ(vecIdx, SIMD::Int(2))) \|
	(value.Int(3) & CmpEQ(vecIdx, SIMD::Int(3)));
	dst.move(0, -((bits >> bitIdx) & SIMD::Int(1)));
	}
	break;

	case spv::OpGroupNonUniformBallotBitCount:
	{
	auto operation = spv::GroupOperation(insn.word(4));
	auto valueId = Object::ID(insn.word(5));
	ASSERT(type.componentCount == 1);
	ASSERT(shader.getObjectType(valueId).componentCount == 4);
	Operand value(shader, *this, valueId);
	switch(operation)
	{
	case spv::GroupOperationReduce:
	dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(15)));
	break;
	case spv::GroupOperationInclusiveScan:
	dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(1, 3, 7, 15)));
	break;
	case spv::GroupOperationExclusiveScan:
	dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(0, 1, 3, 7)));
	break;
	default:
	UNSUPPORTED("GroupOperation %d", int(operation));
	}
	}
	break;

	case spv::OpGroupNonUniformBallotFindLSB:
	{
	auto valueId = Object::ID(insn.word(4));
	ASSERT(type.componentCount == 1);
	ASSERT(shader.getObjectType(valueId).componentCount == 4);
	Operand value(shader, *this, valueId);
	dst.move(0, Cttz(value.UInt(0) & SIMD::UInt(15), true));
	}
	break;

	case spv::OpGroupNonUniformBallotFindMSB:
	{
	auto valueId = Object::ID(insn.word(4));
	ASSERT(type.componentCount == 1);
	ASSERT(shader.getObjectType(valueId).componentCount == 4);
	Operand value(shader, *this, valueId);
	dst.move(0, SIMD::UInt(31) - Ctlz(value.UInt(0) & SIMD::UInt(15), false));
	}
	break;

	case spv::OpGroupNonUniformShuffle:
	{
	Operand value(shader, *this, insn.word(4));
	Operand id(shader, *this, insn.word(5));
	auto x = CmpEQ(SIMD::Int(0), id.Int(0));
	auto y = CmpEQ(SIMD::Int(1), id.Int(0));
	auto z = CmpEQ(SIMD::Int(2), id.Int(0));
	auto w = CmpEQ(SIMD::Int(3), id.Int(0));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::Int v = value.Int(i);
	dst.move(i, (x & v.xxxx) \| (y & v.yyyy) \| (z & v.zzzz) \| (w & v.wwww));
	}
	}
	break;

	case spv::OpGroupNonUniformShuffleXor:
	{
	Operand value(shader, *this, insn.word(4));
	Operand mask(shader, *this, insn.word(5));
	auto x = CmpEQ(SIMD::Int(0), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
	auto y = CmpEQ(SIMD::Int(1), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
	auto z = CmpEQ(SIMD::Int(2), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
	auto w = CmpEQ(SIMD::Int(3), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::Int v = value.Int(i);
	dst.move(i, (x & v.xxxx) \| (y & v.yyyy) \| (z & v.zzzz) \| (w & v.wwww));
	}
	}
	break;

	case spv::OpGroupNonUniformShuffleUp:
	{
	Operand value(shader, *this, insn.word(4));
	Operand delta(shader, *this, insn.word(5));
	auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0));
	auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0));
	auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0));
	auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::Int v = value.Int(i);
	dst.move(i, (d0 & v.xyzw) \| (d1 & v.xxyz) \| (d2 & v.xxxy) \| (d3 & v.xxxx));
	}
	}
	break;

	case spv::OpGroupNonUniformShuffleDown:
	{
	Operand value(shader, *this, insn.word(4));
	Operand delta(shader, *this, insn.word(5));
	auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0));
	auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0));
	auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0));
	auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0));
	for(auto i = 0u; i < type.componentCount; i++)
	{
	SIMD::Int v = value.Int(i);
	dst.move(i, (d0 & v.xyzw) \| (d1 & v.yzww) \| (d2 & v.zwww) \| (d3 & v.wwww));
	}
	}
	break;

	// The remaining instructions are GroupNonUniformArithmetic operations
	default:
	auto &type = shader.getType(Type::ID(insn.word(1)));
	auto operation = static_cast<spv::GroupOperation>(insn.word(4));
	Operand value(shader, *this, insn.word(5));
	auto mask = As<SIMD::UInt>(activeLaneMask()); // Considers helper invocations active. See b/151137030

	for(uint32_t i = 0; i < type.componentCount; i++)
	{
	switch(insn.opcode())
	{
	case spv::OpGroupNonUniformIAdd:
	dst.move(i, BinaryOperation<SIMD::Int>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) { return a + b; }));
	break;
	case spv::OpGroupNonUniformFAdd:
	dst.move(i, BinaryOperation<SIMD::Float>(
	operation, value.UInt(i), mask, 0.0f,
	[](auto a, auto b) { return a + b; }));
	break;

	case spv::OpGroupNonUniformIMul:
	dst.move(i, BinaryOperation<SIMD::Int>(
	operation, value.UInt(i), mask, 1,
	[](auto a, auto b) { return a * b; }));
	break;

	case spv::OpGroupNonUniformFMul:
	dst.move(i, BinaryOperation<SIMD::Float>(
	operation, value.UInt(i), mask, 1.0f,
	[](auto a, auto b) { return a * b; }));
	break;

	case spv::OpGroupNonUniformBitwiseAnd:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, ~0u,
	[](auto a, auto b) { return a & b; }));
	break;

	case spv::OpGroupNonUniformBitwiseOr:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) { return a \| b; }));
	break;

	case spv::OpGroupNonUniformBitwiseXor:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) { return a ^ b; }));
	break;

	case spv::OpGroupNonUniformSMin:
	dst.move(i, BinaryOperation<SIMD::Int>(
	operation, value.UInt(i), mask, INT32_MAX,
	[](auto a, auto b) { return Min(a, b); }));
	break;

	case spv::OpGroupNonUniformUMin:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, ~0u,
	[](auto a, auto b) { return Min(a, b); }));
	break;

	case spv::OpGroupNonUniformFMin:
	dst.move(i, BinaryOperation<SIMD::Float>(
	operation, value.UInt(i), mask, SIMD::Float::infinity(),
	[](auto a, auto b) { return NMin(a, b); }));
	break;

	case spv::OpGroupNonUniformSMax:
	dst.move(i, BinaryOperation<SIMD::Int>(
	operation, value.UInt(i), mask, INT32_MIN,
	[](auto a, auto b) { return Max(a, b); }));
	break;

	case spv::OpGroupNonUniformUMax:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) { return Max(a, b); }));
	break;

	case spv::OpGroupNonUniformFMax:
	dst.move(i, BinaryOperation<SIMD::Float>(
	operation, value.UInt(i), mask, -SIMD::Float::infinity(),
	[](auto a, auto b) { return NMax(a, b); }));
	break;

	case spv::OpGroupNonUniformLogicalAnd:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, ~0u,
	[](auto a, auto b) {
	SIMD::UInt zero = SIMD::UInt(0);
	return CmpNEQ(a, zero) & CmpNEQ(b, zero);
	}));
	break;

	case spv::OpGroupNonUniformLogicalOr:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) {
	SIMD::UInt zero = SIMD::UInt(0);
	return CmpNEQ(a, zero) \| CmpNEQ(b, zero);
	}));
	break;

	case spv::OpGroupNonUniformLogicalXor:
	dst.move(i, BinaryOperation<SIMD::UInt>(
	operation, value.UInt(i), mask, 0,
	[](auto a, auto b) {
	SIMD::UInt zero = SIMD::UInt(0);
	return CmpNEQ(a, zero) ^ CmpNEQ(b, zero);
	}));
	break;

	default:
	UNSUPPORTED("EmitGroupNonUniform op: %s", shader.OpcodeName(type.opcode()));
	}
	}
	break;
	}
	}

	} // namespace sw