| // Copyright 2018 The SwiftShader Authors. All Rights Reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #include "SpirvShader.hpp" |
| |
| #include "SamplerCore.hpp" |
| #include "Reactor/Coroutine.hpp" |
| #include "System/Math.hpp" |
| #include "Vulkan/VkBuffer.hpp" |
| #include "Vulkan/VkBufferView.hpp" |
| #include "Vulkan/VkDebug.hpp" |
| #include "Vulkan/VkDescriptorSet.hpp" |
| #include "Vulkan/VkPipelineLayout.hpp" |
| #include "Vulkan/VkDescriptorSetLayout.hpp" |
| #include "Vulkan/VkRenderPass.hpp" |
| #include "Device/Config.hpp" |
| |
| #include <spirv/unified1/spirv.hpp> |
| #include <spirv/unified1/GLSL.std.450.h> |
| |
| #include <queue> |
| |
| namespace |
| { |
| constexpr float PI = 3.141592653589793f; |
| |
| rr::RValue<rr::Bool> AnyTrue(rr::RValue<sw::SIMD::Int> const &ints) |
| { |
| return rr::SignMask(ints) != 0; |
| } |
| |
| rr::RValue<rr::Bool> AnyFalse(rr::RValue<sw::SIMD::Int> const &ints) |
| { |
| return rr::SignMask(~ints) != 0; |
| } |
| |
| template <typename T> |
| rr::RValue<T> AndAll(rr::RValue<T> const &mask) |
| { |
| T v1 = mask; // [x] [y] [z] [w] |
| T v2 = v1.xzxz & v1.ywyw; // [xy] [zw] [xy] [zw] |
| return v2.xxxx & v2.yyyy; // [xyzw] [xyzw] [xyzw] [xyzw] |
| } |
| |
| template <typename T> |
| rr::RValue<T> OrAll(rr::RValue<T> const &mask) |
| { |
| T v1 = mask; // [x] [y] [z] [w] |
| T v2 = v1.xzxz | v1.ywyw; // [xy] [zw] [xy] [zw] |
| return v2.xxxx | v2.yyyy; // [xyzw] [xyzw] [xyzw] [xyzw] |
| } |
| |
| rr::RValue<sw::SIMD::Float> Sign(rr::RValue<sw::SIMD::Float> const &val) |
| { |
| return rr::As<sw::SIMD::Float>((rr::As<sw::SIMD::UInt>(val) & sw::SIMD::UInt(0x80000000)) | sw::SIMD::UInt(0x3f800000)); |
| } |
| |
| // Returns the <whole, frac> of val. |
| // Both whole and frac will have the same sign as val. |
| std::pair<rr::RValue<sw::SIMD::Float>, rr::RValue<sw::SIMD::Float>> |
| Modf(rr::RValue<sw::SIMD::Float> const &val) |
| { |
| auto abs = Abs(val); |
| auto sign = Sign(val); |
| auto whole = Floor(abs) * sign; |
| auto frac = Frac(abs) * sign; |
| return std::make_pair(whole, frac); |
| } |
| |
| // Returns the number of 1s in bits, per lane. |
| sw::SIMD::UInt CountBits(rr::RValue<sw::SIMD::UInt> const &bits) |
| { |
| // TODO: Add an intrinsic to reactor. Even if there isn't a |
| // single vector instruction, there may be target-dependent |
| // ways to make this faster. |
| // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel |
| sw::SIMD::UInt c = bits - ((bits >> 1) & sw::SIMD::UInt(0x55555555)); |
| c = ((c >> 2) & sw::SIMD::UInt(0x33333333)) + (c & sw::SIMD::UInt(0x33333333)); |
| c = ((c >> 4) + c) & sw::SIMD::UInt(0x0F0F0F0F); |
| c = ((c >> 8) + c) & sw::SIMD::UInt(0x00FF00FF); |
| c = ((c >> 16) + c) & sw::SIMD::UInt(0x0000FFFF); |
| return c; |
| } |
| |
| // Returns 1 << bits. |
| // If the resulting bit overflows a 32 bit integer, 0 is returned. |
| rr::RValue<sw::SIMD::UInt> NthBit32(rr::RValue<sw::SIMD::UInt> const &bits) |
| { |
| return ((sw::SIMD::UInt(1) << bits) & rr::CmpLT(bits, sw::SIMD::UInt(32))); |
| } |
| |
| // Returns bitCount number of of 1's starting from the LSB. |
| rr::RValue<sw::SIMD::UInt> Bitmask32(rr::RValue<sw::SIMD::UInt> const &bitCount) |
| { |
| return NthBit32(bitCount) - sw::SIMD::UInt(1); |
| } |
| |
| // Performs a fused-multiply add, returning a * b + c. |
| rr::RValue<sw::SIMD::Float> FMA( |
| rr::RValue<sw::SIMD::Float> const &a, |
| rr::RValue<sw::SIMD::Float> const &b, |
| rr::RValue<sw::SIMD::Float> const &c) |
| { |
| return a * b + c; |
| } |
| |
| // Returns the exponent of the floating point number f. |
| // Assumes IEEE 754 |
| rr::RValue<sw::SIMD::Int> Exponent(rr::RValue<sw::SIMD::Float> f) |
| { |
| auto v = rr::As<sw::SIMD::UInt>(f); |
| return (sw::SIMD::Int((v >> sw::SIMD::UInt(23)) & sw::SIMD::UInt(0xFF)) - sw::SIMD::Int(126)); |
| } |
| |
| // Returns y if y < x; otherwise result is x. |
| // If one operand is a NaN, the other operand is the result. |
| // If both operands are NaN, the result is a NaN. |
| rr::RValue<sw::SIMD::Float> NMin(rr::RValue<sw::SIMD::Float> const &x, rr::RValue<sw::SIMD::Float> const &y) |
| { |
| using namespace rr; |
| auto xIsNan = IsNan(x); |
| auto yIsNan = IsNan(y); |
| return As<sw::SIMD::Float>( |
| // If neither are NaN, return min |
| ((~xIsNan & ~yIsNan) & As<sw::SIMD::Int>(Min(x, y))) | |
| // If one operand is a NaN, the other operand is the result |
| // If both operands are NaN, the result is a NaN. |
| ((~xIsNan & yIsNan) & As<sw::SIMD::Int>(x)) | |
| (( xIsNan ) & As<sw::SIMD::Int>(y))); |
| } |
| |
| // Returns y if y > x; otherwise result is x. |
| // If one operand is a NaN, the other operand is the result. |
| // If both operands are NaN, the result is a NaN. |
| rr::RValue<sw::SIMD::Float> NMax(rr::RValue<sw::SIMD::Float> const &x, rr::RValue<sw::SIMD::Float> const &y) |
| { |
| using namespace rr; |
| auto xIsNan = IsNan(x); |
| auto yIsNan = IsNan(y); |
| return As<sw::SIMD::Float>( |
| // If neither are NaN, return max |
| ((~xIsNan & ~yIsNan) & As<sw::SIMD::Int>(Max(x, y))) | |
| // If one operand is a NaN, the other operand is the result |
| // If both operands are NaN, the result is a NaN. |
| ((~xIsNan & yIsNan) & As<sw::SIMD::Int>(x)) | |
| (( xIsNan ) & As<sw::SIMD::Int>(y))); |
| } |
| |
| // Returns the determinant of a 2x2 matrix. |
| rr::RValue<sw::SIMD::Float> Determinant( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, |
| rr::RValue<sw::SIMD::Float> const &c, rr::RValue<sw::SIMD::Float> const &d) |
| { |
| return a*d - b*c; |
| } |
| |
| // Returns the determinant of a 3x3 matrix. |
| rr::RValue<sw::SIMD::Float> Determinant( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, rr::RValue<sw::SIMD::Float> const &c, |
| rr::RValue<sw::SIMD::Float> const &d, rr::RValue<sw::SIMD::Float> const &e, rr::RValue<sw::SIMD::Float> const &f, |
| rr::RValue<sw::SIMD::Float> const &g, rr::RValue<sw::SIMD::Float> const &h, rr::RValue<sw::SIMD::Float> const &i) |
| { |
| return a*e*i + b*f*g + c*d*h - c*e*g - b*d*i - a*f*h; |
| } |
| |
| // Returns the determinant of a 4x4 matrix. |
| rr::RValue<sw::SIMD::Float> Determinant( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, rr::RValue<sw::SIMD::Float> const &c, rr::RValue<sw::SIMD::Float> const &d, |
| rr::RValue<sw::SIMD::Float> const &e, rr::RValue<sw::SIMD::Float> const &f, rr::RValue<sw::SIMD::Float> const &g, rr::RValue<sw::SIMD::Float> const &h, |
| rr::RValue<sw::SIMD::Float> const &i, rr::RValue<sw::SIMD::Float> const &j, rr::RValue<sw::SIMD::Float> const &k, rr::RValue<sw::SIMD::Float> const &l, |
| rr::RValue<sw::SIMD::Float> const &m, rr::RValue<sw::SIMD::Float> const &n, rr::RValue<sw::SIMD::Float> const &o, rr::RValue<sw::SIMD::Float> const &p) |
| { |
| return a * Determinant(f, g, h, |
| j, k, l, |
| n, o, p) - |
| b * Determinant(e, g, h, |
| i, k, l, |
| m, o, p) + |
| c * Determinant(e, f, h, |
| i, j, l, |
| m, n, p) - |
| d * Determinant(e, f, g, |
| i, j, k, |
| m, n, o); |
| } |
| |
| // Returns the inverse of a 2x2 matrix. |
| std::array<rr::RValue<sw::SIMD::Float>, 4> MatrixInverse( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, |
| rr::RValue<sw::SIMD::Float> const &c, rr::RValue<sw::SIMD::Float> const &d) |
| { |
| auto s = sw::SIMD::Float(1.0f) / Determinant(a, b, c, d); |
| return {{s*d, -s*b, -s*c, s*a}}; |
| } |
| |
| // Returns the inverse of a 3x3 matrix. |
| std::array<rr::RValue<sw::SIMD::Float>, 9> MatrixInverse( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, rr::RValue<sw::SIMD::Float> const &c, |
| rr::RValue<sw::SIMD::Float> const &d, rr::RValue<sw::SIMD::Float> const &e, rr::RValue<sw::SIMD::Float> const &f, |
| rr::RValue<sw::SIMD::Float> const &g, rr::RValue<sw::SIMD::Float> const &h, rr::RValue<sw::SIMD::Float> const &i) |
| { |
| auto s = sw::SIMD::Float(1.0f) / Determinant( |
| a, b, c, |
| d, e, f, |
| g, h, i); // TODO: duplicate arithmetic calculating the det and below. |
| |
| return {{ |
| s * (e*i - f*h), s * (c*h - b*i), s * (b*f - c*e), |
| s * (f*g - d*i), s * (a*i - c*g), s * (c*d - a*f), |
| s * (d*h - e*g), s * (b*g - a*h), s * (a*e - b*d), |
| }}; |
| } |
| |
| // Returns the inverse of a 4x4 matrix. |
| std::array<rr::RValue<sw::SIMD::Float>, 16> MatrixInverse( |
| rr::RValue<sw::SIMD::Float> const &a, rr::RValue<sw::SIMD::Float> const &b, rr::RValue<sw::SIMD::Float> const &c, rr::RValue<sw::SIMD::Float> const &d, |
| rr::RValue<sw::SIMD::Float> const &e, rr::RValue<sw::SIMD::Float> const &f, rr::RValue<sw::SIMD::Float> const &g, rr::RValue<sw::SIMD::Float> const &h, |
| rr::RValue<sw::SIMD::Float> const &i, rr::RValue<sw::SIMD::Float> const &j, rr::RValue<sw::SIMD::Float> const &k, rr::RValue<sw::SIMD::Float> const &l, |
| rr::RValue<sw::SIMD::Float> const &m, rr::RValue<sw::SIMD::Float> const &n, rr::RValue<sw::SIMD::Float> const &o, rr::RValue<sw::SIMD::Float> const &p) |
| { |
| auto s = sw::SIMD::Float(1.0f) / Determinant( |
| a, b, c, d, |
| e, f, g, h, |
| i, j, k, l, |
| m, n, o, p); // TODO: duplicate arithmetic calculating the det and below. |
| |
| auto kplo = k*p - l*o, jpln = j*p - l*n, jokn = j*o - k*n; |
| auto gpho = g*p - h*o, fphn = f*p - h*n, fogn = f*o - g*n; |
| auto glhk = g*l - h*k, flhj = f*l - h*j, fkgj = f*k - g*j; |
| auto iplm = i*p - l*m, iokm = i*o - k*m, ephm = e*p - h*m; |
| auto eogm = e*o - g*m, elhi = e*l - h*i, ekgi = e*k - g*i; |
| auto injm = i*n - j*m, enfm = e*n - f*m, ejfi = e*j - f*i; |
| |
| return {{ |
| s * ( f * kplo - g * jpln + h * jokn), |
| s * (-b * kplo + c * jpln - d * jokn), |
| s * ( b * gpho - c * fphn + d * fogn), |
| s * (-b * glhk + c * flhj - d * fkgj), |
| |
| s * (-e * kplo + g * iplm - h * iokm), |
| s * ( a * kplo - c * iplm + d * iokm), |
| s * (-a * gpho + c * ephm - d * eogm), |
| s * ( a * glhk - c * elhi + d * ekgi), |
| |
| s * ( e * jpln - f * iplm + h * injm), |
| s * (-a * jpln + b * iplm - d * injm), |
| s * ( a * fphn - b * ephm + d * enfm), |
| s * (-a * flhj + b * elhi - d * ejfi), |
| |
| s * (-e * jokn + f * iokm - g * injm), |
| s * ( a * jokn - b * iokm + c * injm), |
| s * (-a * fogn + b * eogm - c * enfm), |
| s * ( a * fkgj - b * ekgi + c * ejfi), |
| }}; |
| } |
| |
| |
| sw::SIMD::Pointer interleaveByLane(sw::SIMD::Pointer p) |
| { |
| p *= sw::SIMD::Width; |
| p.staticOffsets[0] += 0 * sizeof(float); |
| p.staticOffsets[1] += 1 * sizeof(float); |
| p.staticOffsets[2] += 2 * sizeof(float); |
| p.staticOffsets[3] += 3 * sizeof(float); |
| return p; |
| } |
| |
| VkFormat SpirvFormatToVulkanFormat(spv::ImageFormat format) |
| { |
| switch (format) |
| { |
| case spv::ImageFormatRgba32f: return VK_FORMAT_R32G32B32A32_SFLOAT; |
| case spv::ImageFormatRgba32i: return VK_FORMAT_R32G32B32A32_SINT; |
| case spv::ImageFormatRgba32ui: return VK_FORMAT_R32G32B32A32_UINT; |
| case spv::ImageFormatR32f: return VK_FORMAT_R32_SFLOAT; |
| case spv::ImageFormatR32i: return VK_FORMAT_R32_SINT; |
| case spv::ImageFormatR32ui: return VK_FORMAT_R32_UINT; |
| case spv::ImageFormatRgba8: return VK_FORMAT_R8G8B8A8_UNORM; |
| case spv::ImageFormatRgba8Snorm: return VK_FORMAT_R8G8B8A8_SNORM; |
| case spv::ImageFormatRgba8i: return VK_FORMAT_R8G8B8A8_SINT; |
| case spv::ImageFormatRgba8ui: return VK_FORMAT_R8G8B8A8_UINT; |
| case spv::ImageFormatRgba16f: return VK_FORMAT_R16G16B16A16_SFLOAT; |
| case spv::ImageFormatRgba16i: return VK_FORMAT_R16G16B16A16_SINT; |
| case spv::ImageFormatRgba16ui: return VK_FORMAT_R16G16B16A16_UINT; |
| case spv::ImageFormatRg32f: return VK_FORMAT_R32G32_SFLOAT; |
| case spv::ImageFormatRg32i: return VK_FORMAT_R32G32_SINT; |
| case spv::ImageFormatRg32ui: return VK_FORMAT_R32G32_UINT; |
| |
| default: |
| UNIMPLEMENTED("SPIR-V ImageFormat %u", format); |
| return VK_FORMAT_UNDEFINED; |
| } |
| } |
| |
| sw::SIMD::Float sRGBtoLinear(sw::SIMD::Float c) |
| { |
| sw::SIMD::Float lc = c * sw::SIMD::Float(1.0f / 12.92f); |
| sw::SIMD::Float ec = sw::power((c + sw::SIMD::Float(0.055f)) * sw::SIMD::Float(1.0f / 1.055f), sw::SIMD::Float(2.4f)); |
| |
| sw::SIMD::Int linear = CmpLT(c, sw::SIMD::Float(0.04045f)); |
| |
| return rr::As<sw::SIMD::Float>((linear & rr::As<sw::SIMD::Int>(lc)) | (~linear & rr::As<sw::SIMD::Int>(ec))); // TODO: IfThenElse() |
| } |
| |
| } // anonymous namespace |
| |
| namespace sw |
| { |
| namespace SIMD |
| { |
| |
| template<typename T> |
| T Load(Pointer ptr, OutOfBoundsBehavior robustness, Int mask, bool atomic /* = false */, std::memory_order order /* = std::memory_order_relaxed */, int alignment /* = sizeof(float) */) |
| { |
| using EL = typename Element<T>::type; |
| |
| if (ptr.isStaticallyInBounds(sizeof(float), robustness)) |
| { |
| // All elements are statically known to be in-bounds. |
| // We can avoid costly conditional on masks. |
| |
| if (ptr.hasStaticSequentialOffsets(sizeof(float))) |
| { |
| // Offsets are sequential. Perform regular load. |
| return rr::Load(rr::Pointer<T>(ptr.base + ptr.staticOffsets[0]), alignment, atomic, order); |
| } |
| if (ptr.hasStaticEqualOffsets()) |
| { |
| // Load one, replicate. |
| return T(*rr::Pointer<EL>(ptr.base + ptr.staticOffsets[0], alignment)); |
| } |
| } |
| else |
| { |
| switch(robustness) |
| { |
| case OutOfBoundsBehavior::Nullify: |
| case OutOfBoundsBehavior::RobustBufferAccess: |
| case OutOfBoundsBehavior::UndefinedValue: |
| mask &= ptr.isInBounds(sizeof(float), robustness); // Disable out-of-bounds reads. |
| break; |
| case OutOfBoundsBehavior::UndefinedBehavior: |
| // Nothing to do. Application/compiler must guarantee no out-of-bounds accesses. |
| break; |
| } |
| } |
| |
| auto offsets = ptr.offsets(); |
| |
| if (!atomic && order == std::memory_order_relaxed) |
| { |
| if (ptr.hasStaticEqualOffsets()) |
| { |
| // Load one, replicate. |
| // Be careful of the case where the post-bounds-check mask |
| // is 0, in which case we must not load. |
| T out = T(0); |
| If(AnyTrue(mask)) |
| { |
| EL el = *rr::Pointer<EL>(ptr.base + ptr.staticOffsets[0], alignment); |
| out = T(el); |
| } |
| return out; |
| } |
| |
| bool zeroMaskedLanes = true; |
| switch(robustness) |
| { |
| case OutOfBoundsBehavior::Nullify: |
| case OutOfBoundsBehavior::RobustBufferAccess: // Must either return an in-bounds value, or zero. |
| zeroMaskedLanes = true; |
| break; |
| case OutOfBoundsBehavior::UndefinedValue: |
| case OutOfBoundsBehavior::UndefinedBehavior: |
| zeroMaskedLanes = false; |
| break; |
| } |
| |
| if (ptr.hasStaticSequentialOffsets(sizeof(float))) |
| { |
| return rr::MaskedLoad(rr::Pointer<T>(ptr.base + ptr.staticOffsets[0]), mask, alignment, zeroMaskedLanes); |
| } |
| |
| return rr::Gather(rr::Pointer<EL>(ptr.base), offsets, mask, alignment, zeroMaskedLanes); |
| } |
| else |
| { |
| T out; |
| auto anyLanesDisabled = AnyFalse(mask); |
| If(ptr.hasEqualOffsets() && !anyLanesDisabled) |
| { |
| // Load one, replicate. |
| auto offset = Extract(offsets, 0); |
| out = T(rr::Load(rr::Pointer<EL>(&ptr.base[offset]), alignment, atomic, order)); |
| } |
| Else If(ptr.hasSequentialOffsets(sizeof(float)) && !anyLanesDisabled) |
| { |
| // Load all elements in a single SIMD instruction. |
| auto offset = Extract(offsets, 0); |
| out = rr::Load(rr::Pointer<T>(&ptr.base[offset]), alignment, atomic, order); |
| } |
| Else |
| { |
| // Divergent offsets or masked lanes. |
| out = T(0); |
| for (int i = 0; i < SIMD::Width; i++) |
| { |
| If(Extract(mask, i) != 0) |
| { |
| auto offset = Extract(offsets, i); |
| auto el = rr::Load(rr::Pointer<EL>(&ptr.base[offset]), alignment, atomic, order); |
| out = Insert(out, el, i); |
| } |
| } |
| } |
| return out; |
| } |
| } |
| |
| template<typename T> |
| void Store(Pointer ptr, T val, OutOfBoundsBehavior robustness, Int mask, bool atomic /* = false */, std::memory_order order /* = std::memory_order_relaxed */) |
| { |
| using EL = typename Element<T>::type; |
| constexpr size_t alignment = sizeof(float); |
| auto offsets = ptr.offsets(); |
| |
| switch(robustness) |
| { |
| case OutOfBoundsBehavior::Nullify: |
| case OutOfBoundsBehavior::RobustBufferAccess: // TODO: Allows writing anywhere within bounds. Could be faster than masking. |
| case OutOfBoundsBehavior::UndefinedValue: // Should not be used for store operations. Treat as robust buffer access. |
| mask &= ptr.isInBounds(sizeof(float), robustness); // Disable out-of-bounds writes. |
| break; |
| case OutOfBoundsBehavior::UndefinedBehavior: |
| // Nothing to do. Application/compiler must guarantee no out-of-bounds accesses. |
| break; |
| } |
| |
| if (!atomic && order == std::memory_order_relaxed) |
| { |
| if (ptr.hasStaticEqualOffsets()) |
| { |
| If (AnyTrue(mask)) |
| { |
| // All equal. One of these writes will win -- elect the winning lane. |
| auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); |
| auto elect = mask & ~(v0111 & (mask.xxyz | mask.xxxy | mask.xxxx)); |
| auto maskedVal = As<SIMD::Int>(val) & elect; |
| auto scalarVal = Extract(maskedVal, 0) | |
| Extract(maskedVal, 1) | |
| Extract(maskedVal, 2) | |
| Extract(maskedVal, 3); |
| *rr::Pointer<EL>(ptr.base + ptr.staticOffsets[0], alignment) = As<EL>(scalarVal); |
| } |
| } |
| else if (ptr.hasStaticSequentialOffsets(sizeof(float))) |
| { |
| if (ptr.isStaticallyInBounds(sizeof(float), robustness)) |
| { |
| // Pointer has no elements OOB, and the store is not atomic. |
| // Perform a RMW. |
| auto p = rr::Pointer<SIMD::Int>(ptr.base + ptr.staticOffsets[0], alignment); |
| auto prev = *p; |
| *p = (prev & ~mask) | (As<SIMD::Int>(val) & mask); |
| } |
| else |
| { |
| rr::MaskedStore(rr::Pointer<T>(ptr.base + ptr.staticOffsets[0]), val, mask, alignment); |
| } |
| } |
| else |
| { |
| rr::Scatter(rr::Pointer<EL>(ptr.base), val, offsets, mask, alignment); |
| } |
| } |
| else |
| { |
| auto anyLanesDisabled = AnyFalse(mask); |
| If(ptr.hasSequentialOffsets(sizeof(float)) && !anyLanesDisabled) |
| { |
| // Store all elements in a single SIMD instruction. |
| auto offset = Extract(offsets, 0); |
| Store(val, rr::Pointer<T>(&ptr.base[offset]), alignment, atomic, order); |
| } |
| Else |
| { |
| // Divergent offsets or masked lanes. |
| for (int i = 0; i < SIMD::Width; i++) |
| { |
| If(Extract(mask, i) != 0) |
| { |
| auto offset = Extract(offsets, i); |
| rr::Store(Extract(val, i), rr::Pointer<EL>(&ptr.base[offset]), alignment, atomic, order); |
| } |
| } |
| } |
| } |
| } |
| |
| } // namespace SIMD |
| |
| SpirvShader::SpirvShader( |
| uint32_t codeSerialID, |
| VkShaderStageFlagBits pipelineStage, |
| const char *entryPointName, |
| InsnStore const &insns, |
| const vk::RenderPass *renderPass, |
| uint32_t subpassIndex, |
| bool robustBufferAccess) |
| : insns{insns}, inputs{MAX_INTERFACE_COMPONENTS}, |
| outputs{MAX_INTERFACE_COMPONENTS}, |
| codeSerialID(codeSerialID), |
| robustBufferAccess(robustBufferAccess) |
| { |
| ASSERT(insns.size() > 0); |
| |
| if (renderPass) |
| { |
| // capture formats of any input attachments present |
| auto subpass = renderPass->getSubpass(subpassIndex); |
| inputAttachmentFormats.reserve(subpass.inputAttachmentCount); |
| for (auto i = 0u; i < subpass.inputAttachmentCount; i++) |
| { |
| auto attachmentIndex = subpass.pInputAttachments[i].attachment; |
| inputAttachmentFormats.push_back(attachmentIndex != VK_ATTACHMENT_UNUSED |
| ? renderPass->getAttachment(attachmentIndex).format : VK_FORMAT_UNDEFINED); |
| } |
| } |
| |
| // Simplifying assumptions (to be satisfied by earlier transformations) |
| // - The only input/output OpVariables present are those used by the entrypoint |
| |
| Function::ID currentFunction; |
| Block::ID currentBlock; |
| InsnIterator blockStart; |
| |
| for (auto insn : *this) |
| { |
| spv::Op opcode = insn.opcode(); |
| |
| switch (opcode) |
| { |
| case spv::OpEntryPoint: |
| { |
| executionModel = spv::ExecutionModel(insn.word(1)); |
| auto id = Function::ID(insn.word(2)); |
| auto name = insn.string(3); |
| auto stage = executionModelToStage(executionModel); |
| if (stage == pipelineStage && strcmp(name, entryPointName) == 0) |
| { |
| ASSERT_MSG(entryPoint == 0, "Duplicate entry point with name '%s' and stage %d", name, int(stage)); |
| entryPoint = id; |
| } |
| break; |
| } |
| |
| case spv::OpExecutionMode: |
| ProcessExecutionMode(insn); |
| break; |
| |
| case spv::OpDecorate: |
| { |
| TypeOrObjectID targetId = insn.word(1); |
| auto decoration = static_cast<spv::Decoration>(insn.word(2)); |
| uint32_t value = insn.wordCount() > 3 ? insn.word(3) : 0; |
| |
| decorations[targetId].Apply(decoration, value); |
| |
| switch(decoration) |
| { |
| case spv::DecorationDescriptorSet: |
| descriptorDecorations[targetId].DescriptorSet = value; |
| break; |
| case spv::DecorationBinding: |
| descriptorDecorations[targetId].Binding = value; |
| break; |
| case spv::DecorationInputAttachmentIndex: |
| descriptorDecorations[targetId].InputAttachmentIndex = value; |
| break; |
| default: |
| // Only handling descriptor decorations here. |
| break; |
| } |
| |
| if (decoration == spv::DecorationCentroid) |
| modes.NeedsCentroid = true; |
| break; |
| } |
| |
| case spv::OpMemberDecorate: |
| { |
| Type::ID targetId = insn.word(1); |
| auto memberIndex = insn.word(2); |
| auto decoration = static_cast<spv::Decoration>(insn.word(3)); |
| uint32_t value = insn.wordCount() > 4 ? insn.word(4) : 0; |
| |
| auto &d = memberDecorations[targetId]; |
| if (memberIndex >= d.size()) |
| d.resize(memberIndex + 1); // on demand; exact size would require another pass... |
| |
| d[memberIndex].Apply(decoration, value); |
| |
| if (decoration == spv::DecorationCentroid) |
| modes.NeedsCentroid = true; |
| break; |
| } |
| |
| case spv::OpDecorationGroup: |
| // Nothing to do here. We don't need to record the definition of the group; we'll just have |
| // the bundle of decorations float around. If we were to ever walk the decorations directly, |
| // we might think about introducing this as a real Object. |
| break; |
| |
| case spv::OpGroupDecorate: |
| { |
| uint32_t group = insn.word(1); |
| auto const &groupDecorations = decorations[group]; |
| auto const &descriptorGroupDecorations = descriptorDecorations[group]; |
| for (auto i = 2u; i < insn.wordCount(); i++) |
| { |
| // Remaining operands are targets to apply the group to. |
| uint32_t target = insn.word(i); |
| decorations[target].Apply(groupDecorations); |
| descriptorDecorations[target].Apply(descriptorGroupDecorations); |
| } |
| |
| break; |
| } |
| |
| case spv::OpGroupMemberDecorate: |
| { |
| auto const &srcDecorations = decorations[insn.word(1)]; |
| for (auto i = 2u; i < insn.wordCount(); i += 2) |
| { |
| // remaining operands are pairs of <id>, literal for members to apply to. |
| auto &d = memberDecorations[insn.word(i)]; |
| auto memberIndex = insn.word(i + 1); |
| if (memberIndex >= d.size()) |
| d.resize(memberIndex + 1); // on demand resize, see above... |
| d[memberIndex].Apply(srcDecorations); |
| } |
| break; |
| } |
| |
| case spv::OpLabel: |
| { |
| ASSERT(currentBlock.value() == 0); |
| currentBlock = Block::ID(insn.word(1)); |
| blockStart = insn; |
| break; |
| } |
| |
| // Branch Instructions (subset of Termination Instructions): |
| case spv::OpBranch: |
| case spv::OpBranchConditional: |
| case spv::OpSwitch: |
| case spv::OpReturn: |
| // fallthrough |
| |
| // Termination instruction: |
| case spv::OpKill: |
| case spv::OpUnreachable: |
| { |
| ASSERT(currentBlock.value() != 0); |
| ASSERT(currentFunction.value() != 0); |
| |
| auto blockEnd = insn; blockEnd++; |
| functions[currentFunction].blocks[currentBlock] = Block(blockStart, blockEnd); |
| currentBlock = Block::ID(0); |
| |
| if (opcode == spv::OpKill) |
| { |
| modes.ContainsKill = true; |
| } |
| break; |
| } |
| |
| case spv::OpLoopMerge: |
| case spv::OpSelectionMerge: |
| break; // Nothing to do in analysis pass. |
| |
| case spv::OpTypeVoid: |
| case spv::OpTypeBool: |
| case spv::OpTypeInt: |
| case spv::OpTypeFloat: |
| case spv::OpTypeVector: |
| case spv::OpTypeMatrix: |
| case spv::OpTypeImage: |
| case spv::OpTypeSampler: |
| case spv::OpTypeSampledImage: |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| case spv::OpTypeStruct: |
| case spv::OpTypePointer: |
| case spv::OpTypeFunction: |
| DeclareType(insn); |
| break; |
| |
| case spv::OpVariable: |
| { |
| Type::ID typeId = insn.word(1); |
| Object::ID resultId = insn.word(2); |
| auto storageClass = static_cast<spv::StorageClass>(insn.word(3)); |
| |
| auto &object = defs[resultId]; |
| object.kind = Object::Kind::Pointer; |
| object.definition = insn; |
| object.type = typeId; |
| |
| ASSERT(getType(typeId).definition.opcode() == spv::OpTypePointer); |
| ASSERT(getType(typeId).storageClass == storageClass); |
| |
| switch (storageClass) |
| { |
| case spv::StorageClassInput: |
| case spv::StorageClassOutput: |
| ProcessInterfaceVariable(object); |
| break; |
| |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| object.kind = Object::Kind::DescriptorSet; |
| break; |
| |
| case spv::StorageClassPushConstant: |
| case spv::StorageClassPrivate: |
| case spv::StorageClassFunction: |
| case spv::StorageClassUniformConstant: |
| break; // Correctly handled. |
| |
| case spv::StorageClassWorkgroup: |
| { |
| auto &elTy = getType(getType(typeId).element); |
| auto sizeInBytes = elTy.sizeInComponents * static_cast<uint32_t>(sizeof(float)); |
| workgroupMemory.allocate(resultId, sizeInBytes); |
| object.kind = Object::Kind::Pointer; |
| break; |
| } |
| case spv::StorageClassAtomicCounter: |
| case spv::StorageClassImage: |
| UNIMPLEMENTED("StorageClass %d not yet implemented", (int)storageClass); |
| break; |
| |
| case spv::StorageClassCrossWorkgroup: |
| UNSUPPORTED("SPIR-V OpenCL Execution Model (StorageClassCrossWorkgroup)"); |
| break; |
| |
| case spv::StorageClassGeneric: |
| UNSUPPORTED("SPIR-V GenericPointer Capability (StorageClassGeneric)"); |
| break; |
| |
| default: |
| UNREACHABLE("Unexpected StorageClass %d", storageClass); // See Appendix A of the Vulkan spec. |
| break; |
| } |
| break; |
| } |
| |
| case spv::OpConstant: |
| case spv::OpSpecConstant: |
| CreateConstant(insn).constantValue[0] = insn.word(3); |
| break; |
| case spv::OpConstantFalse: |
| case spv::OpSpecConstantFalse: |
| CreateConstant(insn).constantValue[0] = 0; // Represent Boolean false as zero. |
| break; |
| case spv::OpConstantTrue: |
| case spv::OpSpecConstantTrue: |
| CreateConstant(insn).constantValue[0] = ~0u; // Represent Boolean true as all bits set. |
| break; |
| case spv::OpConstantNull: |
| case spv::OpUndef: |
| { |
| // TODO: consider a real LLVM-level undef. For now, zero is a perfectly good value. |
| // OpConstantNull forms a constant of arbitrary type, all zeros. |
| auto &object = CreateConstant(insn); |
| auto &objectTy = getType(object.type); |
| for (auto i = 0u; i < objectTy.sizeInComponents; i++) |
| { |
| object.constantValue[i] = 0; |
| } |
| break; |
| } |
| case spv::OpConstantComposite: |
| case spv::OpSpecConstantComposite: |
| { |
| auto &object = CreateConstant(insn); |
| auto offset = 0u; |
| for (auto i = 0u; i < insn.wordCount() - 3; i++) |
| { |
| auto &constituent = getObject(insn.word(i + 3)); |
| auto &constituentTy = getType(constituent.type); |
| for (auto j = 0u; j < constituentTy.sizeInComponents; j++) |
| { |
| object.constantValue[offset++] = constituent.constantValue[j]; |
| } |
| } |
| |
| auto objectId = Object::ID(insn.word(2)); |
| auto decorationsIt = decorations.find(objectId); |
| if (decorationsIt != decorations.end() && |
| decorationsIt->second.BuiltIn == spv::BuiltInWorkgroupSize) |
| { |
| // https://www.khronos.org/registry/vulkan/specs/1.1/html/vkspec.html#interfaces-builtin-variables : |
| // Decorating an object with the WorkgroupSize built-in |
| // decoration will make that object contain the dimensions |
| // of a local workgroup. If an object is decorated with the |
| // WorkgroupSize decoration, this must take precedence over |
| // any execution mode set for LocalSize. |
| // The object decorated with WorkgroupSize must be declared |
| // as a three-component vector of 32-bit integers. |
| ASSERT(getType(object.type).sizeInComponents == 3); |
| modes.WorkgroupSizeX = object.constantValue[0]; |
| modes.WorkgroupSizeY = object.constantValue[1]; |
| modes.WorkgroupSizeZ = object.constantValue[2]; |
| } |
| break; |
| } |
| case spv::OpSpecConstantOp: |
| EvalSpecConstantOp(insn); |
| break; |
| |
| case spv::OpCapability: |
| { |
| auto capability = static_cast<spv::Capability>(insn.word(1)); |
| switch (capability) |
| { |
| case spv::CapabilityMatrix: capabilities.Matrix = true; break; |
| case spv::CapabilityShader: capabilities.Shader = true; break; |
| case spv::CapabilityInputAttachment: capabilities.InputAttachment = true; break; |
| case spv::CapabilitySampled1D: capabilities.Sampled1D = true; break; |
| case spv::CapabilityImage1D: capabilities.Image1D = true; break; |
| case spv::CapabilitySampledBuffer: capabilities.SampledBuffer = true; break; |
| case spv::CapabilityImageBuffer: capabilities.ImageBuffer = true; break; |
| case spv::CapabilityImageQuery: capabilities.ImageQuery = true; break; |
| case spv::CapabilityDerivativeControl: capabilities.DerivativeControl = true; break; |
| case spv::CapabilityGroupNonUniform: capabilities.GroupNonUniform = true; break; |
| case spv::CapabilityMultiView: capabilities.MultiView = true; break; |
| case spv::CapabilityDeviceGroup: capabilities.DeviceGroup = true; break; |
| case spv::CapabilityGroupNonUniformVote: capabilities.GroupNonUniformVote = true; break; |
| case spv::CapabilityGroupNonUniformBallot: capabilities.GroupNonUniformBallot = true; break; |
| case spv::CapabilityGroupNonUniformShuffle: capabilities.GroupNonUniformShuffle = true; break; |
| case spv::CapabilityGroupNonUniformShuffleRelative: capabilities.GroupNonUniformShuffleRelative = true; break; |
| case spv::CapabilityStorageImageExtendedFormats: capabilities.StorageImageExtendedFormats = true; break; |
| default: |
| UNSUPPORTED("Unsupported capability %u", insn.word(1)); |
| } |
| break; // Various capabilities will be declared, but none affect our code generation at this point. |
| } |
| |
| case spv::OpMemoryModel: |
| break; // Memory model does not affect our code generation until we decide to do Vulkan Memory Model support. |
| |
| case spv::OpFunction: |
| { |
| auto functionId = Function::ID(insn.word(2)); |
| ASSERT_MSG(currentFunction == 0, "Functions %d and %d overlap", currentFunction.value(), functionId.value()); |
| currentFunction = functionId; |
| auto &function = functions[functionId]; |
| function.result = Type::ID(insn.word(1)); |
| function.type = Type::ID(insn.word(4)); |
| // Scan forward to find the function's label. |
| for (auto it = insn; it != end() && function.entry == 0; it++) |
| { |
| switch (it.opcode()) |
| { |
| case spv::OpFunction: |
| case spv::OpFunctionParameter: |
| break; |
| case spv::OpLabel: |
| function.entry = Block::ID(it.word(1)); |
| break; |
| default: |
| WARN("Unexpected opcode '%s' following OpFunction", OpcodeName(it.opcode()).c_str()); |
| } |
| } |
| ASSERT_MSG(function.entry != 0, "Function<%d> has no label", currentFunction.value()); |
| break; |
| } |
| |
| case spv::OpFunctionEnd: |
| currentFunction = 0; |
| break; |
| |
| case spv::OpExtInstImport: |
| { |
| // We will only support the GLSL 450 extended instruction set, so no point in tracking the ID we assign it. |
| // Valid shaders will not attempt to import any other instruction sets. |
| auto ext = insn.string(2); |
| if (0 != strcmp("GLSL.std.450", ext)) |
| { |
| UNSUPPORTED("SPIR-V Extension: %s", ext); |
| } |
| break; |
| } |
| case spv::OpName: |
| case spv::OpMemberName: |
| case spv::OpSource: |
| case spv::OpSourceContinued: |
| case spv::OpSourceExtension: |
| case spv::OpLine: |
| case spv::OpNoLine: |
| case spv::OpModuleProcessed: |
| case spv::OpString: |
| // No semantic impact |
| break; |
| |
| case spv::OpFunctionParameter: |
| // These should have all been removed by preprocessing passes. If we see them here, |
| // our assumptions are wrong and we will probably generate wrong code. |
| UNREACHABLE("%s should have already been lowered.", OpcodeName(opcode).c_str()); |
| break; |
| |
| case spv::OpFunctionCall: |
| // TODO(b/141246700): Add full support for spv::OpFunctionCall |
| break; |
| |
| case spv::OpFConvert: |
| UNSUPPORTED("SPIR-V Float16 or Float64 Capability (OpFConvert)"); |
| break; |
| |
| case spv::OpSConvert: |
| UNSUPPORTED("SPIR-V Int16 or Int64 Capability (OpSConvert)"); |
| break; |
| |
| case spv::OpUConvert: |
| UNSUPPORTED("SPIR-V Int16 or Int64 Capability (OpUConvert)"); |
| break; |
| |
| case spv::OpLoad: |
| case spv::OpAccessChain: |
| case spv::OpInBoundsAccessChain: |
| case spv::OpSampledImage: |
| case spv::OpImage: |
| { |
| // Propagate the descriptor decorations to the result. |
| Object::ID resultId = insn.word(2); |
| Object::ID pointerId = insn.word(3); |
| const auto &d = descriptorDecorations.find(pointerId); |
| |
| if(d != descriptorDecorations.end()) |
| { |
| descriptorDecorations[resultId] = d->second; |
| } |
| |
| DefineResult(insn); |
| |
| if (opcode == spv::OpAccessChain || opcode == spv::OpInBoundsAccessChain) |
| { |
| Decorations dd{}; |
| ApplyDecorationsForAccessChain(&dd, &descriptorDecorations[resultId], pointerId, insn.wordCount() - 4, insn.wordPointer(4)); |
| // Note: offset is the one thing that does *not* propagate, as the access chain accounts for it. |
| dd.HasOffset = false; |
| decorations[resultId].Apply(dd); |
| } |
| } |
| break; |
| |
| case spv::OpCompositeConstruct: |
| case spv::OpCompositeInsert: |
| case spv::OpCompositeExtract: |
| case spv::OpVectorShuffle: |
| case spv::OpVectorTimesScalar: |
| case spv::OpMatrixTimesScalar: |
| case spv::OpMatrixTimesVector: |
| case spv::OpVectorTimesMatrix: |
| case spv::OpMatrixTimesMatrix: |
| case spv::OpOuterProduct: |
| case spv::OpTranspose: |
| case spv::OpVectorExtractDynamic: |
| case spv::OpVectorInsertDynamic: |
| // Unary ops |
| case spv::OpNot: |
| case spv::OpBitFieldInsert: |
| case spv::OpBitFieldSExtract: |
| case spv::OpBitFieldUExtract: |
| case spv::OpBitReverse: |
| case spv::OpBitCount: |
| case spv::OpSNegate: |
| case spv::OpFNegate: |
| case spv::OpLogicalNot: |
| case spv::OpQuantizeToF16: |
| // Binary ops |
| case spv::OpIAdd: |
| case spv::OpISub: |
| case spv::OpIMul: |
| case spv::OpSDiv: |
| case spv::OpUDiv: |
| case spv::OpFAdd: |
| case spv::OpFSub: |
| case spv::OpFMul: |
| case spv::OpFDiv: |
| case spv::OpFMod: |
| case spv::OpFRem: |
| case spv::OpFOrdEqual: |
| case spv::OpFUnordEqual: |
| case spv::OpFOrdNotEqual: |
| case spv::OpFUnordNotEqual: |
| case spv::OpFOrdLessThan: |
| case spv::OpFUnordLessThan: |
| case spv::OpFOrdGreaterThan: |
| case spv::OpFUnordGreaterThan: |
| case spv::OpFOrdLessThanEqual: |
| case spv::OpFUnordLessThanEqual: |
| case spv::OpFOrdGreaterThanEqual: |
| case spv::OpFUnordGreaterThanEqual: |
| case spv::OpSMod: |
| case spv::OpSRem: |
| case spv::OpUMod: |
| case spv::OpIEqual: |
| case spv::OpINotEqual: |
| case spv::OpUGreaterThan: |
| case spv::OpSGreaterThan: |
| case spv::OpUGreaterThanEqual: |
| case spv::OpSGreaterThanEqual: |
| case spv::OpULessThan: |
| case spv::OpSLessThan: |
| case spv::OpULessThanEqual: |
| case spv::OpSLessThanEqual: |
| case spv::OpShiftRightLogical: |
| case spv::OpShiftRightArithmetic: |
| case spv::OpShiftLeftLogical: |
| case spv::OpBitwiseOr: |
| case spv::OpBitwiseXor: |
| case spv::OpBitwiseAnd: |
| case spv::OpLogicalOr: |
| case spv::OpLogicalAnd: |
| case spv::OpLogicalEqual: |
| case spv::OpLogicalNotEqual: |
| case spv::OpUMulExtended: |
| case spv::OpSMulExtended: |
| case spv::OpIAddCarry: |
| case spv::OpISubBorrow: |
| case spv::OpDot: |
| case spv::OpConvertFToU: |
| case spv::OpConvertFToS: |
| case spv::OpConvertSToF: |
| case spv::OpConvertUToF: |
| case spv::OpBitcast: |
| case spv::OpSelect: |
| case spv::OpExtInst: |
| case spv::OpIsInf: |
| case spv::OpIsNan: |
| case spv::OpAny: |
| case spv::OpAll: |
| case spv::OpDPdx: |
| case spv::OpDPdxCoarse: |
| case spv::OpDPdy: |
| case spv::OpDPdyCoarse: |
| case spv::OpFwidth: |
| case spv::OpFwidthCoarse: |
| case spv::OpDPdxFine: |
| case spv::OpDPdyFine: |
| case spv::OpFwidthFine: |
| case spv::OpAtomicLoad: |
| case spv::OpAtomicIAdd: |
| case spv::OpAtomicISub: |
| case spv::OpAtomicSMin: |
| case spv::OpAtomicSMax: |
| case spv::OpAtomicUMin: |
| case spv::OpAtomicUMax: |
| case spv::OpAtomicAnd: |
| case spv::OpAtomicOr: |
| case spv::OpAtomicXor: |
| case spv::OpAtomicIIncrement: |
| case spv::OpAtomicIDecrement: |
| case spv::OpAtomicExchange: |
| case spv::OpAtomicCompareExchange: |
| case spv::OpPhi: |
| case spv::OpImageSampleImplicitLod: |
| case spv::OpImageSampleExplicitLod: |
| case spv::OpImageSampleDrefImplicitLod: |
| case spv::OpImageSampleDrefExplicitLod: |
| case spv::OpImageSampleProjImplicitLod: |
| case spv::OpImageSampleProjExplicitLod: |
| case spv::OpImageSampleProjDrefImplicitLod: |
| case spv::OpImageSampleProjDrefExplicitLod: |
| case spv::OpImageGather: |
| case spv::OpImageDrefGather: |
| case spv::OpImageFetch: |
| case spv::OpImageQuerySizeLod: |
| case spv::OpImageQuerySize: |
| case spv::OpImageQueryLod: |
| case spv::OpImageQueryLevels: |
| case spv::OpImageQuerySamples: |
| case spv::OpImageRead: |
| case spv::OpImageTexelPointer: |
| case spv::OpGroupNonUniformElect: |
| case spv::OpGroupNonUniformAll: |
| case spv::OpGroupNonUniformAny: |
| case spv::OpGroupNonUniformAllEqual: |
| case spv::OpGroupNonUniformBroadcast: |
| case spv::OpGroupNonUniformBroadcastFirst: |
| case spv::OpGroupNonUniformBallot: |
| case spv::OpGroupNonUniformInverseBallot: |
| case spv::OpGroupNonUniformBallotBitExtract: |
| case spv::OpGroupNonUniformBallotBitCount: |
| case spv::OpGroupNonUniformBallotFindLSB: |
| case spv::OpGroupNonUniformBallotFindMSB: |
| case spv::OpGroupNonUniformShuffle: |
| case spv::OpGroupNonUniformShuffleXor: |
| case spv::OpGroupNonUniformShuffleUp: |
| case spv::OpGroupNonUniformShuffleDown: |
| case spv::OpCopyObject: |
| case spv::OpArrayLength: |
| // Instructions that yield an intermediate value or divergent pointer |
| DefineResult(insn); |
| break; |
| |
| case spv::OpStore: |
| case spv::OpAtomicStore: |
| case spv::OpImageWrite: |
| case spv::OpCopyMemory: |
| case spv::OpMemoryBarrier: |
| // Don't need to do anything during analysis pass |
| break; |
| |
| case spv::OpControlBarrier: |
| modes.ContainsControlBarriers = true; |
| break; |
| |
| case spv::OpExtension: |
| { |
| auto ext = insn.string(1); |
| // Part of core SPIR-V 1.3. Vulkan 1.1 implementations must also accept the pre-1.3 |
| // extension per Appendix A, `Vulkan Environment for SPIR-V`. |
| if (!strcmp(ext, "SPV_KHR_storage_buffer_storage_class")) break; |
| if (!strcmp(ext, "SPV_KHR_shader_draw_parameters")) break; |
| if (!strcmp(ext, "SPV_KHR_16bit_storage")) break; |
| if (!strcmp(ext, "SPV_KHR_variable_pointers")) break; |
| if (!strcmp(ext, "SPV_KHR_device_group")) break; |
| if (!strcmp(ext, "SPV_KHR_multiview")) break; |
| UNSUPPORTED("SPIR-V Extension: %s", ext); |
| break; |
| } |
| |
| default: |
| UNIMPLEMENTED("%s", OpcodeName(opcode).c_str()); |
| } |
| } |
| |
| ASSERT_MSG(entryPoint != 0, "Entry point '%s' not found", entryPointName); |
| for (auto &it : functions) |
| { |
| it.second.AssignBlockFields(); |
| } |
| } |
| |
| void SpirvShader::DeclareType(InsnIterator insn) |
| { |
| Type::ID resultId = insn.word(1); |
| |
| auto &type = types[resultId]; |
| type.definition = insn; |
| type.sizeInComponents = ComputeTypeSize(insn); |
| |
| // A structure is a builtin block if it has a builtin |
| // member. All members of such a structure are builtins. |
| switch (insn.opcode()) |
| { |
| case spv::OpTypeStruct: |
| { |
| auto d = memberDecorations.find(resultId); |
| if (d != memberDecorations.end()) |
| { |
| for (auto &m : d->second) |
| { |
| if (m.HasBuiltIn) |
| { |
| type.isBuiltInBlock = true; |
| break; |
| } |
| } |
| } |
| break; |
| } |
| case spv::OpTypePointer: |
| { |
| Type::ID elementTypeId = insn.word(3); |
| type.element = elementTypeId; |
| type.isBuiltInBlock = getType(elementTypeId).isBuiltInBlock; |
| type.storageClass = static_cast<spv::StorageClass>(insn.word(2)); |
| break; |
| } |
| case spv::OpTypeVector: |
| case spv::OpTypeMatrix: |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| { |
| Type::ID elementTypeId = insn.word(2); |
| type.element = elementTypeId; |
| break; |
| } |
| default: |
| break; |
| } |
| } |
| |
| SpirvShader::Object& SpirvShader::CreateConstant(InsnIterator insn) |
| { |
| Type::ID typeId = insn.word(1); |
| Object::ID resultId = insn.word(2); |
| auto &object = defs[resultId]; |
| auto &objectTy = getType(typeId); |
| object.type = typeId; |
| object.kind = Object::Kind::Constant; |
| object.definition = insn; |
| object.constantValue = std::unique_ptr<uint32_t[]>(new uint32_t[objectTy.sizeInComponents]); |
| return object; |
| } |
| |
| void SpirvShader::ProcessInterfaceVariable(Object &object) |
| { |
| auto &objectTy = getType(object.type); |
| ASSERT(objectTy.storageClass == spv::StorageClassInput || objectTy.storageClass == spv::StorageClassOutput); |
| |
| ASSERT(objectTy.opcode() == spv::OpTypePointer); |
| auto pointeeTy = getType(objectTy.element); |
| |
| auto &builtinInterface = (objectTy.storageClass == spv::StorageClassInput) ? inputBuiltins : outputBuiltins; |
| auto &userDefinedInterface = (objectTy.storageClass == spv::StorageClassInput) ? inputs : outputs; |
| |
| ASSERT(object.opcode() == spv::OpVariable); |
| Object::ID resultId = object.definition.word(2); |
| |
| if (objectTy.isBuiltInBlock) |
| { |
| // walk the builtin block, registering each of its members separately. |
| auto m = memberDecorations.find(objectTy.element); |
| ASSERT(m != memberDecorations.end()); // otherwise we wouldn't have marked the type chain |
| auto &structType = pointeeTy.definition; |
| auto offset = 0u; |
| auto word = 2u; |
| for (auto &member : m->second) |
| { |
| auto &memberType = getType(structType.word(word)); |
| |
| if (member.HasBuiltIn) |
| { |
| builtinInterface[member.BuiltIn] = {resultId, offset, memberType.sizeInComponents}; |
| } |
| |
| offset += memberType.sizeInComponents; |
| ++word; |
| } |
| return; |
| } |
| |
| auto d = decorations.find(resultId); |
| if (d != decorations.end() && d->second.HasBuiltIn) |
| { |
| builtinInterface[d->second.BuiltIn] = {resultId, 0, pointeeTy.sizeInComponents}; |
| } |
| else |
| { |
| object.kind = Object::Kind::InterfaceVariable; |
| VisitInterface(resultId, |
| [&userDefinedInterface](Decorations const &d, AttribType type) { |
| // Populate a single scalar slot in the interface from a collection of decorations and the intended component type. |
| auto scalarSlot = (d.Location << 2) | d.Component; |
| ASSERT(scalarSlot >= 0 && |
| scalarSlot < static_cast<int32_t>(userDefinedInterface.size())); |
| |
| auto &slot = userDefinedInterface[scalarSlot]; |
| slot.Type = type; |
| slot.Flat = d.Flat; |
| slot.NoPerspective = d.NoPerspective; |
| slot.Centroid = d.Centroid; |
| }); |
| } |
| } |
| |
| void SpirvShader::ProcessExecutionMode(InsnIterator insn) |
| { |
| auto mode = static_cast<spv::ExecutionMode>(insn.word(2)); |
| switch (mode) |
| { |
| case spv::ExecutionModeEarlyFragmentTests: |
| modes.EarlyFragmentTests = true; |
| break; |
| case spv::ExecutionModeDepthReplacing: |
| modes.DepthReplacing = true; |
| break; |
| case spv::ExecutionModeDepthGreater: |
| modes.DepthGreater = true; |
| break; |
| case spv::ExecutionModeDepthLess: |
| modes.DepthLess = true; |
| break; |
| case spv::ExecutionModeDepthUnchanged: |
| modes.DepthUnchanged = true; |
| break; |
| case spv::ExecutionModeLocalSize: |
| modes.WorkgroupSizeX = insn.word(3); |
| modes.WorkgroupSizeY = insn.word(4); |
| modes.WorkgroupSizeZ = insn.word(5); |
| break; |
| case spv::ExecutionModeOriginUpperLeft: |
| // This is always the case for a Vulkan shader. Do nothing. |
| break; |
| default: |
| UNREACHABLE("Execution mode: %d", int(mode)); |
| } |
| } |
| |
| uint32_t SpirvShader::ComputeTypeSize(InsnIterator insn) |
| { |
| // Types are always built from the bottom up (with the exception of forward ptrs, which |
| // don't appear in Vulkan shaders. Therefore, we can always assume our component parts have |
| // already been described (and so their sizes determined) |
| switch (insn.opcode()) |
| { |
| case spv::OpTypeVoid: |
| case spv::OpTypeSampler: |
| case spv::OpTypeImage: |
| case spv::OpTypeSampledImage: |
| case spv::OpTypeFunction: |
| case spv::OpTypeRuntimeArray: |
| // Objects that don't consume any space. |
| // Descriptor-backed objects currently only need exist at compile-time. |
| // Runtime arrays don't appear in places where their size would be interesting |
| return 0; |
| |
| case spv::OpTypeBool: |
| case spv::OpTypeFloat: |
| case spv::OpTypeInt: |
| // All the fundamental types are 1 component. If we ever add support for 8/16/64-bit components, |
| // we might need to change this, but only 32 bit components are required for Vulkan 1.1. |
| return 1; |
| |
| case spv::OpTypeVector: |
| case spv::OpTypeMatrix: |
| // Vectors and matrices both consume element count * element size. |
| return getType(insn.word(2)).sizeInComponents * insn.word(3); |
| |
| case spv::OpTypeArray: |
| { |
| // Element count * element size. Array sizes come from constant ids. |
| auto arraySize = GetConstScalarInt(insn.word(3)); |
| return getType(insn.word(2)).sizeInComponents * arraySize; |
| } |
| |
| case spv::OpTypeStruct: |
| { |
| uint32_t size = 0; |
| for (uint32_t i = 2u; i < insn.wordCount(); i++) |
| { |
| size += getType(insn.word(i)).sizeInComponents; |
| } |
| return size; |
| } |
| |
| case spv::OpTypePointer: |
| // Runtime representation of a pointer is a per-lane index. |
| // Note: clients are expected to look through the pointer if they want the pointee size instead. |
| return 1; |
| |
| default: |
| UNREACHABLE("%s", OpcodeName(insn.opcode()).c_str()); |
| return 0; |
| } |
| } |
| |
| bool SpirvShader::StoresInHelperInvocation(spv::StorageClass storageClass) |
| { |
| switch (storageClass) |
| { |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| case spv::StorageClassImage: |
| return false; |
| default: |
| return true; |
| } |
| } |
| |
| bool SpirvShader::IsExplicitLayout(spv::StorageClass storageClass) |
| { |
| switch (storageClass) |
| { |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| case spv::StorageClassPushConstant: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| bool SpirvShader::IsStorageInterleavedByLane(spv::StorageClass storageClass) |
| { |
| switch (storageClass) |
| { |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| case spv::StorageClassPushConstant: |
| case spv::StorageClassWorkgroup: |
| case spv::StorageClassImage: |
| return false; |
| default: |
| return true; |
| } |
| } |
| |
| template<typename F> |
| int SpirvShader::VisitInterfaceInner(Type::ID id, Decorations d, F f) const |
| { |
| // Recursively walks variable definition and its type tree, taking into account |
| // any explicit Location or Component decorations encountered; where explicit |
| // Locations or Components are not specified, assigns them sequentially. |
| // Collected decorations are carried down toward the leaves and across |
| // siblings; Effect of decorations intentionally does not flow back up the tree. |
| // |
| // F is a functor to be called with the effective decoration set for every component. |
| // |
| // Returns the next available location, and calls f(). |
| |
| // This covers the rules in Vulkan 1.1 spec, 14.1.4 Location Assignment. |
| |
| ApplyDecorationsForId(&d, id); |
| |
| auto const &obj = getType(id); |
| switch(obj.opcode()) |
| { |
| case spv::OpTypePointer: |
| return VisitInterfaceInner<F>(obj.definition.word(3), d, f); |
| case spv::OpTypeMatrix: |
| for (auto i = 0u; i < obj.definition.word(3); i++, d.Location++) |
| { |
| // consumes same components of N consecutive locations |
| VisitInterfaceInner<F>(obj.definition.word(2), d, f); |
| } |
| return d.Location; |
| case spv::OpTypeVector: |
| for (auto i = 0u; i < obj.definition.word(3); i++, d.Component++) |
| { |
| // consumes N consecutive components in the same location |
| VisitInterfaceInner<F>(obj.definition.word(2), d, f); |
| } |
| return d.Location + 1; |
| case spv::OpTypeFloat: |
| f(d, ATTRIBTYPE_FLOAT); |
| return d.Location + 1; |
| case spv::OpTypeInt: |
| f(d, obj.definition.word(3) ? ATTRIBTYPE_INT : ATTRIBTYPE_UINT); |
| return d.Location + 1; |
| case spv::OpTypeBool: |
| f(d, ATTRIBTYPE_UINT); |
| return d.Location + 1; |
| case spv::OpTypeStruct: |
| { |
| // iterate over members, which may themselves have Location/Component decorations |
| for (auto i = 0u; i < obj.definition.wordCount() - 2; i++) |
| { |
| ApplyDecorationsForIdMember(&d, id, i); |
| d.Location = VisitInterfaceInner<F>(obj.definition.word(i + 2), d, f); |
| d.Component = 0; // Implicit locations always have component=0 |
| } |
| return d.Location; |
| } |
| case spv::OpTypeArray: |
| { |
| auto arraySize = GetConstScalarInt(obj.definition.word(3)); |
| for (auto i = 0u; i < arraySize; i++) |
| { |
| d.Location = VisitInterfaceInner<F>(obj.definition.word(2), d, f); |
| } |
| return d.Location; |
| } |
| default: |
| // Intentionally partial; most opcodes do not participate in type hierarchies |
| return 0; |
| } |
| } |
| |
| template<typename F> |
| void SpirvShader::VisitInterface(Object::ID id, F f) const |
| { |
| // Walk a variable definition and call f for each component in it. |
| Decorations d{}; |
| ApplyDecorationsForId(&d, id); |
| |
| auto def = getObject(id).definition; |
| ASSERT(def.opcode() == spv::OpVariable); |
| VisitInterfaceInner<F>(def.word(1), d, f); |
| } |
| |
| template<typename F> |
| void SpirvShader::VisitMemoryObjectInner(sw::SpirvShader::Type::ID id, sw::SpirvShader::Decorations d, uint32_t& index, uint32_t offset, F f) const |
| { |
| // Walk a type tree in an explicitly laid out storage class, calling |
| // a functor for each scalar element within the object. |
| |
| // The functor's first parameter is the index of the scalar element; |
| // the second parameter is the offset (in bytes) from the base of the |
| // object. |
| |
| ApplyDecorationsForId(&d, id); |
| auto const &type = getType(id); |
| |
| if (d.HasOffset) |
| { |
| offset += d.Offset; |
| d.HasOffset = false; |
| } |
| |
| switch (type.opcode()) |
| { |
| case spv::OpTypePointer: |
| VisitMemoryObjectInner<F>(type.definition.word(3), d, index, offset, f); |
| break; |
| case spv::OpTypeInt: |
| case spv::OpTypeFloat: |
| f(index++, offset); |
| break; |
| case spv::OpTypeVector: |
| { |
| auto elemStride = (d.InsideMatrix && d.HasRowMajor && d.RowMajor) ? d.MatrixStride : static_cast<int32_t>(sizeof(float)); |
| for (auto i = 0u; i < type.definition.word(3); i++) |
| { |
| VisitMemoryObjectInner(type.definition.word(2), d, index, offset + elemStride * i, f); |
| } |
| break; |
| } |
| case spv::OpTypeMatrix: |
| { |
| auto columnStride = (d.HasRowMajor && d.RowMajor) ? static_cast<int32_t>(sizeof(float)) : d.MatrixStride; |
| d.InsideMatrix = true; |
| for (auto i = 0u; i < type.definition.word(3); i++) |
| { |
| ASSERT(d.HasMatrixStride); |
| VisitMemoryObjectInner(type.definition.word(2), d, index, offset + columnStride * i, f); |
| } |
| break; |
| } |
| case spv::OpTypeStruct: |
| for (auto i = 0u; i < type.definition.wordCount() - 2; i++) |
| { |
| ApplyDecorationsForIdMember(&d, id, i); |
| VisitMemoryObjectInner<F>(type.definition.word(i + 2), d, index, offset, f); |
| } |
| break; |
| case spv::OpTypeArray: |
| { |
| auto arraySize = GetConstScalarInt(type.definition.word(3)); |
| for (auto i = 0u; i < arraySize; i++) |
| { |
| ASSERT(d.HasArrayStride); |
| VisitMemoryObjectInner<F>(type.definition.word(2), d, index, offset + i * d.ArrayStride, f); |
| } |
| break; |
| } |
| default: |
| UNREACHABLE("%s", OpcodeName(type.opcode()).c_str()); |
| } |
| } |
| |
| template<typename F> |
| void SpirvShader::VisitMemoryObject(sw::SpirvShader::Object::ID id, F f) const |
| { |
| auto typeId = getObject(id).type; |
| auto const & type = getType(typeId); |
| if (IsExplicitLayout(type.storageClass)) |
| { |
| Decorations d{}; |
| ApplyDecorationsForId(&d, id); |
| uint32_t index = 0; |
| VisitMemoryObjectInner<F>(typeId, d, index, 0, f); |
| } |
| else |
| { |
| // Objects without explicit layout are tightly packed. |
| for (auto i = 0u; i < getType(type.element).sizeInComponents; i++) |
| { |
| f(i, i * sizeof(float)); |
| } |
| } |
| } |
| |
| SIMD::Pointer SpirvShader::GetPointerToData(Object::ID id, int arrayIndex, EmitState const *state) const |
| { |
| auto routine = state->routine; |
| auto &object = getObject(id); |
| switch (object.kind) |
| { |
| case Object::Kind::Pointer: |
| case Object::Kind::InterfaceVariable: |
| return state->getPointer(id); |
| |
| case Object::Kind::DescriptorSet: |
| { |
| const auto &d = descriptorDecorations.at(id); |
| ASSERT(d.DescriptorSet >= 0 && d.DescriptorSet < vk::MAX_BOUND_DESCRIPTOR_SETS); |
| ASSERT(d.Binding >= 0); |
| |
| auto set = state->getPointer(id); |
| |
| auto setLayout = routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| ASSERT_MSG(setLayout->hasBinding(d.Binding), "Descriptor set %d does not contain binding %d", int(d.DescriptorSet), int(d.Binding)); |
| int bindingOffset = static_cast<int>(setLayout->getBindingOffset(d.Binding, arrayIndex)); |
| |
| Pointer<Byte> descriptor = set.base + bindingOffset; // BufferDescriptor* |
| Pointer<Byte> data = *Pointer<Pointer<Byte>>(descriptor + OFFSET(vk::BufferDescriptor, ptr)); // void* |
| Int size = *Pointer<Int>(descriptor + OFFSET(vk::BufferDescriptor, sizeInBytes)); |
| if (setLayout->isBindingDynamic(d.Binding)) |
| { |
| uint32_t dynamicBindingIndex = |
| routine->pipelineLayout->getDynamicOffsetBase(d.DescriptorSet) + |
| setLayout->getDynamicDescriptorOffset(d.Binding) + |
| arrayIndex; |
| Int offset = routine->descriptorDynamicOffsets[dynamicBindingIndex]; |
| Int robustnessSize = *Pointer<Int>(descriptor + OFFSET(vk::BufferDescriptor, robustnessSize)); |
| return SIMD::Pointer(data + offset, Min(size, robustnessSize - offset)); |
| } |
| else |
| { |
| return SIMD::Pointer(data, size); |
| } |
| } |
| |
| default: |
| UNREACHABLE("Invalid pointer kind %d", int(object.kind)); |
| return SIMD::Pointer(Pointer<Byte>(), 0); |
| } |
| } |
| |
| void SpirvShader::ApplyDecorationsForAccessChain(Decorations *d, DescriptorDecorations *dd, Object::ID baseId, uint32_t numIndexes, uint32_t const *indexIds) const |
| { |
| ApplyDecorationsForId(d, baseId); |
| auto &baseObject = getObject(baseId); |
| ApplyDecorationsForId(d, baseObject.type); |
| auto typeId = getType(baseObject.type).element; |
| |
| for (auto i = 0u; i < numIndexes; i++) |
| { |
| ApplyDecorationsForId(d, typeId); |
| auto & type = getType(typeId); |
| switch (type.opcode()) |
| { |
| case spv::OpTypeStruct: |
| { |
| int memberIndex = GetConstScalarInt(indexIds[i]); |
| ApplyDecorationsForIdMember(d, typeId, memberIndex); |
| typeId = type.definition.word(2u + memberIndex); |
| break; |
| } |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| if (dd->InputAttachmentIndex >= 0) |
| { |
| dd->InputAttachmentIndex += GetConstScalarInt(indexIds[i]); |
| } |
| typeId = type.element; |
| break; |
| case spv::OpTypeVector: |
| typeId = type.element; |
| break; |
| case spv::OpTypeMatrix: |
| typeId = type.element; |
| d->InsideMatrix = true; |
| break; |
| default: |
| UNREACHABLE("%s", OpcodeName(type.definition.opcode()).c_str()); |
| } |
| } |
| } |
| |
| SIMD::Pointer SpirvShader::WalkExplicitLayoutAccessChain(Object::ID baseId, uint32_t numIndexes, uint32_t const *indexIds, EmitState const *state) const |
| { |
| // Produce a offset into external memory in sizeof(float) units |
| |
| auto &baseObject = getObject(baseId); |
| Type::ID typeId = getType(baseObject.type).element; |
| Decorations d = {}; |
| ApplyDecorationsForId(&d, baseObject.type); |
| |
| uint32_t arrayIndex = 0; |
| if (baseObject.kind == Object::Kind::DescriptorSet) |
| { |
| auto type = getType(typeId).definition.opcode(); |
| if (type == spv::OpTypeArray || type == spv::OpTypeRuntimeArray) |
| { |
| ASSERT(getObject(indexIds[0]).kind == Object::Kind::Constant); |
| arrayIndex = GetConstScalarInt(indexIds[0]); |
| |
| numIndexes--; |
| indexIds++; |
| typeId = getType(typeId).element; |
| } |
| } |
| |
| auto ptr = GetPointerToData(baseId, arrayIndex, state); |
| |
| int constantOffset = 0; |
| |
| for (auto i = 0u; i < numIndexes; i++) |
| { |
| auto & type = getType(typeId); |
| ApplyDecorationsForId(&d, typeId); |
| |
| switch (type.definition.opcode()) |
| { |
| case spv::OpTypeStruct: |
| { |
| int memberIndex = GetConstScalarInt(indexIds[i]); |
| ApplyDecorationsForIdMember(&d, typeId, memberIndex); |
| ASSERT(d.HasOffset); |
| constantOffset += d.Offset; |
| typeId = type.definition.word(2u + memberIndex); |
| break; |
| } |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| { |
| // TODO: b/127950082: Check bounds. |
| ASSERT(d.HasArrayStride); |
| auto & obj = getObject(indexIds[i]); |
| if (obj.kind == Object::Kind::Constant) |
| { |
| constantOffset += d.ArrayStride * GetConstScalarInt(indexIds[i]); |
| } |
| else |
| { |
| ptr += SIMD::Int(d.ArrayStride) * state->getIntermediate(indexIds[i]).Int(0); |
| } |
| typeId = type.element; |
| break; |
| } |
| case spv::OpTypeMatrix: |
| { |
| // TODO: b/127950082: Check bounds. |
| ASSERT(d.HasMatrixStride); |
| d.InsideMatrix = true; |
| auto columnStride = (d.HasRowMajor && d.RowMajor) ? static_cast<int32_t>(sizeof(float)) : d.MatrixStride; |
| auto & obj = getObject(indexIds[i]); |
| if (obj.kind == Object::Kind::Constant) |
| { |
| constantOffset += columnStride * GetConstScalarInt(indexIds[i]); |
| } |
| else |
| { |
| ptr += SIMD::Int(columnStride) * state->getIntermediate(indexIds[i]).Int(0); |
| } |
| typeId = type.element; |
| break; |
| } |
| case spv::OpTypeVector: |
| { |
| auto elemStride = (d.InsideMatrix && d.HasRowMajor && d.RowMajor) ? d.MatrixStride : static_cast<int32_t>(sizeof(float)); |
| auto & obj = getObject(indexIds[i]); |
| if (obj.kind == Object::Kind::Constant) |
| { |
| constantOffset += elemStride * GetConstScalarInt(indexIds[i]); |
| } |
| else |
| { |
| ptr += SIMD::Int(elemStride) * state->getIntermediate(indexIds[i]).Int(0); |
| } |
| typeId = type.element; |
| break; |
| } |
| default: |
| UNREACHABLE("%s", OpcodeName(type.definition.opcode()).c_str()); |
| } |
| } |
| |
| ptr += constantOffset; |
| return ptr; |
| } |
| |
| SIMD::Pointer SpirvShader::WalkAccessChain(Object::ID baseId, uint32_t numIndexes, uint32_t const *indexIds, EmitState const *state) const |
| { |
| // TODO: avoid doing per-lane work in some cases if we can? |
| auto routine = state->routine; |
| auto &baseObject = getObject(baseId); |
| Type::ID typeId = getType(baseObject.type).element; |
| |
| auto ptr = state->getPointer(baseId); |
| |
| int constantOffset = 0; |
| |
| for (auto i = 0u; i < numIndexes; i++) |
| { |
| auto & type = getType(typeId); |
| switch(type.opcode()) |
| { |
| case spv::OpTypeStruct: |
| { |
| int memberIndex = GetConstScalarInt(indexIds[i]); |
| int offsetIntoStruct = 0; |
| for (auto j = 0; j < memberIndex; j++) { |
| auto memberType = type.definition.word(2u + j); |
| offsetIntoStruct += getType(memberType).sizeInComponents * sizeof(float); |
| } |
| constantOffset += offsetIntoStruct; |
| typeId = type.definition.word(2u + memberIndex); |
| break; |
| } |
| |
| case spv::OpTypeVector: |
| case spv::OpTypeMatrix: |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| { |
| // TODO: b/127950082: Check bounds. |
| if (getType(baseObject.type).storageClass == spv::StorageClassUniformConstant) |
| { |
| // indexing into an array of descriptors. |
| auto &obj = getObject(indexIds[i]); |
| if (obj.kind != Object::Kind::Constant) |
| { |
| UNSUPPORTED("SPIR-V SampledImageArrayDynamicIndexing Capability"); |
| } |
| |
| auto d = descriptorDecorations.at(baseId); |
| ASSERT(d.DescriptorSet >= 0); |
| ASSERT(d.Binding >= 0); |
| auto setLayout = routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| auto stride = static_cast<uint32_t>(setLayout->getBindingStride(d.Binding)); |
| ptr.base += stride * GetConstScalarInt(indexIds[i]); |
| } |
| else |
| { |
| auto stride = getType(type.element).sizeInComponents * static_cast<uint32_t>(sizeof(float)); |
| auto & obj = getObject(indexIds[i]); |
| if (obj.kind == Object::Kind::Constant) |
| { |
| ptr += stride * GetConstScalarInt(indexIds[i]); |
| } |
| else |
| { |
| ptr += SIMD::Int(stride) * state->getIntermediate(indexIds[i]).Int(0); |
| } |
| } |
| typeId = type.element; |
| break; |
| } |
| |
| default: |
| UNREACHABLE("%s", OpcodeName(type.opcode()).c_str()); |
| } |
| } |
| |
| if (constantOffset != 0) |
| { |
| ptr += constantOffset; |
| } |
| return ptr; |
| } |
| |
| uint32_t SpirvShader::WalkLiteralAccessChain(Type::ID typeId, uint32_t numIndexes, uint32_t const *indexes) const |
| { |
| uint32_t componentOffset = 0; |
| |
| for (auto i = 0u; i < numIndexes; i++) |
| { |
| auto & type = getType(typeId); |
| switch(type.opcode()) |
| { |
| case spv::OpTypeStruct: |
| { |
| int memberIndex = indexes[i]; |
| int offsetIntoStruct = 0; |
| for (auto j = 0; j < memberIndex; j++) { |
| auto memberType = type.definition.word(2u + j); |
| offsetIntoStruct += getType(memberType).sizeInComponents; |
| } |
| componentOffset += offsetIntoStruct; |
| typeId = type.definition.word(2u + memberIndex); |
| break; |
| } |
| |
| case spv::OpTypeVector: |
| case spv::OpTypeMatrix: |
| case spv::OpTypeArray: |
| { |
| auto elementType = type.definition.word(2); |
| auto stride = getType(elementType).sizeInComponents; |
| componentOffset += stride * indexes[i]; |
| typeId = elementType; |
| break; |
| } |
| |
| default: |
| UNREACHABLE("%s", OpcodeName(type.opcode()).c_str()); |
| } |
| } |
| |
| return componentOffset; |
| } |
| |
| void SpirvShader::Decorations::Apply(spv::Decoration decoration, uint32_t arg) |
| { |
| switch (decoration) |
| { |
| case spv::DecorationLocation: |
| HasLocation = true; |
| Location = static_cast<int32_t>(arg); |
| break; |
| case spv::DecorationComponent: |
| HasComponent = true; |
| Component = arg; |
| break; |
| case spv::DecorationBuiltIn: |
| HasBuiltIn = true; |
| BuiltIn = static_cast<spv::BuiltIn>(arg); |
| break; |
| case spv::DecorationFlat: |
| Flat = true; |
| break; |
| case spv::DecorationNoPerspective: |
| NoPerspective = true; |
| break; |
| case spv::DecorationCentroid: |
| Centroid = true; |
| break; |
| case spv::DecorationBlock: |
| Block = true; |
| break; |
| case spv::DecorationBufferBlock: |
| BufferBlock = true; |
| break; |
| case spv::DecorationOffset: |
| HasOffset = true; |
| Offset = static_cast<int32_t>(arg); |
| break; |
| case spv::DecorationArrayStride: |
| HasArrayStride = true; |
| ArrayStride = static_cast<int32_t>(arg); |
| break; |
| case spv::DecorationMatrixStride: |
| HasMatrixStride = true; |
| MatrixStride = static_cast<int32_t>(arg); |
| break; |
| case spv::DecorationRelaxedPrecision: |
| RelaxedPrecision = true; |
| break; |
| case spv::DecorationRowMajor: |
| HasRowMajor = true; |
| RowMajor = true; |
| break; |
| case spv::DecorationColMajor: |
| HasRowMajor = true; |
| RowMajor = false; |
| default: |
| // Intentionally partial, there are many decorations we just don't care about. |
| break; |
| } |
| } |
| |
| void SpirvShader::Decorations::Apply(const sw::SpirvShader::Decorations &src) |
| { |
| // Apply a decoration group to this set of decorations |
| if (src.HasBuiltIn) |
| { |
| HasBuiltIn = true; |
| BuiltIn = src.BuiltIn; |
| } |
| |
| if (src.HasLocation) |
| { |
| HasLocation = true; |
| Location = src.Location; |
| } |
| |
| if (src.HasComponent) |
| { |
| HasComponent = true; |
| Component = src.Component; |
| } |
| |
| if (src.HasOffset) |
| { |
| HasOffset = true; |
| Offset = src.Offset; |
| } |
| |
| if (src.HasArrayStride) |
| { |
| HasArrayStride = true; |
| ArrayStride = src.ArrayStride; |
| } |
| |
| if (src.HasMatrixStride) |
| { |
| HasMatrixStride = true; |
| MatrixStride = src.MatrixStride; |
| } |
| |
| if (src.HasRowMajor) |
| { |
| HasRowMajor = true; |
| RowMajor = src.RowMajor; |
| } |
| |
| Flat |= src.Flat; |
| NoPerspective |= src.NoPerspective; |
| Centroid |= src.Centroid; |
| Block |= src.Block; |
| BufferBlock |= src.BufferBlock; |
| RelaxedPrecision |= src.RelaxedPrecision; |
| InsideMatrix |= src.InsideMatrix; |
| } |
| |
| void SpirvShader::DescriptorDecorations::Apply(const sw::SpirvShader::DescriptorDecorations &src) |
| { |
| if(src.DescriptorSet >= 0) |
| { |
| DescriptorSet = src.DescriptorSet; |
| } |
| |
| if(src.Binding >= 0) |
| { |
| Binding = src.Binding; |
| } |
| |
| if (src.InputAttachmentIndex >= 0) |
| { |
| InputAttachmentIndex = src.InputAttachmentIndex; |
| } |
| } |
| |
| void SpirvShader::ApplyDecorationsForId(Decorations *d, TypeOrObjectID id) const |
| { |
| auto it = decorations.find(id); |
| if (it != decorations.end()) |
| d->Apply(it->second); |
| } |
| |
| void SpirvShader::ApplyDecorationsForIdMember(Decorations *d, Type::ID id, uint32_t member) const |
| { |
| auto it = memberDecorations.find(id); |
| if (it != memberDecorations.end() && member < it->second.size()) |
| { |
| d->Apply(it->second[member]); |
| } |
| } |
| |
| void SpirvShader::DefineResult(const InsnIterator &insn) |
| { |
| Type::ID typeId = insn.word(1); |
| Object::ID resultId = insn.word(2); |
| auto &object = defs[resultId]; |
| object.type = typeId; |
| |
| switch (getType(typeId).opcode()) |
| { |
| case spv::OpTypePointer: |
| case spv::OpTypeImage: |
| case spv::OpTypeSampledImage: |
| case spv::OpTypeSampler: |
| object.kind = Object::Kind::Pointer; |
| break; |
| |
| default: |
| object.kind = Object::Kind::Intermediate; |
| } |
| |
| object.definition = insn; |
| } |
| |
| OutOfBoundsBehavior SpirvShader::EmitState::getOutOfBoundsBehavior(spv::StorageClass storageClass) const |
| { |
| switch(storageClass) |
| { |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| // Buffer resource access. robustBufferAccess feature applies. |
| return robustBufferAccess ? OutOfBoundsBehavior::RobustBufferAccess |
| : OutOfBoundsBehavior::UndefinedBehavior; |
| |
| case spv::StorageClassImage: |
| return OutOfBoundsBehavior::UndefinedValue; // "The value returned by a read of an invalid texel is undefined" |
| |
| case spv::StorageClassInput: |
| if(executionModel == spv::ExecutionModelVertex) |
| { |
| // Vertex attributes follow robustBufferAccess rules. |
| return robustBufferAccess ? OutOfBoundsBehavior::RobustBufferAccess |
| : OutOfBoundsBehavior::UndefinedBehavior; |
| } |
| // Fall through to default case. |
| default: |
| // TODO(b/137183137): Optimize if the pointer resulted from OpInBoundsAccessChain. |
| // TODO(b/131224163): Optimize cases statically known to be within bounds. |
| return OutOfBoundsBehavior::UndefinedValue; |
| } |
| |
| return OutOfBoundsBehavior::Nullify; |
| } |
| |
| // emit-time |
| |
| void SpirvShader::emitProlog(SpirvRoutine *routine) const |
| { |
| for (auto insn : *this) |
| { |
| switch (insn.opcode()) |
| { |
| case spv::OpVariable: |
| { |
| Type::ID resultPointerTypeId = insn.word(1); |
| auto resultPointerType = getType(resultPointerTypeId); |
| auto pointeeType = getType(resultPointerType.element); |
| |
| if(pointeeType.sizeInComponents > 0) // TODO: what to do about zero-slot objects? |
| { |
| Object::ID resultId = insn.word(2); |
| routine->createVariable(resultId, pointeeType.sizeInComponents); |
| } |
| break; |
| } |
| case spv::OpPhi: |
| { |
| auto type = getType(insn.word(1)); |
| Object::ID resultId = insn.word(2); |
| routine->phis.emplace(resultId, SpirvRoutine::Variable(type.sizeInComponents)); |
| break; |
| } |
| |
| case spv::OpImageDrefGather: |
| case spv::OpImageFetch: |
| case spv::OpImageGather: |
| case spv::OpImageQueryLod: |
| case spv::OpImageSampleDrefExplicitLod: |
| case spv::OpImageSampleDrefImplicitLod: |
| case spv::OpImageSampleExplicitLod: |
| case spv::OpImageSampleImplicitLod: |
| case spv::OpImageSampleProjDrefExplicitLod: |
| case spv::OpImageSampleProjDrefImplicitLod: |
| case spv::OpImageSampleProjExplicitLod: |
| case spv::OpImageSampleProjImplicitLod: |
| { |
| Object::ID resultId = insn.word(2); |
| routine->samplerCache.emplace(resultId, SpirvRoutine::SamplerCache{}); |
| break; |
| } |
| |
| default: |
| // Nothing else produces interface variables, so can all be safely ignored. |
| break; |
| } |
| } |
| } |
| |
| void SpirvShader::emit(SpirvRoutine *routine, RValue<SIMD::Int> const &activeLaneMask, RValue<SIMD::Int> const &storesAndAtomicsMask, const vk::DescriptorSet::Bindings &descriptorSets) const |
| { |
| EmitState state(routine, entryPoint, activeLaneMask, storesAndAtomicsMask, descriptorSets, robustBufferAccess, executionModel); |
| |
| // Emit everything up to the first label |
| // TODO: Separate out dispatch of block from non-block instructions? |
| for (auto insn : *this) |
| { |
| if (insn.opcode() == spv::OpLabel) |
| { |
| break; |
| } |
| EmitInstruction(insn, &state); |
| } |
| |
| // Emit all the blocks starting from entryPoint. |
| EmitBlocks(getFunction(entryPoint).entry, &state); |
| } |
| |
| void SpirvShader::EmitBlocks(Block::ID id, EmitState *state, Block::ID ignore /* = 0 */) const |
| { |
| auto oldPending = state->pending; |
| auto &function = getFunction(state->function); |
| |
| std::deque<Block::ID> pending; |
| state->pending = &pending; |
| pending.push_front(id); |
| while (pending.size() > 0) |
| { |
| auto id = pending.front(); |
| |
| auto const &block = function.getBlock(id); |
| if (id == ignore) |
| { |
| pending.pop_front(); |
| continue; |
| } |
| |
| // Ensure all dependency blocks have been generated. |
| auto depsDone = true; |
| function.ForeachBlockDependency(id, [&](Block::ID dep) |
| { |
| if (state->visited.count(dep) == 0) |
| { |
| state->pending->push_front(dep); |
| depsDone = false; |
| } |
| }); |
| |
| if (!depsDone) |
| { |
| continue; |
| } |
| |
| pending.pop_front(); |
| |
| state->block = id; |
| |
| switch (block.kind) |
| { |
| case Block::Simple: |
| case Block::StructuredBranchConditional: |
| case Block::UnstructuredBranchConditional: |
| case Block::StructuredSwitch: |
| case Block::UnstructuredSwitch: |
| EmitNonLoop(state); |
| break; |
| |
| case Block::Loop: |
| EmitLoop(state); |
| break; |
| |
| default: |
| UNREACHABLE("Unexpected Block Kind: %d", int(block.kind)); |
| } |
| } |
| |
| state->pending = oldPending; |
| } |
| |
| void SpirvShader::EmitInstructions(InsnIterator begin, InsnIterator end, EmitState *state) const |
| { |
| for (auto insn = begin; insn != end; insn++) |
| { |
| auto res = EmitInstruction(insn, state); |
| switch (res) |
| { |
| case EmitResult::Continue: |
| continue; |
| case EmitResult::Terminator: |
| break; |
| default: |
| UNREACHABLE("Unexpected EmitResult %d", int(res)); |
| break; |
| } |
| } |
| } |
| |
| void SpirvShader::EmitNonLoop(EmitState *state) const |
| { |
| auto &function = getFunction(state->function); |
| auto blockId = state->block; |
| auto block = function.getBlock(blockId); |
| |
| if (!state->visited.emplace(blockId).second) |
| { |
| return; // Already generated this block. |
| } |
| |
| if (blockId != function.entry) |
| { |
| // Set the activeLaneMask. |
| SIMD::Int activeLaneMask(0); |
| for (auto in : block.ins) |
| { |
| auto inMask = GetActiveLaneMaskEdge(state, in, blockId); |
| activeLaneMask |= inMask; |
| } |
| state->setActiveLaneMask(activeLaneMask); |
| } |
| |
| EmitInstructions(block.begin(), block.end(), state); |
| |
| for (auto out : block.outs) |
| { |
| if (state->visited.count(out) == 0) |
| { |
| state->pending->push_back(out); |
| } |
| } |
| } |
| |
| void SpirvShader::EmitLoop(EmitState *state) const |
| { |
| auto &function = getFunction(state->function); |
| auto blockId = state->block; |
| auto &block = function.getBlock(blockId); |
| auto mergeBlockId = block.mergeBlock; |
| auto &mergeBlock = function.getBlock(mergeBlockId); |
| |
| if (!state->visited.emplace(blockId).second) |
| { |
| return; // Already emitted this loop. |
| } |
| |
| // Gather all the blocks that make up the loop. |
| std::unordered_set<Block::ID> loopBlocks; |
| loopBlocks.emplace(block.mergeBlock); |
| function.TraverseReachableBlocks(blockId, loopBlocks); |
| |
| // incomingBlocks are block ins that are not back-edges. |
| std::unordered_set<Block::ID> incomingBlocks; |
| for (auto in : block.ins) |
| { |
| if (loopBlocks.count(in) == 0) |
| { |
| incomingBlocks.emplace(in); |
| } |
| } |
| |
| // Emit the loop phi instructions, and initialize them with a value from |
| // the incoming blocks. |
| for (auto insn = block.begin(); insn != block.mergeInstruction; insn++) |
| { |
| if (insn.opcode() == spv::OpPhi) |
| { |
| StorePhi(blockId, insn, state, incomingBlocks); |
| } |
| } |
| |
| // loopActiveLaneMask is the mask of lanes that are continuing to loop. |
| // This is initialized with the incoming active lane masks. |
| SIMD::Int loopActiveLaneMask = SIMD::Int(0); |
| for (auto in : incomingBlocks) |
| { |
| loopActiveLaneMask |= GetActiveLaneMaskEdge(state, in, blockId); |
| } |
| |
| // mergeActiveLaneMasks contains edge lane masks for the merge block. |
| // This is the union of all edge masks across all iterations of the loop. |
| std::unordered_map<Block::ID, SIMD::Int> mergeActiveLaneMasks; |
| for (auto in : function.getBlock(mergeBlockId).ins) |
| { |
| mergeActiveLaneMasks.emplace(in, SIMD::Int(0)); |
| } |
| |
| // Create the loop basic blocks |
| auto headerBasicBlock = Nucleus::createBasicBlock(); |
| auto mergeBasicBlock = Nucleus::createBasicBlock(); |
| |
| // Start emitting code inside the loop. |
| Nucleus::createBr(headerBasicBlock); |
| Nucleus::setInsertBlock(headerBasicBlock); |
| |
| // Load the active lane mask. |
| state->setActiveLaneMask(loopActiveLaneMask); |
| |
| // Emit the non-phi loop header block's instructions. |
| for (auto insn = block.begin(); insn != block.end(); insn++) |
| { |
| if (insn.opcode() == spv::OpPhi) |
| { |
| LoadPhi(insn, state); |
| } |
| else |
| { |
| EmitInstruction(insn, state); |
| } |
| } |
| |
| // Emit all blocks between the loop header and the merge block, but |
| // don't emit the merge block yet. |
| for (auto out : block.outs) |
| { |
| EmitBlocks(out, state, mergeBlockId); |
| } |
| |
| // Restore current block id after emitting loop blocks. |
| state->block = blockId; |
| |
| // Rebuild the loopActiveLaneMask from the loop back edges. |
| loopActiveLaneMask = SIMD::Int(0); |
| for (auto in : block.ins) |
| { |
| if (function.ExistsPath(blockId, in, mergeBlockId)) |
| { |
| loopActiveLaneMask |= GetActiveLaneMaskEdge(state, in, blockId); |
| } |
| } |
| |
| // Add active lanes to the merge lane mask. |
| for (auto in : function.getBlock(mergeBlockId).ins) |
| { |
| auto edge = Block::Edge{in, mergeBlockId}; |
| auto it = state->edgeActiveLaneMasks.find(edge); |
| if (it != state->edgeActiveLaneMasks.end()) |
| { |
| mergeActiveLaneMasks[in] |= it->second; |
| } |
| } |
| |
| // Update loop phi values. |
| for (auto insn = block.begin(); insn != block.mergeInstruction; insn++) |
| { |
| if (insn.opcode() == spv::OpPhi) |
| { |
| StorePhi(blockId, insn, state, loopBlocks); |
| } |
| } |
| |
| // Use the [loop -> merge] active lane masks to update the phi values in |
| // the merge block. We need to do this to handle divergent control flow |
| // in the loop. |
| // |
| // Consider the following: |
| // |
| // int phi_source = 0; |
| // for (uint i = 0; i < 4; i++) |
| // { |
| // phi_source = 0; |
| // if (gl_GlobalInvocationID.x % 4 == i) // divergent control flow |
| // { |
| // phi_source = 42; // single lane assignment. |
| // break; // activeLaneMask for [loop->merge] is active for a single lane. |
| // } |
| // // -- we are here -- |
| // } |
| // // merge block |
| // int phi = phi_source; // OpPhi |
| // |
| // In this example, with each iteration of the loop, phi_source will |
| // only have a single lane assigned. However by 'phi' value in the merge |
| // block needs to be assigned the union of all the per-lane assignments |
| // of phi_source when that lane exited the loop. |
| for (auto insn = mergeBlock.begin(); insn != mergeBlock.end(); insn++) |
| { |
| if (insn.opcode() == spv::OpPhi) |
| { |
| StorePhi(mergeBlockId, insn, state, loopBlocks); |
| } |
| } |
| |
| // Loop body now done. |
| // If any lanes are still active, jump back to the loop header, |
| // otherwise jump to the merge block. |
| Nucleus::createCondBr(AnyTrue(loopActiveLaneMask).value, headerBasicBlock, mergeBasicBlock); |
| |
| // Continue emitting from the merge block. |
| Nucleus::setInsertBlock(mergeBasicBlock); |
| state->pending->push_back(mergeBlockId); |
| for (auto it : mergeActiveLaneMasks) |
| { |
| state->addActiveLaneMaskEdge(it.first, mergeBlockId, it.second); |
| } |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitInstruction(InsnIterator insn, EmitState *state) const |
| { |
| auto opcode = insn.opcode(); |
| |
| switch (opcode) |
| { |
| case spv::OpTypeVoid: |
| case spv::OpTypeInt: |
| case spv::OpTypeFloat: |
| case spv::OpTypeBool: |
| case spv::OpTypeVector: |
| case spv::OpTypeArray: |
| case spv::OpTypeRuntimeArray: |
| case spv::OpTypeMatrix: |
| case spv::OpTypeStruct: |
| case spv::OpTypePointer: |
| case spv::OpTypeFunction: |
| case spv::OpTypeImage: |
| case spv::OpTypeSampledImage: |
| case spv::OpTypeSampler: |
| case spv::OpExecutionMode: |
| case spv::OpMemoryModel: |
| case spv::OpFunction: |
| case spv::OpFunctionEnd: |
| case spv::OpConstant: |
| case spv::OpConstantNull: |
| case spv::OpConstantTrue: |
| case spv::OpConstantFalse: |
| case spv::OpConstantComposite: |
| case spv::OpSpecConstant: |
| case spv::OpSpecConstantTrue: |
| case spv::OpSpecConstantFalse: |
| case spv::OpSpecConstantComposite: |
| case spv::OpSpecConstantOp: |
| case spv::OpUndef: |
| case spv::OpExtension: |
| case spv::OpCapability: |
| case spv::OpEntryPoint: |
| case spv::OpExtInstImport: |
| case spv::OpDecorate: |
| case spv::OpMemberDecorate: |
| case spv::OpGroupDecorate: |
| case spv::OpGroupMemberDecorate: |
| case spv::OpDecorationGroup: |
| case spv::OpName: |
| case spv::OpMemberName: |
| case spv::OpSource: |
| case spv::OpSourceContinued: |
| case spv::OpSourceExtension: |
| case spv::OpLine: |
| case spv::OpNoLine: |
| case spv::OpModuleProcessed: |
| case spv::OpString: |
| // Nothing to do at emit time. These are either fully handled at analysis time, |
| // or don't require any work at all. |
| return EmitResult::Continue; |
| |
| case spv::OpLabel: |
| return EmitResult::Continue; |
| |
| case spv::OpVariable: |
| return EmitVariable(insn, state); |
| |
| case spv::OpLoad: |
| case spv::OpAtomicLoad: |
| return EmitLoad(insn, state); |
| |
| case spv::OpStore: |
| case spv::OpAtomicStore: |
| return EmitStore(insn, state); |
| |
| case spv::OpAtomicIAdd: |
| case spv::OpAtomicISub: |
| case spv::OpAtomicSMin: |
| case spv::OpAtomicSMax: |
| case spv::OpAtomicUMin: |
| case spv::OpAtomicUMax: |
| case spv::OpAtomicAnd: |
| case spv::OpAtomicOr: |
| case spv::OpAtomicXor: |
| case spv::OpAtomicIIncrement: |
| case spv::OpAtomicIDecrement: |
| case spv::OpAtomicExchange: |
| return EmitAtomicOp(insn, state); |
| |
| case spv::OpAtomicCompareExchange: |
| return EmitAtomicCompareExchange(insn, state); |
| |
| case spv::OpAccessChain: |
| case spv::OpInBoundsAccessChain: |
| return EmitAccessChain(insn, state); |
| |
| case spv::OpCompositeConstruct: |
| return EmitCompositeConstruct(insn, state); |
| |
| case spv::OpCompositeInsert: |
| return EmitCompositeInsert(insn, state); |
| |
| case spv::OpCompositeExtract: |
| return EmitCompositeExtract(insn, state); |
| |
| case spv::OpVectorShuffle: |
| return EmitVectorShuffle(insn, state); |
| |
| case spv::OpVectorExtractDynamic: |
| return EmitVectorExtractDynamic(insn, state); |
| |
| case spv::OpVectorInsertDynamic: |
| return EmitVectorInsertDynamic(insn, state); |
| |
| case spv::OpVectorTimesScalar: |
| case spv::OpMatrixTimesScalar: |
| return EmitVectorTimesScalar(insn, state); |
| |
| case spv::OpMatrixTimesVector: |
| return EmitMatrixTimesVector(insn, state); |
| |
| case spv::OpVectorTimesMatrix: |
| return EmitVectorTimesMatrix(insn, state); |
| |
| case spv::OpMatrixTimesMatrix: |
| return EmitMatrixTimesMatrix(insn, state); |
| |
| case spv::OpOuterProduct: |
| return EmitOuterProduct(insn, state); |
| |
| case spv::OpTranspose: |
| return EmitTranspose(insn, state); |
| |
| case spv::OpNot: |
| case spv::OpBitFieldInsert: |
| case spv::OpBitFieldSExtract: |
| case spv::OpBitFieldUExtract: |
| case spv::OpBitReverse: |
| case spv::OpBitCount: |
| case spv::OpSNegate: |
| case spv::OpFNegate: |
| case spv::OpLogicalNot: |
| case spv::OpConvertFToU: |
| case spv::OpConvertFToS: |
| case spv::OpConvertSToF: |
| case spv::OpConvertUToF: |
| case spv::OpBitcast: |
| case spv::OpIsInf: |
| case spv::OpIsNan: |
| case spv::OpDPdx: |
| case spv::OpDPdxCoarse: |
| case spv::OpDPdy: |
| case spv::OpDPdyCoarse: |
| case spv::OpFwidth: |
| case spv::OpFwidthCoarse: |
| case spv::OpDPdxFine: |
| case spv::OpDPdyFine: |
| case spv::OpFwidthFine: |
| case spv::OpQuantizeToF16: |
| return EmitUnaryOp(insn, state); |
| |
| case spv::OpIAdd: |
| case spv::OpISub: |
| case spv::OpIMul: |
| case spv::OpSDiv: |
| case spv::OpUDiv: |
| case spv::OpFAdd: |
| case spv::OpFSub: |
| case spv::OpFMul: |
| case spv::OpFDiv: |
| case spv::OpFMod: |
| case spv::OpFRem: |
| case spv::OpFOrdEqual: |
| case spv::OpFUnordEqual: |
| case spv::OpFOrdNotEqual: |
| case spv::OpFUnordNotEqual: |
| case spv::OpFOrdLessThan: |
| case spv::OpFUnordLessThan: |
| case spv::OpFOrdGreaterThan: |
| case spv::OpFUnordGreaterThan: |
| case spv::OpFOrdLessThanEqual: |
| case spv::OpFUnordLessThanEqual: |
| case spv::OpFOrdGreaterThanEqual: |
| case spv::OpFUnordGreaterThanEqual: |
| case spv::OpSMod: |
| case spv::OpSRem: |
| case spv::OpUMod: |
| case spv::OpIEqual: |
| case spv::OpINotEqual: |
| case spv::OpUGreaterThan: |
| case spv::OpSGreaterThan: |
| case spv::OpUGreaterThanEqual: |
| case spv::OpSGreaterThanEqual: |
| case spv::OpULessThan: |
| case spv::OpSLessThan: |
| case spv::OpULessThanEqual: |
| case spv::OpSLessThanEqual: |
| case spv::OpShiftRightLogical: |
| case spv::OpShiftRightArithmetic: |
| case spv::OpShiftLeftLogical: |
| case spv::OpBitwiseOr: |
| case spv::OpBitwiseXor: |
| case spv::OpBitwiseAnd: |
| case spv::OpLogicalOr: |
| case spv::OpLogicalAnd: |
| case spv::OpLogicalEqual: |
| case spv::OpLogicalNotEqual: |
| case spv::OpUMulExtended: |
| case spv::OpSMulExtended: |
| case spv::OpIAddCarry: |
| case spv::OpISubBorrow: |
| return EmitBinaryOp(insn, state); |
| |
| case spv::OpDot: |
| return EmitDot(insn, state); |
| |
| case spv::OpSelect: |
| return EmitSelect(insn, state); |
| |
| case spv::OpExtInst: |
| return EmitExtendedInstruction(insn, state); |
| |
| case spv::OpAny: |
| return EmitAny(insn, state); |
| |
| case spv::OpAll: |
| return EmitAll(insn, state); |
| |
| case spv::OpBranch: |
| return EmitBranch(insn, state); |
| |
| case spv::OpPhi: |
| return EmitPhi(insn, state); |
| |
| case spv::OpSelectionMerge: |
| case spv::OpLoopMerge: |
| return EmitResult::Continue; |
| |
| case spv::OpBranchConditional: |
| return EmitBranchConditional(insn, state); |
| |
| case spv::OpSwitch: |
| return EmitSwitch(insn, state); |
| |
| case spv::OpUnreachable: |
| return EmitUnreachable(insn, state); |
| |
| case spv::OpReturn: |
| return EmitReturn(insn, state); |
| |
| case spv::OpFunctionCall: |
| return EmitFunctionCall(insn, state); |
| |
| case spv::OpKill: |
| return EmitKill(insn, state); |
| |
| case spv::OpImageSampleImplicitLod: |
| return EmitImageSampleImplicitLod(None, insn, state); |
| |
| case spv::OpImageSampleExplicitLod: |
| return EmitImageSampleExplicitLod(None, insn, state); |
| |
| case spv::OpImageSampleDrefImplicitLod: |
| return EmitImageSampleImplicitLod(Dref, insn, state); |
| |
| case spv::OpImageSampleDrefExplicitLod: |
| return EmitImageSampleExplicitLod(Dref, insn, state); |
| |
| case spv::OpImageSampleProjImplicitLod: |
| return EmitImageSampleImplicitLod(Proj, insn, state); |
| |
| case spv::OpImageSampleProjExplicitLod: |
| return EmitImageSampleExplicitLod(Proj, insn, state); |
| |
| case spv::OpImageSampleProjDrefImplicitLod: |
| return EmitImageSampleImplicitLod(ProjDref, insn, state); |
| |
| case spv::OpImageSampleProjDrefExplicitLod: |
| return EmitImageSampleExplicitLod(ProjDref, insn, state); |
| |
| case spv::OpImageGather: |
| return EmitImageGather(None, insn, state); |
| |
| case spv::OpImageDrefGather: |
| return EmitImageGather(Dref, insn, state); |
| |
| case spv::OpImageFetch: |
| return EmitImageFetch(insn, state); |
| |
| case spv::OpImageQuerySizeLod: |
| return EmitImageQuerySizeLod(insn, state); |
| |
| case spv::OpImageQuerySize: |
| return EmitImageQuerySize(insn, state); |
| |
| case spv::OpImageQueryLod: |
| return EmitImageQueryLod(insn, state); |
| |
| case spv::OpImageQueryLevels: |
| return EmitImageQueryLevels(insn, state); |
| |
| case spv::OpImageQuerySamples: |
| return EmitImageQuerySamples(insn, state); |
| |
| case spv::OpImageRead: |
| return EmitImageRead(insn, state); |
| |
| case spv::OpImageWrite: |
| return EmitImageWrite(insn, state); |
| |
| case spv::OpImageTexelPointer: |
| return EmitImageTexelPointer(insn, state); |
| |
| case spv::OpSampledImage: |
| case spv::OpImage: |
| return EmitSampledImageCombineOrSplit(insn, state); |
| |
| case spv::OpCopyObject: |
| return EmitCopyObject(insn, state); |
| |
| case spv::OpCopyMemory: |
| return EmitCopyMemory(insn, state); |
| |
| case spv::OpControlBarrier: |
| return EmitControlBarrier(insn, state); |
| |
| case spv::OpMemoryBarrier: |
| return EmitMemoryBarrier(insn, state); |
| |
| case spv::OpGroupNonUniformElect: |
| case spv::OpGroupNonUniformAll: |
| case spv::OpGroupNonUniformAny: |
| case spv::OpGroupNonUniformAllEqual: |
| case spv::OpGroupNonUniformBroadcast: |
| case spv::OpGroupNonUniformBroadcastFirst: |
| case spv::OpGroupNonUniformBallot: |
| case spv::OpGroupNonUniformInverseBallot: |
| case spv::OpGroupNonUniformBallotBitExtract: |
| case spv::OpGroupNonUniformBallotBitCount: |
| case spv::OpGroupNonUniformBallotFindLSB: |
| case spv::OpGroupNonUniformBallotFindMSB: |
| case spv::OpGroupNonUniformShuffle: |
| case spv::OpGroupNonUniformShuffleXor: |
| case spv::OpGroupNonUniformShuffleUp: |
| case spv::OpGroupNonUniformShuffleDown: |
| return EmitGroupNonUniform(insn, state); |
| |
| case spv::OpArrayLength: |
| return EmitArrayLength(insn, state); |
| |
| default: |
| UNREACHABLE("%s", OpcodeName(opcode).c_str()); |
| break; |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVariable(InsnIterator insn, EmitState *state) const |
| { |
| auto routine = state->routine; |
| Object::ID resultId = insn.word(2); |
| auto &object = getObject(resultId); |
| auto &objectTy = getType(object.type); |
| |
| switch (objectTy.storageClass) |
| { |
| case spv::StorageClassOutput: |
| case spv::StorageClassPrivate: |
| case spv::StorageClassFunction: |
| { |
| ASSERT(objectTy.opcode() == spv::OpTypePointer); |
| auto base = &routine->getVariable(resultId)[0]; |
| auto elementTy = getType(objectTy.element); |
| auto size = elementTy.sizeInComponents * static_cast<uint32_t>(sizeof(float)) * SIMD::Width; |
| state->createPointer(resultId, SIMD::Pointer(base, size)); |
| break; |
| } |
| case spv::StorageClassWorkgroup: |
| { |
| ASSERT(objectTy.opcode() == spv::OpTypePointer); |
| auto base = &routine->workgroupMemory[0]; |
| auto size = workgroupMemory.size(); |
| state->createPointer(resultId, SIMD::Pointer(base, size, workgroupMemory.offsetOf(resultId))); |
| break; |
| } |
| case spv::StorageClassInput: |
| { |
| if (object.kind == Object::Kind::InterfaceVariable) |
| { |
| auto &dst = routine->getVariable(resultId); |
| int offset = 0; |
| VisitInterface(resultId, |
| [&](Decorations const &d, AttribType type) { |
| auto scalarSlot = d.Location << 2 | d.Component; |
| dst[offset++] = routine->inputs[scalarSlot]; |
| }); |
| } |
| ASSERT(objectTy.opcode() == spv::OpTypePointer); |
| auto base = &routine->getVariable(resultId)[0]; |
| auto elementTy = getType(objectTy.element); |
| auto size = elementTy.sizeInComponents * static_cast<uint32_t>(sizeof(float)) * SIMD::Width; |
| state->createPointer(resultId, SIMD::Pointer(base, size)); |
| break; |
| } |
| case spv::StorageClassUniformConstant: |
| { |
| const auto &d = descriptorDecorations.at(resultId); |
| ASSERT(d.DescriptorSet >= 0); |
| ASSERT(d.Binding >= 0); |
| |
| uint32_t arrayIndex = 0; // TODO(b/129523279) |
| auto setLayout = routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| if (setLayout->hasBinding(d.Binding)) |
| { |
| uint32_t bindingOffset = static_cast<uint32_t>(setLayout->getBindingOffset(d.Binding, arrayIndex)); |
| Pointer<Byte> set = routine->descriptorSets[d.DescriptorSet]; // DescriptorSet* |
| Pointer<Byte> binding = Pointer<Byte>(set + bindingOffset); // vk::SampledImageDescriptor* |
| auto size = 0; // Not required as this pointer is not directly used by SIMD::Read or SIMD::Write. |
| state->createPointer(resultId, SIMD::Pointer(binding, size)); |
| } |
| else |
| { |
| // TODO: Error if the variable with the non-existant binding is |
| // used? Or perhaps strip these unused variable declarations as |
| // a preprocess on the SPIR-V? |
| } |
| break; |
| } |
| case spv::StorageClassUniform: |
| case spv::StorageClassStorageBuffer: |
| { |
| const auto &d = descriptorDecorations.at(resultId); |
| ASSERT(d.DescriptorSet >= 0 && d.DescriptorSet < vk::MAX_BOUND_DESCRIPTOR_SETS); |
| auto size = 0; // Not required as this pointer is not directly used by SIMD::Read or SIMD::Write. |
| state->createPointer(resultId, SIMD::Pointer(routine->descriptorSets[d.DescriptorSet], size)); |
| break; |
| } |
| case spv::StorageClassPushConstant: |
| { |
| state->createPointer(resultId, SIMD::Pointer(routine->pushConstants, vk::MAX_PUSH_CONSTANT_SIZE)); |
| break; |
| } |
| default: |
| UNREACHABLE("Storage class %d", objectTy.storageClass); |
| break; |
| } |
| |
| if (insn.wordCount() > 4) |
| { |
| Object::ID initializerId = insn.word(4); |
| if (getObject(initializerId).kind != Object::Kind::Constant) |
| { |
| UNIMPLEMENTED("Non-constant initializers not yet implemented"); |
| } |
| switch (objectTy.storageClass) |
| { |
| case spv::StorageClassOutput: |
| case spv::StorageClassPrivate: |
| case spv::StorageClassFunction: |
| { |
| bool interleavedByLane = IsStorageInterleavedByLane(objectTy.storageClass); |
| auto ptr = GetPointerToData(resultId, 0, state); |
| GenericValue initialValue(this, state, initializerId); |
| VisitMemoryObject(resultId, [&](uint32_t i, uint32_t offset) |
| { |
| auto p = ptr + offset; |
| if (interleavedByLane) { p = interleaveByLane(p); } |
| auto robustness = OutOfBoundsBehavior::UndefinedBehavior; // Local variables are always within bounds. |
| SIMD::Store(p, initialValue.Float(i), robustness, state->activeLaneMask()); |
| }); |
| break; |
| } |
| default: |
| ASSERT_MSG(initializerId == 0, "Vulkan does not permit variables of storage class %d to have initializers", int(objectTy.storageClass)); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitLoad(InsnIterator insn, EmitState *state) const |
| { |
| bool atomic = (insn.opcode() == spv::OpAtomicLoad); |
| Object::ID resultId = insn.word(2); |
| Object::ID pointerId = insn.word(3); |
| auto &result = getObject(resultId); |
| auto &resultTy = getType(result.type); |
| auto &pointer = getObject(pointerId); |
| auto &pointerTy = getType(pointer.type); |
| std::memory_order memoryOrder = std::memory_order_relaxed; |
| |
| ASSERT(getType(pointer.type).element == result.type); |
| ASSERT(Type::ID(insn.word(1)) == result.type); |
| ASSERT(!atomic || getType(getType(pointer.type).element).opcode() == spv::OpTypeInt); // Vulkan 1.1: "Atomic instructions must declare a scalar 32-bit integer type, for the value pointed to by Pointer." |
| |
| if(pointerTy.storageClass == spv::StorageClassUniformConstant) |
| { |
| // Just propagate the pointer. |
| auto &ptr = state->getPointer(pointerId); |
| state->createPointer(resultId, ptr); |
| return EmitResult::Continue; |
| } |
| |
| if(atomic) |
| { |
| Object::ID semanticsId = insn.word(5); |
| auto memorySemantics = static_cast<spv::MemorySemanticsMask>(getObject(semanticsId).constantValue[0]); |
| memoryOrder = MemoryOrder(memorySemantics); |
| } |
| |
| auto ptr = GetPointerToData(pointerId, 0, state); |
| bool interleavedByLane = IsStorageInterleavedByLane(pointerTy.storageClass); |
| auto &dst = state->createIntermediate(resultId, resultTy.sizeInComponents); |
| auto robustness = state->getOutOfBoundsBehavior(pointerTy.storageClass); |
| |
| VisitMemoryObject(pointerId, [&](uint32_t i, uint32_t offset) |
| { |
| auto p = ptr + offset; |
| if (interleavedByLane) { p = interleaveByLane(p); } // TODO: Interleave once, then add offset? |
| dst.move(i, SIMD::Load<SIMD::Float>(p, robustness, state->activeLaneMask(), atomic, memoryOrder)); |
| }); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitStore(InsnIterator insn, EmitState *state) const |
| { |
| bool atomic = (insn.opcode() == spv::OpAtomicStore); |
| Object::ID pointerId = insn.word(1); |
| Object::ID objectId = insn.word(atomic ? 4 : 2); |
| auto &object = getObject(objectId); |
| auto &pointer = getObject(pointerId); |
| auto &pointerTy = getType(pointer.type); |
| auto &elementTy = getType(pointerTy.element); |
| std::memory_order memoryOrder = std::memory_order_relaxed; |
| |
| if(atomic) |
| { |
| Object::ID semanticsId = insn.word(3); |
| auto memorySemantics = static_cast<spv::MemorySemanticsMask>(getObject(semanticsId).constantValue[0]); |
| memoryOrder = MemoryOrder(memorySemantics); |
| } |
| |
| ASSERT(!atomic || elementTy.opcode() == spv::OpTypeInt); // Vulkan 1.1: "Atomic instructions must declare a scalar 32-bit integer type, for the value pointed to by Pointer." |
| |
| auto ptr = GetPointerToData(pointerId, 0, state); |
| bool interleavedByLane = IsStorageInterleavedByLane(pointerTy.storageClass); |
| auto robustness = state->getOutOfBoundsBehavior(pointerTy.storageClass); |
| |
| SIMD::Int mask = state->activeLaneMask(); |
| if (!StoresInHelperInvocation(pointerTy.storageClass)) |
| { |
| mask = mask & state->storesAndAtomicsMask(); |
| } |
| |
| if (object.kind == Object::Kind::Constant) |
| { |
| // Constant source data. |
| const uint32_t *src = object.constantValue.get(); |
| VisitMemoryObject(pointerId, [&](uint32_t i, uint32_t offset) |
| { |
| auto p = ptr + offset; |
| if (interleavedByLane) { p = interleaveByLane(p); } |
| SIMD::Store(p, SIMD::Int(src[i]), robustness, mask, atomic, memoryOrder); |
| }); |
| } |
| else |
| { |
| // Intermediate source data. |
| auto &src = state->getIntermediate(objectId); |
| VisitMemoryObject(pointerId, [&](uint32_t i, uint32_t offset) |
| { |
| auto p = ptr + offset; |
| if (interleavedByLane) { p = interleaveByLane(p); } |
| SIMD::Store(p, src.Float(i), robustness, mask, atomic, memoryOrder); |
| }); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitAccessChain(InsnIterator insn, EmitState *state) const |
| { |
| Type::ID typeId = insn.word(1); |
| Object::ID resultId = insn.word(2); |
| Object::ID baseId = insn.word(3); |
| uint32_t numIndexes = insn.wordCount() - 4; |
| const uint32_t *indexes = insn.wordPointer(4); |
| auto &type = getType(typeId); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getObject(resultId).kind == Object::Kind::Pointer); |
| |
| if(type.storageClass == spv::StorageClassPushConstant || |
| type.storageClass == spv::StorageClassUniform || |
| type.storageClass == spv::StorageClassStorageBuffer) |
| { |
| auto ptr = WalkExplicitLayoutAccessChain(baseId, numIndexes, indexes, state); |
| state->createPointer(resultId, ptr); |
| } |
| else |
| { |
| auto ptr = WalkAccessChain(baseId, numIndexes, indexes, state); |
| state->createPointer(resultId, ptr); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitCompositeConstruct(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto offset = 0u; |
| |
| for (auto i = 0u; i < insn.wordCount() - 3; i++) |
| { |
| Object::ID srcObjectId = insn.word(3u + i); |
| auto & srcObject = getObject(srcObjectId); |
| auto & srcObjectTy = getType(srcObject.type); |
| GenericValue srcObjectAccess(this, state, srcObjectId); |
| |
| for (auto j = 0u; j < srcObjectTy.sizeInComponents; j++) |
| { |
| dst.move(offset++, srcObjectAccess.Float(j)); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitCompositeInsert(InsnIterator insn, EmitState *state) const |
| { |
| Type::ID resultTypeId = insn.word(1); |
| auto &type = getType(resultTypeId); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &newPartObject = getObject(insn.word(3)); |
| auto &newPartObjectTy = getType(newPartObject.type); |
| auto firstNewComponent = WalkLiteralAccessChain(resultTypeId, insn.wordCount() - 5, insn.wordPointer(5)); |
| |
| GenericValue srcObjectAccess(this, state, insn.word(4)); |
| GenericValue newPartObjectAccess(this, state, insn.word(3)); |
| |
| // old components before |
| for (auto i = 0u; i < firstNewComponent; i++) |
| { |
| dst.move(i, srcObjectAccess.Float(i)); |
| } |
| // new part |
| for (auto i = 0u; i < newPartObjectTy.sizeInComponents; i++) |
| { |
| dst.move(firstNewComponent + i, newPartObjectAccess.Float(i)); |
| } |
| // old components after |
| for (auto i = firstNewComponent + newPartObjectTy.sizeInComponents; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, srcObjectAccess.Float(i)); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitCompositeExtract(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &compositeObject = getObject(insn.word(3)); |
| Type::ID compositeTypeId = compositeObject.definition.word(1); |
| auto firstComponent = WalkLiteralAccessChain(compositeTypeId, insn.wordCount() - 4, insn.wordPointer(4)); |
| |
| GenericValue compositeObjectAccess(this, state, insn.word(3)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, compositeObjectAccess.Float(firstComponent + i)); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVectorShuffle(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| |
| // Note: number of components in result type, first half type, and second |
| // half type are all independent. |
| auto &firstHalfType = getType(getObject(insn.word(3)).type); |
| |
| GenericValue firstHalfAccess(this, state, insn.word(3)); |
| GenericValue secondHalfAccess(this, state, insn.word(4)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto selector = insn.word(5 + i); |
| if (selector == static_cast<uint32_t>(-1)) |
| { |
| // Undefined value. Until we decide to do real undef values, zero is as good |
| // a value as any |
| dst.move(i, RValue<SIMD::Float>(0.0f)); |
| } |
| else if (selector < firstHalfType.sizeInComponents) |
| { |
| dst.move(i, firstHalfAccess.Float(selector)); |
| } |
| else |
| { |
| dst.move(i, secondHalfAccess.Float(selector - firstHalfType.sizeInComponents)); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVectorExtractDynamic(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &srcType = getType(getObject(insn.word(3)).type); |
| |
| GenericValue src(this, state, insn.word(3)); |
| GenericValue index(this, state, insn.word(4)); |
| |
| SIMD::UInt v = SIMD::UInt(0); |
| |
| for (auto i = 0u; i < srcType.sizeInComponents; i++) |
| { |
| v |= CmpEQ(index.UInt(0), SIMD::UInt(i)) & src.UInt(i); |
| } |
| |
| dst.move(0, v); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVectorInsertDynamic(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| |
| GenericValue src(this, state, insn.word(3)); |
| GenericValue component(this, state, insn.word(4)); |
| GenericValue index(this, state, insn.word(5)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::UInt mask = CmpEQ(SIMD::UInt(i), index.UInt(0)); |
| dst.move(i, (src.UInt(i) & ~mask) | (component.UInt(0) & mask)); |
| } |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVectorTimesScalar(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, lhs.Float(i) * rhs.Float(0)); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitMatrixTimesVector(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| auto rhsType = getType(rhs.type); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Float v = lhs.Float(i) * rhs.Float(0); |
| for (auto j = 1u; j < rhsType.sizeInComponents; j++) |
| { |
| v += lhs.Float(i + type.sizeInComponents * j) * rhs.Float(j); |
| } |
| dst.move(i, v); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitVectorTimesMatrix(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| auto lhsType = getType(lhs.type); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Float v = lhs.Float(0) * rhs.Float(i * lhsType.sizeInComponents); |
| for (auto j = 1u; j < lhsType.sizeInComponents; j++) |
| { |
| v += lhs.Float(j) * rhs.Float(i * lhsType.sizeInComponents + j); |
| } |
| dst.move(i, v); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitMatrixTimesMatrix(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| |
| auto numColumns = type.definition.word(3); |
| auto numRows = getType(type.definition.word(2)).definition.word(3); |
| auto numAdds = getType(getObject(insn.word(3)).type).definition.word(3); |
| |
| for (auto row = 0u; row < numRows; row++) |
| { |
| for (auto col = 0u; col < numColumns; col++) |
| { |
| SIMD::Float v = SIMD::Float(0); |
| for (auto i = 0u; i < numAdds; i++) |
| { |
| v += lhs.Float(i * numRows + row) * rhs.Float(col * numAdds + i); |
| } |
| dst.move(numRows * col + row, v); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitOuterProduct(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| auto &lhsType = getType(lhs.type); |
| auto &rhsType = getType(rhs.type); |
| |
| ASSERT(type.definition.opcode() == spv::OpTypeMatrix); |
| ASSERT(lhsType.definition.opcode() == spv::OpTypeVector); |
| ASSERT(rhsType.definition.opcode() == spv::OpTypeVector); |
| ASSERT(getType(lhsType.element).opcode() == spv::OpTypeFloat); |
| ASSERT(getType(rhsType.element).opcode() == spv::OpTypeFloat); |
| |
| auto numRows = lhsType.definition.word(3); |
| auto numCols = rhsType.definition.word(3); |
| |
| for (auto col = 0u; col < numCols; col++) |
| { |
| for (auto row = 0u; row < numRows; row++) |
| { |
| dst.move(col * numRows + row, lhs.Float(row) * rhs.Float(col)); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitTranspose(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto mat = GenericValue(this, state, insn.word(3)); |
| |
| auto numCols = type.definition.word(3); |
| auto numRows = getType(type.definition.word(2)).sizeInComponents; |
| |
| for (auto col = 0u; col < numCols; col++) |
| { |
| for (auto row = 0u; row < numRows; row++) |
| { |
| dst.move(col * numRows + row, mat.Float(row * numCols + col)); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitUnaryOp(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto src = GenericValue(this, state, insn.word(3)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| switch (insn.opcode()) |
| { |
| case spv::OpNot: |
| case spv::OpLogicalNot: // logical not == bitwise not due to all-bits boolean representation |
| dst.move(i, ~src.UInt(i)); |
| break; |
| case spv::OpBitFieldInsert: |
| { |
| auto insert = GenericValue(this, state, insn.word(4)).UInt(i); |
| auto offset = GenericValue(this, state, insn.word(5)).UInt(0); |
| auto count = GenericValue(this, state, insn.word(6)).UInt(0); |
| auto one = SIMD::UInt(1); |
| auto v = src.UInt(i); |
| auto mask = Bitmask32(offset + count) ^ Bitmask32(offset); |
| dst.move(i, (v & ~mask) | ((insert << offset) & mask)); |
| break; |
| } |
| case spv::OpBitFieldSExtract: |
| case spv::OpBitFieldUExtract: |
| { |
| auto offset = GenericValue(this, state, insn.word(4)).UInt(0); |
| auto count = GenericValue(this, state, insn.word(5)).UInt(0); |
| auto one = SIMD::UInt(1); |
| auto v = src.UInt(i); |
| SIMD::UInt out = (v >> offset) & Bitmask32(count); |
| if (insn.opcode() == spv::OpBitFieldSExtract) |
| { |
| auto sign = out & NthBit32(count - one); |
| auto sext = ~(sign - one); |
| out |= sext; |
| } |
| dst.move(i, out); |
| break; |
| } |
| case spv::OpBitReverse: |
| { |
| // TODO: Add an intrinsic to reactor. Even if there isn't a |
| // single vector instruction, there may be target-dependent |
| // ways to make this faster. |
| // https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel |
| SIMD::UInt v = src.UInt(i); |
| v = ((v >> 1) & SIMD::UInt(0x55555555)) | ((v & SIMD::UInt(0x55555555)) << 1); |
| v = ((v >> 2) & SIMD::UInt(0x33333333)) | ((v & SIMD::UInt(0x33333333)) << 2); |
| v = ((v >> 4) & SIMD::UInt(0x0F0F0F0F)) | ((v & SIMD::UInt(0x0F0F0F0F)) << 4); |
| v = ((v >> 8) & SIMD::UInt(0x00FF00FF)) | ((v & SIMD::UInt(0x00FF00FF)) << 8); |
| v = (v >> 16) | (v << 16); |
| dst.move(i, v); |
| break; |
| } |
| case spv::OpBitCount: |
| dst.move(i, CountBits(src.UInt(i))); |
| break; |
| case spv::OpSNegate: |
| dst.move(i, -src.Int(i)); |
| break; |
| case spv::OpFNegate: |
| dst.move(i, -src.Float(i)); |
| break; |
| case spv::OpConvertFToU: |
| dst.move(i, SIMD::UInt(src.Float(i))); |
| break; |
| case spv::OpConvertFToS: |
| dst.move(i, SIMD::Int(src.Float(i))); |
| break; |
| case spv::OpConvertSToF: |
| dst.move(i, SIMD::Float(src.Int(i))); |
| break; |
| case spv::OpConvertUToF: |
| dst.move(i, SIMD::Float(src.UInt(i))); |
| break; |
| case spv::OpBitcast: |
| dst.move(i, src.Float(i)); |
| break; |
| case spv::OpIsInf: |
| dst.move(i, IsInf(src.Float(i))); |
| break; |
| case spv::OpIsNan: |
| dst.move(i, IsNan(src.Float(i))); |
| break; |
| case spv::OpDPdx: |
| case spv::OpDPdxCoarse: |
| // Derivative instructions: FS invocations are laid out like so: |
| // 0 1 |
| // 2 3 |
| static_assert(SIMD::Width == 4, "All cross-lane instructions will need care when using a different width"); |
| dst.move(i, SIMD::Float(Extract(src.Float(i), 1) - Extract(src.Float(i), 0))); |
| break; |
| case spv::OpDPdy: |
| case spv::OpDPdyCoarse: |
| dst.move(i, SIMD::Float(Extract(src.Float(i), 2) - Extract(src.Float(i), 0))); |
| break; |
| case spv::OpFwidth: |
| case spv::OpFwidthCoarse: |
| dst.move(i, SIMD::Float(Abs(Extract(src.Float(i), 1) - Extract(src.Float(i), 0)) |
| + Abs(Extract(src.Float(i), 2) - Extract(src.Float(i), 0)))); |
| break; |
| case spv::OpDPdxFine: |
| { |
| auto firstRow = Extract(src.Float(i), 1) - Extract(src.Float(i), 0); |
| auto secondRow = Extract(src.Float(i), 3) - Extract(src.Float(i), 2); |
| SIMD::Float v = SIMD::Float(firstRow); |
| v = Insert(v, secondRow, 2); |
| v = Insert(v, secondRow, 3); |
| dst.move(i, v); |
| break; |
| } |
| case spv::OpDPdyFine: |
| { |
| auto firstColumn = Extract(src.Float(i), 2) - Extract(src.Float(i), 0); |
| auto secondColumn = Extract(src.Float(i), 3) - Extract(src.Float(i), 1); |
| SIMD::Float v = SIMD::Float(firstColumn); |
| v = Insert(v, secondColumn, 1); |
| v = Insert(v, secondColumn, 3); |
| dst.move(i, v); |
| break; |
| } |
| case spv::OpFwidthFine: |
| { |
| auto firstRow = Extract(src.Float(i), 1) - Extract(src.Float(i), 0); |
| auto secondRow = Extract(src.Float(i), 3) - Extract(src.Float(i), 2); |
| SIMD::Float dpdx = SIMD::Float(firstRow); |
| dpdx = Insert(dpdx, secondRow, 2); |
| dpdx = Insert(dpdx, secondRow, 3); |
| auto firstColumn = Extract(src.Float(i), 2) - Extract(src.Float(i), 0); |
| auto secondColumn = Extract(src.Float(i), 3) - Extract(src.Float(i), 1); |
| SIMD::Float dpdy = SIMD::Float(firstColumn); |
| dpdy = Insert(dpdy, secondColumn, 1); |
| dpdy = Insert(dpdy, secondColumn, 3); |
| dst.move(i, Abs(dpdx) + Abs(dpdy)); |
| break; |
| } |
| case spv::OpQuantizeToF16: |
| { |
| // Note: keep in sync with the specialization constant version in EvalSpecConstantUnaryOp |
| auto abs = Abs(src.Float(i)); |
| auto sign = src.Int(i) & SIMD::Int(0x80000000); |
| auto isZero = CmpLT(abs, SIMD::Float(0.000061035f)); |
| auto isInf = CmpGT(abs, SIMD::Float(65504.0f)); |
| auto isNaN = IsNan(abs); |
| auto isInfOrNan = isInf | isNaN; |
| SIMD::Int v = src.Int(i) & SIMD::Int(0xFFFFE000); |
| v &= ~isZero | SIMD::Int(0x80000000); |
| v = sign | (isInfOrNan & SIMD::Int(0x7F800000)) | (~isInfOrNan & v); |
| v |= isNaN & SIMD::Int(0x400000); |
| dst.move(i, v); |
| break; |
| } |
| default: |
| UNREACHABLE("%s", OpcodeName(insn.opcode()).c_str()); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitBinaryOp(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &lhsType = getType(getObject(insn.word(3)).type); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| |
| for (auto i = 0u; i < lhsType.sizeInComponents; i++) |
| { |
| switch (insn.opcode()) |
| { |
| case spv::OpIAdd: |
| dst.move(i, lhs.Int(i) + rhs.Int(i)); |
| break; |
| case spv::OpISub: |
| dst.move(i, lhs.Int(i) - rhs.Int(i)); |
| break; |
| case spv::OpIMul: |
| dst.move(i, lhs.Int(i) * rhs.Int(i)); |
| break; |
| case spv::OpSDiv: |
| { |
| SIMD::Int a = lhs.Int(i); |
| SIMD::Int b = rhs.Int(i); |
| b = b | CmpEQ(b, SIMD::Int(0)); // prevent divide-by-zero |
| a = a | (CmpEQ(a, SIMD::Int(0x80000000)) & CmpEQ(b, SIMD::Int(-1))); // prevent integer overflow |
| dst.move(i, a / b); |
| break; |
| } |
| case spv::OpUDiv: |
| { |
| auto zeroMask = As<SIMD::UInt>(CmpEQ(rhs.Int(i), SIMD::Int(0))); |
| dst.move(i, lhs.UInt(i) / (rhs.UInt(i) | zeroMask)); |
| break; |
| } |
| case spv::OpSRem: |
| { |
| SIMD::Int a = lhs.Int(i); |
| SIMD::Int b = rhs.Int(i); |
| b = b | CmpEQ(b, SIMD::Int(0)); // prevent divide-by-zero |
| a = a | (CmpEQ(a, SIMD::Int(0x80000000)) & CmpEQ(b, SIMD::Int(-1))); // prevent integer overflow |
| dst.move(i, a % b); |
| break; |
| } |
| case spv::OpSMod: |
| { |
| SIMD::Int a = lhs.Int(i); |
| SIMD::Int b = rhs.Int(i); |
| b = b | CmpEQ(b, SIMD::Int(0)); // prevent divide-by-zero |
| a = a | (CmpEQ(a, SIMD::Int(0x80000000)) & CmpEQ(b, SIMD::Int(-1))); // prevent integer overflow |
| auto mod = a % b; |
| // If a and b have opposite signs, the remainder operation takes |
| // the sign from a but OpSMod is supposed to take the sign of b. |
| // Adding b will ensure that the result has the correct sign and |
| // that it is still congruent to a modulo b. |
| // |
| // See also http://mathforum.org/library/drmath/view/52343.html |
| auto signDiff = CmpNEQ(CmpGE(a, SIMD::Int(0)), CmpGE(b, SIMD::Int(0))); |
| auto fixedMod = mod + (b & CmpNEQ(mod, SIMD::Int(0)) & signDiff); |
| dst.move(i, As<SIMD::Float>(fixedMod)); |
| break; |
| } |
| case spv::OpUMod: |
| { |
| auto zeroMask = As<SIMD::UInt>(CmpEQ(rhs.Int(i), SIMD::Int(0))); |
| dst.move(i, lhs.UInt(i) % (rhs.UInt(i) | zeroMask)); |
| break; |
| } |
| case spv::OpIEqual: |
| case spv::OpLogicalEqual: |
| dst.move(i, CmpEQ(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpINotEqual: |
| case spv::OpLogicalNotEqual: |
| dst.move(i, CmpNEQ(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpUGreaterThan: |
| dst.move(i, CmpGT(lhs.UInt(i), rhs.UInt(i))); |
| break; |
| case spv::OpSGreaterThan: |
| dst.move(i, CmpGT(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpUGreaterThanEqual: |
| dst.move(i, CmpGE(lhs.UInt(i), rhs.UInt(i))); |
| break; |
| case spv::OpSGreaterThanEqual: |
| dst.move(i, CmpGE(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpULessThan: |
| dst.move(i, CmpLT(lhs.UInt(i), rhs.UInt(i))); |
| break; |
| case spv::OpSLessThan: |
| dst.move(i, CmpLT(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpULessThanEqual: |
| dst.move(i, CmpLE(lhs.UInt(i), rhs.UInt(i))); |
| break; |
| case spv::OpSLessThanEqual: |
| dst.move(i, CmpLE(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpFAdd: |
| dst.move(i, lhs.Float(i) + rhs.Float(i)); |
| break; |
| case spv::OpFSub: |
| dst.move(i, lhs.Float(i) - rhs.Float(i)); |
| break; |
| case spv::OpFMul: |
| dst.move(i, lhs.Float(i) * rhs.Float(i)); |
| break; |
| case spv::OpFDiv: |
| dst.move(i, lhs.Float(i) / rhs.Float(i)); |
| break; |
| case spv::OpFMod: |
| // TODO(b/126873455): inaccurate for values greater than 2^24 |
| dst.move(i, lhs.Float(i) - rhs.Float(i) * Floor(lhs.Float(i) / rhs.Float(i))); |
| break; |
| case spv::OpFRem: |
| dst.move(i, lhs.Float(i) % rhs.Float(i)); |
| break; |
| case spv::OpFOrdEqual: |
| dst.move(i, CmpEQ(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordEqual: |
| dst.move(i, CmpUEQ(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFOrdNotEqual: |
| dst.move(i, CmpNEQ(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordNotEqual: |
| dst.move(i, CmpUNEQ(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFOrdLessThan: |
| dst.move(i, CmpLT(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordLessThan: |
| dst.move(i, CmpULT(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFOrdGreaterThan: |
| dst.move(i, CmpGT(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordGreaterThan: |
| dst.move(i, CmpUGT(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFOrdLessThanEqual: |
| dst.move(i, CmpLE(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordLessThanEqual: |
| dst.move(i, CmpULE(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFOrdGreaterThanEqual: |
| dst.move(i, CmpGE(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpFUnordGreaterThanEqual: |
| dst.move(i, CmpUGE(lhs.Float(i), rhs.Float(i))); |
| break; |
| case spv::OpShiftRightLogical: |
| dst.move(i, lhs.UInt(i) >> rhs.UInt(i)); |
| break; |
| case spv::OpShiftRightArithmetic: |
| dst.move(i, lhs.Int(i) >> rhs.Int(i)); |
| break; |
| case spv::OpShiftLeftLogical: |
| dst.move(i, lhs.UInt(i) << rhs.UInt(i)); |
| break; |
| case spv::OpBitwiseOr: |
| case spv::OpLogicalOr: |
| dst.move(i, lhs.UInt(i) | rhs.UInt(i)); |
| break; |
| case spv::OpBitwiseXor: |
| dst.move(i, lhs.UInt(i) ^ rhs.UInt(i)); |
| break; |
| case spv::OpBitwiseAnd: |
| case spv::OpLogicalAnd: |
| dst.move(i, lhs.UInt(i) & rhs.UInt(i)); |
| break; |
| case spv::OpSMulExtended: |
| // Extended ops: result is a structure containing two members of the same type as lhs & rhs. |
| // In our flat view then, component i is the i'th component of the first member; |
| // component i + N is the i'th component of the second member. |
| dst.move(i, lhs.Int(i) * rhs.Int(i)); |
| dst.move(i + lhsType.sizeInComponents, MulHigh(lhs.Int(i), rhs.Int(i))); |
| break; |
| case spv::OpUMulExtended: |
| dst.move(i, lhs.UInt(i) * rhs.UInt(i)); |
| dst.move(i + lhsType.sizeInComponents, MulHigh(lhs.UInt(i), rhs.UInt(i))); |
| break; |
| case spv::OpIAddCarry: |
| dst.move(i, lhs.UInt(i) + rhs.UInt(i)); |
| dst.move(i + lhsType.sizeInComponents, CmpLT(dst.UInt(i), lhs.UInt(i)) >> 31); |
| break; |
| case spv::OpISubBorrow: |
| dst.move(i, lhs.UInt(i) - rhs.UInt(i)); |
| dst.move(i + lhsType.sizeInComponents, CmpLT(lhs.UInt(i), rhs.UInt(i)) >> 31); |
| break; |
| default: |
| UNREACHABLE("%s", OpcodeName(insn.opcode()).c_str()); |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitDot(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| ASSERT(type.sizeInComponents == 1); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &lhsType = getType(getObject(insn.word(3)).type); |
| auto lhs = GenericValue(this, state, insn.word(3)); |
| auto rhs = GenericValue(this, state, insn.word(4)); |
| |
| dst.move(0, Dot(lhsType.sizeInComponents, lhs, rhs)); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitSelect(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto cond = GenericValue(this, state, insn.word(3)); |
| auto condIsScalar = (getType(cond.type).sizeInComponents == 1); |
| auto lhs = GenericValue(this, state, insn.word(4)); |
| auto rhs = GenericValue(this, state, insn.word(5)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto sel = cond.Int(condIsScalar ? 0 : i); |
| dst.move(i, (sel & lhs.Int(i)) | (~sel & rhs.Int(i))); // TODO: IfThenElse() |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitExtendedInstruction(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto extInstIndex = static_cast<GLSLstd450>(insn.word(4)); |
| |
| switch (extInstIndex) |
| { |
| case GLSLstd450FAbs: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Abs(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450SAbs: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Abs(src.Int(i))); |
| } |
| break; |
| } |
| case GLSLstd450Cross: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| dst.move(0, lhs.Float(1) * rhs.Float(2) - rhs.Float(1) * lhs.Float(2)); |
| dst.move(1, lhs.Float(2) * rhs.Float(0) - rhs.Float(2) * lhs.Float(0)); |
| dst.move(2, lhs.Float(0) * rhs.Float(1) - rhs.Float(0) * lhs.Float(1)); |
| break; |
| } |
| case GLSLstd450Floor: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Floor(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Trunc: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Trunc(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Ceil: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Ceil(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Fract: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Frac(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Round: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Round(src.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450RoundEven: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto x = Round(src.Float(i)); |
| // dst = round(src) + ((round(src) < src) * 2 - 1) * (fract(src) == 0.5) * isOdd(round(src)); |
| dst.move(i, x + ((SIMD::Float(CmpLT(x, src.Float(i)) & SIMD::Int(1)) * SIMD::Float(2.0f)) - SIMD::Float(1.0f)) * |
| SIMD::Float(CmpEQ(Frac(src.Float(i)), SIMD::Float(0.5f)) & SIMD::Int(1)) * SIMD::Float(Int4(x) & SIMD::Int(1))); |
| } |
| break; |
| } |
| case GLSLstd450FMin: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(lhs.Float(i), rhs.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450FMax: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Max(lhs.Float(i), rhs.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450SMin: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(lhs.Int(i), rhs.Int(i))); |
| } |
| break; |
| } |
| case GLSLstd450SMax: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Max(lhs.Int(i), rhs.Int(i))); |
| } |
| break; |
| } |
| case GLSLstd450UMin: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(lhs.UInt(i), rhs.UInt(i))); |
| } |
| break; |
| } |
| case GLSLstd450UMax: |
| { |
| auto lhs = GenericValue(this, state, insn.word(5)); |
| auto rhs = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Max(lhs.UInt(i), rhs.UInt(i))); |
| } |
| break; |
| } |
| case GLSLstd450Step: |
| { |
| auto edge = GenericValue(this, state, insn.word(5)); |
| auto x = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, CmpNLT(x.Float(i), edge.Float(i)) & As<SIMD::Int>(SIMD::Float(1.0f))); |
| } |
| break; |
| } |
| case GLSLstd450SmoothStep: |
| { |
| auto edge0 = GenericValue(this, state, insn.word(5)); |
| auto edge1 = GenericValue(this, state, insn.word(6)); |
| auto x = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto tx = Min(Max((x.Float(i) - edge0.Float(i)) / |
| (edge1.Float(i) - edge0.Float(i)), SIMD::Float(0.0f)), SIMD::Float(1.0f)); |
| dst.move(i, tx * tx * (Float4(3.0f) - Float4(2.0f) * tx)); |
| } |
| break; |
| } |
| case GLSLstd450FMix: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto y = GenericValue(this, state, insn.word(6)); |
| auto a = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, a.Float(i) * (y.Float(i) - x.Float(i)) + x.Float(i)); |
| } |
| break; |
| } |
| case GLSLstd450FClamp: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto minVal = GenericValue(this, state, insn.word(6)); |
| auto maxVal = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(Max(x.Float(i), minVal.Float(i)), maxVal.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450SClamp: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto minVal = GenericValue(this, state, insn.word(6)); |
| auto maxVal = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(Max(x.Int(i), minVal.Int(i)), maxVal.Int(i))); |
| } |
| break; |
| } |
| case GLSLstd450UClamp: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto minVal = GenericValue(this, state, insn.word(6)); |
| auto maxVal = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Min(Max(x.UInt(i), minVal.UInt(i)), maxVal.UInt(i))); |
| } |
| break; |
| } |
| case GLSLstd450FSign: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto neg = As<SIMD::Int>(CmpLT(src.Float(i), SIMD::Float(-0.0f))) & As<SIMD::Int>(SIMD::Float(-1.0f)); |
| auto pos = As<SIMD::Int>(CmpNLE(src.Float(i), SIMD::Float(+0.0f))) & As<SIMD::Int>(SIMD::Float(1.0f)); |
| dst.move(i, neg | pos); |
| } |
| break; |
| } |
| case GLSLstd450SSign: |
| { |
| auto src = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto neg = CmpLT(src.Int(i), SIMD::Int(0)) & SIMD::Int(-1); |
| auto pos = CmpNLE(src.Int(i), SIMD::Int(0)) & SIMD::Int(1); |
| dst.move(i, neg | pos); |
| } |
| break; |
| } |
| case GLSLstd450Reflect: |
| { |
| auto I = GenericValue(this, state, insn.word(5)); |
| auto N = GenericValue(this, state, insn.word(6)); |
| |
| SIMD::Float d = Dot(type.sizeInComponents, I, N); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, I.Float(i) - SIMD::Float(2.0f) * d * N.Float(i)); |
| } |
| break; |
| } |
| case GLSLstd450Refract: |
| { |
| auto I = GenericValue(this, state, insn.word(5)); |
| auto N = GenericValue(this, state, insn.word(6)); |
| auto eta = GenericValue(this, state, insn.word(7)); |
| |
| SIMD::Float d = Dot(type.sizeInComponents, I, N); |
| SIMD::Float k = SIMD::Float(1.0f) - eta.Float(0) * eta.Float(0) * (SIMD::Float(1.0f) - d * d); |
| SIMD::Int pos = CmpNLT(k, SIMD::Float(0.0f)); |
| SIMD::Float t = (eta.Float(0) * d + Sqrt(k)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, pos & As<SIMD::Int>(eta.Float(0) * I.Float(i) - t * N.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450FaceForward: |
| { |
| auto N = GenericValue(this, state, insn.word(5)); |
| auto I = GenericValue(this, state, insn.word(6)); |
| auto Nref = GenericValue(this, state, insn.word(7)); |
| |
| SIMD::Float d = Dot(type.sizeInComponents, I, Nref); |
| SIMD::Int neg = CmpLT(d, SIMD::Float(0.0f)); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto n = N.Float(i); |
| dst.move(i, (neg & As<SIMD::Int>(n)) | (~neg & As<SIMD::Int>(-n))); |
| } |
| break; |
| } |
| case GLSLstd450Length: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| SIMD::Float d = Dot(getType(getObject(insn.word(5)).type).sizeInComponents, x, x); |
| |
| dst.move(0, Sqrt(d)); |
| break; |
| } |
| case GLSLstd450Normalize: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| SIMD::Float d = Dot(getType(getObject(insn.word(5)).type).sizeInComponents, x, x); |
| SIMD::Float invLength = SIMD::Float(1.0f) / Sqrt(d); |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, invLength * x.Float(i)); |
| } |
| break; |
| } |
| case GLSLstd450Distance: |
| { |
| auto p0 = GenericValue(this, state, insn.word(5)); |
| auto p1 = GenericValue(this, state, insn.word(6)); |
| auto p0Type = getType(p0.type); |
| |
| // sqrt(dot(p0-p1, p0-p1)) |
| SIMD::Float d = (p0.Float(0) - p1.Float(0)) * (p0.Float(0) - p1.Float(0)); |
| |
| for (auto i = 1u; i < p0Type.sizeInComponents; i++) |
| { |
| d += (p0.Float(i) - p1.Float(i)) * (p0.Float(i) - p1.Float(i)); |
| } |
| |
| dst.move(0, Sqrt(d)); |
| break; |
| } |
| case GLSLstd450Modf: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| auto ptrId = Object::ID(insn.word(6)); |
| auto ptrTy = getType(getObject(ptrId).type); |
| auto ptr = GetPointerToData(ptrId, 0, state); |
| bool interleavedByLane = IsStorageInterleavedByLane(ptrTy.storageClass); |
| // TODO: GLSL modf() takes an output parameter and thus the pointer is assumed |
| // to be in bounds even for inactive lanes. |
| // - Clarify the SPIR-V spec. |
| // - Eliminate lane masking and assume interleaving. |
| auto robustness = OutOfBoundsBehavior::UndefinedBehavior; |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Float whole, frac; |
| std::tie(whole, frac) = Modf(val.Float(i)); |
| dst.move(i, frac); |
| auto p = ptr + (i * sizeof(float)); |
| if (interleavedByLane) { p = interleaveByLane(p); } |
| SIMD::Store(p, whole, robustness, state->activeLaneMask()); |
| } |
| break; |
| } |
| case GLSLstd450ModfStruct: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| auto valTy = getType(val.type); |
| |
| for (auto i = 0u; i < valTy.sizeInComponents; i++) |
| { |
| SIMD::Float whole, frac; |
| std::tie(whole, frac) = Modf(val.Float(i)); |
| dst.move(i, frac); |
| dst.move(i + valTy.sizeInComponents, whole); |
| } |
| break; |
| } |
| case GLSLstd450PackSnorm4x8: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, (SIMD::Int(Round(Min(Max(val.Float(0), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) | |
| ((SIMD::Int(Round(Min(Max(val.Float(1), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 8) | |
| ((SIMD::Int(Round(Min(Max(val.Float(2), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 16) | |
| ((SIMD::Int(Round(Min(Max(val.Float(3), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 24)); |
| break; |
| } |
| case GLSLstd450PackUnorm4x8: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, (SIMD::UInt(Round(Min(Max(val.Float(0), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) | |
| ((SIMD::UInt(Round(Min(Max(val.Float(1), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 8) | |
| ((SIMD::UInt(Round(Min(Max(val.Float(2), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 16) | |
| ((SIMD::UInt(Round(Min(Max(val.Float(3), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 24)); |
| break; |
| } |
| case GLSLstd450PackSnorm2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, (SIMD::Int(Round(Min(Max(val.Float(0), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(32767.0f))) & |
| SIMD::Int(0xFFFF)) | |
| ((SIMD::Int(Round(Min(Max(val.Float(1), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(32767.0f))) & |
| SIMD::Int(0xFFFF)) << 16)); |
| break; |
| } |
| case GLSLstd450PackUnorm2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, (SIMD::UInt(Round(Min(Max(val.Float(0), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(65535.0f))) & |
| SIMD::UInt(0xFFFF)) | |
| ((SIMD::UInt(Round(Min(Max(val.Float(1), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(65535.0f))) & |
| SIMD::UInt(0xFFFF)) << 16)); |
| break; |
| } |
| case GLSLstd450PackHalf2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, FloatToHalfBits(val.UInt(0), false) | FloatToHalfBits(val.UInt(1), true)); |
| break; |
| } |
| case GLSLstd450UnpackSnorm4x8: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, Min(Max(SIMD::Float(((val.Int(0)<<24) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(1, Min(Max(SIMD::Float(((val.Int(0)<<16) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(2, Min(Max(SIMD::Float(((val.Int(0)<<8) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(3, Min(Max(SIMD::Float(((val.Int(0)) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| break; |
| } |
| case GLSLstd450UnpackUnorm4x8: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, SIMD::Float((val.UInt(0) & SIMD::UInt(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(1, SIMD::Float(((val.UInt(0)>>8) & SIMD::UInt(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(2, SIMD::Float(((val.UInt(0)>>16) & SIMD::UInt(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(3, SIMD::Float(((val.UInt(0)>>24) & SIMD::UInt(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| break; |
| } |
| case GLSLstd450UnpackSnorm2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| // clamp(f / 32767.0, -1.0, 1.0) |
| dst.move(0, Min(Max(SIMD::Float(As<SIMD::Int>((val.UInt(0) & SIMD::UInt(0x0000FFFF)) << 16)) * |
| SIMD::Float(1.0f / float(0x7FFF0000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(1, Min(Max(SIMD::Float(As<SIMD::Int>(val.UInt(0) & SIMD::UInt(0xFFFF0000))) * SIMD::Float(1.0f / float(0x7FFF0000)), |
| SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| break; |
| } |
| case GLSLstd450UnpackUnorm2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| // f / 65535.0 |
| dst.move(0, SIMD::Float((val.UInt(0) & SIMD::UInt(0x0000FFFF)) << 16) * SIMD::Float(1.0f / float(0xFFFF0000))); |
| dst.move(1, SIMD::Float(val.UInt(0) & SIMD::UInt(0xFFFF0000)) * SIMD::Float(1.0f / float(0xFFFF0000))); |
| break; |
| } |
| case GLSLstd450UnpackHalf2x16: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| dst.move(0, halfToFloatBits(val.UInt(0) & SIMD::UInt(0x0000FFFF))); |
| dst.move(1, halfToFloatBits((val.UInt(0) & SIMD::UInt(0xFFFF0000)) >> 16)); |
| break; |
| } |
| case GLSLstd450Fma: |
| { |
| auto a = GenericValue(this, state, insn.word(5)); |
| auto b = GenericValue(this, state, insn.word(6)); |
| auto c = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, FMA(a.Float(i), b.Float(i), c.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Frexp: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| auto ptrId = Object::ID(insn.word(6)); |
| auto ptrTy = getType(getObject(ptrId).type); |
| auto ptr = GetPointerToData(ptrId, 0, state); |
| bool interleavedByLane = IsStorageInterleavedByLane(ptrTy.storageClass); |
| // TODO: GLSL frexp() takes an output parameter and thus the pointer is assumed |
| // to be in bounds even for inactive lanes. |
| // - Clarify the SPIR-V spec. |
| // - Eliminate lane masking and assume interleaving. |
| auto robustness = OutOfBoundsBehavior::UndefinedBehavior; |
| |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Float significand; |
| SIMD::Int exponent; |
| std::tie(significand, exponent) = Frexp(val.Float(i)); |
| |
| dst.move(i, significand); |
| |
| auto p = ptr + (i * sizeof(float)); |
| if (interleavedByLane) { p = interleaveByLane(p); } |
| SIMD::Store(p, exponent, robustness, state->activeLaneMask()); |
| } |
| break; |
| } |
| case GLSLstd450FrexpStruct: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| auto numComponents = getType(val.type).sizeInComponents; |
| for (auto i = 0u; i < numComponents; i++) |
| { |
| auto significandAndExponent = Frexp(val.Float(i)); |
| dst.move(i, significandAndExponent.first); |
| dst.move(i + numComponents, significandAndExponent.second); |
| } |
| break; |
| } |
| case GLSLstd450Ldexp: |
| { |
| auto significand = GenericValue(this, state, insn.word(5)); |
| auto exponent = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| // Assumes IEEE 754 |
| auto in = significand.Float(i); |
| auto significandExponent = Exponent(in); |
| auto combinedExponent = exponent.Int(i) + significandExponent; |
| auto isSignificandZero = SIMD::UInt(CmpEQ(significand.Int(i), SIMD::Int(0))); |
| auto isSignificandInf = SIMD::UInt(IsInf(in)); |
| auto isSignificandNaN = SIMD::UInt(IsNan(in)); |
| auto isExponentNotTooSmall = SIMD::UInt(CmpGE(combinedExponent, SIMD::Int(-126))); |
| auto isExponentNotTooLarge = SIMD::UInt(CmpLE(combinedExponent, SIMD::Int(128))); |
| auto isExponentInBounds = isExponentNotTooSmall & isExponentNotTooLarge; |
| |
| SIMD::UInt v; |
| v = significand.UInt(i) & SIMD::UInt(0x7FFFFF); // Add significand. |
| v |= (SIMD::UInt(combinedExponent + SIMD::Int(126)) << SIMD::UInt(23)); // Add exponent. |
| v &= isExponentInBounds; // Clear v if the exponent is OOB. |
| |
| v |= significand.UInt(i) & SIMD::UInt(0x80000000); // Add sign bit. |
| v |= ~isExponentNotTooLarge & SIMD::UInt(0x7F800000); // Mark as inf if the exponent is too great. |
| |
| // If the input significand is zero, inf or nan, just return the |
| // input significand. |
| auto passthrough = isSignificandZero | isSignificandInf | isSignificandNaN; |
| v = (v & ~passthrough) | (significand.UInt(i) & passthrough); |
| |
| dst.move(i, As<SIMD::Float>(v)); |
| } |
| break; |
| } |
| case GLSLstd450Radians: |
| { |
| auto degrees = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, degrees.Float(i) * SIMD::Float(PI / 180.0f)); |
| } |
| break; |
| } |
| case GLSLstd450Degrees: |
| { |
| auto radians = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, radians.Float(i) * SIMD::Float(180.0f / PI)); |
| } |
| break; |
| } |
| case GLSLstd450Sin: |
| { |
| auto radians = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Sin(radians.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Cos: |
| { |
| auto radians = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Cos(radians.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Tan: |
| { |
| auto radians = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Tan(radians.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Asin: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Asin(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Acos: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Acos(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Atan: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Atan(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Sinh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Sinh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Cosh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Cosh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Tanh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Tanh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Asinh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Asinh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Acosh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Acosh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Atanh: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Atanh(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Atan2: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto y = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Atan2(x.Float(i), y.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Pow: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto y = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Pow(x.Float(i), y.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Exp: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Exp(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Log: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Log(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Exp2: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Exp2(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Log2: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Log2(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450Sqrt: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, Sqrt(val.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450InverseSqrt: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| Decorations d; |
| ApplyDecorationsForId(&d, insn.word(5)); |
| if (d.RelaxedPrecision) |
| { |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, RcpSqrt_pp(val.Float(i))); |
| } |
| } |
| else |
| { |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, SIMD::Float(1.0f) / Sqrt(val.Float(i))); |
| } |
| } |
| break; |
| } |
| case GLSLstd450Determinant: |
| { |
| auto mat = GenericValue(this, state, insn.word(5)); |
| auto numComponents = getType(mat.type).sizeInComponents; |
| switch (numComponents) |
| { |
| case 4: // 2x2 |
| dst.move(0, Determinant( |
| mat.Float(0), mat.Float(1), |
| mat.Float(2), mat.Float(3))); |
| break; |
| case 9: // 3x3 |
| dst.move(0, Determinant( |
| mat.Float(0), mat.Float(1), mat.Float(2), |
| mat.Float(3), mat.Float(4), mat.Float(5), |
| mat.Float(6), mat.Float(7), mat.Float(8))); |
| break; |
| case 16: // 4x4 |
| dst.move(0, Determinant( |
| mat.Float(0), mat.Float(1), mat.Float(2), mat.Float(3), |
| mat.Float(4), mat.Float(5), mat.Float(6), mat.Float(7), |
| mat.Float(8), mat.Float(9), mat.Float(10), mat.Float(11), |
| mat.Float(12), mat.Float(13), mat.Float(14), mat.Float(15))); |
| break; |
| default: |
| UNREACHABLE("GLSLstd450Determinant can only operate with square matrices. Got %d elements", int(numComponents)); |
| } |
| break; |
| } |
| case GLSLstd450MatrixInverse: |
| { |
| auto mat = GenericValue(this, state, insn.word(5)); |
| auto numComponents = getType(mat.type).sizeInComponents; |
| switch (numComponents) |
| { |
| case 4: // 2x2 |
| { |
| auto inv = MatrixInverse( |
| mat.Float(0), mat.Float(1), |
| mat.Float(2), mat.Float(3)); |
| for (uint32_t i = 0; i < inv.size(); i++) |
| { |
| dst.move(i, inv[i]); |
| } |
| break; |
| } |
| case 9: // 3x3 |
| { |
| auto inv = MatrixInverse( |
| mat.Float(0), mat.Float(1), mat.Float(2), |
| mat.Float(3), mat.Float(4), mat.Float(5), |
| mat.Float(6), mat.Float(7), mat.Float(8)); |
| for (uint32_t i = 0; i < inv.size(); i++) |
| { |
| dst.move(i, inv[i]); |
| } |
| break; |
| } |
| case 16: // 4x4 |
| { |
| auto inv = MatrixInverse( |
| mat.Float(0), mat.Float(1), mat.Float(2), mat.Float(3), |
| mat.Float(4), mat.Float(5), mat.Float(6), mat.Float(7), |
| mat.Float(8), mat.Float(9), mat.Float(10), mat.Float(11), |
| mat.Float(12), mat.Float(13), mat.Float(14), mat.Float(15)); |
| for (uint32_t i = 0; i < inv.size(); i++) |
| { |
| dst.move(i, inv[i]); |
| } |
| break; |
| } |
| default: |
| UNREACHABLE("GLSLstd450MatrixInverse can only operate with square matrices. Got %d elements", int(numComponents)); |
| } |
| break; |
| } |
| case GLSLstd450IMix: |
| { |
| UNREACHABLE("GLSLstd450IMix has been removed from the specification"); |
| break; |
| } |
| case GLSLstd450PackDouble2x32: |
| { |
| UNSUPPORTED("SPIR-V Float64 Capability (GLSLstd450PackDouble2x32)"); |
| break; |
| } |
| case GLSLstd450UnpackDouble2x32: |
| { |
| UNSUPPORTED("SPIR-V Float64 Capability (GLSLstd450UnpackDouble2x32)"); |
| break; |
| } |
| case GLSLstd450FindILsb: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto v = val.UInt(i); |
| dst.move(i, Cttz(v, true) | CmpEQ(v, SIMD::UInt(0))); |
| } |
| break; |
| } |
| case GLSLstd450FindSMsb: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto v = val.UInt(i) ^ As<SIMD::UInt>(CmpLT(val.Int(i), SIMD::Int(0))); |
| dst.move(i, SIMD::UInt(31) - Ctlz(v, false)); |
| } |
| break; |
| } |
| case GLSLstd450FindUMsb: |
| { |
| auto val = GenericValue(this, state, insn.word(5)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, SIMD::UInt(31) - Ctlz(val.UInt(i), false)); |
| } |
| break; |
| } |
| case GLSLstd450InterpolateAtCentroid: |
| { |
| UNSUPPORTED("SPIR-V SampleRateShading Capability (GLSLstd450InterpolateAtCentroid)"); |
| break; |
| } |
| case GLSLstd450InterpolateAtSample: |
| { |
| UNSUPPORTED("SPIR-V SampleRateShading Capability (GLSLstd450InterpolateAtCentroid)"); |
| break; |
| } |
| case GLSLstd450InterpolateAtOffset: |
| { |
| UNSUPPORTED("SPIR-V SampleRateShading Capability (GLSLstd450InterpolateAtCentroid)"); |
| break; |
| } |
| case GLSLstd450NMin: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto y = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, NMin(x.Float(i), y.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450NMax: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto y = GenericValue(this, state, insn.word(6)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, NMax(x.Float(i), y.Float(i))); |
| } |
| break; |
| } |
| case GLSLstd450NClamp: |
| { |
| auto x = GenericValue(this, state, insn.word(5)); |
| auto minVal = GenericValue(this, state, insn.word(6)); |
| auto maxVal = GenericValue(this, state, insn.word(7)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| auto clamp = NMin(NMax(x.Float(i), minVal.Float(i)), maxVal.Float(i)); |
| dst.move(i, clamp); |
| } |
| break; |
| } |
| default: |
| UNREACHABLE("ExtInst %d", int(extInstIndex)); |
| break; |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| std::memory_order SpirvShader::MemoryOrder(spv::MemorySemanticsMask memorySemantics) |
| { |
| auto control = static_cast<uint32_t>(memorySemantics) & static_cast<uint32_t>( |
| spv::MemorySemanticsAcquireMask | |
| spv::MemorySemanticsReleaseMask | |
| spv::MemorySemanticsAcquireReleaseMask | |
| spv::MemorySemanticsSequentiallyConsistentMask |
| ); |
| switch (control) |
| { |
| case spv::MemorySemanticsMaskNone: return std::memory_order_relaxed; |
| case spv::MemorySemanticsAcquireMask: return std::memory_order_acquire; |
| case spv::MemorySemanticsReleaseMask: return std::memory_order_release; |
| case spv::MemorySemanticsAcquireReleaseMask: return std::memory_order_acq_rel; |
| case spv::MemorySemanticsSequentiallyConsistentMask: return std::memory_order_acq_rel; // Vulkan 1.1: "SequentiallyConsistent is treated as AcquireRelease" |
| default: |
| // "it is invalid for more than one of these four bits to be set: |
| // Acquire, Release, AcquireRelease, or SequentiallyConsistent." |
| UNREACHABLE("MemorySemanticsMask: %x", int(control)); |
| return std::memory_order_acq_rel; |
| } |
| } |
| |
| SIMD::Float SpirvShader::Dot(unsigned numComponents, GenericValue const & x, GenericValue const & y) const |
| { |
| SIMD::Float d = x.Float(0) * y.Float(0); |
| |
| for (auto i = 1u; i < numComponents; i++) |
| { |
| d += x.Float(i) * y.Float(i); |
| } |
| |
| return d; |
| } |
| |
| SIMD::UInt SpirvShader::FloatToHalfBits(SIMD::UInt floatBits, bool storeInUpperBits) const |
| { |
| static const uint32_t mask_sign = 0x80000000u; |
| static const uint32_t mask_round = ~0xfffu; |
| static const uint32_t c_f32infty = 255 << 23; |
| static const uint32_t c_magic = 15 << 23; |
| static const uint32_t c_nanbit = 0x200; |
| static const uint32_t c_infty_as_fp16 = 0x7c00; |
| static const uint32_t c_clamp = (31 << 23) - 0x1000; |
| |
| SIMD::UInt justsign = SIMD::UInt(mask_sign) & floatBits; |
| SIMD::UInt absf = floatBits ^ justsign; |
| SIMD::UInt b_isnormal = CmpNLE(SIMD::UInt(c_f32infty), absf); |
| |
| // Note: this version doesn't round to the nearest even in case of a tie as defined by IEEE 754-2008, it rounds to +inf |
| // instead of nearest even, since that's fine for GLSL ES 3.0's needs (see section 2.1.1 Floating-Point Computation) |
| SIMD::UInt joined = ((((As<SIMD::UInt>(Min(As<SIMD::Float>(absf & SIMD::UInt(mask_round)) * As<SIMD::Float>(SIMD::UInt(c_magic)), |
| As<SIMD::Float>(SIMD::UInt(c_clamp))))) - SIMD::UInt(mask_round)) >> 13) & b_isnormal) | |
| ((b_isnormal ^ SIMD::UInt(0xFFFFFFFF)) & ((CmpNLE(absf, SIMD::UInt(c_f32infty)) & SIMD::UInt(c_nanbit)) | |
| SIMD::UInt(c_infty_as_fp16))); |
| |
| return storeInUpperBits ? ((joined << 16) | justsign) : joined | (justsign >> 16); |
| } |
| |
| std::pair<SIMD::Float, SIMD::Int> SpirvShader::Frexp(RValue<SIMD::Float> val) const |
| { |
| // Assumes IEEE 754 |
| auto v = As<SIMD::UInt>(val); |
| auto isNotZero = CmpNEQ(v & SIMD::UInt(0x7FFFFFFF), SIMD::UInt(0)); |
| auto zeroSign = v & SIMD::UInt(0x80000000) & ~isNotZero; |
| auto significand = As<SIMD::Float>((((v & SIMD::UInt(0x807FFFFF)) | SIMD::UInt(0x3F000000)) & isNotZero) | zeroSign); |
| auto exponent = Exponent(val) & SIMD::Int(isNotZero); |
| return std::make_pair(significand, exponent); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitAny(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| ASSERT(type.sizeInComponents == 1); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &srcType = getType(getObject(insn.word(3)).type); |
| auto src = GenericValue(this, state, insn.word(3)); |
| |
| SIMD::UInt result = src.UInt(0); |
| |
| for (auto i = 1u; i < srcType.sizeInComponents; i++) |
| { |
| result |= src.UInt(i); |
| } |
| |
| dst.move(0, result); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitAll(InsnIterator insn, EmitState *state) const |
| { |
| auto &type = getType(insn.word(1)); |
| ASSERT(type.sizeInComponents == 1); |
| auto &dst = state->createIntermediate(insn.word(2), type.sizeInComponents); |
| auto &srcType = getType(getObject(insn.word(3)).type); |
| auto src = GenericValue(this, state, insn.word(3)); |
| |
| SIMD::UInt result = src.UInt(0); |
| |
| for (auto i = 1u; i < srcType.sizeInComponents; i++) |
| { |
| result &= src.UInt(i); |
| } |
| |
| dst.move(0, result); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitBranch(InsnIterator insn, EmitState *state) const |
| { |
| auto target = Block::ID(insn.word(1)); |
| state->addActiveLaneMaskEdge(state->block, target, state->activeLaneMask()); |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitBranchConditional(InsnIterator insn, EmitState *state) const |
| { |
| auto &function = getFunction(state->function); |
| auto block = function.getBlock(state->block); |
| ASSERT(block.branchInstruction == insn); |
| |
| auto condId = Object::ID(block.branchInstruction.word(1)); |
| auto trueBlockId = Block::ID(block.branchInstruction.word(2)); |
| auto falseBlockId = Block::ID(block.branchInstruction.word(3)); |
| |
| auto cond = GenericValue(this, state, condId); |
| ASSERT_MSG(getType(cond.type).sizeInComponents == 1, "Condition must be a Boolean type scalar"); |
| |
| // TODO: Optimize for case where all lanes take same path. |
| |
| state->addOutputActiveLaneMaskEdge(trueBlockId, cond.Int(0)); |
| state->addOutputActiveLaneMaskEdge(falseBlockId, ~cond.Int(0)); |
| |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitSwitch(InsnIterator insn, EmitState *state) const |
| { |
| auto &function = getFunction(state->function); |
| auto block = function.getBlock(state->block); |
| ASSERT(block.branchInstruction == insn); |
| |
| auto selId = Object::ID(block.branchInstruction.word(1)); |
| |
| auto sel = GenericValue(this, state, selId); |
| ASSERT_MSG(getType(sel.type).sizeInComponents == 1, "Selector must be a scalar"); |
| |
| auto numCases = (block.branchInstruction.wordCount() - 3) / 2; |
| |
| // TODO: Optimize for case where all lanes take same path. |
| |
| SIMD::Int defaultLaneMask = state->activeLaneMask(); |
| |
| // Gather up the case label matches and calculate defaultLaneMask. |
| std::vector<RValue<SIMD::Int>> caseLabelMatches; |
| caseLabelMatches.reserve(numCases); |
| for (uint32_t i = 0; i < numCases; i++) |
| { |
| auto label = block.branchInstruction.word(i * 2 + 3); |
| auto caseBlockId = Block::ID(block.branchInstruction.word(i * 2 + 4)); |
| auto caseLabelMatch = CmpEQ(sel.Int(0), SIMD::Int(label)); |
| state->addOutputActiveLaneMaskEdge(caseBlockId, caseLabelMatch); |
| defaultLaneMask &= ~caseLabelMatch; |
| } |
| |
| auto defaultBlockId = Block::ID(block.branchInstruction.word(2)); |
| state->addOutputActiveLaneMaskEdge(defaultBlockId, defaultLaneMask); |
| |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitUnreachable(InsnIterator insn, EmitState *state) const |
| { |
| // TODO: Log something in this case? |
| state->setActiveLaneMask(SIMD::Int(0)); |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitReturn(InsnIterator insn, EmitState *state) const |
| { |
| state->setActiveLaneMask(SIMD::Int(0)); |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitKill(InsnIterator insn, EmitState *state) const |
| { |
| state->routine->killMask |= SignMask(state->activeLaneMask()); |
| state->setActiveLaneMask(SIMD::Int(0)); |
| return EmitResult::Terminator; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitFunctionCall(InsnIterator insn, EmitState *state) const |
| { |
| auto functionId = Function::ID(insn.word(3)); |
| const auto& functionIt = functions.find(functionId); |
| ASSERT(functionIt != functions.end()); |
| auto& function = functionIt->second; |
| |
| // TODO(b/141246700): Add full support for spv::OpFunctionCall |
| // The only supported function is a single OpKill wrapped in a |
| // function, as a result of the "wrap OpKill" SPIRV-Tools pass |
| ASSERT(function.blocks.size() == 1); |
| spv::Op wrapOpKill[] = { spv::OpLabel, spv::OpKill }; |
| |
| for (auto block : function.blocks) |
| { |
| int insnNumber = 0; |
| for (auto blockInsn : block.second) |
| { |
| if (insnNumber > 1) |
| { |
| UNIMPLEMENTED("Function block number of instructions: %d", insnNumber); |
| return EmitResult::Continue; |
| } |
| if (blockInsn.opcode() != wrapOpKill[insnNumber++]) |
| { |
| UNIMPLEMENTED("Function block instruction %d : %s", insnNumber - 1, OpcodeName(blockInsn.opcode()).c_str()); |
| return EmitResult::Continue; |
| } |
| if (blockInsn.opcode() == spv::OpKill) |
| { |
| EmitInstruction(blockInsn, state); |
| } |
| } |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitPhi(InsnIterator insn, EmitState *state) const |
| { |
| auto &function = getFunction(state->function); |
| auto currentBlock = function.getBlock(state->block); |
| if (!currentBlock.isLoopMerge) |
| { |
| // If this is a loop merge block, then don't attempt to update the |
| // phi values from the ins. EmitLoop() has had to take special care |
| // of this phi in order to correctly deal with divergent lanes. |
| StorePhi(state->block, insn, state, currentBlock.ins); |
| } |
| LoadPhi(insn, state); |
| return EmitResult::Continue; |
| } |
| |
| void SpirvShader::LoadPhi(InsnIterator insn, EmitState *state) const |
| { |
| auto typeId = Type::ID(insn.word(1)); |
| auto type = getType(typeId); |
| auto objectId = Object::ID(insn.word(2)); |
| |
| auto storageIt = state->routine->phis.find(objectId); |
| ASSERT(storageIt != state->routine->phis.end()); |
| auto &storage = storageIt->second; |
| |
| auto &dst = state->createIntermediate(objectId, type.sizeInComponents); |
| for(uint32_t i = 0; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, storage[i]); |
| } |
| } |
| |
| void SpirvShader::StorePhi(Block::ID currentBlock, InsnIterator insn, EmitState *state, std::unordered_set<SpirvShader::Block::ID> const& filter) const |
| { |
| auto typeId = Type::ID(insn.word(1)); |
| auto type = getType(typeId); |
| auto objectId = Object::ID(insn.word(2)); |
| |
| auto storageIt = state->routine->phis.find(objectId); |
| ASSERT(storageIt != state->routine->phis.end()); |
| auto &storage = storageIt->second; |
| |
| for (uint32_t w = 3; w < insn.wordCount(); w += 2) |
| { |
| auto varId = Object::ID(insn.word(w + 0)); |
| auto blockId = Block::ID(insn.word(w + 1)); |
| |
| if (filter.count(blockId) == 0) |
| { |
| continue; |
| } |
| |
| auto mask = GetActiveLaneMaskEdge(state, blockId, currentBlock); |
| auto in = GenericValue(this, state, varId); |
| |
| for (uint32_t i = 0; i < type.sizeInComponents; i++) |
| { |
| storage[i] = As<SIMD::Float>((As<SIMD::Int>(storage[i]) & ~mask) | (in.Int(i) & mask)); |
| } |
| } |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageSampleImplicitLod(Variant variant, InsnIterator insn, EmitState *state) const |
| { |
| return EmitImageSample({variant, Implicit}, insn, state); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageGather(Variant variant, InsnIterator insn, EmitState *state) const |
| { |
| ImageInstruction instruction = {variant, Gather}; |
| instruction.gatherComponent = !instruction.isDref() ? getObject(insn.word(5)).constantValue[0] : 0; |
| |
| return EmitImageSample(instruction, insn, state); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageSampleExplicitLod(Variant variant, InsnIterator insn, EmitState *state) const |
| { |
| auto isDref = (variant == Dref) || (variant == ProjDref); |
| uint32_t imageOperands = static_cast<spv::ImageOperandsMask>(insn.word(isDref ? 6 : 5)); |
| imageOperands &= ~spv::ImageOperandsConstOffsetMask; // Dealt with later. |
| |
| if((imageOperands & spv::ImageOperandsLodMask) == imageOperands) |
| { |
| return EmitImageSample({variant, Lod}, insn, state); |
| } |
| else if((imageOperands & spv::ImageOperandsGradMask) == imageOperands) |
| { |
| return EmitImageSample({variant, Grad}, insn, state); |
| } |
| else UNIMPLEMENTED("Image Operands %x", imageOperands); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageFetch(InsnIterator insn, EmitState *state) const |
| { |
| return EmitImageSample({None, Fetch}, insn, state); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageSample(ImageInstruction instruction, InsnIterator insn, EmitState *state) const |
| { |
| Type::ID resultTypeId = insn.word(1); |
| Object::ID resultId = insn.word(2); |
| Object::ID sampledImageId = insn.word(3); // For OpImageFetch this is just an Image, not a SampledImage. |
| Object::ID coordinateId = insn.word(4); |
| auto &resultType = getType(resultTypeId); |
| |
| auto &result = state->createIntermediate(resultId, resultType.sizeInComponents); |
| auto imageDescriptor = state->getPointer(sampledImageId).base; // vk::SampledImageDescriptor* |
| |
| // If using a separate sampler, look through the OpSampledImage instruction to find the sampler descriptor |
| auto &sampledImage = getObject(sampledImageId); |
| auto samplerDescriptor = (sampledImage.opcode() == spv::OpSampledImage) ? |
| state->getPointer(sampledImage.definition.word(4)).base : imageDescriptor; |
| |
| auto coordinate = GenericValue(this, state, coordinateId); |
| auto &coordinateType = getType(coordinate.type); |
| |
| Pointer<Byte> sampler = samplerDescriptor + OFFSET(vk::SampledImageDescriptor, sampler); // vk::Sampler* |
| Pointer<Byte> texture = imageDescriptor + OFFSET(vk::SampledImageDescriptor, texture); // sw::Texture* |
| |
| // Above we assumed that if the SampledImage operand is not the result of an OpSampledImage, |
| // it must be a combined image sampler loaded straight from the descriptor set. For OpImageFetch |
| // it's just an Image operand, so there's no sampler descriptor data. |
| if(getType(sampledImage.type).opcode() != spv::OpTypeSampledImage) |
| { |
| sampler = Pointer<Byte>(nullptr); |
| } |
| |
| uint32_t imageOperands = spv::ImageOperandsMaskNone; |
| bool lodOrBias = false; |
| Object::ID lodOrBiasId = 0; |
| bool grad = false; |
| Object::ID gradDxId = 0; |
| Object::ID gradDyId = 0; |
| bool constOffset = false; |
| Object::ID offsetId = 0; |
| bool sample = false; |
| Object::ID sampleId = 0; |
| |
| uint32_t operand = (instruction.isDref() || instruction.samplerMethod == Gather) ? 6 : 5; |
| |
| if(insn.wordCount() > operand) |
| { |
| imageOperands = static_cast<spv::ImageOperandsMask>(insn.word(operand++)); |
| |
| if(imageOperands & spv::ImageOperandsBiasMask) |
| { |
| lodOrBias = true; |
| lodOrBiasId = insn.word(operand); |
| operand++; |
| imageOperands &= ~spv::ImageOperandsBiasMask; |
| |
| ASSERT(instruction.samplerMethod == Implicit); |
| instruction.samplerMethod = Bias; |
| } |
| |
| if(imageOperands & spv::ImageOperandsLodMask) |
| { |
| lodOrBias = true; |
| lodOrBiasId = insn.word(operand); |
| operand++; |
| imageOperands &= ~spv::ImageOperandsLodMask; |
| } |
| |
| if(imageOperands & spv::ImageOperandsGradMask) |
| { |
| ASSERT(!lodOrBias); // SPIR-V 1.3: "It is invalid to set both the Lod and Grad bits." Bias is for ImplicitLod, Grad for ExplicitLod. |
| grad = true; |
| gradDxId = insn.word(operand + 0); |
| gradDyId = insn.word(operand + 1); |
| operand += 2; |
| imageOperands &= ~spv::ImageOperandsGradMask; |
| } |
| |
| if(imageOperands & spv::ImageOperandsConstOffsetMask) |
| { |
| constOffset = true; |
| offsetId = insn.word(operand); |
| operand++; |
| imageOperands &= ~spv::ImageOperandsConstOffsetMask; |
| } |
| |
| if(imageOperands & spv::ImageOperandsSampleMask) |
| { |
| sample = true; |
| sampleId = insn.word(operand); |
| imageOperands &= ~spv::ImageOperandsSampleMask; |
| |
| ASSERT(instruction.samplerMethod == Fetch); |
| instruction.sample = true; |
| } |
| |
| if(imageOperands != 0) |
| { |
| UNSUPPORTED("Image operand %x", imageOperands); |
| } |
| } |
| |
| Array<SIMD::Float> in(16); // Maximum 16 input parameter components. |
| |
| uint32_t coordinates = coordinateType.sizeInComponents - instruction.isProj(); |
| instruction.coordinates = coordinates; |
| |
| uint32_t i = 0; |
| for( ; i < coordinates; i++) |
| { |
| if(instruction.isProj()) |
| { |
| in[i] = coordinate.Float(i) / coordinate.Float(coordinates); // TODO(b/129523279): Optimize using reciprocal. |
| } |
| else |
| { |
| in[i] = coordinate.Float(i); |
| } |
| } |
| |
| if(instruction.isDref()) |
| { |
| auto drefValue = GenericValue(this, state, insn.word(5)); |
| |
| if(instruction.isProj()) |
| { |
| in[i] = drefValue.Float(0) / coordinate.Float(coordinates); // TODO(b/129523279): Optimize using reciprocal. |
| } |
| else |
| { |
| in[i] = drefValue.Float(0); |
| } |
| |
| i++; |
| } |
| |
| if(lodOrBias) |
| { |
| auto lodValue = GenericValue(this, state, lodOrBiasId); |
| in[i] = lodValue.Float(0); |
| i++; |
| } |
| else if(grad) |
| { |
| auto dxValue = GenericValue(this, state, gradDxId); |
| auto dyValue = GenericValue(this, state, gradDyId); |
| auto &dxyType = getType(dxValue.type); |
| ASSERT(dxyType.sizeInComponents == getType(dyValue.type).sizeInComponents); |
| |
| instruction.grad = dxyType.sizeInComponents; |
| |
| for(uint32_t j = 0; j < dxyType.sizeInComponents; j++, i++) |
| { |
| in[i] = dxValue.Float(j); |
| } |
| |
| for(uint32_t j = 0; j < dxyType.sizeInComponents; j++, i++) |
| { |
| in[i] = dyValue.Float(j); |
| } |
| } |
| else if (instruction.samplerMethod == Fetch) |
| { |
| // The instruction didn't provide a lod operand, but the sampler's Fetch |
| // function requires one to be present. If no lod is supplied, the default |
| // is zero. |
| in[i] = As<SIMD::Float>(SIMD::Int(0)); |
| i++; |
| } |
| |
| if(constOffset) |
| { |
| auto offsetValue = GenericValue(this, state, offsetId); |
| auto &offsetType = getType(offsetValue.type); |
| |
| instruction.offset = offsetType.sizeInComponents; |
| |
| for(uint32_t j = 0; j < offsetType.sizeInComponents; j++, i++) |
| { |
| in[i] = As<SIMD::Float>(offsetValue.Int(j)); // Integer values, but transfered as float. |
| } |
| } |
| |
| if(sample) |
| { |
| auto sampleValue = GenericValue(this, state, sampleId); |
| in[i] = sampleValue.Float(0); |
| } |
| |
| auto cacheIt = state->routine->samplerCache.find(resultId); |
| ASSERT(cacheIt != state->routine->samplerCache.end()); |
| auto &cache = cacheIt->second; |
| auto cacheHit = cache.imageDescriptor == imageDescriptor && cache.sampler == sampler; |
| |
| If(!cacheHit) |
| { |
| cache.function = Call(getImageSampler, instruction.parameters, imageDescriptor, sampler); |
| cache.imageDescriptor = imageDescriptor; |
| cache.sampler = sampler; |
| } |
| |
| Array<SIMD::Float> out(4); |
| Call<ImageSampler>(cache.function, texture, sampler, &in[0], &out[0], state->routine->constants); |
| |
| for (auto i = 0u; i < resultType.sizeInComponents; i++) { result.move(i, out[i]); } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageQuerySizeLod(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultTy = getType(Type::ID(insn.word(1))); |
| auto resultId = Object::ID(insn.word(2)); |
| auto imageId = Object::ID(insn.word(3)); |
| auto lodId = Object::ID(insn.word(4)); |
| |
| auto &dst = state->createIntermediate(resultId, resultTy.sizeInComponents); |
| GetImageDimensions(state, resultTy, imageId, lodId, dst); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageQuerySize(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultTy = getType(Type::ID(insn.word(1))); |
| auto resultId = Object::ID(insn.word(2)); |
| auto imageId = Object::ID(insn.word(3)); |
| auto lodId = Object::ID(0); |
| |
| auto &dst = state->createIntermediate(resultId, resultTy.sizeInComponents); |
| GetImageDimensions(state, resultTy, imageId, lodId, dst); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageQueryLod(InsnIterator insn, EmitState *state) const |
| { |
| return EmitImageSample({None, Query}, insn, state); |
| } |
| |
| void SpirvShader::GetImageDimensions(EmitState const *state, Type const &resultTy, Object::ID imageId, Object::ID lodId, Intermediate &dst) const |
| { |
| auto routine = state->routine; |
| auto &image = getObject(imageId); |
| auto &imageType = getType(image.type); |
| |
| ASSERT(imageType.definition.opcode() == spv::OpTypeImage); |
| bool isArrayed = imageType.definition.word(5) != 0; |
| bool isCubeMap = imageType.definition.word(3) == spv::DimCube; |
| |
| const DescriptorDecorations &d = descriptorDecorations.at(imageId); |
| auto setLayout = routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| auto &bindingLayout = setLayout->getBindingLayout(d.Binding); |
| |
| Pointer<Byte> descriptor = state->getPointer(imageId).base; |
| |
| Pointer<Int> extent; |
| Int arrayLayers; |
| |
| switch (bindingLayout.descriptorType) |
| { |
| case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: |
| case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: |
| { |
| extent = descriptor + OFFSET(vk::StorageImageDescriptor, extent); // int[3]* |
| arrayLayers = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, arrayLayers)); // uint32_t |
| break; |
| } |
| case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: |
| case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: |
| case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: |
| { |
| extent = descriptor + OFFSET(vk::SampledImageDescriptor, extent); // int[3]* |
| arrayLayers = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, arrayLayers)); // uint32_t |
| break; |
| } |
| default: |
| UNREACHABLE("Image descriptorType: %d", int(bindingLayout.descriptorType)); |
| } |
| |
| auto dimensions = resultTy.sizeInComponents - (isArrayed ? 1 : 0); |
| std::vector<Int> out; |
| if (lodId != 0) |
| { |
| auto lodVal = GenericValue(this, state, lodId); |
| ASSERT(getType(lodVal.type).sizeInComponents == 1); |
| auto lod = lodVal.Int(0); |
| auto one = SIMD::Int(1); |
| for (uint32_t i = 0; i < dimensions; i++) |
| { |
| dst.move(i, Max(SIMD::Int(extent[i]) >> lod, one)); |
| } |
| } |
| else |
| { |
| for (uint32_t i = 0; i < dimensions; i++) |
| { |
| dst.move(i, SIMD::Int(extent[i])); |
| } |
| } |
| |
| if (isArrayed) |
| { |
| auto numElements = isCubeMap ? (arrayLayers / 6) : RValue<Int>(arrayLayers); |
| dst.move(dimensions, SIMD::Int(numElements)); |
| } |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageQueryLevels(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultTy = getType(Type::ID(insn.word(1))); |
| ASSERT(resultTy.sizeInComponents == 1); |
| auto resultId = Object::ID(insn.word(2)); |
| auto imageId = Object::ID(insn.word(3)); |
| |
| const DescriptorDecorations &d = descriptorDecorations.at(imageId); |
| auto setLayout = state->routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| auto &bindingLayout = setLayout->getBindingLayout(d.Binding); |
| |
| Pointer<Byte> descriptor = state->getPointer(imageId).base; |
| Int mipLevels = 0; |
| switch (bindingLayout.descriptorType) |
| { |
| case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: |
| case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: |
| case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: |
| mipLevels = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, mipLevels)); // uint32_t |
| break; |
| default: |
| UNREACHABLE("Image descriptorType: %d", int(bindingLayout.descriptorType)); |
| } |
| |
| auto &dst = state->createIntermediate(resultId, 1); |
| dst.move(0, SIMD::Int(mipLevels)); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageQuerySamples(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultTy = getType(Type::ID(insn.word(1))); |
| ASSERT(resultTy.sizeInComponents == 1); |
| auto resultId = Object::ID(insn.word(2)); |
| auto imageId = Object::ID(insn.word(3)); |
| auto imageTy = getType(getObject(imageId).type); |
| ASSERT(imageTy.definition.opcode() == spv::OpTypeImage); |
| ASSERT(imageTy.definition.word(3) == spv::Dim2D); |
| ASSERT(imageTy.definition.word(6 /* MS */) == 1); |
| |
| const DescriptorDecorations &d = descriptorDecorations.at(imageId); |
| auto setLayout = state->routine->pipelineLayout->getDescriptorSetLayout(d.DescriptorSet); |
| auto &bindingLayout = setLayout->getBindingLayout(d.Binding); |
| |
| Pointer<Byte> descriptor = state->getPointer(imageId).base; |
| Int sampleCount = 0; |
| switch (bindingLayout.descriptorType) |
| { |
| case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: |
| sampleCount = *Pointer<Int>(descriptor + OFFSET(vk::StorageImageDescriptor, sampleCount)); // uint32_t |
| break; |
| case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: |
| case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: |
| case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: |
| sampleCount = *Pointer<Int>(descriptor + OFFSET(vk::SampledImageDescriptor, sampleCount)); // uint32_t |
| break; |
| default: |
| UNREACHABLE("Image descriptorType: %d", int(bindingLayout.descriptorType)); |
| } |
| |
| auto &dst = state->createIntermediate(resultId, 1); |
| dst.move(0, SIMD::Int(sampleCount)); |
| |
| return EmitResult::Continue; |
| } |
| |
| SIMD::Pointer SpirvShader::GetTexelAddress(EmitState const *state, SIMD::Pointer ptr, GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize, Object::ID sampleId, bool useStencilAspect) const |
| { |
| auto routine = state->routine; |
| bool isArrayed = imageType.definition.word(5) != 0; |
| auto dim = static_cast<spv::Dim>(imageType.definition.word(3)); |
| int dims = getType(coordinate.type).sizeInComponents - (isArrayed ? 1 : 0); |
| |
| SIMD::Int u = coordinate.Int(0); |
| SIMD::Int v = SIMD::Int(0); |
| |
| if (getType(coordinate.type).sizeInComponents > 1) |
| { |
| v = coordinate.Int(1); |
| } |
| |
| if (dim == spv::DimSubpassData) |
| { |
| u += routine->windowSpacePosition[0]; |
| v += routine->windowSpacePosition[1]; |
| } |
| |
| if (useStencilAspect) |
| { |
| // Adjust addressing for quad layout. Pitches are already correct for the stencil aspect. |
| // In the quad-layout block, pixel order is [x0,y0 x1,y0 x0,y1 x1,y1] |
| u = ((v & SIMD::Int(1)) << 1) | ((u << 1) - (u & SIMD::Int(1))); |
| v &= SIMD::Int(~1); |
| } |
| |
| auto rowPitch = SIMD::Int(*Pointer<Int>(descriptor + (useStencilAspect |
| ? OFFSET(vk::StorageImageDescriptor, stencilRowPitchBytes) |
| : OFFSET(vk::StorageImageDescriptor, rowPitchBytes)))); |
| auto slicePitch = SIMD::Int( |
| *Pointer<Int>(descriptor + (useStencilAspect |
| ? OFFSET(vk::StorageImageDescriptor, stencilSlicePitchBytes) |
| : OFFSET(vk::StorageImageDescriptor, slicePitchBytes)))); |
| auto samplePitch = SIMD::Int( |
| *Pointer<Int>(descriptor + (useStencilAspect |
| ? OFFSET(vk::StorageImageDescriptor, stencilSamplePitchBytes) |
| : OFFSET(vk::StorageImageDescriptor, samplePitchBytes)))); |
| |
| ptr += u * SIMD::Int(texelSize); |
| if (dims > 1) |
| { |
| ptr += v * rowPitch; |
| } |
| if (dims > 2) |
| { |
| ptr += coordinate.Int(2) * slicePitch; |
| } |
| if (isArrayed) |
| { |
| ptr += coordinate.Int(dims) * slicePitch; |
| } |
| |
| if (dim == spv::DimSubpassData) |
| { |
| // Multiview input attachment access is to the layer corresponding to the current view |
| ptr += SIMD::Int(routine->viewID) * slicePitch; |
| } |
| |
| if (sampleId.value()) |
| { |
| GenericValue sample(this, state, sampleId); |
| ptr += sample.Int(0) * samplePitch; |
| } |
| |
| return ptr; |
| } |
| |
| void SpirvShader::Yield(YieldResult res) const |
| { |
| rr::Yield(RValue<Int>(int(res))); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageRead(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultType = getType(Type::ID(insn.word(1))); |
| auto imageId = Object::ID(insn.word(3)); |
| auto &image = getObject(imageId); |
| auto &imageType = getType(image.type); |
| Object::ID resultId = insn.word(2); |
| |
| Object::ID sampleId = 0; |
| |
| if (insn.wordCount() > 5) |
| { |
| int operand = 6; |
| auto imageOperands = insn.word(5); |
| if (imageOperands & spv::ImageOperandsSampleMask) |
| { |
| sampleId = insn.word(operand++); |
| imageOperands &= ~spv::ImageOperandsSampleMask; |
| } |
| |
| // Should be no remaining image operands. |
| ASSERT(!imageOperands); |
| } |
| |
| ASSERT(imageType.definition.opcode() == spv::OpTypeImage); |
| auto dim = static_cast<spv::Dim>(imageType.definition.word(3)); |
| |
| auto coordinate = GenericValue(this, state, insn.word(4)); |
| const DescriptorDecorations &d = descriptorDecorations.at(imageId); |
| |
| // For subpass data, format in the instruction is spv::ImageFormatUnknown. Get it from |
| // the renderpass data instead. In all other cases, we can use the format in the instruction. |
| auto vkFormat = (dim == spv::DimSubpassData) |
| ? inputAttachmentFormats[d.InputAttachmentIndex] |
| : SpirvFormatToVulkanFormat(static_cast<spv::ImageFormat>(imageType.definition.word(8))); |
| |
| // Depth+Stencil image attachments select aspect based on the Sampled Type of the |
| // OpTypeImage. If float, then we want the depth aspect. If int, we want the stencil aspect. |
| auto useStencilAspect = (vkFormat == VK_FORMAT_D32_SFLOAT_S8_UINT && |
| getType(imageType.definition.word(2)).opcode() == spv::OpTypeInt); |
| |
| if (useStencilAspect) |
| { |
| vkFormat = VK_FORMAT_S8_UINT; |
| } |
| |
| auto pointer = state->getPointer(imageId); |
| Pointer<Byte> binding = pointer.base; |
| Pointer<Byte> imageBase = *Pointer<Pointer<Byte>>(binding + (useStencilAspect |
| ? OFFSET(vk::StorageImageDescriptor, stencilPtr) |
| : OFFSET(vk::StorageImageDescriptor, ptr))); |
| |
| auto imageSizeInBytes = *Pointer<Int>(binding + OFFSET(vk::StorageImageDescriptor, sizeInBytes)); |
| |
| auto &dst = state->createIntermediate(resultId, resultType.sizeInComponents); |
| |
| auto texelSize = vk::Format(vkFormat).bytes(); |
| auto basePtr = SIMD::Pointer(imageBase, imageSizeInBytes); |
| auto texelPtr = GetTexelAddress(state, basePtr, coordinate, imageType, binding, texelSize, sampleId, useStencilAspect); |
| |
| // "The value returned by a read of an invalid texel is undefined, |
| // unless that read operation is from a buffer resource and the robustBufferAccess feature is enabled." |
| // TODO: Don't always assume a buffer resource. |
| auto robustness = OutOfBoundsBehavior::RobustBufferAccess; |
| |
| SIMD::Int packed[4]; |
| // Round up texel size: for formats smaller than 32 bits per texel, we will emit a bunch |
| // of (overlapping) 32b loads here, and each lane will pick out what it needs from the low bits. |
| // TODO: specialize for small formats? |
| for (auto i = 0; i < (texelSize + 3)/4; i++) |
| { |
| packed[i] = SIMD::Load<SIMD::Int>(texelPtr, robustness, state->activeLaneMask(), false, std::memory_order_relaxed, std::min(texelSize, 4)); |
| texelPtr += sizeof(float); |
| } |
| |
| // Format support requirements here come from two sources: |
| // - Minimum required set of formats for loads from storage images |
| // - Any format supported as a color or depth/stencil attachment, for input attachments |
| switch(vkFormat) |
| { |
| case VK_FORMAT_R32G32B32A32_SFLOAT: |
| case VK_FORMAT_R32G32B32A32_SINT: |
| case VK_FORMAT_R32G32B32A32_UINT: |
| dst.move(0, packed[0]); |
| dst.move(1, packed[1]); |
| dst.move(2, packed[2]); |
| dst.move(3, packed[3]); |
| break; |
| case VK_FORMAT_R32_SINT: |
| case VK_FORMAT_R32_UINT: |
| dst.move(0, packed[0]); |
| // Fill remaining channels with 0,0,1 (of the correct type) |
| dst.move(1, SIMD::Int(0)); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R32_SFLOAT: |
| case VK_FORMAT_D32_SFLOAT: |
| case VK_FORMAT_D32_SFLOAT_S8_UINT: |
| dst.move(0, packed[0]); |
| // Fill remaining channels with 0,0,1 (of the correct type) |
| dst.move(1, SIMD::Float(0)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_D16_UNORM: |
| dst.move(0, SIMD::Float(packed[0] & SIMD::Int(0xffff)) * SIMD::Float(1.0f / 65535.0f)); |
| dst.move(1, SIMD::Float(0)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_R16G16B16A16_SINT: |
| dst.move(0, (packed[0] << 16) >> 16); |
| dst.move(1, (packed[0]) >> 16); |
| dst.move(2, (packed[1] << 16) >> 16); |
| dst.move(3, (packed[1]) >> 16); |
| break; |
| case VK_FORMAT_R16G16B16A16_UINT: |
| dst.move(0, packed[0] & SIMD::Int(0xffff)); |
| dst.move(1, (packed[0] >> 16) & SIMD::Int(0xffff)); |
| dst.move(2, packed[1] & SIMD::Int(0xffff)); |
| dst.move(3, (packed[1] >> 16) & SIMD::Int(0xffff)); |
| break; |
| case VK_FORMAT_R16G16B16A16_SFLOAT: |
| dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF))); |
| dst.move(1, halfToFloatBits((As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFFFF0000)) >> 16)); |
| dst.move(2, halfToFloatBits(As<SIMD::UInt>(packed[1]) & SIMD::UInt(0x0000FFFF))); |
| dst.move(3, halfToFloatBits((As<SIMD::UInt>(packed[1]) & SIMD::UInt(0xFFFF0000)) >> 16)); |
| break; |
| case VK_FORMAT_R8G8B8A8_SNORM: |
| dst.move(0, Min(Max(SIMD::Float(((packed[0]<<24) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(1, Min(Max(SIMD::Float(((packed[0]<<16) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(2, Min(Max(SIMD::Float(((packed[0]<<8) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| dst.move(3, Min(Max(SIMD::Float(((packed[0]) & SIMD::Int(0xFF000000))) * SIMD::Float(1.0f / float(0x7f000000)), SIMD::Float(-1.0f)), SIMD::Float(1.0f))); |
| break; |
| case VK_FORMAT_R8G8B8A8_UNORM: |
| case VK_FORMAT_A8B8G8R8_UNORM_PACK32: |
| dst.move(0, SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(1, SIMD::Float(((packed[0]>>8) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(2, SIMD::Float(((packed[0]>>16) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(3, SIMD::Float(((packed[0]>>24) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| break; |
| case VK_FORMAT_R8G8B8A8_SRGB: |
| case VK_FORMAT_A8B8G8R8_SRGB_PACK32: |
| dst.move(0, ::sRGBtoLinear(SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(1, ::sRGBtoLinear(SIMD::Float(((packed[0]>>8) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(2, ::sRGBtoLinear(SIMD::Float(((packed[0]>>16) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(3, SIMD::Float(((packed[0]>>24) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| break; |
| case VK_FORMAT_B8G8R8A8_UNORM: |
| dst.move(0, SIMD::Float(((packed[0]>>16) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(1, SIMD::Float(((packed[0]>>8) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(2, SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(3, SIMD::Float(((packed[0]>>24) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| break; |
| case VK_FORMAT_B8G8R8A8_SRGB: |
| dst.move(0, ::sRGBtoLinear(SIMD::Float(((packed[0]>>16) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(1, ::sRGBtoLinear(SIMD::Float(((packed[0]>>8) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(2, ::sRGBtoLinear(SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f))); |
| dst.move(3, SIMD::Float(((packed[0]>>24) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| break; |
| case VK_FORMAT_R8G8B8A8_UINT: |
| case VK_FORMAT_A8B8G8R8_UINT_PACK32: |
| dst.move(0, (As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF))); |
| dst.move(1, ((As<SIMD::UInt>(packed[0])>>8) & SIMD::UInt(0xFF))); |
| dst.move(2, ((As<SIMD::UInt>(packed[0])>>16) & SIMD::UInt(0xFF))); |
| dst.move(3, ((As<SIMD::UInt>(packed[0])>>24) & SIMD::UInt(0xFF))); |
| break; |
| case VK_FORMAT_R8G8B8A8_SINT: |
| case VK_FORMAT_A8B8G8R8_SINT_PACK32: |
| dst.move(0, (packed[0] << 24) >> 24); |
| dst.move(1, (packed[0] << 16) >> 24); |
| dst.move(2, (packed[0] << 8) >> 24); |
| dst.move(3, (packed[0]) >> 24); |
| break; |
| case VK_FORMAT_R8_UNORM: |
| dst.move(0, SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(1, SIMD::Float(0)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_R8_UINT: |
| case VK_FORMAT_S8_UINT: |
| dst.move(0, (As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF))); |
| dst.move(1, SIMD::UInt(0)); |
| dst.move(2, SIMD::UInt(0)); |
| dst.move(3, SIMD::UInt(1)); |
| break; |
| case VK_FORMAT_R8_SINT: |
| dst.move(0, (packed[0] << 24) >> 24); |
| dst.move(1, SIMD::Int(0)); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R8G8_UNORM: |
| dst.move(0, SIMD::Float((packed[0] & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(1, SIMD::Float(((packed[0]>>8) & SIMD::Int(0xFF))) * SIMD::Float(1.0f / 255.f)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_R8G8_UINT: |
| dst.move(0, (As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFF))); |
| dst.move(1, ((As<SIMD::UInt>(packed[0])>>8) & SIMD::UInt(0xFF))); |
| dst.move(2, SIMD::UInt(0)); |
| dst.move(3, SIMD::UInt(1)); |
| break; |
| case VK_FORMAT_R8G8_SINT: |
| dst.move(0, (packed[0] << 24) >> 24); |
| dst.move(1, (packed[0] << 16) >> 24); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R16_SFLOAT: |
| dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF))); |
| dst.move(1, SIMD::Float(0)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_R16_UINT: |
| dst.move(0, packed[0] & SIMD::Int(0xffff)); |
| dst.move(1, SIMD::UInt(0)); |
| dst.move(2, SIMD::UInt(0)); |
| dst.move(3, SIMD::UInt(1)); |
| break; |
| case VK_FORMAT_R16_SINT: |
| dst.move(0, (packed[0] << 16) >> 16); |
| dst.move(1, SIMD::Int(0)); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R16G16_SFLOAT: |
| dst.move(0, halfToFloatBits(As<SIMD::UInt>(packed[0]) & SIMD::UInt(0x0000FFFF))); |
| dst.move(1, halfToFloatBits((As<SIMD::UInt>(packed[0]) & SIMD::UInt(0xFFFF0000)) >> 16)); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_R16G16_UINT: |
| dst.move(0, packed[0] & SIMD::Int(0xffff)); |
| dst.move(1, (packed[0] >> 16) & SIMD::Int(0xffff)); |
| dst.move(2, SIMD::UInt(0)); |
| dst.move(3, SIMD::UInt(1)); |
| break; |
| case VK_FORMAT_R16G16_SINT: |
| dst.move(0, (packed[0] << 16) >> 16); |
| dst.move(1, (packed[0]) >> 16); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R32G32_SINT: |
| case VK_FORMAT_R32G32_UINT: |
| dst.move(0, packed[0]); |
| dst.move(1, packed[1]); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(1)); |
| break; |
| case VK_FORMAT_R32G32_SFLOAT: |
| dst.move(0, packed[0]); |
| dst.move(1, packed[1]); |
| dst.move(2, SIMD::Float(0)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_A2B10G10R10_UINT_PACK32: |
| dst.move(0, (packed[0]) & SIMD::Int(0x3FF)); |
| dst.move(1, (packed[0] >> 10) & SIMD::Int(0x3FF)); |
| dst.move(2, (packed[0] >> 20) & SIMD::Int(0x3FF)); |
| dst.move(3, (packed[0] >> 30) & SIMD::Int(0x3)); |
| break; |
| case VK_FORMAT_A2B10G10R10_UNORM_PACK32: |
| dst.move(0, SIMD::Float((packed[0]) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF)); |
| dst.move(1, SIMD::Float((packed[0] >> 10) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF)); |
| dst.move(2, SIMD::Float((packed[0] >> 20) & SIMD::Int(0x3FF)) * SIMD::Float(1.0f / 0x3FF)); |
| dst.move(3, SIMD::Float((packed[0] >> 30) & SIMD::Int(0x3)) * SIMD::Float(1.0f / 0x3)); |
| break; |
| case VK_FORMAT_R5G6B5_UNORM_PACK16: |
| dst.move(0, SIMD::Float((packed[0] >> 11) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F)); |
| dst.move(1, SIMD::Float((packed[0] >> 5) & SIMD::Int(0x3F)) * SIMD::Float(1.0f / 0x3F)); |
| dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F)); |
| dst.move(3, SIMD::Float(1)); |
| break; |
| case VK_FORMAT_A1R5G5B5_UNORM_PACK16: |
| dst.move(0, SIMD::Float((packed[0] >> 10) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F)); |
| dst.move(1, SIMD::Float((packed[0] >> 5) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F)); |
| dst.move(2, SIMD::Float((packed[0]) & SIMD::Int(0x1F)) * SIMD::Float(1.0f / 0x1F)); |
| dst.move(3, SIMD::Float((packed[0] >> 15) & SIMD::Int(0x1))); |
| break; |
| default: |
| UNIMPLEMENTED("VkFormat %d", int(vkFormat)); |
| break; |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageWrite(InsnIterator insn, EmitState *state) const |
| { |
| auto imageId = Object::ID(insn.word(1)); |
| auto &image = getObject(imageId); |
| auto &imageType = getType(image.type); |
| |
| ASSERT(imageType.definition.opcode() == spv::OpTypeImage); |
| |
| // TODO(b/131171141): Not handling any image operands yet. |
| ASSERT(insn.wordCount() == 4); |
| |
| auto coordinate = GenericValue(this, state, insn.word(2)); |
| auto texel = GenericValue(this, state, insn.word(3)); |
| |
| Pointer<Byte> binding = state->getPointer(imageId).base; |
| Pointer<Byte> imageBase = *Pointer<Pointer<Byte>>(binding + OFFSET(vk::StorageImageDescriptor, ptr)); |
| auto imageSizeInBytes = *Pointer<Int>(binding + OFFSET(vk::StorageImageDescriptor, sizeInBytes)); |
| |
| SIMD::Int packed[4]; |
| auto numPackedElements = 0u; |
| int texelSize = 0; |
| auto format = static_cast<spv::ImageFormat>(imageType.definition.word(8)); |
| switch (format) |
| { |
| case spv::ImageFormatRgba32f: |
| case spv::ImageFormatRgba32i: |
| case spv::ImageFormatRgba32ui: |
| texelSize = 16; |
| packed[0] = texel.Int(0); |
| packed[1] = texel.Int(1); |
| packed[2] = texel.Int(2); |
| packed[3] = texel.Int(3); |
| numPackedElements = 4; |
| break; |
| case spv::ImageFormatR32f: |
| case spv::ImageFormatR32i: |
| case spv::ImageFormatR32ui: |
| texelSize = 4; |
| packed[0] = texel.Int(0); |
| numPackedElements = 1; |
| break; |
| case spv::ImageFormatRgba8: |
| texelSize = 4; |
| packed[0] = (SIMD::UInt(Round(Min(Max(texel.Float(0), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) | |
| ((SIMD::UInt(Round(Min(Max(texel.Float(1), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 8) | |
| ((SIMD::UInt(Round(Min(Max(texel.Float(2), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 16) | |
| ((SIMD::UInt(Round(Min(Max(texel.Float(3), SIMD::Float(0.0f)), SIMD::Float(1.0f)) * SIMD::Float(255.0f)))) << 24); |
| numPackedElements = 1; |
| break; |
| case spv::ImageFormatRgba8Snorm: |
| texelSize = 4; |
| packed[0] = (SIMD::Int(Round(Min(Max(texel.Float(0), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) | |
| ((SIMD::Int(Round(Min(Max(texel.Float(1), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 8) | |
| ((SIMD::Int(Round(Min(Max(texel.Float(2), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 16) | |
| ((SIMD::Int(Round(Min(Max(texel.Float(3), SIMD::Float(-1.0f)), SIMD::Float(1.0f)) * SIMD::Float(127.0f))) & |
| SIMD::Int(0xFF)) << 24); |
| numPackedElements = 1; |
| break; |
| case spv::ImageFormatRgba8i: |
| case spv::ImageFormatRgba8ui: |
| texelSize = 4; |
| packed[0] = (SIMD::UInt(texel.UInt(0) & SIMD::UInt(0xff))) | |
| (SIMD::UInt(texel.UInt(1) & SIMD::UInt(0xff)) << 8) | |
| (SIMD::UInt(texel.UInt(2) & SIMD::UInt(0xff)) << 16) | |
| (SIMD::UInt(texel.UInt(3) & SIMD::UInt(0xff)) << 24); |
| numPackedElements = 1; |
| break; |
| case spv::ImageFormatRgba16f: |
| texelSize = 8; |
| packed[0] = FloatToHalfBits(texel.UInt(0), false) | FloatToHalfBits(texel.UInt(1), true); |
| packed[1] = FloatToHalfBits(texel.UInt(2), false) | FloatToHalfBits(texel.UInt(3), true); |
| numPackedElements = 2; |
| break; |
| case spv::ImageFormatRgba16i: |
| case spv::ImageFormatRgba16ui: |
| texelSize = 8; |
| packed[0] = SIMD::UInt(texel.UInt(0) & SIMD::UInt(0xffff)) | (SIMD::UInt(texel.UInt(1) & SIMD::UInt(0xffff)) << 16); |
| packed[1] = SIMD::UInt(texel.UInt(2) & SIMD::UInt(0xffff)) | (SIMD::UInt(texel.UInt(3) & SIMD::UInt(0xffff)) << 16); |
| numPackedElements = 2; |
| break; |
| case spv::ImageFormatRg32f: |
| case spv::ImageFormatRg32i: |
| case spv::ImageFormatRg32ui: |
| texelSize = 8; |
| packed[0] = texel.Int(0); |
| packed[1] = texel.Int(1); |
| numPackedElements = 2; |
| break; |
| |
| case spv::ImageFormatRg16f: |
| case spv::ImageFormatR11fG11fB10f: |
| case spv::ImageFormatR16f: |
| case spv::ImageFormatRgba16: |
| case spv::ImageFormatRgb10A2: |
| case spv::ImageFormatRg16: |
| case spv::ImageFormatRg8: |
| case spv::ImageFormatR16: |
| case spv::ImageFormatR8: |
| case spv::ImageFormatRgba16Snorm: |
| case spv::ImageFormatRg16Snorm: |
| case spv::ImageFormatRg8Snorm: |
| case spv::ImageFormatR16Snorm: |
| case spv::ImageFormatR8Snorm: |
| case spv::ImageFormatRg16i: |
| case spv::ImageFormatRg8i: |
| case spv::ImageFormatR16i: |
| case spv::ImageFormatR8i: |
| case spv::ImageFormatRgb10a2ui: |
| case spv::ImageFormatRg16ui: |
| case spv::ImageFormatRg8ui: |
| case spv::ImageFormatR16ui: |
| case spv::ImageFormatR8ui: |
| UNIMPLEMENTED("spv::ImageFormat %d", int(format)); |
| break; |
| |
| default: |
| UNREACHABLE("spv::ImageFormat %d", int(format)); |
| break; |
| } |
| |
| auto basePtr = SIMD::Pointer(imageBase, imageSizeInBytes); |
| auto texelPtr = GetTexelAddress(state, basePtr, coordinate, imageType, binding, texelSize, 0, false); |
| |
| // SPIR-V 1.4: "If the coordinates are outside the image, the memory location that is accessed is undefined." |
| auto robustness = OutOfBoundsBehavior::UndefinedValue; |
| |
| for (auto i = 0u; i < numPackedElements; i++) |
| { |
| SIMD::Store(texelPtr, packed[i], robustness, state->activeLaneMask()); |
| texelPtr += sizeof(float); |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitImageTexelPointer(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultType = getType(Type::ID(insn.word(1))); |
| auto imageId = Object::ID(insn.word(3)); |
| auto &image = getObject(imageId); |
| // Note: OpImageTexelPointer is unusual in that the image is passed by pointer. |
| // Look through to get the actual image type. |
| auto &imageType = getType(getType(image.type).element); |
| Object::ID resultId = insn.word(2); |
| |
| ASSERT(imageType.opcode() == spv::OpTypeImage); |
| ASSERT(resultType.storageClass == spv::StorageClassImage); |
| ASSERT(getType(resultType.element).opcode() == spv::OpTypeInt); |
| |
| auto coordinate = GenericValue(this, state, insn.word(4)); |
| |
| Pointer<Byte> binding = state->getPointer(imageId).base; |
| Pointer<Byte> imageBase = *Pointer<Pointer<Byte>>(binding + OFFSET(vk::StorageImageDescriptor, ptr)); |
| auto imageSizeInBytes = *Pointer<Int>(binding + OFFSET(vk::StorageImageDescriptor, sizeInBytes)); |
| |
| auto basePtr = SIMD::Pointer(imageBase, imageSizeInBytes); |
| auto ptr = GetTexelAddress(state, basePtr, coordinate, imageType, binding, sizeof(uint32_t), 0, false); |
| |
| state->createPointer(resultId, ptr); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitSampledImageCombineOrSplit(InsnIterator insn, EmitState *state) const |
| { |
| // Propagate the image pointer in both cases. |
| // Consumers of OpSampledImage will look through to find the sampler pointer. |
| |
| Object::ID resultId = insn.word(2); |
| Object::ID imageId = insn.word(3); |
| |
| state->createPointer(resultId, state->getPointer(imageId)); |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitAtomicOp(InsnIterator insn, EmitState *state) const |
| { |
| auto &resultType = getType(Type::ID(insn.word(1))); |
| Object::ID resultId = insn.word(2); |
| Object::ID semanticsId = insn.word(5); |
| auto memorySemantics = static_cast<spv::MemorySemanticsMask>(getObject(semanticsId).constantValue[0]); |
| auto memoryOrder = MemoryOrder(memorySemantics); |
| // Where no value is provided (increment/decrement) use an implicit value of 1. |
| auto value = (insn.wordCount() == 7) ? GenericValue(this, state, insn.word(6)).UInt(0) : RValue<SIMD::UInt>(1); |
| auto &dst = state->createIntermediate(resultId, resultType.sizeInComponents); |
| auto ptr = state->getPointer(insn.word(3)); |
| auto ptrOffsets = ptr.offsets(); |
| |
| SIMD::UInt x(0); |
| auto mask = state->activeLaneMask() & state->storesAndAtomicsMask(); |
| for (int j = 0; j < SIMD::Width; j++) |
| { |
| If(Extract(mask, j) != 0) |
| { |
| auto offset = Extract(ptrOffsets, j); |
| auto laneValue = Extract(value, j); |
| UInt v; |
| switch (insn.opcode()) |
| { |
| case spv::OpAtomicIAdd: |
| case spv::OpAtomicIIncrement: |
| v = AddAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicISub: |
| case spv::OpAtomicIDecrement: |
| v = SubAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicAnd: |
| v = AndAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicOr: |
| v = OrAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicXor: |
| v = XorAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicSMin: |
| v = As<UInt>(MinAtomic(Pointer<Int>(&ptr.base[offset]), As<Int>(laneValue), memoryOrder)); |
| break; |
| case spv::OpAtomicSMax: |
| v = As<UInt>(MaxAtomic(Pointer<Int>(&ptr.base[offset]), As<Int>(laneValue), memoryOrder)); |
| break; |
| case spv::OpAtomicUMin: |
| v = MinAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicUMax: |
| v = MaxAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| case spv::OpAtomicExchange: |
| v = ExchangeAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, memoryOrder); |
| break; |
| default: |
| UNREACHABLE("%s", OpcodeName(insn.opcode()).c_str()); |
| break; |
| } |
| x = Insert(x, v, j); |
| } |
| } |
| |
| dst.move(0, x); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const |
| { |
| // Separate from EmitAtomicOp due to different instruction encoding |
| auto &resultType = getType(Type::ID(insn.word(1))); |
| Object::ID resultId = insn.word(2); |
| |
| auto memorySemanticsEqual = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(5)).constantValue[0]); |
| auto memoryOrderEqual = MemoryOrder(memorySemanticsEqual); |
| auto memorySemanticsUnequal = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(6)).constantValue[0]); |
| auto memoryOrderUnequal = MemoryOrder(memorySemanticsUnequal); |
| |
| auto value = GenericValue(this, state, insn.word(7)); |
| auto comparator = GenericValue(this, state, insn.word(8)); |
| auto &dst = state->createIntermediate(resultId, resultType.sizeInComponents); |
| auto ptr = state->getPointer(insn.word(3)); |
| auto ptrOffsets = ptr.offsets(); |
| |
| SIMD::UInt x(0); |
| auto mask = state->activeLaneMask() & state->storesAndAtomicsMask(); |
| for (int j = 0; j < SIMD::Width; j++) |
| { |
| If(Extract(mask, j) != 0) |
| { |
| auto offset = Extract(ptrOffsets, j); |
| auto laneValue = Extract(value.UInt(0), j); |
| auto laneComparator = Extract(comparator.UInt(0), j); |
| UInt v = CompareExchangeAtomic(Pointer<UInt>(&ptr.base[offset]), laneValue, laneComparator, memoryOrderEqual, memoryOrderUnequal); |
| x = Insert(x, v, j); |
| } |
| } |
| |
| dst.move(0, x); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitCopyObject(InsnIterator insn, EmitState *state) const |
| { |
| auto ty = getType(insn.word(1)); |
| auto &dst = state->createIntermediate(insn.word(2), ty.sizeInComponents); |
| auto src = GenericValue(this, state, insn.word(3)); |
| for (uint32_t i = 0; i < ty.sizeInComponents; i++) |
| { |
| dst.move(i, src.Int(i)); |
| } |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitCopyMemory(InsnIterator insn, EmitState *state) const |
| { |
| Object::ID dstPtrId = insn.word(1); |
| Object::ID srcPtrId = insn.word(2); |
| auto &dstPtrTy = getType(getObject(dstPtrId).type); |
| auto &srcPtrTy = getType(getObject(srcPtrId).type); |
| ASSERT(dstPtrTy.element == srcPtrTy.element); |
| |
| bool dstInterleavedByLane = IsStorageInterleavedByLane(dstPtrTy.storageClass); |
| bool srcInterleavedByLane = IsStorageInterleavedByLane(srcPtrTy.storageClass); |
| auto dstPtr = GetPointerToData(dstPtrId, 0, state); |
| auto srcPtr = GetPointerToData(srcPtrId, 0, state); |
| |
| std::unordered_map<uint32_t, uint32_t> srcOffsets; |
| |
| VisitMemoryObject(srcPtrId, [&](uint32_t i, uint32_t srcOffset) { srcOffsets[i] = srcOffset; }); |
| |
| VisitMemoryObject(dstPtrId, [&](uint32_t i, uint32_t dstOffset) |
| { |
| auto it = srcOffsets.find(i); |
| ASSERT(it != srcOffsets.end()); |
| auto srcOffset = it->second; |
| |
| auto dst = dstPtr + dstOffset; |
| auto src = srcPtr + srcOffset; |
| if (dstInterleavedByLane) { dst = interleaveByLane(dst); } |
| if (srcInterleavedByLane) { src = interleaveByLane(src); } |
| |
| // TODO(b/131224163): Optimize based on src/dst storage classes. |
| auto robustness = OutOfBoundsBehavior::RobustBufferAccess; |
| |
| auto value = SIMD::Load<SIMD::Float>(src, robustness, state->activeLaneMask()); |
| SIMD::Store(dst, value, robustness, state->activeLaneMask()); |
| }); |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitControlBarrier(InsnIterator insn, EmitState *state) const |
| { |
| auto executionScope = spv::Scope(GetConstScalarInt(insn.word(1))); |
| auto semantics = spv::MemorySemanticsMask(GetConstScalarInt(insn.word(3))); |
| // TODO: We probably want to consider the memory scope here. For now, |
| // just always emit the full fence. |
| Fence(semantics); |
| |
| switch (executionScope) |
| { |
| case spv::ScopeWorkgroup: |
| Yield(YieldResult::ControlBarrier); |
| break; |
| case spv::ScopeSubgroup: |
| break; |
| default: |
| // See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module. |
| UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup"); |
| break; |
| } |
| |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitMemoryBarrier(InsnIterator insn, EmitState *state) const |
| { |
| auto semantics = spv::MemorySemanticsMask(GetConstScalarInt(insn.word(2))); |
| // TODO: We probably want to consider the memory scope here. For now, |
| // just always emit the full fence. |
| Fence(semantics); |
| return EmitResult::Continue; |
| } |
| |
| void SpirvShader::Fence(spv::MemorySemanticsMask semantics) const |
| { |
| if (semantics == spv::MemorySemanticsMaskNone) |
| { |
| return; //no-op |
| } |
| rr::Fence(MemoryOrder(semantics)); |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitGroupNonUniform(InsnIterator insn, EmitState *state) const |
| { |
| static_assert(SIMD::Width == 4, "EmitGroupNonUniform makes many assumptions that the SIMD vector width is 4"); |
| |
| auto &type = getType(Type::ID(insn.word(1))); |
| Object::ID resultId = insn.word(2); |
| auto scope = spv::Scope(GetConstScalarInt(insn.word(3))); |
| ASSERT_MSG(scope == spv::ScopeSubgroup, "Scope for Non Uniform Group Operations must be Subgroup for Vulkan 1.1"); |
| |
| auto &dst = state->createIntermediate(resultId, type.sizeInComponents); |
| |
| switch (insn.opcode()) |
| { |
| case spv::OpGroupNonUniformElect: |
| { |
| // Result is true only in the active invocation with the lowest id |
| // in the group, otherwise result is false. |
| SIMD::Int active = state->activeLaneMask(); |
| // TODO: Would be nice if we could write this as: |
| // elect = active & ~(active.Oxyz | active.OOxy | active.OOOx) |
| auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); |
| auto elect = active & ~(v0111 & (active.xxyz | active.xxxy | active.xxxx)); |
| dst.move(0, elect); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformAll: |
| { |
| GenericValue predicate(this, state, insn.word(4)); |
| dst.move(0, AndAll(predicate.UInt(0) | ~As<SIMD::UInt>(state->activeLaneMask()))); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformAny: |
| { |
| GenericValue predicate(this, state, insn.word(4)); |
| dst.move(0, OrAll(predicate.UInt(0) & As<SIMD::UInt>(state->activeLaneMask()))); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformAllEqual: |
| { |
| GenericValue value(this, state, insn.word(4)); |
| auto res = SIMD::UInt(0xffffffff); |
| SIMD::UInt active = As<SIMD::UInt>(state->activeLaneMask()); |
| SIMD::UInt inactive = ~active; |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::UInt v = value.UInt(i) & active; |
| SIMD::UInt filled = v; |
| for (int j = 0; j < SIMD::Width - 1; j++) |
| { |
| filled |= filled.yzwx & inactive; // Populate inactive 'holes' with a live value |
| } |
| res &= AndAll(CmpEQ(filled.xyzw, filled.yzwx)); |
| } |
| dst.move(0, res); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBroadcast: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| auto id = SIMD::Int(GetConstScalarInt(insn.word(5))); |
| GenericValue value(this, state, valueId); |
| auto mask = CmpEQ(id, SIMD::Int(0, 1, 2, 3)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, OrAll(value.Int(i) & mask)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBroadcastFirst: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| GenericValue value(this, state, valueId); |
| // Result is true only in the active invocation with the lowest id |
| // in the group, otherwise result is false. |
| SIMD::Int active = state->activeLaneMask(); |
| // TODO: Would be nice if we could write this as: |
| // elect = active & ~(active.Oxyz | active.OOxy | active.OOOx) |
| auto v0111 = SIMD::Int(0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); |
| auto elect = active & ~(v0111 & (active.xxyz | active.xxxy | active.xxxx)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| dst.move(i, OrAll(value.Int(i) & elect)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBallot: |
| { |
| ASSERT(type.sizeInComponents == 4); |
| GenericValue predicate(this, state, insn.word(4)); |
| dst.move(0, SIMD::Int(SignMask(state->activeLaneMask() & predicate.Int(0)))); |
| dst.move(1, SIMD::Int(0)); |
| dst.move(2, SIMD::Int(0)); |
| dst.move(3, SIMD::Int(0)); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformInverseBallot: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getType(getObject(valueId).type).sizeInComponents == 4); |
| GenericValue value(this, state, valueId); |
| auto bit = (value.Int(0) >> SIMD::Int(0, 1, 2, 3)) & SIMD::Int(1); |
| dst.move(0, -bit); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBallotBitExtract: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| auto indexId = Object::ID(insn.word(5)); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getType(getObject(valueId).type).sizeInComponents == 4); |
| ASSERT(getType(getObject(indexId).type).sizeInComponents == 1); |
| GenericValue value(this, state, valueId); |
| GenericValue index(this, state, indexId); |
| auto vecIdx = index.Int(0) / SIMD::Int(32); |
| auto bitIdx = index.Int(0) & SIMD::Int(31); |
| auto bits = (value.Int(0) & CmpEQ(vecIdx, SIMD::Int(0))) | |
| (value.Int(1) & CmpEQ(vecIdx, SIMD::Int(1))) | |
| (value.Int(2) & CmpEQ(vecIdx, SIMD::Int(2))) | |
| (value.Int(3) & CmpEQ(vecIdx, SIMD::Int(3))); |
| dst.move(0, -((bits >> bitIdx) & SIMD::Int(1))); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBallotBitCount: |
| { |
| auto operation = spv::GroupOperation(insn.word(4)); |
| auto valueId = Object::ID(insn.word(5)); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getType(getObject(valueId).type).sizeInComponents == 4); |
| GenericValue value(this, state, valueId); |
| switch (operation) |
| { |
| case spv::GroupOperationReduce: |
| dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(15))); |
| break; |
| case spv::GroupOperationInclusiveScan: |
| dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(1, 3, 7, 15))); |
| break; |
| case spv::GroupOperationExclusiveScan: |
| dst.move(0, CountBits(value.UInt(0) & SIMD::UInt(0, 1, 3, 7))); |
| break; |
| default: |
| UNSUPPORTED("GroupOperation %d", int(operation)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBallotFindLSB: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getType(getObject(valueId).type).sizeInComponents == 4); |
| GenericValue value(this, state, valueId); |
| dst.move(0, Cttz(value.UInt(0) & SIMD::UInt(15), true)); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformBallotFindMSB: |
| { |
| auto valueId = Object::ID(insn.word(4)); |
| ASSERT(type.sizeInComponents == 1); |
| ASSERT(getType(getObject(valueId).type).sizeInComponents == 4); |
| GenericValue value(this, state, valueId); |
| dst.move(0, SIMD::UInt(31) - Ctlz(value.UInt(0) & SIMD::UInt(15), false)); |
| break; |
| } |
| |
| case spv::OpGroupNonUniformShuffle: |
| { |
| GenericValue value(this, state, insn.word(4)); |
| GenericValue id(this, state, insn.word(5)); |
| auto x = CmpEQ(SIMD::Int(0), id.Int(0)); |
| auto y = CmpEQ(SIMD::Int(1), id.Int(0)); |
| auto z = CmpEQ(SIMD::Int(2), id.Int(0)); |
| auto w = CmpEQ(SIMD::Int(3), id.Int(0)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Int v = value.Int(i); |
| dst.move(i, (x & v.xxxx) | (y & v.yyyy) | (z & v.zzzz) | (w & v.wwww)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformShuffleXor: |
| { |
| GenericValue value(this, state, insn.word(4)); |
| GenericValue mask(this, state, insn.word(5)); |
| auto x = CmpEQ(SIMD::Int(0), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0)); |
| auto y = CmpEQ(SIMD::Int(1), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0)); |
| auto z = CmpEQ(SIMD::Int(2), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0)); |
| auto w = CmpEQ(SIMD::Int(3), SIMD::Int(0, 1, 2, 3) ^ mask.Int(0)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Int v = value.Int(i); |
| dst.move(i, (x & v.xxxx) | (y & v.yyyy) | (z & v.zzzz) | (w & v.wwww)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformShuffleUp: |
| { |
| GenericValue value(this, state, insn.word(4)); |
| GenericValue delta(this, state, insn.word(5)); |
| auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0)); |
| auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0)); |
| auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0)); |
| auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Int v = value.Int(i); |
| dst.move(i, (d0 & v.xyzw) | (d1 & v.xxyz) | (d2 & v.xxxy) | (d3 & v.xxxx)); |
| } |
| break; |
| } |
| |
| case spv::OpGroupNonUniformShuffleDown: |
| { |
| GenericValue value(this, state, insn.word(4)); |
| GenericValue delta(this, state, insn.word(5)); |
| auto d0 = CmpEQ(SIMD::Int(0), delta.Int(0)); |
| auto d1 = CmpEQ(SIMD::Int(1), delta.Int(0)); |
| auto d2 = CmpEQ(SIMD::Int(2), delta.Int(0)); |
| auto d3 = CmpEQ(SIMD::Int(3), delta.Int(0)); |
| for (auto i = 0u; i < type.sizeInComponents; i++) |
| { |
| SIMD::Int v = value.Int(i); |
| dst.move(i, (d0 & v.xyzw) | (d1 & v.yzww) | (d2 & v.zwww) | (d3 & v.wwww)); |
| } |
| break; |
| } |
| |
| default: |
| UNIMPLEMENTED("EmitGroupNonUniform op: %s", OpcodeName(type.opcode()).c_str()); |
| } |
| return EmitResult::Continue; |
| } |
| |
| SpirvShader::EmitResult SpirvShader::EmitArrayLength(InsnIterator insn, EmitState *state) const |
| { |
| auto resultTyId = Type::ID(insn.word(1)); |
| auto resultId = Object::ID(insn.word(2)); |
| auto structPtrId = Object::ID(insn.word(3)); |
| auto arrayFieldIdx = insn.word(4); |
| |
| auto &resultType = getType(resultTyId); |
| ASSERT(resultType.sizeInComponents == 1); |
| ASSERT(resultType.definition.opcode() == spv::OpTypeInt); |
| |
| auto &structPtrTy = getType(getObject(structPtrId).type); |
| auto &structTy = getType(structPtrTy.element); |
| auto &arrayTy = getType(structTy.definition.word(2 + arrayFieldIdx)); |
| ASSERT(arrayTy.definition.opcode() == spv::OpTypeRuntimeArray); |
| auto &arrayElTy = getType(arrayTy.element); |
| |
| auto &result = state->createIntermediate(resultId, 1); |
| auto structBase = GetPointerToData(structPtrId, 0, state); |
| |
| Decorations d = {}; |
| ApplyDecorationsForIdMember(&d, structPtrTy.element, arrayFieldIdx); |
| ASSERT(d.HasOffset); |
| |
| auto arrayBase = structBase + d.Offset; |
| auto arraySizeInBytes = SIMD::Int(arrayBase.limit()) - arrayBase.offsets(); |
| auto arrayLength = arraySizeInBytes / SIMD::Int(arrayElTy.sizeInComponents * sizeof(float)); |
| |
| result.move(0, SIMD::Int(arrayLength)); |
| |
| return EmitResult::Continue; |
| } |
| |
| uint32_t SpirvShader::GetConstScalarInt(Object::ID id) const |
| { |
| auto &scopeObj = getObject(id); |
| ASSERT(scopeObj.kind == Object::Kind::Constant); |
| ASSERT(getType(scopeObj.type).sizeInComponents == 1); |
| return scopeObj.constantValue[0]; |
| } |
| |
| void SpirvShader::EvalSpecConstantOp(InsnIterator insn) |
| { |
| auto opcode = static_cast<spv::Op>(insn.word(3)); |
| |
| switch (opcode) |
| { |
| case spv::OpIAdd: |
| case spv::OpISub: |
| case spv::OpIMul: |
| case spv::OpUDiv: |
| case spv::OpSDiv: |
| case spv::OpUMod: |
| case spv::OpSMod: |
| case spv::OpSRem: |
| case spv::OpShiftRightLogical: |
| case spv::OpShiftRightArithmetic: |
| case spv::OpShiftLeftLogical: |
| case spv::OpBitwiseOr: |
| case spv::OpLogicalOr: |
| case spv::OpBitwiseAnd: |
| case spv::OpLogicalAnd: |
| case spv::OpBitwiseXor: |
| case spv::OpLogicalEqual: |
| case spv::OpIEqual: |
| case spv::OpLogicalNotEqual: |
| case spv::OpINotEqual: |
| case spv::OpULessThan: |
| case spv::OpSLessThan: |
| case spv::OpUGreaterThan: |
| case spv::OpSGreaterThan: |
| case spv::OpULessThanEqual: |
| case spv::OpSLessThanEqual: |
| case spv::OpUGreaterThanEqual: |
| case spv::OpSGreaterThanEqual: |
| EvalSpecConstantBinaryOp(insn); |
| break; |
| |
| case spv::OpSConvert: |
| case spv::OpFConvert: |
| case spv::OpUConvert: |
| case spv::OpSNegate: |
| case spv::OpNot: |
| case spv::OpLogicalNot: |
| case spv::OpQuantizeToF16: |
| EvalSpecConstantUnaryOp(insn); |
| break; |
| |
| case spv::OpSelect: |
| { |
| auto &result = CreateConstant(insn); |
| auto const &cond = getObject(insn.word(4)); |
| auto condIsScalar = (getType(cond.type).sizeInComponents == 1); |
| auto const &left = getObject(insn.word(5)); |
| auto const &right = getObject(insn.word(6)); |
| |
| for (auto i = 0u; i < getType(result.type).sizeInComponents; i++) |
| { |
| auto sel = cond.constantValue[condIsScalar ? 0 : i]; |
| result.constantValue[i] = sel ? left.constantValue[i] : right.constantValue[i]; |
| } |
| break; |
| } |
| |
| case spv::OpCompositeExtract: |
| { |
| auto &result = CreateConstant(insn); |
| auto const &compositeObject = getObject(insn.word(4)); |
| auto firstComponent = WalkLiteralAccessChain(compositeObject.type, insn.wordCount() - 5, insn.wordPointer(5)); |
| |
| for (auto i = 0u; i < getType(result.type).sizeInComponents; i++) |
| { |
| result.constantValue[i] = compositeObject.constantValue[firstComponent + i]; |
| } |
| break; |
| } |
| |
| case spv::OpCompositeInsert: |
| { |
| auto &result = CreateConstant(insn); |
| auto const &newPart = getObject(insn.word(4)); |
| auto const &oldObject = getObject(insn.word(5)); |
| auto firstNewComponent = WalkLiteralAccessChain(result.type, insn.wordCount() - 6, insn.wordPointer(6)); |
| |
| // old components before |
| for (auto i = 0u; i < firstNewComponent; i++) |
| { |
| result.constantValue[i] = oldObject.constantValue[i]; |
| } |
| // new part |
| for (auto i = 0u; i < getType(newPart.type).sizeInComponents; i++) |
| { |
| result.constantValue[firstNewComponent + i] = newPart.constantValue[i]; |
| } |
| // old components after |
| for (auto i = firstNewComponent + getType(newPart.type).sizeInComponents; i < getType(result.type).sizeInComponents; i++) |
| { |
| result.constantValue[i] = oldObject.constantValue[i]; |
| } |
| break; |
| } |
| |
| case spv::OpVectorShuffle: |
| { |
| auto &result = CreateConstant(insn); |
| auto const &firstHalf = getObject(insn.word(4)); |
| auto const &secondHalf = getObject(insn.word(5)); |
| |
| for (auto i = 0u; i < getType(result.type).sizeInComponents; i++) |
| { |
| auto selector = insn.word(6 + i); |
| if (selector == static_cast<uint32_t>(-1)) |
| { |
| // Undefined value, we'll use zero |
| result.constantValue[i] = 0; |
| } |
| else if (selector < getType(firstHalf.type).sizeInComponents) |
| { |
| result.constantValue[i] = firstHalf.constantValue[selector]; |
| } |
| else |
| { |
| result.constantValue[i] = secondHalf.constantValue[selector - getType(firstHalf.type).sizeInComponents]; |
| } |
| } |
| break; |
| } |
| |
| default: |
| // Other spec constant ops are possible, but require capabilities that are |
| // not exposed in our Vulkan implementation (eg Kernel), so we should never |
| // get here for correct shaders. |
| UNSUPPORTED("EvalSpecConstantOp op: %s", OpcodeName(opcode).c_str()); |
| } |
| } |
| |
| void SpirvShader::EvalSpecConstantUnaryOp(InsnIterator insn) |
| { |
| auto &result = CreateConstant(insn); |
| |
| auto opcode = static_cast<spv::Op>(insn.word(3)); |
| auto const &lhs = getObject(insn.word(4)); |
| auto size = getType(lhs.type).sizeInComponents; |
| |
| for (auto i = 0u; i < size; i++) |
| { |
| auto &v = result.constantValue[i]; |
| auto l = lhs.constantValue[i]; |
| |
| switch (opcode) |
| { |
| case spv::OpSConvert: |
| case spv::OpFConvert: |
| case spv::OpUConvert: |
| UNREACHABLE("Not possible until we have multiple bit widths"); |
| break; |
| |
| case spv::OpSNegate: |
| v = -(int)l; |
| break; |
| case spv::OpNot: |
| case spv::OpLogicalNot: |
| v = ~l; |
| break; |
| |
| case spv::OpQuantizeToF16: |
| { |
| // Can do this nicer with host code, but want to perfectly mirror the reactor code we emit. |
| auto abs = bit_cast<float>(l & 0x7FFFFFFF); |
| auto sign = l & 0x80000000; |
| auto isZero = abs < 0.000061035f ? ~0u : 0u; |
| auto isInf = abs > 65504.0f ? ~0u : 0u; |
| auto isNaN = (abs != abs) ? ~0u : 0u; |
| auto isInfOrNan = isInf | isNaN; |
| v = l & 0xFFFFE000; |
| v &= ~isZero | 0x80000000; |
| v = sign | (isInfOrNan & 0x7F800000) | (~isInfOrNan & v); |
| v |= isNaN & 0x400000; |
| break; |
| } |
| default: |
| UNREACHABLE("EvalSpecConstantUnaryOp op: %s", OpcodeName(opcode).c_str()); |
| } |
| } |
| } |
| |
| void SpirvShader::EvalSpecConstantBinaryOp(InsnIterator insn) |
| { |
| auto &result = CreateConstant(insn); |
| |
| auto opcode = static_cast<spv::Op>(insn.word(3)); |
| auto const &lhs = getObject(insn.word(4)); |
| auto const &rhs = getObject(insn.word(5)); |
| auto size = getType(lhs.type).sizeInComponents; |
| |
| for (auto i = 0u; i < size; i++) |
| { |
| auto &v = result.constantValue[i]; |
| auto l = lhs.constantValue[i]; |
| auto r = rhs.constantValue[i]; |
| |
| switch (opcode) |
| { |
| case spv::OpIAdd: |
| v = l + r; |
| break; |
| case spv::OpISub: |
| v = l - r; |
| break; |
| case spv::OpIMul: |
| v = l * r; |
| break; |
| case spv::OpUDiv: |
| v = (r == 0) ? 0 : l / r; |
| break; |
| case spv::OpUMod: |
| v = (r == 0) ? 0 : l % r; |
| break; |
| case spv::OpSDiv: |
| if (r == 0) r = UINT32_MAX; |
| if (l == static_cast<uint32_t>(INT32_MIN)) l = UINT32_MAX; |
| v = static_cast<int32_t>(l) / static_cast<int32_t>(r); |
| break; |
| case spv::OpSRem: |
| if (r == 0) r = UINT32_MAX; |
| if (l == static_cast<uint32_t>(INT32_MIN)) l = UINT32_MAX; |
| v = static_cast<int32_t>(l) % static_cast<int32_t>(r); |
| break; |
| case spv::OpSMod: |
| if (r == 0) r = UINT32_MAX; |
| if (l == static_cast<uint32_t>(INT32_MIN)) l = UINT32_MAX; |
| // Test if a signed-multiply would be negative. |
| v = static_cast<int32_t>(l) % static_cast<int32_t>(r); |
| if ((v & 0x80000000) != (r & 0x80000000)) |
| v += r; |
| break; |
| case spv::OpShiftRightLogical: |
| v = l >> r; |
| break; |
| case spv::OpShiftRightArithmetic: |
| v = static_cast<int32_t>(l) >> r; |
| break; |
| case spv::OpShiftLeftLogical: |
| v = l << r; |
| break; |
| case spv::OpBitwiseOr: |
| case spv::OpLogicalOr: |
| v = l | r; |
| break; |
| case spv::OpBitwiseAnd: |
| case spv::OpLogicalAnd: |
| v = l & r; |
| break; |
| case spv::OpBitwiseXor: |
| v = l ^ r; |
| break; |
| case spv::OpLogicalEqual: |
| case spv::OpIEqual: |
| v = (l == r) ? ~0u : 0u; |
| break; |
| case spv::OpLogicalNotEqual: |
| case spv::OpINotEqual: |
| v = (l != r) ? ~0u : 0u; |
| break; |
| case spv::OpULessThan: |
| v = l < r ? ~0u : 0u; |
| break; |
| case spv::OpSLessThan: |
| v = static_cast<int32_t>(l) < static_cast<int32_t>(r) ? ~0u : 0u; |
| break; |
| case spv::OpUGreaterThan: |
| v = l > r ? ~0u : 0u; |
| break; |
| case spv::OpSGreaterThan: |
| v = static_cast<int32_t>(l) > static_cast<int32_t>(r) ? ~0u : 0u; |
| break; |
| case spv::OpULessThanEqual: |
| v = l <= r ? ~0u : 0u; |
| break; |
| case spv::OpSLessThanEqual: |
| v = static_cast<int32_t>(l) <= static_cast<int32_t>(r) ? ~0u : 0u; |
| break; |
| case spv::OpUGreaterThanEqual: |
| v = l >= r ? ~0u : 0u; |
| break; |
| case spv::OpSGreaterThanEqual: |
| v = static_cast<int32_t>(l) >= static_cast<int32_t>(r) ? ~0u : 0u; |
| break; |
| default: |
| UNREACHABLE("EvalSpecConstantBinaryOp op: %s", OpcodeName(opcode).c_str()); |
| } |
| } |
| } |
| |
| void SpirvShader::emitEpilog(SpirvRoutine *routine) const |
| { |
| for (auto insn : *this) |
| { |
| switch (insn.opcode()) |
| { |
| case spv::OpVariable: |
| { |
| Object::ID resultId = insn.word(2); |
| auto &object = getObject(resultId); |
| auto &objectTy = getType(object.type); |
| if (object.kind == Object::Kind::InterfaceVariable && objectTy.storageClass == spv::StorageClassOutput) |
| { |
| auto &dst = routine->getVariable(resultId); |
| int offset = 0; |
| VisitInterface(resultId, |
| [&](Decorations const &d, AttribType type) { |
| auto scalarSlot = d.Location << 2 | d.Component; |
| routine->outputs[scalarSlot] = dst[offset++]; |
| }); |
| } |
| break; |
| } |
| default: |
| break; |
| } |
| } |
| |
| // Clear phis that are no longer used. This serves two purposes: |
| // (1) The phi rr::Variables are destructed, preventing pointless |
| // materialization. |
| // (2) Frees memory that will never be used again. |
| routine->phis.clear(); |
| } |
| |
| SpirvShader::Block::Block(InsnIterator begin, InsnIterator end) : begin_(begin), end_(end) |
| { |
| // Default to a Simple, this may change later. |
| kind = Block::Simple; |
| |
| // Walk the instructions to find the last two of the block. |
| InsnIterator insns[2]; |
| for (auto insn : *this) |
| { |
| insns[0] = insns[1]; |
| insns[1] = insn; |
| } |
| |
| switch (insns[1].opcode()) |
| { |
| case spv::OpBranch: |
| branchInstruction = insns[1]; |
| outs.emplace(Block::ID(branchInstruction.word(1))); |
| |
| switch (insns[0].opcode()) |
| { |
| case spv::OpLoopMerge: |
| kind = Loop; |
| mergeInstruction = insns[0]; |
| mergeBlock = Block::ID(mergeInstruction.word(1)); |
| continueTarget = Block::ID(mergeInstruction.word(2)); |
| break; |
| |
| default: |
| kind = Block::Simple; |
| break; |
| } |
| break; |
| |
| case spv::OpBranchConditional: |
| branchInstruction = insns[1]; |
| outs.emplace(Block::ID(branchInstruction.word(2))); |
| outs.emplace(Block::ID(branchInstruction.word(3))); |
| |
| switch (insns[0].opcode()) |
| { |
| case spv::OpSelectionMerge: |
| kind = StructuredBranchConditional; |
| mergeInstruction = insns[0]; |
| mergeBlock = Block::ID(mergeInstruction.word(1)); |
| break; |
| |
| case spv::OpLoopMerge: |
| kind = Loop; |
| mergeInstruction = insns[0]; |
| mergeBlock = Block::ID(mergeInstruction.word(1)); |
| continueTarget = Block::ID(mergeInstruction.word(2)); |
| break; |
| |
| default: |
| kind = UnstructuredBranchConditional; |
| break; |
| } |
| break; |
| |
| case spv::OpSwitch: |
| branchInstruction = insns[1]; |
| outs.emplace(Block::ID(branchInstruction.word(2))); |
| for (uint32_t w = 4; w < branchInstruction.wordCount(); w += 2) |
| { |
| outs.emplace(Block::ID(branchInstruction.word(w))); |
| } |
| |
| switch (insns[0].opcode()) |
| { |
| case spv::OpSelectionMerge: |
| kind = StructuredSwitch; |
| mergeInstruction = insns[0]; |
| mergeBlock = Block::ID(mergeInstruction.word(1)); |
| break; |
| |
| default: |
| kind = UnstructuredSwitch; |
| break; |
| } |
| break; |
| |
| default: |
| break; |
| } |
| } |
| |
| void SpirvShader::Function::TraverseReachableBlocks(Block::ID id, SpirvShader::Block::Set& reachable) const |
| { |
| if (reachable.count(id) == 0) |
| { |
| reachable.emplace(id); |
| for (auto out : getBlock(id).outs) |
| { |
| TraverseReachableBlocks(out, reachable); |
| } |
| } |
| } |
| |
| void SpirvShader::Function::AssignBlockFields() |
| { |
| Block::Set reachable; |
| TraverseReachableBlocks(entry, reachable); |
| |
| for (auto &it : blocks) |
| { |
| auto &blockId = it.first; |
| auto &block = it.second; |
| if (reachable.count(blockId) > 0) |
| { |
| for (auto &outId : it.second.outs) |
| { |
| auto outIt = blocks.find(outId); |
| ASSERT_MSG(outIt != blocks.end(), "Block %d has a non-existent out %d", blockId.value(), outId.value()); |
| auto &out = outIt->second; |
| out.ins.emplace(blockId); |
| } |
| if (block.kind == Block::Loop) |
| { |
| auto mergeIt = blocks.find(block.mergeBlock); |
| ASSERT_MSG(mergeIt != blocks.end(), "Loop block %d has a non-existent merge block %d", blockId.value(), block.mergeBlock.value()); |
| mergeIt->second.isLoopMerge = true; |
| } |
| } |
| } |
| } |
| |
| void SpirvShader::Function::ForeachBlockDependency(Block::ID blockId, std::function<void(Block::ID)> f) const |
| { |
| auto block = getBlock(blockId); |
| for (auto dep : block.ins) |
| { |
| if (block.kind != Block::Loop || // if not a loop... |
| !ExistsPath(blockId, dep, block.mergeBlock)) // or a loop and not a loop back edge |
| { |
| f(dep); |
| } |
| } |
| } |
| |
| bool SpirvShader::Function::ExistsPath(Block::ID from, Block::ID to, Block::ID notPassingThrough) const |
| { |
| // TODO: Optimize: This can be cached on the block. |
| Block::Set seen; |
| seen.emplace(notPassingThrough); |
| |
| std::queue<Block::ID> pending; |
| pending.emplace(from); |
| |
| while (pending.size() > 0) |
| { |
| auto id = pending.front(); |
| pending.pop(); |
| for (auto out : getBlock(id).outs) |
| { |
| if (seen.count(out) != 0) { continue; } |
| if (out == to) { return true; } |
| pending.emplace(out); |
| } |
| seen.emplace(id); |
| } |
| |
| return false; |
| } |
| |
| void SpirvShader::EmitState::addOutputActiveLaneMaskEdge(Block::ID to, RValue<SIMD::Int> mask) |
| { |
| addActiveLaneMaskEdge(block, to, mask & activeLaneMask()); |
| } |
| |
| void SpirvShader::EmitState::addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask) |
| { |
| auto edge = Block::Edge{from, to}; |
| auto it = edgeActiveLaneMasks.find(edge); |
| if (it == edgeActiveLaneMasks.end()) |
| { |
| edgeActiveLaneMasks.emplace(edge, mask); |
| } |
| else |
| { |
| auto combined = it->second | mask; |
| edgeActiveLaneMasks.erase(edge); |
| edgeActiveLaneMasks.emplace(edge, combined); |
| } |
| } |
| |
| RValue<SIMD::Int> SpirvShader::GetActiveLaneMaskEdge(EmitState *state, Block::ID from, Block::ID to) const |
| { |
| auto edge = Block::Edge{from, to}; |
| auto it = state->edgeActiveLaneMasks.find(edge); |
| ASSERT_MSG(it != state->edgeActiveLaneMasks.end(), "Could not find edge %d -> %d", from.value(), to.value()); |
| return it->second; |
| } |
| |
| VkShaderStageFlagBits SpirvShader::executionModelToStage(spv::ExecutionModel model) |
| { |
| switch (model) |
| { |
| case spv::ExecutionModelVertex: return VK_SHADER_STAGE_VERTEX_BIT; |
| // case spv::ExecutionModelTessellationControl: return VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; |
| // case spv::ExecutionModelTessellationEvaluation: return VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT; |
| // case spv::ExecutionModelGeometry: return VK_SHADER_STAGE_GEOMETRY_BIT; |
| case spv::ExecutionModelFragment: return VK_SHADER_STAGE_FRAGMENT_BIT; |
| case spv::ExecutionModelGLCompute: return VK_SHADER_STAGE_COMPUTE_BIT; |
| // case spv::ExecutionModelKernel: return VkShaderStageFlagBits(0); // Not supported by vulkan. |
| // case spv::ExecutionModelTaskNV: return VK_SHADER_STAGE_TASK_BIT_NV; |
| // case spv::ExecutionModelMeshNV: return VK_SHADER_STAGE_MESH_BIT_NV; |
| // case spv::ExecutionModelRayGenerationNV: return VK_SHADER_STAGE_RAYGEN_BIT_NV; |
| // case spv::ExecutionModelIntersectionNV: return VK_SHADER_STAGE_INTERSECTION_BIT_NV; |
| // case spv::ExecutionModelAnyHitNV: return VK_SHADER_STAGE_ANY_HIT_BIT_NV; |
| // case spv::ExecutionModelClosestHitNV: return VK_SHADER_STAGE_CLOSEST_HIT_BIT_NV; |
| // case spv::ExecutionModelMissNV: return VK_SHADER_STAGE_MISS_BIT_NV; |
| // case spv::ExecutionModelCallableNV: return VK_SHADER_STAGE_CALLABLE_BIT_NV; |
| default: |
| UNSUPPORTED("ExecutionModel: %d", int(model)); |
| return VkShaderStageFlagBits(0); |
| } |
| } |
| |
| SpirvShader::GenericValue::GenericValue(SpirvShader const *shader, EmitState const *state, SpirvShader::Object::ID objId) : |
| obj(shader->getObject(objId)), |
| intermediate(obj.kind == SpirvShader::Object::Kind::Intermediate ? &state->getIntermediate(objId) : nullptr), |
| type(obj.type) {} |
| |
| SpirvRoutine::SpirvRoutine(vk::PipelineLayout const *pipelineLayout) : |
| pipelineLayout(pipelineLayout) |
| { |
| } |
| |
| void SpirvRoutine::setImmutableInputBuiltins(SpirvShader const *shader) |
| { |
| setInputBuiltin(shader, spv::BuiltInSubgroupLocalInvocationId, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 1); |
| value[builtin.FirstComponent] = As<SIMD::Float>(SIMD::Int(0, 1, 2, 3)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInSubgroupEqMask, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 4); |
| value[builtin.FirstComponent + 0] = As<SIMD::Float>(SIMD::Int(1, 2, 4, 8)); |
| value[builtin.FirstComponent + 1] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 2] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 3] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInSubgroupGeMask, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 4); |
| value[builtin.FirstComponent + 0] = As<SIMD::Float>(SIMD::Int(15, 14, 12, 8)); |
| value[builtin.FirstComponent + 1] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 2] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 3] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInSubgroupGtMask, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 4); |
| value[builtin.FirstComponent + 0] = As<SIMD::Float>(SIMD::Int(14, 12, 8, 0)); |
| value[builtin.FirstComponent + 1] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 2] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 3] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInSubgroupLeMask, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 4); |
| value[builtin.FirstComponent + 0] = As<SIMD::Float>(SIMD::Int(1, 3, 7, 15)); |
| value[builtin.FirstComponent + 1] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 2] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 3] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInSubgroupLtMask, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 4); |
| value[builtin.FirstComponent + 0] = As<SIMD::Float>(SIMD::Int(0, 1, 3, 7)); |
| value[builtin.FirstComponent + 1] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 2] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| value[builtin.FirstComponent + 3] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| |
| setInputBuiltin(shader, spv::BuiltInDeviceIndex, [&](const SpirvShader::BuiltinMapping& builtin, Array<SIMD::Float>& value) |
| { |
| ASSERT(builtin.SizeInComponents == 1); |
| // Only a single physical device is supported. |
| value[builtin.FirstComponent] = As<SIMD::Float>(SIMD::Int(0, 0, 0, 0)); |
| }); |
| } |
| } |