Assert attachment format supports blending when blending is enabled

The Vulkan spec states that "blendEnable [...] must be VK_FALSE if the
attached image’s format features does not contain
VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT"

Previously we would silently ignore blending of integer formats. It was
determined to be undefined behavior in https://gitlab.khronos.org/vulkan/vulkan/issues/2098

Bug: b/155147929
Change-Id: I01500d8c39d7f2c9a484944b4a93c6004e938c05
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/45708
Presubmit-Ready: Nicolas Capens <nicolascapens@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>
Tested-by: Nicolas Capens <nicolascapens@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
diff --git a/src/Device/PixelProcessor.hpp b/src/Device/PixelProcessor.hpp
index ba190a6..3db3b5f 100644
--- a/src/Device/PixelProcessor.hpp
+++ b/src/Device/PixelProcessor.hpp
@@ -18,6 +18,7 @@
 #include "Context.hpp"
 #include "Memset.hpp"
 #include "RoutineCache.hpp"
+#include "Vulkan/VkFormat.hpp"
 
 #include <memory>
 
@@ -84,7 +85,7 @@
 		BlendState blendState[RENDERTARGETS];
 
 		unsigned int colorWriteMask;
-		VkFormat targetFormat[RENDERTARGETS];
+		vk::Format targetFormat[RENDERTARGETS];
 		unsigned int multiSampleCount;
 		unsigned int multiSampleMask;
 		bool enableMultiSampling;
diff --git a/src/Pipeline/PixelRoutine.cpp b/src/Pipeline/PixelRoutine.cpp
index 5c3f0c9..9a7a61c 100644
--- a/src/Pipeline/PixelRoutine.cpp
+++ b/src/Pipeline/PixelRoutine.cpp
@@ -1098,7 +1098,7 @@
 		}
 		break;
 		default:
-			UNSUPPORTED("VkFormat %d", state.targetFormat[index]);
+			UNSUPPORTED("VkFormat %d", int(state.targetFormat[index]));
 	}
 
 	if(isSRGB(index))
@@ -1109,11 +1109,13 @@
 
 void PixelRoutine::alphaBlend(int index, const Pointer<Byte> &cBuffer, Vector4s &current, const Int &x)
 {
-	if(!state.blendState[index].alphaBlendEnable || vk::Format(state.targetFormat[index]).isUnnormalizedInteger())
+	if(!state.blendState[index].alphaBlendEnable)
 	{
 		return;
 	}
 
+	ASSERT(state.targetFormat[index].supportsColorAttachmentBlend());
+
 	Vector4s pixel;
 	readPixel(index, cBuffer, x, pixel);
 
@@ -1869,11 +1871,14 @@
 
 void PixelRoutine::alphaBlend(int index, const Pointer<Byte> &cBuffer, Vector4f &oC, const Int &x)
 {
-	if(!state.blendState[index].alphaBlendEnable || vk::Format(state.targetFormat[index]).isUnnormalizedInteger())
+	if(!state.blendState[index].alphaBlendEnable)
 	{
 		return;
 	}
 
+	vk::Format format = state.targetFormat[index];
+	ASSERT(format.supportsColorAttachmentBlend());
+
 	Pointer<Byte> buffer = cBuffer;
 	Int pitchB = *Pointer<Int>(data + OFFSET(DrawData, colorPitchB[index]));
 
@@ -1888,7 +1893,6 @@
 	Short4 c23;
 
 	Float4 one;
-	vk::Format format(state.targetFormat[index]);
 	if(format.isFloatFormat())
 	{
 		one = Float4(1.0f);
diff --git a/src/Vulkan/VkFormat.cpp b/src/Vulkan/VkFormat.cpp
index 3053716..33fc133 100644
--- a/src/Vulkan/VkFormat.cpp
+++ b/src/Vulkan/VkFormat.cpp
@@ -2228,6 +2228,37 @@
 	return sw::float4(1.0f, 1.0f, 1.0f, 1.0f);
 }
 
+bool Format::supportsColorAttachmentBlend() const
+{
+	switch(format)
+	{
+		// Vulkan 1.1 mandatory
+		case VK_FORMAT_R5G6B5_UNORM_PACK16:
+		case VK_FORMAT_A1R5G5B5_UNORM_PACK16:
+		case VK_FORMAT_R8_UNORM:
+		case VK_FORMAT_R8G8_UNORM:
+		case VK_FORMAT_R8G8B8A8_UNORM:
+		case VK_FORMAT_R8G8B8A8_SRGB:
+		case VK_FORMAT_B8G8R8A8_UNORM:
+		case VK_FORMAT_B8G8R8A8_SRGB:
+		case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
+		case VK_FORMAT_A8B8G8R8_SRGB_PACK32:
+		case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
+		case VK_FORMAT_R16_SFLOAT:
+		case VK_FORMAT_R16G16_SFLOAT:
+		case VK_FORMAT_R16G16B16A16_SFLOAT:
+		// Optional
+		case VK_FORMAT_A2R10G10B10_UNORM_PACK32:
+		case VK_FORMAT_R32_SFLOAT:
+		case VK_FORMAT_R32G32_SFLOAT:
+		case VK_FORMAT_R32G32B32A32_SFLOAT:
+		case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
+			return true;
+		default:
+			return false;
+	}
+}
+
 bool Format::has16bitPackedTextureFormat() const
 {
 	if(bytes() != 2)
diff --git a/src/Vulkan/VkFormat.hpp b/src/Vulkan/VkFormat.hpp
index d1334b2..128e38b 100644
--- a/src/Vulkan/VkFormat.hpp
+++ b/src/Vulkan/VkFormat.hpp
@@ -60,6 +60,8 @@
 
 	sw::float4 getScale() const;
 
+	bool supportsColorAttachmentBlend() const;
+
 	// Texture sampling utilities
 	bool has16bitPackedTextureFormat() const;
 	bool has8bitTextureComponents() const;
diff --git a/src/Vulkan/VkPhysicalDevice.cpp b/src/Vulkan/VkPhysicalDevice.cpp
index 0ac4a5c..77cefb6 100644
--- a/src/Vulkan/VkPhysicalDevice.cpp
+++ b/src/Vulkan/VkPhysicalDevice.cpp
@@ -685,9 +685,6 @@
 		case VK_FORMAT_R32G32_SFLOAT:
 		case VK_FORMAT_R32G32B32A32_SFLOAT:
 		case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
-			pFormatProperties->optimalTilingFeatures |=
-			    VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT;
-			// [[fallthrough]]
 		case VK_FORMAT_R8_UINT:
 		case VK_FORMAT_R8_SINT:
 		case VK_FORMAT_R8G8_UINT:
@@ -725,6 +722,12 @@
 			break;
 	}
 
+	if(format.supportsColorAttachmentBlend())
+	{
+		pFormatProperties->optimalTilingFeatures |=
+		    VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT;
+	}
+
 	switch(format)
 	{
 		case VK_FORMAT_R8_UNORM: