Refactor image sample normalization

Previously the normalization factor (i.e. the 16-bit integer value
representing 1.0 for normalized texel formats) was computed even for
formats that don't require normalization after filtering (e.g floating-
point formats, integer formats, and D16_UNORM if depth compare is
performed by the sampler). This caused UndefinedBehaviorSanitizer
errors if components have more than 16 bits. Note these were benign
since the normalization factors computed by it weren't actually used.

This change performs the normalization in one spot after filtering, and
only computes the scaling factors when needed.

Bug: b/204709464
Change-Id: I634854e167f85af001f551e5b805bd6a6201fa29
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/60748
Tested-by: Nicolas Capens <nicolascapens@google.com>
Reviewed-by: Alexis Hétu <sugoi@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>

src/Pipeline/SamplerCore.cpp

diff --git a/src/Pipeline/SamplerCore.cpp b/src/Pipeline/SamplerCore.cpp
index b79532d..35d2731 100644
--- a/src/Pipeline/SamplerCore.cpp
+++ b/src/Pipeline/SamplerCore.cpp
@@ -142,21 +142,11 @@
 	bool use32BitFiltering = hasFloatTexture() || hasUnnormalizedIntegerTexture() || force32BitFiltering ||
 	                         state.isCube() || state.unnormalizedCoordinates || state.compareEnable ||
 	                         borderModeActive() || (function == Gather) || (function == Fetch);
-	const sw::float4 compScale = getComponentScale();
-	int gatherComponent = (function == Gather) ? getGatherComponent() : 0;
 	int numComponents = (function == Gather) ? 4 : textureComponentCount();
 
 	if(use32BitFiltering)
 	{
 		c = sampleFloatFilter(texture, u, v, w, a, dRef, offset, sample, lod, anisotropy, uDelta, vDelta, function);
-
-		if(!hasFloatTexture() && !hasUnnormalizedIntegerTexture() && !state.compareEnable)
-		{
-			for(int component = 0; component < numComponents; component++)
-			{
-				c[component] *= Float4(1.0f / compScale[(function == Gather) ? gatherComponent : component]);
-			}
-		}
 	}
 	else  // 16-bit filtering.
 	{
@@ -172,8 +162,17 @@
 			{
 				c[component] = Float4(cs[component]);
 			}
+		}
+	}
 
-			c[component] *= Float4(1.0f / compScale[(function == Gather) ? gatherComponent : component]);
+	if(hasNormalizedFormat() && !state.compareEnable)
+	{
+		sw::float4 scale = getComponentScale();
+
+		for(int component = 0; component < numComponents; component++)
+		{
+			int texelComponent = (function == Gather) ? getGatherComponent() : component;
+			c[component] *= Float4(1.0f / scale[texelComponent]);
 		}
 	}
 
@@ -2072,7 +2071,7 @@
 {
 	Vector4i border;
 
-	const bool scaled = !hasFloatTexture() && !hasUnnormalizedIntegerTexture() && !state.compareEnable;
+	const bool scaled = hasNormalizedFormat() && !state.compareEnable;
 	const sw::float4 scaleComp = scaled ? getComponentScale() : sw::float4(1.0f, 1.0f, 1.0f, 1.0f);
 
 	switch(state.border)
@@ -2502,6 +2501,11 @@
 	c = Insert(c, *Pointer<Short>(LUT + 2 * Int(Extract(c, 3))), 3);
 }
 
+bool SamplerCore::hasNormalizedFormat() const
+{
+	return state.textureFormat.isSignedNormalized() || state.textureFormat.isUnsignedNormalized();
+}
+
 bool SamplerCore::hasFloatTexture() const
 {
 	return state.textureFormat.isFloatFormat();

src/Pipeline/SamplerCore.hpp

diff --git a/src/Pipeline/SamplerCore.hpp b/src/Pipeline/SamplerCore.hpp
index 27e89bb..cda7ca2 100644
--- a/src/Pipeline/SamplerCore.hpp
+++ b/src/Pipeline/SamplerCore.hpp
@@ -96,6 +96,7 @@
 	Int4 computeFilterOffset(Float &lod);
 	void sRGBtoLinearFF00(Short4 &c);
 
+	bool hasNormalizedFormat() const;
 	bool hasFloatTexture() const;
 	bool hasUnnormalizedIntegerTexture() const;
 	bool hasUnsignedTextureComponent(int component) const;