Fix integer overflow in image size calculation

GLsizei is historically defined as a 32-bit signed integer, which is
not allowed to be negative. But doing arithmetic with them can lead to
overflows producing undefined values, including negative ones.

Use size_t instead for these image data size calculations. It's defined
as an unsigned integer large enough to fit valid memory buffer sizes.
In practice that's 32-bit on 32-bit platforms, and 64-bit on 64-bit
ones, which is large enough to avoid overflow in the OutOfMemory test.

Bug: b/140421004
Change-Id: Id05dd64d470f579455fdc9925b0ab303ecaf37fb
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/36968
Tested-by: Nicolas Capens <nicolascapens@google.com>
Reviewed-by: Alexis Hétu <sugoi@google.com>

src/OpenGL/libGLESv2/Context.cpp

diff --git a/src/OpenGL/libGLESv2/Context.cpp b/src/OpenGL/libGLESv2/Context.cpp
index e03abc2..35ec488 100644
--- a/src/OpenGL/libGLESv2/Context.cpp
+++ b/src/OpenGL/libGLESv2/Context.cpp
@@ -1567,15 +1567,16 @@
 	return mState.genericUniformBuffer;
 }
 
-GLsizei Context::getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const
+size_t Context::getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const
 {
 	GLsizei inputWidth = (mState.unpackParameters.rowLength == 0) ? width : mState.unpackParameters.rowLength;
 	GLsizei inputPitch = gl::ComputePitch(inputWidth, format, type, mState.unpackParameters.alignment);
 	GLsizei inputHeight = (mState.unpackParameters.imageHeight == 0) ? height : mState.unpackParameters.imageHeight;
-	return inputPitch * inputHeight * depth;
+
+	return static_cast<size_t>(inputPitch) * inputHeight * depth;
 }
 
-GLenum Context::getPixels(const GLvoid **pixels, GLenum type, GLsizei imageSize) const
+GLenum Context::getPixels(const GLvoid **pixels, GLenum type, size_t imageSize) const
 {
 	if(mState.pixelUnpackBuffer)
 	{
@@ -1598,7 +1599,7 @@
 			return GL_INVALID_OPERATION;
 		}
 
-		if(mState.pixelUnpackBuffer->size() - offset < static_cast<size_t>(imageSize))
+		if(mState.pixelUnpackBuffer->size() - offset < imageSize)
 		{
 			return GL_INVALID_OPERATION;
 		}

src/OpenGL/libGLESv2/Context.h

diff --git a/src/OpenGL/libGLESv2/Context.h b/src/OpenGL/libGLESv2/Context.h
index 4b37709..5eb28b5 100644
--- a/src/OpenGL/libGLESv2/Context.h
+++ b/src/OpenGL/libGLESv2/Context.h
@@ -637,8 +637,8 @@
 	Buffer *getPixelPackBuffer() const;
 	Buffer *getPixelUnpackBuffer() const;
 	Buffer *getGenericUniformBuffer() const;
-	GLsizei getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const;
-	GLenum getPixels(const GLvoid **data, GLenum type, GLsizei imageSize) const;
+	size_t getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const;
+	GLenum getPixels(const GLvoid **data, GLenum type, size_t imageSize) const;
 	bool getBuffer(GLenum target, es2::Buffer **buffer) const;
 	Program *getCurrentProgram() const;
 	Texture *getTargetTexture(GLenum target) const;