Validate allocation size in PoolAllocator

BUG=chromium:8662046

Change-Id: I3f8371b586b3a497c871a56bdf3c2b489e383b7d
Reviewed-on: https://swiftshader-review.googlesource.com/1360
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Nicolas Capens <nicolascapens@google.com>

src/OpenGL/compiler/PoolAlloc.cpp

diff --git a/src/OpenGL/compiler/PoolAlloc.cpp b/src/OpenGL/compiler/PoolAlloc.cpp
index c0ae1ec..9cdbeaa 100644
--- a/src/OpenGL/compiler/PoolAlloc.cpp
+++ b/src/OpenGL/compiler/PoolAlloc.cpp
@@ -227,24 +227,27 @@
 
 void* TPoolAllocator::allocate(size_t numBytes)
 {
-    // If we are using guard blocks, all allocations are bracketed by
-    // them: [guardblock][allocation][guardblock].  numBytes is how
-    // much memory the caller asked for.  allocationSize is the total
-    // size including guard blocks.  In release build,
-    // guardBlockSize=0 and this all gets optimized away.
-    size_t allocationSize = TAllocation::allocationSize(numBytes);
-    
     //
     // Just keep some interesting statistics.
     //
     ++numCalls;
     totalBytes += numBytes;
 
+    // If we are using guard blocks, all allocations are bracketed by
+    // them: [guardblock][allocation][guardblock].  numBytes is how
+    // much memory the caller asked for.  allocationSize is the total
+    // size including guard blocks.  In release build,
+    // guardBlockSize=0 and this all gets optimized away.
+    size_t allocationSize = TAllocation::allocationSize(numBytes);
+    // Detect integer overflow.
+    if (allocationSize < numBytes)
+        return 0;
+
     //
     // Do the allocation, most likely case first, for efficiency.
     // This step could be moved to be inline sometime.
     //
-    if (currentPageOffset + allocationSize <= pageSize) {
+    if (allocationSize <= pageSize - currentPageOffset) {
         //
         // Safe to allocate from currentPageOffset.
         //
@@ -255,12 +258,16 @@
         return initializeAllocation(inUseList, memory, numBytes);
     }
 
-    if (allocationSize + headerSkip > pageSize) {
+    if (allocationSize > pageSize - headerSkip) {
         //
         // Do a multi-page allocation.  Don't mix these with the others.
         // The OS is efficient and allocating and free-ing multiple pages.
         //
         size_t numBytesToAlloc = allocationSize + headerSkip;
+        // Detect integer overflow.
+        if (numBytesToAlloc < allocationSize)
+            return 0;
+
         tHeader* memory = reinterpret_cast<tHeader*>(::new char[numBytesToAlloc]);
         if (memory == 0)
             return 0;

src/Radiance/compiler/PoolAlloc.cpp

diff --git a/src/Radiance/compiler/PoolAlloc.cpp b/src/Radiance/compiler/PoolAlloc.cpp
index c0ae1ec..9cdbeaa 100644
--- a/src/Radiance/compiler/PoolAlloc.cpp
+++ b/src/Radiance/compiler/PoolAlloc.cpp
@@ -227,24 +227,27 @@
 
 void* TPoolAllocator::allocate(size_t numBytes)
 {
-    // If we are using guard blocks, all allocations are bracketed by
-    // them: [guardblock][allocation][guardblock].  numBytes is how
-    // much memory the caller asked for.  allocationSize is the total
-    // size including guard blocks.  In release build,
-    // guardBlockSize=0 and this all gets optimized away.
-    size_t allocationSize = TAllocation::allocationSize(numBytes);
-    
     //
     // Just keep some interesting statistics.
     //
     ++numCalls;
     totalBytes += numBytes;
 
+    // If we are using guard blocks, all allocations are bracketed by
+    // them: [guardblock][allocation][guardblock].  numBytes is how
+    // much memory the caller asked for.  allocationSize is the total
+    // size including guard blocks.  In release build,
+    // guardBlockSize=0 and this all gets optimized away.
+    size_t allocationSize = TAllocation::allocationSize(numBytes);
+    // Detect integer overflow.
+    if (allocationSize < numBytes)
+        return 0;
+
     //
     // Do the allocation, most likely case first, for efficiency.
     // This step could be moved to be inline sometime.
     //
-    if (currentPageOffset + allocationSize <= pageSize) {
+    if (allocationSize <= pageSize - currentPageOffset) {
         //
         // Safe to allocate from currentPageOffset.
         //
@@ -255,12 +258,16 @@
         return initializeAllocation(inUseList, memory, numBytes);
     }
 
-    if (allocationSize + headerSkip > pageSize) {
+    if (allocationSize > pageSize - headerSkip) {
         //
         // Do a multi-page allocation.  Don't mix these with the others.
         // The OS is efficient and allocating and free-ing multiple pages.
         //
         size_t numBytesToAlloc = allocationSize + headerSkip;
+        // Detect integer overflow.
+        if (numBytesToAlloc < allocationSize)
+            return 0;
+
         tHeader* memory = reinterpret_cast<tHeader*>(::new char[numBytesToAlloc]);
         if (memory == 0)
             return 0;