Fixed recursion analysis

Recursion analysis was broken, because assigning
the error value UINT_MAX was then automatically
increased by 1 at the caller site, resulting in
a 0, and recursions would go undetected.

Change-Id: I8ab9990c12d827d8eac2d6084f9170096ad2aef2
Reviewed-on: https://swiftshader-review.googlesource.com/3552
Tested-by: Alexis Hétu <sugoi@google.com>
Reviewed-by: Nicolas Capens <capn@google.com>
diff --git a/src/OpenGL/compiler/AnalyzeCallDepth.cpp b/src/OpenGL/compiler/AnalyzeCallDepth.cpp
index 94c3540..d4f30fa 100644
--- a/src/OpenGL/compiler/AnalyzeCallDepth.cpp
+++ b/src/OpenGL/compiler/AnalyzeCallDepth.cpp
@@ -40,21 +40,24 @@
 

     for(size_t i = 0; i < callees.size(); i++)

 	{

+		unsigned int calleeDepth = 0;

         switch(callees[i]->visit)

 		{

         case InVisit:

             // Cycle detected (recursion)

             return UINT_MAX;

         case PostVisit:

-			callDepth = std::max(callDepth, 1 + callees[i]->getLastDepth());

+			calleeDepth = callees[i]->getLastDepth();

             break;

         case PreVisit:

-			callDepth = std::max(callDepth, 1 + callees[i]->analyzeCallDepth(analyzeCallDepth));

+			calleeDepth = callees[i]->analyzeCallDepth(analyzeCallDepth);

 			break;

         default:

             UNREACHABLE(callees[i]->visit);

             break;

         }

+		if(calleeDepth != UINT_MAX) ++calleeDepth;

+		callDepth = std::max(callDepth, calleeDepth);

     }

 

     visit = PostVisit;

@@ -156,11 +159,13 @@
 		return 0;

 	}

 

-    unsigned int depth = 1 + main->analyzeCallDepth(this);

+    unsigned int depth = main->analyzeCallDepth(this);

+	if(depth != UINT_MAX) ++depth;

 

 	for(FunctionSet::iterator globalCall = globalFunctionCalls.begin(); globalCall != globalFunctionCalls.end(); globalCall++)

 	{

-		unsigned int globalDepth = 1 + (*globalCall)->analyzeCallDepth(this);

+		unsigned int globalDepth = (*globalCall)->analyzeCallDepth(this);

+		if(globalDepth != UINT_MAX) ++globalDepth;

 

 		if(globalDepth > depth)

 		{