tests/regres/deqp/deqp.go - SwiftShader.git - Git at Google

 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // Package deqp provides functions for running dEQP, as well as loading and storing the results.
 package deqp

 import (
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"

 	"swiftshader.googlesource.com/SwiftShader/tests/regres/cov"
 	"swiftshader.googlesource.com/SwiftShader/tests/regres/shell"
 	"swiftshader.googlesource.com/SwiftShader/tests/regres/testlist"
 	"swiftshader.googlesource.com/SwiftShader/tests/regres/util"
 )

 const dataVersion = 1

 var (
 	// Regular expression to parse the output of a dEQP test.
 	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
 	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
 	unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNSUPPORTED()
 	unsupportedRE = regexp.MustCompile(`[^\n]*UNSUPPORTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNREACHABLE()
 	unreachableRE = regexp.MustCompile(`[^\n]*UNREACHABLE:[^\n]*`)
 	// Regular expression to parse a test that failed due to ASSERT()
 	assertRE = regexp.MustCompile(`[^\n]*ASSERT\([^\)]*\)[^\n]*`)
 	// Regular expression to parse a test that failed due to ABORT()
 	abortRE = regexp.MustCompile(`[^\n]*ABORT:[^\n]*`)
 	// Regular expression to parse individual test names and output
 	caseOutputRE = regexp.MustCompile("Test case '([^']*)'..")
 )

 // Config contains the inputs required for running dEQP on a group of test lists.
 type Config struct {
 	ExeEgl           string
 	ExeGles2         string
 	ExeGles3         string
 	ExeVulkan        string
 	TempDir          string // Directory for temporary log files, coverage output.
 	TestLists        testlist.Lists
 	Env              []string
 	LogReplacements  map[string]string
 	NumParallelTests int
 	MaxTestsPerProc  int
 	CoverageEnv      *cov.Env
 	TestTimeout      time.Duration
 	ValidationLayer  bool
 }

 // Results holds the results of tests across all APIs.
 // The Results structure may be serialized to cache results.
 type Results struct {
 	Version  int
 	Error    string
 	Tests    map[string]TestResult
 	Coverage *cov.Tree
 	Duration time.Duration
 }

 // TestResult holds the results of a single dEQP test.
 type TestResult struct {
 	Test      string
 	Status    testlist.Status
 	TimeTaken time.Duration
 	Err       string `json:",omitempty"`
 	Coverage  *cov.Coverage
 }

 func (r TestResult) String() string {
 	if r.Err != "" {
 		return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
 	}
 	return fmt.Sprintf("%s: %s", r.Test, r.Status)
 }

 // LoadResults loads cached test results from disk.
 func LoadResults(path string) (*Results, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open '%s' for loading test results: %w", path, err)
 	}
 	defer f.Close()

 	var out Results
 	if err := json.NewDecoder(f).Decode(&out); err != nil {
 		return nil, err
 	}
 	if out.Version != dataVersion {
 		return nil, errors.New("Data is from an old version")
 	}
 	return &out, nil
 }

 // Save saves (caches) test results to disk.
 func (r *Results) Save(path string) error {
 	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
 		return fmt.Errorf("failed to make '%s' for saving test results: %w", filepath.Dir(path), err)
 	}

 	f, err := os.Create(path)
 	if err != nil {
 		return fmt.Errorf("failed to open '%s' for saving test results: %w", path, err)
 	}
 	defer f.Close()

 	enc := json.NewEncoder(f)
 	enc.SetIndent("", "  ")
 	if err := enc.Encode(r); err != nil {
 		return fmt.Errorf("failed to encode test results: %w", err)
 	}

 	return nil
 }

 // Run runs all the tests.
 func (c *Config) Run() (*Results, error) {
 	start := time.Now()

 	if c.TempDir == "" {
 		dir, err := ioutil.TempDir("", "deqp")
 		if err != nil {
 			return nil, fmt.Errorf("failed to generate temporary directory: %w", err)
 		}
 		c.TempDir = dir
 	}

 	// Wait group that completes once all the tests have finished.
 	wg := sync.WaitGroup{}
 	results := make(chan TestResult, 256)

 	numTests := 0

 	goroutineIndex := 0

 	// For each API that we are testing
 	for _, list := range c.TestLists {
 		// Resolve the test runner
 		exe, supportsCoverage := "", false

 		switch list.API {
 		case testlist.EGL:
 			exe = c.ExeEgl
 		case testlist.GLES2:
 			exe = c.ExeGles2
 		case testlist.GLES3:
 			exe = c.ExeGles3
 		case testlist.Vulkan:
 			exe, supportsCoverage = c.ExeVulkan, true
 		default:
 			return nil, fmt.Errorf("Unknown API '%v'", list.API)
 		}
 		if !util.IsFile(exe) {
 			return nil, fmt.Errorf("failed to find dEQP executable at '%s'", exe)
 		}

 		// Build a chan for the test names to be run.
 		tests := make(chan string, len(list.Tests))

 		numParallelTests := c.NumParallelTests
 		if list.API != testlist.Vulkan {
 			// OpenGL tests attempt to open lots of X11 display connections,
 			// which may cause us to run out of handles. This maximum was
 			// determined experimentally on a 72-core system.
 			maxParallelGLTests := 16

 			if numParallelTests > maxParallelGLTests {
 				numParallelTests = maxParallelGLTests
 			}
 		}

 		// Start a number of go routines to run the tests.
 		wg.Add(numParallelTests)
 		for i := 0; i < numParallelTests; i++ {
 			go func(index int) {
 				c.TestRoutine(exe, tests, results, index, supportsCoverage)
 				wg.Done()
 			}(goroutineIndex)
 			goroutineIndex++
 		}

 		// Shuffle the test list.
 		// This attempts to mix heavy-load tests with lighter ones.
 		shuffled := make([]string, len(list.Tests))
 		for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
 			shuffled[i] = list.Tests[j]
 		}

 		// Hand the tests to the TestRoutines.
 		for _, t := range shuffled {
 			tests <- t
 		}

 		// Close the tests chan to indicate that there are no more tests to run.
 		// The TestRoutine functions will return once all tests have been
 		// run.
 		close(tests)

 		numTests += len(list.Tests)
 	}

 	out := Results{
 		Version: dataVersion,
 		Tests:   map[string]TestResult{},
 	}

 	if c.CoverageEnv != nil {
 		out.Coverage = &cov.Tree{}
 		out.Coverage.Add(cov.Path{}, c.CoverageEnv.AllSourceFiles())
 	}

 	// Collect the results.
 	finished := make(chan struct{})
 	lastUpdate := time.Now()
 	go func() {
 		start, i := time.Now(), 0
 		for r := range results {
 			i++
 			if time.Since(lastUpdate) > time.Minute {
 				lastUpdate = time.Now()
 				remaining := numTests - i
 				log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
 					i, numTests, util.Percent(i, numTests),
 					(time.Since(start)/time.Duration(i))*time.Duration(remaining))
 			}
 			out.Tests[r.Test] = r
 			if r.Coverage != nil {
 				path := strings.Split(r.Test, ".")
 				out.Coverage.Add(cov.Path(path), r.Coverage)
 				r.Coverage = nil // Free memory
 			}
 		}
 		close(finished)
 	}()

 	wg.Wait()      // Block until all the deqpTestRoutines have finished.
 	close(results) // Signal no more results.
 	<-finished     // And wait for the result collecting go-routine to finish.

 	out.Duration = time.Since(start)

 	return &out, nil
 }

 // TestRoutine repeatedly runs the dEQP test executable exe with the tests
 // taken from tests. The output of the dEQP test is parsed, and the test result
 // is written to results.
 // TestRoutine only returns once the tests chan has been closed.
 // TestRoutine does not close the results chan.
 func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int, supportsCoverage bool) {
 	// Context for the GCOV_PREFIX environment variable:
 	// If you compile SwiftShader with gcc and the --coverage flag, the build will contain coverage instrumentation.
 	// We can use this to get the code coverage of SwiftShader from running dEQP.
 	// The coverage instrumentation reads the existing coverage files on start-up (at a hardcoded path alongside the
 	// SwiftShader build), updates coverage info as the programs runs, then (over)writes the coverage files on exit.
 	// Thus, multiple parallel processes will race when updating coverage information. The GCOV_PREFIX environment
 	// variable adds a prefix to the hardcoded paths.
 	// E.g. Given GCOV_PREFIX=/tmp/coverage, the hardcoded path /ss/build/a.gcno becomes /tmp/coverage/ss/build/a.gcno.
 	// This is mainly intended for running the target program on a different machine where the hardcoded paths don't
 	// make sense. It can also be used to avoid races. It would be trivial to avoid races if the GCOV_PREFIX variable
 	// supported macro variables like the Clang code coverage "%p" variable that expands to the process ID; in this
 	// case, we could use GCOV_PREFIX=/tmp/coverage/%p to avoid races. Unfortunately, gcc does not support this.
 	// Furthermore, processing coverage information from many directories can be slow; we start a lot of dEQP child
 	// processes, each of which will likely get a unique process ID. In practice, we only need one directory per go
 	// routine.

 	// If GCOV_PREFIX is in Env, replace occurrences of "PROC_ID" in GCOV_PREFIX with goroutineIndex.
 	// This avoids races between parallel child processes reading and writing coverage output files.
 	// For example, GCOV_PREFIX="/tmp/gcov_output/PROC_ID" becomes GCOV_PREFIX="/tmp/gcov_output/1" in the first go routine.
 	// You might expect PROC_ID to be the process ID of some process, but the only real requirement is that
 	// it is a unique ID between the *parallel* child processes.
 	env := make([]string, 0, len(c.Env))
 	for _, v := range c.Env {
 		if strings.HasPrefix(v, "GCOV_PREFIX=") {
 			v = strings.ReplaceAll(v, "PROC_ID", strconv.Itoa(goroutineIndex))
 		}
 		env = append(env, v)
 	}

 	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
 	if supportsCoverage {
 		if c.CoverageEnv != nil {
 			env = cov.AppendRuntimeEnv(env, coverageFile)
 		}
 	}

 	logPath := "/dev/null" // TODO(bclayton): Try "nul" on windows.
 	if !util.IsFile(logPath) {
 		logPath = filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
 	}

 	testNames := []string{}
 	for name := range tests {
 		testNames = append(testNames, name)
 		if len(testNames) >= c.MaxTestsPerProc {
 			c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
 			// Clear list of test names
 			testNames = testNames[:0]
 		}
 	}
 	if len(testNames) > 0 {
 		c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
 	}
 }

 func (c *Config) PerformTests(exe string, env []string, coverageFile string, logPath string, testNames []string, supportsCoverage bool, results chan<- TestResult) {
 	// log.Printf("Running test(s) '%s'\n", testNames)

 	start := time.Now()
 	// Set validation layer according to flag.
 	validation := "disable"
 	if c.ValidationLayer {
 		validation = "enable"
 	}

 	// The list of test names will be passed to stdin, since the deqp-stdin-caselist option is used
 	stdin := strings.Join(testNames, "\n") + "\n"

 	numTests := len(testNames)
 	timeout := c.TestTimeout * time.Duration(numTests)
 	outRaw, deqpErr := shell.Exec(timeout, exe, filepath.Dir(exe), env, stdin,
 		"--deqp-validation="+validation,
 		"--deqp-surface-type=pbuffer",
 		"--deqp-shadercache=disable",
 		"--deqp-log-images=disable",
 		"--deqp-log-shader-sources=disable",
 		"--deqp-log-decompiled-spirv=disable",
 		"--deqp-log-empty-loginfo=disable",
 		"--deqp-log-flush=disable",
 		"--deqp-log-filename="+logPath,
 		"--deqp-stdin-caselist")
 	duration := time.Since(start)
 	out := string(outRaw)
 	out = strings.ReplaceAll(out, exe, "<dEQP>")
 	for k, v := range c.LogReplacements {
 		out = strings.ReplaceAll(out, k, v)
 	}

 	var coverage *cov.Coverage
 	if c.CoverageEnv != nil && supportsCoverage {
 		var covErr error
 		coverage, covErr = c.CoverageEnv.Import(coverageFile)
 		if covErr != nil {
 			log.Printf("Warning: Failed to process test coverage for test '%v'. %v", testNames, covErr)
 		}
 		os.Remove(coverageFile)
 	}

 	if numTests > 1 {
 		// Separate output per test case
 		caseOutputs := caseOutputRE.Split(out, -1)

 		// If the output isn't as expected, a crash may have happened
 		isCrash := (len(caseOutputs) != (numTests + 1))

 		// Verify the exit code to see if a crash has happened
 		var exitErr *exec.ExitError
 		if errors.As(deqpErr, &exitErr) {
 			if exitErr.ExitCode() == 255 {
 				isCrash = true
 			}
 		}

 		// If a crash has happened, re-run tests separately
 		if isCrash {
 			// Re-run tests one by one
 			for _, testName := range testNames {
 				singleTest := []string{testName}
 				c.PerformTests(exe, env, coverageFile, logPath, singleTest, supportsCoverage, results)
 			}
 		} else {
 			caseOutputs = caseOutputs[1:] // Ignore text up to first "Test case '...'"
 			caseNameMatches := caseOutputRE.FindAllStringSubmatch(out, -1)
 			caseNames := make([]string, len(caseNameMatches))
 			for i, m := range caseNameMatches {
 				caseNames[i] = m[1]
 			}

 			averageDuration := duration / time.Duration(numTests)
 			for i, caseOutput := range caseOutputs {
 				results <- c.AnalyzeOutput(caseNames[i], caseOutput, averageDuration, coverage, deqpErr)
 			}
 		}
 	} else {
 		results <- c.AnalyzeOutput(testNames[0], out, duration, coverage, deqpErr)
 	}
 }

 func (c *Config) AnalyzeOutput(name string, out string, duration time.Duration, coverage *cov.Coverage, err error) TestResult {
 	for _, test := range []struct {
 		re *regexp.Regexp
 		s  testlist.Status
 	}{
 		{unimplementedRE, testlist.Unimplemented},
 		{unsupportedRE, testlist.Unsupported},
 		{unreachableRE, testlist.Unreachable},
 		{assertRE, testlist.Assert},
 		{abortRE, testlist.Abort},
 	} {
 		if s := test.re.FindString(out); s != "" {
 			return TestResult{
 				Test:      name,
 				Status:    test.s,
 				TimeTaken: duration,
 				Err:       s,
 				Coverage:  coverage,
 			}
 		}
 	}

 	// Don't treat non-zero error codes as crashes.
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) {
 		if exitErr.ExitCode() != 255 {
 			out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
 			err = nil
 		}
 	}

 	switch err.(type) {
 	default:
 		return TestResult{
 			Test:      name,
 			Status:    testlist.Crash,
 			TimeTaken: duration,
 			Err:       out,
 			Coverage:  coverage,
 		}
 	case shell.ErrTimeout:
 		log.Printf("Timeout for test '%v'\n", name)
 		return TestResult{
 			Test:      name,
 			Status:    testlist.Timeout,
 			TimeTaken: duration,
 			Coverage:  coverage,
 		}
 	case nil:
 		toks := deqpRE.FindStringSubmatch(out)
 		if len(toks) < 3 {
 			return TestResult{Test: name, Status: testlist.Unknown, TimeTaken: duration, Coverage: coverage}
 		}
 		switch toks[1] {
 		case "Pass":
 			return TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
 		case "NotSupported":
 			return TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
 		case "CompatibilityWarning":
 			return TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
 		case "QualityWarning":
 			return TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
 		case "Fail":
 			var err string
 			if toks[2] != "Fail" {
 				err = toks[2]
 			}
 			return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 		case "InternalError":
 			var err string
 			if toks[2] != "InternalError" {
 				err = toks[2]
 			}
 			return TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
 		default:
 			err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 			log.Println("Warning: ", err)
 			return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
 		}
 	}
 }
	// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// Package deqp provides functions for running dEQP, as well as loading and storing the results.
	package deqp

	import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"swiftshader.googlesource.com/SwiftShader/tests/regres/cov"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/shell"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/testlist"
	"swiftshader.googlesource.com/SwiftShader/tests/regres/util"
	)

	const dataVersion = 1

	var (
	// Regular expression to parse the output of a dEQP test.
	deqpRE = regexp.MustCompile(`(Fail\|Pass\|NotSupported\|CompatibilityWarning\|QualityWarning\|InternalError) \(([^\)]*)\)`)
	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
	unimplementedRE = regexp.MustCompile(`[^\n]UNIMPLEMENTED:[^\n]`)
	// Regular expression to parse a test that failed due to UNSUPPORTED()
	unsupportedRE = regexp.MustCompile(`[^\n]UNSUPPORTED:[^\n]`)
	// Regular expression to parse a test that failed due to UNREACHABLE()
	unreachableRE = regexp.MustCompile(`[^\n]UNREACHABLE:[^\n]`)
	// Regular expression to parse a test that failed due to ASSERT()
	assertRE = regexp.MustCompile(`[^\n]ASSERT\([^\)]\)[^\n]*`)
	// Regular expression to parse a test that failed due to ABORT()
	abortRE = regexp.MustCompile(`[^\n]ABORT:[^\n]`)
	// Regular expression to parse individual test names and output
	caseOutputRE = regexp.MustCompile("Test case '([^']*)'..")
	)

	// Config contains the inputs required for running dEQP on a group of test lists.
	type Config struct {
	ExeEgl string
	ExeGles2 string
	ExeGles3 string
	ExeVulkan string
	TempDir string // Directory for temporary log files, coverage output.
	TestLists testlist.Lists
	Env []string
	LogReplacements map[string]string
	NumParallelTests int
	MaxTestsPerProc int
	CoverageEnv *cov.Env
	TestTimeout time.Duration
	ValidationLayer bool
	}

	// Results holds the results of tests across all APIs.
	// The Results structure may be serialized to cache results.
	type Results struct {
	Version int
	Error string
	Tests map[string]TestResult
	Coverage *cov.Tree
	Duration time.Duration
	}

	// TestResult holds the results of a single dEQP test.
	type TestResult struct {
	Test string
	Status testlist.Status
	TimeTaken time.Duration
	Err string `json:",omitempty"`
	Coverage *cov.Coverage
	}

	func (r TestResult) String() string {
	if r.Err != "" {
	return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
	}
	return fmt.Sprintf("%s: %s", r.Test, r.Status)
	}

	// LoadResults loads cached test results from disk.
	func LoadResults(path string) (*Results, error) {
	f, err := os.Open(path)
	if err != nil {
	return nil, fmt.Errorf("failed to open '%s' for loading test results: %w", path, err)
	}
	defer f.Close()

	var out Results
	if err := json.NewDecoder(f).Decode(&out); err != nil {
	return nil, err
	}
	if out.Version != dataVersion {
	return nil, errors.New("Data is from an old version")
	}
	return &out, nil
	}

	// Save saves (caches) test results to disk.
	func (r *Results) Save(path string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
	return fmt.Errorf("failed to make '%s' for saving test results: %w", filepath.Dir(path), err)
	}

	f, err := os.Create(path)
	if err != nil {
	return fmt.Errorf("failed to open '%s' for saving test results: %w", path, err)
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	enc.SetIndent("", " ")
	if err := enc.Encode(r); err != nil {
	return fmt.Errorf("failed to encode test results: %w", err)
	}

	return nil
	}

	// Run runs all the tests.
	func (c Config) Run() (Results, error) {
	start := time.Now()

	if c.TempDir == "" {
	dir, err := ioutil.TempDir("", "deqp")
	if err != nil {
	return nil, fmt.Errorf("failed to generate temporary directory: %w", err)
	}
	c.TempDir = dir
	}

	// Wait group that completes once all the tests have finished.
	wg := sync.WaitGroup{}
	results := make(chan TestResult, 256)

	numTests := 0

	goroutineIndex := 0

	// For each API that we are testing
	for _, list := range c.TestLists {
	// Resolve the test runner
	exe, supportsCoverage := "", false

	switch list.API {
	case testlist.EGL:
	exe = c.ExeEgl
	case testlist.GLES2:
	exe = c.ExeGles2
	case testlist.GLES3:
	exe = c.ExeGles3
	case testlist.Vulkan:
	exe, supportsCoverage = c.ExeVulkan, true
	default:
	return nil, fmt.Errorf("Unknown API '%v'", list.API)
	}
	if !util.IsFile(exe) {
	return nil, fmt.Errorf("failed to find dEQP executable at '%s'", exe)
	}

	// Build a chan for the test names to be run.
	tests := make(chan string, len(list.Tests))

	numParallelTests := c.NumParallelTests
	if list.API != testlist.Vulkan {
	// OpenGL tests attempt to open lots of X11 display connections,
	// which may cause us to run out of handles. This maximum was
	// determined experimentally on a 72-core system.
	maxParallelGLTests := 16

	if numParallelTests > maxParallelGLTests {
	numParallelTests = maxParallelGLTests
	}
	}

	// Start a number of go routines to run the tests.
	wg.Add(numParallelTests)
	for i := 0; i < numParallelTests; i++ {
	go func(index int) {
	c.TestRoutine(exe, tests, results, index, supportsCoverage)
	wg.Done()
	}(goroutineIndex)
	goroutineIndex++
	}

	// Shuffle the test list.
	// This attempts to mix heavy-load tests with lighter ones.
	shuffled := make([]string, len(list.Tests))
	for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
	shuffled[i] = list.Tests[j]
	}

	// Hand the tests to the TestRoutines.
	for _, t := range shuffled {
	tests <- t
	}

	// Close the tests chan to indicate that there are no more tests to run.
	// The TestRoutine functions will return once all tests have been
	// run.
	close(tests)

	numTests += len(list.Tests)
	}

	out := Results{
	Version: dataVersion,
	Tests: map[string]TestResult{},
	}

	if c.CoverageEnv != nil {
	out.Coverage = &cov.Tree{}
	out.Coverage.Add(cov.Path{}, c.CoverageEnv.AllSourceFiles())
	}

	// Collect the results.
	finished := make(chan struct{})
	lastUpdate := time.Now()
	go func() {
	start, i := time.Now(), 0
	for r := range results {
	i++
	if time.Since(lastUpdate) > time.Minute {
	lastUpdate = time.Now()
	remaining := numTests - i
	log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
	i, numTests, util.Percent(i, numTests),
	(time.Since(start)/time.Duration(i))*time.Duration(remaining))
	}
	out.Tests[r.Test] = r
	if r.Coverage != nil {
	path := strings.Split(r.Test, ".")
	out.Coverage.Add(cov.Path(path), r.Coverage)
	r.Coverage = nil // Free memory
	}
	}
	close(finished)
	}()

	wg.Wait() // Block until all the deqpTestRoutines have finished.
	close(results) // Signal no more results.
	<-finished // And wait for the result collecting go-routine to finish.

	out.Duration = time.Since(start)

	return &out, nil
	}

	// TestRoutine repeatedly runs the dEQP test executable exe with the tests
	// taken from tests. The output of the dEQP test is parsed, and the test result
	// is written to results.
	// TestRoutine only returns once the tests chan has been closed.
	// TestRoutine does not close the results chan.
	func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int, supportsCoverage bool) {
	// Context for the GCOV_PREFIX environment variable:
	// If you compile SwiftShader with gcc and the --coverage flag, the build will contain coverage instrumentation.
	// We can use this to get the code coverage of SwiftShader from running dEQP.
	// The coverage instrumentation reads the existing coverage files on start-up (at a hardcoded path alongside the
	// SwiftShader build), updates coverage info as the programs runs, then (over)writes the coverage files on exit.
	// Thus, multiple parallel processes will race when updating coverage information. The GCOV_PREFIX environment
	// variable adds a prefix to the hardcoded paths.
	// E.g. Given GCOV_PREFIX=/tmp/coverage, the hardcoded path /ss/build/a.gcno becomes /tmp/coverage/ss/build/a.gcno.
	// This is mainly intended for running the target program on a different machine where the hardcoded paths don't
	// make sense. It can also be used to avoid races. It would be trivial to avoid races if the GCOV_PREFIX variable
	// supported macro variables like the Clang code coverage "%p" variable that expands to the process ID; in this
	// case, we could use GCOV_PREFIX=/tmp/coverage/%p to avoid races. Unfortunately, gcc does not support this.
	// Furthermore, processing coverage information from many directories can be slow; we start a lot of dEQP child
	// processes, each of which will likely get a unique process ID. In practice, we only need one directory per go
	// routine.

	// If GCOV_PREFIX is in Env, replace occurrences of "PROC_ID" in GCOV_PREFIX with goroutineIndex.
	// This avoids races between parallel child processes reading and writing coverage output files.
	// For example, GCOV_PREFIX="/tmp/gcov_output/PROC_ID" becomes GCOV_PREFIX="/tmp/gcov_output/1" in the first go routine.
	// You might expect PROC_ID to be the process ID of some process, but the only real requirement is that
	// it is a unique ID between the parallel child processes.
	env := make([]string, 0, len(c.Env))
	for _, v := range c.Env {
	if strings.HasPrefix(v, "GCOV_PREFIX=") {
	v = strings.ReplaceAll(v, "PROC_ID", strconv.Itoa(goroutineIndex))
	}
	env = append(env, v)
	}

	coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
	if supportsCoverage {
	if c.CoverageEnv != nil {
	env = cov.AppendRuntimeEnv(env, coverageFile)
	}
	}

	logPath := "/dev/null" // TODO(bclayton): Try "nul" on windows.
	if !util.IsFile(logPath) {
	logPath = filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
	}

	testNames := []string{}
	for name := range tests {
	testNames = append(testNames, name)
	if len(testNames) >= c.MaxTestsPerProc {
	c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
	// Clear list of test names
	testNames = testNames[:0]
	}
	}
	if len(testNames) > 0 {
	c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
	}
	}

	func (c *Config) PerformTests(exe string, env []string, coverageFile string, logPath string, testNames []string, supportsCoverage bool, results chan<- TestResult) {
	// log.Printf("Running test(s) '%s'\n", testNames)

	start := time.Now()
	// Set validation layer according to flag.
	validation := "disable"
	if c.ValidationLayer {
	validation = "enable"
	}

	// The list of test names will be passed to stdin, since the deqp-stdin-caselist option is used
	stdin := strings.Join(testNames, "\n") + "\n"

	numTests := len(testNames)
	timeout := c.TestTimeout * time.Duration(numTests)
	outRaw, deqpErr := shell.Exec(timeout, exe, filepath.Dir(exe), env, stdin,
	"--deqp-validation="+validation,
	"--deqp-surface-type=pbuffer",
	"--deqp-shadercache=disable",
	"--deqp-log-images=disable",
	"--deqp-log-shader-sources=disable",
	"--deqp-log-decompiled-spirv=disable",
	"--deqp-log-empty-loginfo=disable",
	"--deqp-log-flush=disable",
	"--deqp-log-filename="+logPath,
	"--deqp-stdin-caselist")
	duration := time.Since(start)
	out := string(outRaw)
	out = strings.ReplaceAll(out, exe, "<dEQP>")
	for k, v := range c.LogReplacements {
	out = strings.ReplaceAll(out, k, v)
	}

	var coverage *cov.Coverage
	if c.CoverageEnv != nil && supportsCoverage {
	var covErr error
	coverage, covErr = c.CoverageEnv.Import(coverageFile)
	if covErr != nil {
	log.Printf("Warning: Failed to process test coverage for test '%v'. %v", testNames, covErr)
	}
	os.Remove(coverageFile)
	}

	if numTests > 1 {
	// Separate output per test case
	caseOutputs := caseOutputRE.Split(out, -1)

	// If the output isn't as expected, a crash may have happened
	isCrash := (len(caseOutputs) != (numTests + 1))

	// Verify the exit code to see if a crash has happened
	var exitErr *exec.ExitError
	if errors.As(deqpErr, &exitErr) {
	if exitErr.ExitCode() == 255 {
	isCrash = true
	}
	}

	// If a crash has happened, re-run tests separately
	if isCrash {
	// Re-run tests one by one
	for _, testName := range testNames {
	singleTest := []string{testName}
	c.PerformTests(exe, env, coverageFile, logPath, singleTest, supportsCoverage, results)
	}
	} else {
	caseOutputs = caseOutputs[1:] // Ignore text up to first "Test case '...'"
	caseNameMatches := caseOutputRE.FindAllStringSubmatch(out, -1)
	caseNames := make([]string, len(caseNameMatches))
	for i, m := range caseNameMatches {
	caseNames[i] = m[1]
	}

	averageDuration := duration / time.Duration(numTests)
	for i, caseOutput := range caseOutputs {
	results <- c.AnalyzeOutput(caseNames[i], caseOutput, averageDuration, coverage, deqpErr)
	}
	}
	} else {
	results <- c.AnalyzeOutput(testNames[0], out, duration, coverage, deqpErr)
	}
	}

	func (c Config) AnalyzeOutput(name string, out string, duration time.Duration, coverage cov.Coverage, err error) TestResult {
	for _, test := range []struct {
	re *regexp.Regexp
	s testlist.Status
	}{
	{unimplementedRE, testlist.Unimplemented},
	{unsupportedRE, testlist.Unsupported},
	{unreachableRE, testlist.Unreachable},
	{assertRE, testlist.Assert},
	{abortRE, testlist.Abort},
	} {
	if s := test.re.FindString(out); s != "" {
	return TestResult{
	Test: name,
	Status: test.s,
	TimeTaken: duration,
	Err: s,
	Coverage: coverage,
	}
	}
	}

	// Don't treat non-zero error codes as crashes.
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
	if exitErr.ExitCode() != 255 {
	out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
	err = nil
	}
	}

	switch err.(type) {
	default:
	return TestResult{
	Test: name,
	Status: testlist.Crash,
	TimeTaken: duration,
	Err: out,
	Coverage: coverage,
	}
	case shell.ErrTimeout:
	log.Printf("Timeout for test '%v'\n", name)
	return TestResult{
	Test: name,
	Status: testlist.Timeout,
	TimeTaken: duration,
	Coverage: coverage,
	}
	case nil:
	toks := deqpRE.FindStringSubmatch(out)
	if len(toks) < 3 {
	return TestResult{Test: name, Status: testlist.Unknown, TimeTaken: duration, Coverage: coverage}
	}
	switch toks[1] {
	case "Pass":
	return TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
	case "NotSupported":
	return TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
	case "CompatibilityWarning":
	return TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
	case "QualityWarning":
	return TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
	case "Fail":
	var err string
	if toks[2] != "Fail" {
	err = toks[2]
	}
	return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
	case "InternalError":
	var err string
	if toks[2] != "InternalError" {
	err = toks[2]
	}
	return TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
	default:
	err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
	log.Println("Warning: ", err)
	return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
	}
	}
	}