tests/regres/deqp/deqp.go - SwiftShader - Git at Google

 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // Package deqp provides functions for running dEQP, as well as loading and storing the results.
 package deqp

 import (
 	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
 	"time"

 	"../cause"
 	"../shell"
 	"../testlist"
 	"../util"
 )

 const dataVersion = 1

 var (
 	// Regular expression to parse the output of a dEQP test.
 	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning) \(([^\)]*)\)`)
 	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
 	unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNSUPPORTED()
 	unsupportedRE = regexp.MustCompile(`[^\n]*UNSUPPORTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNREACHABLE()
 	unreachableRE = regexp.MustCompile(`[^\n]*UNREACHABLE:[^\n]*`)
 	// Regular expression to parse a test that failed due to ASSERT()
 	assertRE = regexp.MustCompile(`[^\n]*ASSERT\([^\)]*\)[^\n]*`)
 	// Regular expression to parse a test that failed due to ABORT()
 	abortRE = regexp.MustCompile(`[^\n]*ABORT:[^\n]*`)
 )

 // Config contains the inputs required for running dEQP on a group of test lists.
 type Config struct {
 	ExeEgl           string
 	ExeGles2         string
 	ExeGles3         string
 	ExeVulkan        string
 	TestLists        testlist.Lists
 	Env              []string
 	LogReplacements  map[string]string
 	NumParallelTests int
 	TestTimeout      time.Duration
 }

 // Results holds the results of tests across all APIs.
 // The Results structure may be serialized to cache results.
 type Results struct {
 	Version  int
 	Error    string
 	Tests    map[string]TestResult
 	Duration time.Duration
 }

 // TestResult holds the results of a single dEQP test.
 type TestResult struct {
 	Test      string
 	Status    testlist.Status
 	TimeTaken time.Duration
 	Err       string `json:",omitempty"`
 }

 func (r TestResult) String() string {
 	if r.Err != "" {
 		return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
 	}
 	return fmt.Sprintf("%s: %s", r.Test, r.Status)
 }

 // LoadResults loads cached test results from disk.
 func LoadResults(path string) (*Results, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, cause.Wrap(err, "Couldn't open '%s' for loading test results", path)
 	}
 	defer f.Close()

 	var out Results
 	if err := json.NewDecoder(f).Decode(&out); err != nil {
 		return nil, err
 	}
 	if out.Version != dataVersion {
 		return nil, errors.New("Data is from an old version")
 	}
 	return &out, nil
 }

 // Save saves (caches) test results to disk.
 func (r *Results) Save(path string) error {
 	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
 		return cause.Wrap(err, "couldn't make '%s' for saving test results", filepath.Dir(path))
 	}

 	f, err := os.Create(path)
 	if err != nil {
 		return cause.Wrap(err, "Couldn't open '%s' for saving test results", path)
 	}
 	defer f.Close()

 	enc := json.NewEncoder(f)
 	enc.SetIndent("", "  ")
 	if err := enc.Encode(r); err != nil {
 		return cause.Wrap(err, "Couldn't encode test results")
 	}

 	return nil
 }

 // Run runs all the tests.
 func (c *Config) Run() (*Results, error) {

 	start := time.Now()

 	// Wait group that completes once all the tests have finished.
 	wg := sync.WaitGroup{}
 	results := make(chan TestResult, 256)

 	numTests := 0

 	// For each API that we are testing
 	for _, list := range c.TestLists {
 		// Resolve the test runner
 		var exe string
 		switch list.API {
 		case testlist.EGL:
 			exe = c.ExeEgl
 		case testlist.GLES2:
 			exe = c.ExeGles2
 		case testlist.GLES3:
 			exe = c.ExeGles3
 		case testlist.Vulkan:
 			exe = c.ExeVulkan
 		default:
 			return nil, fmt.Errorf("Unknown API '%v'", list.API)
 		}
 		if !util.IsFile(exe) {
 			return nil, fmt.Errorf("Couldn't find dEQP executable at '%s'", exe)
 		}

 		// Build a chan for the test names to be run.
 		tests := make(chan string, len(list.Tests))

 		// Start a number of go routines to run the tests.
 		wg.Add(c.NumParallelTests)
 		for i := 0; i < c.NumParallelTests; i++ {
 			go func() {
 				c.TestRoutine(exe, tests, results)
 				wg.Done()
 			}()
 		}

 		// Shuffle the test list.
 		// This attempts to mix heavy-load tests with lighter ones.
 		shuffled := make([]string, len(list.Tests))
 		for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
 			shuffled[i] = list.Tests[j]
 		}

 		// Hand the tests to the TestRoutines.
 		for _, t := range shuffled {
 			tests <- t
 		}

 		// Close the tests chan to indicate that there are no more tests to run.
 		// The TestRoutine functions will return once all tests have been
 		// run.
 		close(tests)

 		numTests += len(list.Tests)
 	}

 	out := Results{
 		Version: dataVersion,
 		Tests:   map[string]TestResult{},
 	}

 	// Collect the results.
 	finished := make(chan struct{})
 	lastUpdate := time.Now()
 	go func() {
 		start, i := time.Now(), 0
 		for r := range results {
 			i++
 			out.Tests[r.Test] = r
 			if time.Since(lastUpdate) > time.Minute {
 				lastUpdate = time.Now()
 				remaining := numTests - i
 				log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
 					i, numTests, util.Percent(i, numTests),
 					(time.Since(start)/time.Duration(i))*time.Duration(remaining))
 			}
 		}
 		close(finished)
 	}()

 	wg.Wait()      // Block until all the deqpTestRoutines have finished.
 	close(results) // Signal no more results.
 	<-finished     // And wait for the result collecting go-routine to finish.

 	out.Duration = time.Since(start)

 	return &out, nil
 }

 // TestRoutine repeatedly runs the dEQP test executable exe with the tests
 // taken from tests. The output of the dEQP test is parsed, and the test result
 // is written to results.
 // TestRoutine only returns once the tests chan has been closed.
 // TestRoutine does not close the results chan.
 func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult) {
 nextTest:
 	for name := range tests {
 		// log.Printf("Running test '%s'\n", name)

 		start := time.Now()
 		outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), c.Env,
 			"--deqp-surface-type=pbuffer",
 			"--deqp-shadercache=disable",
 			"--deqp-log-images=disable",
 			"--deqp-log-shader-sources=disable",
 			"--deqp-log-flush=disable",
 			"-n="+name)
 		duration := time.Since(start)
 		out := string(outRaw)
 		out = strings.ReplaceAll(out, exe, "<dEQP>")
 		for k, v := range c.LogReplacements {
 			out = strings.ReplaceAll(out, k, v)
 		}

 		// Don't treat non-zero error codes as crashes.
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
 			if exitErr.ExitCode() != -1 {
 				out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
 				err = nil
 			}
 		}

 		switch err.(type) {
 		default:
 			for _, test := range []struct {
 				re *regexp.Regexp
 				s  testlist.Status
 			}{
 				{unimplementedRE, testlist.Unimplemented},
 				{unsupportedRE, testlist.Unsupported},
 				{unreachableRE, testlist.Unreachable},
 				{assertRE, testlist.Assert},
 				{abortRE, testlist.Abort},
 			} {
 				if s := test.re.FindString(out); s != "" {
 					results <- TestResult{
 						Test:      name,
 						Status:    test.s,
 						TimeTaken: duration,
 						Err:       s,
 					}
 					continue nextTest
 				}
 			}
 			results <- TestResult{
 				Test:      name,
 				Status:    testlist.Crash,
 				TimeTaken: duration,
 				Err:       out,
 			}
 		case shell.ErrTimeout:
 			log.Printf("Timeout for test '%v'\n", name)
 			results <- TestResult{
 				Test:      name,
 				Status:    testlist.Timeout,
 				TimeTaken: duration,
 			}
 		case nil:
 			toks := deqpRE.FindStringSubmatch(out)
 			if len(toks) < 3 {
 				err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
 				log.Println("Warning: ", err)
 				results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
 				continue
 			}
 			switch toks[1] {
 			case "Pass":
 				results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration}
 			case "NotSupported":
 				results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration}
 			case "CompatibilityWarning":
 				results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration}
 			case "QualityWarning":
 				results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration}
 			case "Fail":
 				var err string
 				if toks[2] != "Fail" {
 					err = toks[2]
 				}
 				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 				log.Println("Warning: ", err)
 				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
 			}
 		}
 	}
 }
	// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// Package deqp provides functions for running dEQP, as well as loading and storing the results.
	package deqp

	import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"../cause"
	"../shell"
	"../testlist"
	"../util"
	)

	const dataVersion = 1

	var (
	// Regular expression to parse the output of a dEQP test.
	deqpRE = regexp.MustCompile(`(Fail\|Pass\|NotSupported\|CompatibilityWarning\|QualityWarning) \(([^\)]*)\)`)
	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
	unimplementedRE = regexp.MustCompile(`[^\n]UNIMPLEMENTED:[^\n]`)
	// Regular expression to parse a test that failed due to UNSUPPORTED()
	unsupportedRE = regexp.MustCompile(`[^\n]UNSUPPORTED:[^\n]`)
	// Regular expression to parse a test that failed due to UNREACHABLE()
	unreachableRE = regexp.MustCompile(`[^\n]UNREACHABLE:[^\n]`)
	// Regular expression to parse a test that failed due to ASSERT()
	assertRE = regexp.MustCompile(`[^\n]ASSERT\([^\)]\)[^\n]*`)
	// Regular expression to parse a test that failed due to ABORT()
	abortRE = regexp.MustCompile(`[^\n]ABORT:[^\n]`)
	)

	// Config contains the inputs required for running dEQP on a group of test lists.
	type Config struct {
	ExeEgl string
	ExeGles2 string
	ExeGles3 string
	ExeVulkan string
	TestLists testlist.Lists
	Env []string
	LogReplacements map[string]string
	NumParallelTests int
	TestTimeout time.Duration
	}

	// Results holds the results of tests across all APIs.
	// The Results structure may be serialized to cache results.
	type Results struct {
	Version int
	Error string
	Tests map[string]TestResult
	Duration time.Duration
	}

	// TestResult holds the results of a single dEQP test.
	type TestResult struct {
	Test string
	Status testlist.Status
	TimeTaken time.Duration
	Err string `json:",omitempty"`
	}

	func (r TestResult) String() string {
	if r.Err != "" {
	return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
	}
	return fmt.Sprintf("%s: %s", r.Test, r.Status)
	}

	// LoadResults loads cached test results from disk.
	func LoadResults(path string) (*Results, error) {
	f, err := os.Open(path)
	if err != nil {
	return nil, cause.Wrap(err, "Couldn't open '%s' for loading test results", path)
	}
	defer f.Close()

	var out Results
	if err := json.NewDecoder(f).Decode(&out); err != nil {
	return nil, err
	}
	if out.Version != dataVersion {
	return nil, errors.New("Data is from an old version")
	}
	return &out, nil
	}

	// Save saves (caches) test results to disk.
	func (r *Results) Save(path string) error {
	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
	return cause.Wrap(err, "couldn't make '%s' for saving test results", filepath.Dir(path))
	}

	f, err := os.Create(path)
	if err != nil {
	return cause.Wrap(err, "Couldn't open '%s' for saving test results", path)
	}
	defer f.Close()

	enc := json.NewEncoder(f)
	enc.SetIndent("", " ")
	if err := enc.Encode(r); err != nil {
	return cause.Wrap(err, "Couldn't encode test results")
	}

	return nil
	}

	// Run runs all the tests.
	func (c Config) Run() (Results, error) {

	start := time.Now()

	// Wait group that completes once all the tests have finished.
	wg := sync.WaitGroup{}
	results := make(chan TestResult, 256)

	numTests := 0

	// For each API that we are testing
	for _, list := range c.TestLists {
	// Resolve the test runner
	var exe string
	switch list.API {
	case testlist.EGL:
	exe = c.ExeEgl
	case testlist.GLES2:
	exe = c.ExeGles2
	case testlist.GLES3:
	exe = c.ExeGles3
	case testlist.Vulkan:
	exe = c.ExeVulkan
	default:
	return nil, fmt.Errorf("Unknown API '%v'", list.API)
	}
	if !util.IsFile(exe) {
	return nil, fmt.Errorf("Couldn't find dEQP executable at '%s'", exe)
	}

	// Build a chan for the test names to be run.
	tests := make(chan string, len(list.Tests))

	// Start a number of go routines to run the tests.
	wg.Add(c.NumParallelTests)
	for i := 0; i < c.NumParallelTests; i++ {
	go func() {
	c.TestRoutine(exe, tests, results)
	wg.Done()
	}()
	}

	// Shuffle the test list.
	// This attempts to mix heavy-load tests with lighter ones.
	shuffled := make([]string, len(list.Tests))
	for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
	shuffled[i] = list.Tests[j]
	}

	// Hand the tests to the TestRoutines.
	for _, t := range shuffled {
	tests <- t
	}

	// Close the tests chan to indicate that there are no more tests to run.
	// The TestRoutine functions will return once all tests have been
	// run.
	close(tests)

	numTests += len(list.Tests)
	}

	out := Results{
	Version: dataVersion,
	Tests: map[string]TestResult{},
	}

	// Collect the results.
	finished := make(chan struct{})
	lastUpdate := time.Now()
	go func() {
	start, i := time.Now(), 0
	for r := range results {
	i++
	out.Tests[r.Test] = r
	if time.Since(lastUpdate) > time.Minute {
	lastUpdate = time.Now()
	remaining := numTests - i
	log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
	i, numTests, util.Percent(i, numTests),
	(time.Since(start)/time.Duration(i))*time.Duration(remaining))
	}
	}
	close(finished)
	}()

	wg.Wait() // Block until all the deqpTestRoutines have finished.
	close(results) // Signal no more results.
	<-finished // And wait for the result collecting go-routine to finish.

	out.Duration = time.Since(start)

	return &out, nil
	}

	// TestRoutine repeatedly runs the dEQP test executable exe with the tests
	// taken from tests. The output of the dEQP test is parsed, and the test result
	// is written to results.
	// TestRoutine only returns once the tests chan has been closed.
	// TestRoutine does not close the results chan.
	func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult) {
	nextTest:
	for name := range tests {
	// log.Printf("Running test '%s'\n", name)

	start := time.Now()
	outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), c.Env,
	"--deqp-surface-type=pbuffer",
	"--deqp-shadercache=disable",
	"--deqp-log-images=disable",
	"--deqp-log-shader-sources=disable",
	"--deqp-log-flush=disable",
	"-n="+name)
	duration := time.Since(start)
	out := string(outRaw)
	out = strings.ReplaceAll(out, exe, "<dEQP>")
	for k, v := range c.LogReplacements {
	out = strings.ReplaceAll(out, k, v)
	}

	// Don't treat non-zero error codes as crashes.
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
	if exitErr.ExitCode() != -1 {
	out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
	err = nil
	}
	}

	switch err.(type) {
	default:
	for _, test := range []struct {
	re *regexp.Regexp
	s testlist.Status
	}{
	{unimplementedRE, testlist.Unimplemented},
	{unsupportedRE, testlist.Unsupported},
	{unreachableRE, testlist.Unreachable},
	{assertRE, testlist.Assert},
	{abortRE, testlist.Abort},
	} {
	if s := test.re.FindString(out); s != "" {
	results <- TestResult{
	Test: name,
	Status: test.s,
	TimeTaken: duration,
	Err: s,
	}
	continue nextTest
	}
	}
	results <- TestResult{
	Test: name,
	Status: testlist.Crash,
	TimeTaken: duration,
	Err: out,
	}
	case shell.ErrTimeout:
	log.Printf("Timeout for test '%v'\n", name)
	results <- TestResult{
	Test: name,
	Status: testlist.Timeout,
	TimeTaken: duration,
	}
	case nil:
	toks := deqpRE.FindStringSubmatch(out)
	if len(toks) < 3 {
	err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
	log.Println("Warning: ", err)
	results <- TestResult{Test: name, Status: testlist.Fail, Err: err}
	continue
	}
	switch toks[1] {
	case "Pass":
	results <- TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration}
	case "NotSupported":
	results <- TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration}
	case "CompatibilityWarning":
	results <- TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration}
	case "QualityWarning":
	results <- TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration}
	case "Fail":
	var err string
	if toks[2] != "Fail" {
	err = toks[2]
	}
	results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
	default:
	err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
	log.Println("Warning: ", err)
	results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration}
	}
	}
	}
	}