blob: b40e17df9578983b00863dd825a0bea60a02315a [file] [log] [blame]
// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package deqp provides functions for running dEQP, as well as loading and storing the results.
package deqp
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"math/rand"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
"swiftshader.googlesource.com/SwiftShader/tests/regres/cov"
"swiftshader.googlesource.com/SwiftShader/tests/regres/shell"
"swiftshader.googlesource.com/SwiftShader/tests/regres/testlist"
"swiftshader.googlesource.com/SwiftShader/tests/regres/util"
)
const dataVersion = 1
var (
// Regular expression to parse the output of a dEQP test.
deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
// Regular expression to parse a test that failed due to UNIMPLEMENTED()
unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
// Regular expression to parse a test that failed due to UNSUPPORTED()
unsupportedRE = regexp.MustCompile(`[^\n]*UNSUPPORTED:[^\n]*`)
// Regular expression to parse a test that failed due to UNREACHABLE()
unreachableRE = regexp.MustCompile(`[^\n]*UNREACHABLE:[^\n]*`)
// Regular expression to parse a test that failed due to ASSERT()
assertRE = regexp.MustCompile(`[^\n]*ASSERT\([^\)]*\)[^\n]*`)
// Regular expression to parse a test that failed due to ABORT()
abortRE = regexp.MustCompile(`[^\n]*ABORT:[^\n]*`)
// Regular expression to parse individual test names and output
caseOutputRE = regexp.MustCompile("Test case '([^']*)'..")
)
// Config contains the inputs required for running dEQP on a group of test lists.
type Config struct {
ExeEgl string
ExeGles2 string
ExeGles3 string
ExeVulkan string
TempDir string // Directory for temporary log files, coverage output.
TestLists testlist.Lists
Env []string
LogReplacements map[string]string
NumParallelTests int
MaxTestsPerProc int
CoverageEnv *cov.Env
TestTimeout time.Duration
ValidationLayer bool
}
// Results holds the results of tests across all APIs.
// The Results structure may be serialized to cache results.
type Results struct {
Version int
Error string
Tests map[string]TestResult
Coverage *cov.Tree
Duration time.Duration
}
// TestResult holds the results of a single dEQP test.
type TestResult struct {
Test string
Status testlist.Status
TimeTaken time.Duration
Err string `json:",omitempty"`
Coverage *cov.Coverage
}
func (r TestResult) String() string {
if r.Err != "" {
return fmt.Sprintf("%s: %s (%s)", r.Test, r.Status, r.Err)
}
return fmt.Sprintf("%s: %s", r.Test, r.Status)
}
// LoadResults loads cached test results from disk.
func LoadResults(path string) (*Results, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("failed to open '%s' for loading test results: %w", path, err)
}
defer f.Close()
var out Results
if err := json.NewDecoder(f).Decode(&out); err != nil {
return nil, err
}
if out.Version != dataVersion {
return nil, errors.New("Data is from an old version")
}
return &out, nil
}
// Save saves (caches) test results to disk.
func (r *Results) Save(path string) error {
if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
return fmt.Errorf("failed to make '%s' for saving test results: %w", filepath.Dir(path), err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("failed to open '%s' for saving test results: %w", path, err)
}
defer f.Close()
enc := json.NewEncoder(f)
enc.SetIndent("", " ")
if err := enc.Encode(r); err != nil {
return fmt.Errorf("failed to encode test results: %w", err)
}
return nil
}
// Run runs all the tests.
func (c *Config) Run() (*Results, error) {
start := time.Now()
if c.TempDir == "" {
dir, err := ioutil.TempDir("", "deqp")
if err != nil {
return nil, fmt.Errorf("failed to generate temporary directory: %w", err)
}
c.TempDir = dir
}
// Wait group that completes once all the tests have finished.
wg := sync.WaitGroup{}
results := make(chan TestResult, 256)
numTests := 0
goroutineIndex := 0
// For each API that we are testing
for _, list := range c.TestLists {
// Resolve the test runner
exe, supportsCoverage := "", false
switch list.API {
case testlist.EGL:
exe = c.ExeEgl
case testlist.GLES2:
exe = c.ExeGles2
case testlist.GLES3:
exe = c.ExeGles3
case testlist.Vulkan:
exe, supportsCoverage = c.ExeVulkan, true
default:
return nil, fmt.Errorf("Unknown API '%v'", list.API)
}
if !util.IsFile(exe) {
return nil, fmt.Errorf("failed to find dEQP executable at '%s'", exe)
}
// Build a chan for the test names to be run.
tests := make(chan string, len(list.Tests))
numParallelTests := c.NumParallelTests
if list.API != testlist.Vulkan {
// OpenGL tests attempt to open lots of X11 display connections,
// which may cause us to run out of handles. This maximum was
// determined experimentally on a 72-core system.
maxParallelGLTests := 16
if numParallelTests > maxParallelGLTests {
numParallelTests = maxParallelGLTests
}
}
// Start a number of go routines to run the tests.
wg.Add(numParallelTests)
for i := 0; i < numParallelTests; i++ {
go func(index int) {
c.TestRoutine(exe, tests, results, index, supportsCoverage)
wg.Done()
}(goroutineIndex)
goroutineIndex++
}
// Shuffle the test list.
// This attempts to mix heavy-load tests with lighter ones.
shuffled := make([]string, len(list.Tests))
for i, j := range rand.New(rand.NewSource(42)).Perm(len(list.Tests)) {
shuffled[i] = list.Tests[j]
}
// Hand the tests to the TestRoutines.
for _, t := range shuffled {
tests <- t
}
// Close the tests chan to indicate that there are no more tests to run.
// The TestRoutine functions will return once all tests have been
// run.
close(tests)
numTests += len(list.Tests)
}
out := Results{
Version: dataVersion,
Tests: map[string]TestResult{},
}
if c.CoverageEnv != nil {
out.Coverage = &cov.Tree{}
out.Coverage.Add(cov.Path{}, c.CoverageEnv.AllSourceFiles())
}
// Collect the results.
finished := make(chan struct{})
lastUpdate := time.Now()
go func() {
start, i := time.Now(), 0
for r := range results {
i++
if time.Since(lastUpdate) > time.Minute {
lastUpdate = time.Now()
remaining := numTests - i
log.Printf("Ran %d/%d tests (%v%%). Estimated completion in %v.\n",
i, numTests, util.Percent(i, numTests),
(time.Since(start)/time.Duration(i))*time.Duration(remaining))
}
out.Tests[r.Test] = r
if r.Coverage != nil {
path := strings.Split(r.Test, ".")
out.Coverage.Add(cov.Path(path), r.Coverage)
r.Coverage = nil // Free memory
}
}
close(finished)
}()
wg.Wait() // Block until all the deqpTestRoutines have finished.
close(results) // Signal no more results.
<-finished // And wait for the result collecting go-routine to finish.
out.Duration = time.Since(start)
return &out, nil
}
// TestRoutine repeatedly runs the dEQP test executable exe with the tests
// taken from tests. The output of the dEQP test is parsed, and the test result
// is written to results.
// TestRoutine only returns once the tests chan has been closed.
// TestRoutine does not close the results chan.
func (c *Config) TestRoutine(exe string, tests <-chan string, results chan<- TestResult, goroutineIndex int, supportsCoverage bool) {
// Context for the GCOV_PREFIX environment variable:
// If you compile SwiftShader with gcc and the --coverage flag, the build will contain coverage instrumentation.
// We can use this to get the code coverage of SwiftShader from running dEQP.
// The coverage instrumentation reads the existing coverage files on start-up (at a hardcoded path alongside the
// SwiftShader build), updates coverage info as the programs runs, then (over)writes the coverage files on exit.
// Thus, multiple parallel processes will race when updating coverage information. The GCOV_PREFIX environment
// variable adds a prefix to the hardcoded paths.
// E.g. Given GCOV_PREFIX=/tmp/coverage, the hardcoded path /ss/build/a.gcno becomes /tmp/coverage/ss/build/a.gcno.
// This is mainly intended for running the target program on a different machine where the hardcoded paths don't
// make sense. It can also be used to avoid races. It would be trivial to avoid races if the GCOV_PREFIX variable
// supported macro variables like the Clang code coverage "%p" variable that expands to the process ID; in this
// case, we could use GCOV_PREFIX=/tmp/coverage/%p to avoid races. Unfortunately, gcc does not support this.
// Furthermore, processing coverage information from many directories can be slow; we start a lot of dEQP child
// processes, each of which will likely get a unique process ID. In practice, we only need one directory per go
// routine.
// If GCOV_PREFIX is in Env, replace occurrences of "PROC_ID" in GCOV_PREFIX with goroutineIndex.
// This avoids races between parallel child processes reading and writing coverage output files.
// For example, GCOV_PREFIX="/tmp/gcov_output/PROC_ID" becomes GCOV_PREFIX="/tmp/gcov_output/1" in the first go routine.
// You might expect PROC_ID to be the process ID of some process, but the only real requirement is that
// it is a unique ID between the *parallel* child processes.
env := make([]string, 0, len(c.Env))
for _, v := range c.Env {
if strings.HasPrefix(v, "GCOV_PREFIX=") {
v = strings.ReplaceAll(v, "PROC_ID", strconv.Itoa(goroutineIndex))
}
env = append(env, v)
}
coverageFile := filepath.Join(c.TempDir, fmt.Sprintf("%v.profraw", goroutineIndex))
if supportsCoverage {
if c.CoverageEnv != nil {
env = cov.AppendRuntimeEnv(env, coverageFile)
}
}
logPath := "/dev/null" // TODO(bclayton): Try "nul" on windows.
if !util.IsFile(logPath) {
logPath = filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
}
testNames := []string{}
for name := range tests {
testNames = append(testNames, name)
if len(testNames) >= c.MaxTestsPerProc {
c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
// Clear list of test names
testNames = testNames[:0]
}
}
if len(testNames) > 0 {
c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
}
}
func (c *Config) PerformTests(exe string, env []string, coverageFile string, logPath string, testNames []string, supportsCoverage bool, results chan<- TestResult) {
// log.Printf("Running test(s) '%s'\n", testNames)
start := time.Now()
// Set validation layer according to flag.
validation := "disable"
if c.ValidationLayer {
validation = "enable"
}
// The list of test names will be passed to stdin, since the deqp-stdin-caselist option is used
stdin := strings.Join(testNames, "\n") + "\n"
numTests := len(testNames)
timeout := c.TestTimeout * time.Duration(numTests)
outRaw, deqpErr := shell.Exec(timeout, exe, filepath.Dir(exe), env, stdin,
"--deqp-validation="+validation,
"--deqp-surface-type=pbuffer",
"--deqp-shadercache=disable",
"--deqp-log-images=disable",
"--deqp-log-shader-sources=disable",
"--deqp-log-decompiled-spirv=disable",
"--deqp-log-empty-loginfo=disable",
"--deqp-log-flush=disable",
"--deqp-log-filename="+logPath,
"--deqp-stdin-caselist")
duration := time.Since(start)
out := string(outRaw)
out = strings.ReplaceAll(out, exe, "<dEQP>")
for k, v := range c.LogReplacements {
out = strings.ReplaceAll(out, k, v)
}
var coverage *cov.Coverage
if c.CoverageEnv != nil && supportsCoverage {
var covErr error
coverage, covErr = c.CoverageEnv.Import(coverageFile)
if covErr != nil {
log.Printf("Warning: Failed to process test coverage for test '%v'. %v", testNames, covErr)
}
os.Remove(coverageFile)
}
if numTests > 1 {
// Separate output per test case
caseOutputs := caseOutputRE.Split(out, -1)
// If the output isn't as expected, a crash may have happened
isCrash := (len(caseOutputs) != (numTests + 1))
// Verify the exit code to see if a crash has happened
var exitErr *exec.ExitError
if errors.As(deqpErr, &exitErr) {
if exitErr.ExitCode() == 255 {
isCrash = true
}
}
// If a crash has happened, re-run tests separately
if isCrash {
// Re-run tests one by one
for _, testName := range testNames {
singleTest := []string{testName}
c.PerformTests(exe, env, coverageFile, logPath, singleTest, supportsCoverage, results)
}
} else {
caseOutputs = caseOutputs[1:] // Ignore text up to first "Test case '...'"
caseNameMatches := caseOutputRE.FindAllStringSubmatch(out, -1)
caseNames := make([]string, len(caseNameMatches))
for i, m := range caseNameMatches {
caseNames[i] = m[1]
}
averageDuration := duration / time.Duration(numTests)
for i, caseOutput := range caseOutputs {
results <- c.AnalyzeOutput(caseNames[i], caseOutput, averageDuration, coverage, deqpErr)
}
}
} else {
results <- c.AnalyzeOutput(testNames[0], out, duration, coverage, deqpErr)
}
}
func (c *Config) AnalyzeOutput(name string, out string, duration time.Duration, coverage *cov.Coverage, err error) TestResult {
for _, test := range []struct {
re *regexp.Regexp
s testlist.Status
}{
{unimplementedRE, testlist.Unimplemented},
{unsupportedRE, testlist.Unsupported},
{unreachableRE, testlist.Unreachable},
{assertRE, testlist.Assert},
{abortRE, testlist.Abort},
} {
if s := test.re.FindString(out); s != "" {
return TestResult{
Test: name,
Status: test.s,
TimeTaken: duration,
Err: s,
Coverage: coverage,
}
}
}
// Don't treat non-zero error codes as crashes.
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
if exitErr.ExitCode() != 255 {
out += fmt.Sprintf("\nProcess terminated with code %d", exitErr.ExitCode())
err = nil
}
}
switch err.(type) {
default:
return TestResult{
Test: name,
Status: testlist.Crash,
TimeTaken: duration,
Err: out,
Coverage: coverage,
}
case shell.ErrTimeout:
log.Printf("Timeout for test '%v'\n", name)
return TestResult{
Test: name,
Status: testlist.Timeout,
TimeTaken: duration,
Coverage: coverage,
}
case nil:
toks := deqpRE.FindStringSubmatch(out)
if len(toks) < 3 {
return TestResult{Test: name, Status: testlist.Unknown, TimeTaken: duration, Coverage: coverage}
}
switch toks[1] {
case "Pass":
return TestResult{Test: name, Status: testlist.Pass, TimeTaken: duration, Coverage: coverage}
case "NotSupported":
return TestResult{Test: name, Status: testlist.NotSupported, TimeTaken: duration, Coverage: coverage}
case "CompatibilityWarning":
return TestResult{Test: name, Status: testlist.CompatibilityWarning, TimeTaken: duration, Coverage: coverage}
case "QualityWarning":
return TestResult{Test: name, Status: testlist.QualityWarning, TimeTaken: duration, Coverage: coverage}
case "Fail":
var err string
if toks[2] != "Fail" {
err = toks[2]
}
return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
case "InternalError":
var err string
if toks[2] != "InternalError" {
err = toks[2]
}
return TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
default:
err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
log.Println("Warning: ", err)
return TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
}
}
}