blob: 98e385ad8a5d9dce0e36ae7a3a2fcc900692a806 [file] [log] [blame]
// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package testlist provides utilities for handling test lists.
package testlist
import (
"bytes"
"crypto/sha1"
"encoding/gob"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"sort"
"strings"
)
// API is an enumerator of graphics APIs.
type API string
// Graphics APIs.
const (
EGL = API("egl")
GLES2 = API("gles2")
GLES3 = API("gles3")
Vulkan = API("vulkan")
)
// Group is a list of tests to be run for a single API.
type Group struct {
Name string
File string
API API
Tests []string
}
// Load loads the test list file and appends all tests to the Group.
func (g *Group) Load() error {
return g.LoadFile(g.File)
}
func (g *Group) LoadFile(file string) error {
dir, _ := filepath.Split(file)
tests, err := ioutil.ReadFile(file)
if err != nil {
return fmt.Errorf("failed to read '%s': %w", file, err)
}
for _, line := range strings.Split(string(tests), "\n") {
line = strings.TrimSpace(line)
// The test list file can contain references to other .txt files
// containing the individual tests.
if strings.HasSuffix(line, ".txt") {
g.LoadFile(filepath.Join(dir, line))
} else if line != "" && !strings.HasPrefix(line, "#") {
g.Tests = append(g.Tests, line)
}
}
sort.Strings(g.Tests)
return nil
}
// Filter returns a new Group that contains only tests that match the predicate.
func (g Group) Filter(pred func(string) bool) Group {
out := Group{
Name: g.Name,
File: g.File,
API: g.API,
}
for _, test := range g.Tests {
if pred(test) {
out.Tests = append(out.Tests, test)
}
}
return out
}
// Limit returns a new Group that contains a maximum of limit tests.
func (g Group) Limit(limit int) Group {
out := Group{
Name: g.Name,
File: g.File,
API: g.API,
Tests: g.Tests,
}
if len(g.Tests) > limit {
out.Tests = g.Tests[:limit]
}
return out
}
// Lists is the full list of tests to be run.
type Lists []Group
// Filter returns a new Lists that contains only tests that match the predicate.
func (l Lists) Filter(pred func(string) bool) Lists {
out := Lists{}
for _, group := range l {
filtered := group.Filter(pred)
if len(filtered.Tests) > 0 {
out = append(out, filtered)
}
}
return out
}
// Hash returns a SHA1 hash of the set of tests.
func (l Lists) Hash() string {
h := sha1.New()
if err := gob.NewEncoder(h).Encode(l); err != nil {
panic(fmt.Errorf("failed to encode testlist to produce hash: %w", err))
}
return hex.EncodeToString(h.Sum(nil))
}
// Load loads the test list json file and returns the full set of tests.
func Load(root, jsonPath string) (Lists, error) {
root, err := filepath.Abs(root)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path of '%s': %w", root, err)
}
jsonPath, err = filepath.Abs(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path of '%s': %w", jsonPath, err)
}
i, err := ioutil.ReadFile(jsonPath)
if err != nil {
return nil, fmt.Errorf("failed to read test list from '%s': %w", jsonPath, err)
}
var jsonGroups []struct {
Name string
API string
TestFile string `json:"tests"`
}
if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
return nil, fmt.Errorf("failed to parse '%s': %w", jsonPath, err)
}
dir := filepath.Dir(jsonPath)
out := make(Lists, len(jsonGroups))
for i, jsonGroup := range jsonGroups {
group := Group{
Name: jsonGroup.Name,
File: filepath.Join(dir, jsonGroup.TestFile),
API: API(jsonGroup.API),
}
if err := group.Load(); err != nil {
return nil, err
}
// Make the path relative before displaying it to the world.
relPath, err := filepath.Rel(root, group.File)
if err != nil {
return nil, fmt.Errorf("failed to get relative path for '%s': %w", group.File, err)
}
group.File = relPath
out[i] = group
}
return out, nil
}
// Status is an enumerator of test results.
type Status string
const (
// Pass is the status of a successful test.
Pass = Status("PASS")
// Fail is the status of a failed test.
Fail = Status("FAIL")
// Timeout is the status of a test that failed to complete in the alloted
// time.
Timeout = Status("TIMEOUT")
// Crash is the status of a test that crashed.
Crash = Status("CRASH")
// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
Unimplemented = Status("UNIMPLEMENTED")
// Unsupported is the status of a test that failed with UNSUPPORTED().
Unsupported = Status("UNSUPPORTED")
// Unreachable is the status of a test that failed with UNREACHABLE().
Unreachable = Status("UNREACHABLE")
// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
Assert = Status("ASSERT")
// Abort is the status of a test that failed with ABORT().
Abort = Status("ABORT")
// NotSupported is the status of a test feature not supported by the driver.
NotSupported = Status("NOT_SUPPORTED")
// CompatibilityWarning is the status passing test with a warning.
CompatibilityWarning = Status("COMPATIBILITY_WARNING")
// QualityWarning is the status passing test with a warning.
QualityWarning = Status("QUALITY_WARNING")
// InternalError is the status of a test that failed on an API usage error.
InternalError = Status("INTERNAL_ERROR")
)
// Statuses is the full list of status types
var Statuses = []Status{
Pass,
Fail,
Timeout,
Crash,
Unimplemented,
Unsupported,
Unreachable,
Assert,
Abort,
NotSupported,
CompatibilityWarning,
QualityWarning,
InternalError,
}
// Failing returns true if the task status requires fixing.
func (s Status) Failing() bool {
switch s {
case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
return true
case Unsupported:
// This may seem surprising that this should be a failure, however these
// should not be reached, as dEQP should not be using features that are
// not advertised.
return true
default:
return false
}
}
// Passing returns true if the task status is considered a pass.
func (s Status) Passing() bool {
switch s {
case Pass, CompatibilityWarning, QualityWarning:
return true
default:
return false
}
}
// FilePathWithStatus returns the path to the test list file with the status
// appended before the file extension.
func FilePathWithStatus(listPath string, status Status) string {
ext := filepath.Ext(listPath)
name := listPath[:len(listPath)-len(ext)]
return name + "-" + string(status) + ext
}