tests/regres/testlist/testlist.go - SwiftShader - Git at Google

 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // Package testlist provides utilities for handling test lists.
 package testlist

 import (
 	"bytes"
 	"crypto/sha1"
 	"encoding/gob"
 	"encoding/hex"
 	"encoding/json"
 	"io/ioutil"
 	"path/filepath"
 	"sort"
 	"strings"

 	"../cause"
 )

 // API is an enumerator of graphics APIs.
 type API string

 // Graphics APIs.
 const (
 	EGL    = API("egl")
 	GLES2  = API("gles2")
 	GLES3  = API("gles3")
 	Vulkan = API("vulkan")
 )

 // Group is a list of tests to be run for a single API.
 type Group struct {
 	Name  string
 	File  string
 	API   API
 	Tests []string
 }

 // Load loads the test list file and appends all tests to the Group.
 func (g *Group) Load() error {
 	tests, err := ioutil.ReadFile(g.File)
 	if err != nil {
 		return cause.Wrap(err, "Couldn't read '%s'", tests)
 	}
 	for _, line := range strings.Split(string(tests), "\n") {
 		line = strings.TrimSpace(line)
 		if line != "" && !strings.HasPrefix(line, "#") {
 			g.Tests = append(g.Tests, line)
 		}
 	}
 	sort.Strings(g.Tests)
 	return nil
 }

 // Filter returns a new Group that contains only tests that match the predicate.
 func (g Group) Filter(pred func(string) bool) Group {
 	out := Group{
 		Name: g.Name,
 		File: g.File,
 		API:  g.API,
 	}
 	for _, test := range g.Tests {
 		if pred(test) {
 			out.Tests = append(out.Tests, test)
 		}
 	}
 	return out
 }

 // Limit returns a new Group that contains a maximum of limit tests.
 func (g Group) Limit(limit int) Group {
 	out := Group{
 		Name:  g.Name,
 		File:  g.File,
 		API:   g.API,
 		Tests: g.Tests,
 	}
 	if len(g.Tests) > limit {
 		out.Tests = g.Tests[:limit]
 	}
 	return out
 }

 // Lists is the full list of tests to be run.
 type Lists []Group

 // Filter returns a new Lists that contains only tests that match the predicate.
 func (l Lists) Filter(pred func(string) bool) Lists {
 	out := Lists{}
 	for _, group := range l {
 		filtered := group.Filter(pred)
 		if len(filtered.Tests) > 0 {
 			out = append(out, filtered)
 		}
 	}
 	return out
 }

 // Hash returns a SHA1 hash of the set of tests.
 func (l Lists) Hash() string {
 	h := sha1.New()
 	if err := gob.NewEncoder(h).Encode(l); err != nil {
 		panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
 	}
 	return hex.EncodeToString(h.Sum(nil))
 }

 // Load loads the test list json file and returns the full set of tests.
 func Load(root, jsonPath string) (Lists, error) {
 	root, err := filepath.Abs(root)
 	if err != nil {
 		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
 	}

 	jsonPath, err = filepath.Abs(jsonPath)
 	if err != nil {
 		return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
 	}

 	i, err := ioutil.ReadFile(jsonPath)
 	if err != nil {
 		return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
 	}

 	var jsonGroups []struct {
 		Name     string
 		API      string
 		TestFile string `json:"tests"`
 	}
 	if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
 		return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
 	}

 	dir := filepath.Dir(jsonPath)

 	out := make(Lists, len(jsonGroups))
 	for i, jsonGroup := range jsonGroups {
 		group := Group{
 			Name: jsonGroup.Name,
 			File: filepath.Join(dir, jsonGroup.TestFile),
 			API:  API(jsonGroup.API),
 		}
 		if err := group.Load(); err != nil {
 			return nil, err
 		}

 		// Make the path relative before displaying it to the world.
 		relPath, err := filepath.Rel(root, group.File)
 		if err != nil {
 			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
 		}
 		group.File = relPath

 		out[i] = group
 	}

 	return out, nil
 }

 // Status is an enumerator of test results.
 type Status string

 const (
 	// Pass is the status of a successful test.
 	Pass = Status("PASS")
 	// Fail is the status of a failed test.
 	Fail = Status("FAIL")
 	// Timeout is the status of a test that failed to complete in the alloted
 	// time.
 	Timeout = Status("TIMEOUT")
 	// Crash is the status of a test that crashed.
 	Crash = Status("CRASH")
 	// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
 	Unimplemented = Status("UNIMPLEMENTED")
 	// Unsupported is the status of a test that failed with UNSUPPORTED().
 	Unsupported = Status("UNSUPPORTED")
 	// Unreachable is the status of a test that failed with UNREACHABLE().
 	Unreachable = Status("UNREACHABLE")
 	// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
 	Assert = Status("ASSERT")
 	// Abort is the status of a test that failed with ABORT().
 	Abort = Status("ABORT")
 	// NotSupported is the status of a test feature not supported by the driver.
 	NotSupported = Status("NOT_SUPPORTED")
 	// CompatibilityWarning is the status passing test with a warning.
 	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
 	// QualityWarning is the status passing test with a warning.
 	QualityWarning = Status("QUALITY_WARNING")
 	// InternalError is the status of a test that failed on an API usage error.
 	InternalError = Status("INTERNAL_ERROR")
 )

 // Statuses is the full list of status types
 var Statuses = []Status{
 	Pass,
 	Fail,
 	Timeout,
 	Crash,
 	Unimplemented,
 	Unsupported,
 	Unreachable,
 	Assert,
 	Abort,
 	NotSupported,
 	CompatibilityWarning,
 	QualityWarning,
 	InternalError,
 }

 // Failing returns true if the task status requires fixing.
 func (s Status) Failing() bool {
 	switch s {
 	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
 		return true
 	case Unsupported:
 		// This may seem surprising that this should be a failure, however these
 		// should not be reached, as dEQP should not be using features that are
 		// not advertised.
 		return true
 	default:
 		return false
 	}
 }

 // Passing returns true if the task status is considered a pass.
 func (s Status) Passing() bool {
 	switch s {
 	case Pass, CompatibilityWarning, QualityWarning:
 		return true
 	default:
 		return false
 	}
 }

 // FilePathWithStatus returns the path to the test list file with the status
 // appended before the file extension.
 func FilePathWithStatus(listPath string, status Status) string {
 	ext := filepath.Ext(listPath)
 	name := listPath[:len(listPath)-len(ext)]
 	return name + "-" + string(status) + ext
 }
	// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// Package testlist provides utilities for handling test lists.
	package testlist

	import (
	"bytes"
	"crypto/sha1"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strings"

	"../cause"
	)

	// API is an enumerator of graphics APIs.
	type API string

	// Graphics APIs.
	const (
	EGL = API("egl")
	GLES2 = API("gles2")
	GLES3 = API("gles3")
	Vulkan = API("vulkan")
	)

	// Group is a list of tests to be run for a single API.
	type Group struct {
	Name string
	File string
	API API
	Tests []string
	}

	// Load loads the test list file and appends all tests to the Group.
	func (g *Group) Load() error {
	tests, err := ioutil.ReadFile(g.File)
	if err != nil {
	return cause.Wrap(err, "Couldn't read '%s'", tests)
	}
	for _, line := range strings.Split(string(tests), "\n") {
	line = strings.TrimSpace(line)
	if line != "" && !strings.HasPrefix(line, "#") {
	g.Tests = append(g.Tests, line)
	}
	}
	sort.Strings(g.Tests)
	return nil
	}

	// Filter returns a new Group that contains only tests that match the predicate.
	func (g Group) Filter(pred func(string) bool) Group {
	out := Group{
	Name: g.Name,
	File: g.File,
	API: g.API,
	}
	for _, test := range g.Tests {
	if pred(test) {
	out.Tests = append(out.Tests, test)
	}
	}
	return out
	}

	// Limit returns a new Group that contains a maximum of limit tests.
	func (g Group) Limit(limit int) Group {
	out := Group{
	Name: g.Name,
	File: g.File,
	API: g.API,
	Tests: g.Tests,
	}
	if len(g.Tests) > limit {
	out.Tests = g.Tests[:limit]
	}
	return out
	}

	// Lists is the full list of tests to be run.
	type Lists []Group

	// Filter returns a new Lists that contains only tests that match the predicate.
	func (l Lists) Filter(pred func(string) bool) Lists {
	out := Lists{}
	for _, group := range l {
	filtered := group.Filter(pred)
	if len(filtered.Tests) > 0 {
	out = append(out, filtered)
	}
	}
	return out
	}

	// Hash returns a SHA1 hash of the set of tests.
	func (l Lists) Hash() string {
	h := sha1.New()
	if err := gob.NewEncoder(h).Encode(l); err != nil {
	panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
	}
	return hex.EncodeToString(h.Sum(nil))
	}

	// Load loads the test list json file and returns the full set of tests.
	func Load(root, jsonPath string) (Lists, error) {
	root, err := filepath.Abs(root)
	if err != nil {
	return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", root)
	}

	jsonPath, err = filepath.Abs(jsonPath)
	if err != nil {
	return nil, cause.Wrap(err, "Couldn't get absolute path of '%s'", jsonPath)
	}

	i, err := ioutil.ReadFile(jsonPath)
	if err != nil {
	return nil, cause.Wrap(err, "Couldn't read test list from '%s'", jsonPath)
	}

	var jsonGroups []struct {
	Name string
	API string
	TestFile string `json:"tests"`
	}
	if err := json.NewDecoder(bytes.NewReader(i)).Decode(&jsonGroups); err != nil {
	return nil, cause.Wrap(err, "Couldn't parse '%s'", jsonPath)
	}

	dir := filepath.Dir(jsonPath)

	out := make(Lists, len(jsonGroups))
	for i, jsonGroup := range jsonGroups {
	group := Group{
	Name: jsonGroup.Name,
	File: filepath.Join(dir, jsonGroup.TestFile),
	API: API(jsonGroup.API),
	}
	if err := group.Load(); err != nil {
	return nil, err
	}

	// Make the path relative before displaying it to the world.
	relPath, err := filepath.Rel(root, group.File)
	if err != nil {
	return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
	}
	group.File = relPath

	out[i] = group
	}

	return out, nil
	}

	// Status is an enumerator of test results.
	type Status string

	const (
	// Pass is the status of a successful test.
	Pass = Status("PASS")
	// Fail is the status of a failed test.
	Fail = Status("FAIL")
	// Timeout is the status of a test that failed to complete in the alloted
	// time.
	Timeout = Status("TIMEOUT")
	// Crash is the status of a test that crashed.
	Crash = Status("CRASH")
	// Unimplemented is the status of a test that failed with UNIMPLEMENTED().
	Unimplemented = Status("UNIMPLEMENTED")
	// Unsupported is the status of a test that failed with UNSUPPORTED().
	Unsupported = Status("UNSUPPORTED")
	// Unreachable is the status of a test that failed with UNREACHABLE().
	Unreachable = Status("UNREACHABLE")
	// Assert is the status of a test that failed with ASSERT() or ASSERT_MSG().
	Assert = Status("ASSERT")
	// Abort is the status of a test that failed with ABORT().
	Abort = Status("ABORT")
	// NotSupported is the status of a test feature not supported by the driver.
	NotSupported = Status("NOT_SUPPORTED")
	// CompatibilityWarning is the status passing test with a warning.
	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
	// QualityWarning is the status passing test with a warning.
	QualityWarning = Status("QUALITY_WARNING")
	// InternalError is the status of a test that failed on an API usage error.
	InternalError = Status("INTERNAL_ERROR")
	)

	// Statuses is the full list of status types
	var Statuses = []Status{
	Pass,
	Fail,
	Timeout,
	Crash,
	Unimplemented,
	Unsupported,
	Unreachable,
	Assert,
	Abort,
	NotSupported,
	CompatibilityWarning,
	QualityWarning,
	InternalError,
	}

	// Failing returns true if the task status requires fixing.
	func (s Status) Failing() bool {
	switch s {
	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
	return true
	case Unsupported:
	// This may seem surprising that this should be a failure, however these
	// should not be reached, as dEQP should not be using features that are
	// not advertised.
	return true
	default:
	return false
	}
	}

	// Passing returns true if the task status is considered a pass.
	func (s Status) Passing() bool {
	switch s {
	case Pass, CompatibilityWarning, QualityWarning:
	return true
	default:
	return false
	}
	}

	// FilePathWithStatus returns the path to the test list file with the status
	// appended before the file extension.
	func FilePathWithStatus(listPath string, status Status) string {
	ext := filepath.Ext(listPath)
	name := listPath[:len(listPath)-len(ext)]
	return name + "-" + string(status) + ext
	}