Regres: Simplify running of local tests
Add `filter` flag to run_testlist so that tests can be filtered by wildcard or regex.
Update `run_testlist.sh` and `run_testlist.bat` to automatically use the vk-master.txt test list.
Change-Id: I94ef1d2e9220f18bdf50555db8c291b3736ec3f3
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/42089
Reviewed-by: Paul Thomson <paulthomson@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 212aef8..520695c 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -24,14 +24,14 @@
"errors"
"flag"
"fmt"
- "io/ioutil"
+ "log"
"os"
+ "path/filepath"
+ "regexp"
"runtime"
- "sort"
"strings"
"time"
- "../../cause"
"../../deqp"
"../../shell"
"../../testlist"
@@ -43,27 +43,32 @@
numThreads = flag.Int("num-threads", runtime.NumCPU(), "number of parallel test runner processes")
maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
output = flag.String("output", "results.json", "path to an output JSON results file")
+ filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
)
const testTimeout = time.Minute * 2
-func runTests() error {
- tests, err := ioutil.ReadFile(*testList)
- if err != nil {
- return cause.Wrap(err, "Couldn't read '%s'", *testList)
- }
+func run() error {
group := testlist.Group{
Name: "",
- File: "",
+ File: *testList,
API: testlist.Vulkan,
}
- for _, line := range strings.Split(string(tests), "\n") {
- line = strings.TrimSpace(line)
- if line != "" && !strings.HasPrefix(line, "#") {
- group.Tests = append(group.Tests, line)
+ if err := group.Load(); err != nil {
+ return err
+ }
+
+ if *filter != "" {
+ if strings.HasPrefix(*filter, "/") {
+ re := regexp.MustCompile((*filter)[1:])
+ group = group.Filter(re.MatchString)
+ } else {
+ group = group.Filter(func(name string) bool {
+ ok, _ := filepath.Match(*filter, name)
+ return ok
+ })
}
}
- sort.Strings(group.Tests)
testLists := testlist.Lists{group}
@@ -80,11 +85,23 @@
TestTimeout: testTimeout,
}
+ log.Printf("Running %d tests...\n", len(group.Tests))
+
res, err := config.Run()
if err != nil {
return err
}
+ counts := map[testlist.Status]int{}
+ for _, r := range res.Tests {
+ counts[r.Status] = counts[r.Status] + 1
+ }
+ for _, s := range testlist.Statuses {
+ if count := counts[s]; count > 0 {
+ log.Printf("%s: %d\n", string(s), count)
+ }
+ }
+
err = res.Save(*output)
if err != nil {
return err
@@ -96,7 +113,7 @@
func main() {
flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
flag.Parse()
- if err := runTests(); err != nil {
+ if err := run(); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(-1)
}
diff --git a/tests/regres/run_testlist.bat b/tests/regres/run_testlist.bat
index ae9aaf1..a83b16f 100644
--- a/tests/regres/run_testlist.bat
+++ b/tests/regres/run_testlist.bat
@@ -1 +1 @@
-go run %~dp0cmd\run_testlist\main.go %*
\ No newline at end of file
+go run %~dp0cmd\run_testlist\main.go --test-list=%~dp0testlists\vk-master.txt %*
diff --git a/tests/regres/run_testlist.sh b/tests/regres/run_testlist.sh
new file mode 100755
index 0000000..04d4d10
--- /dev/null
+++ b/tests/regres/run_testlist.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+
+go run $ROOT_DIR/cmd/run_testlist/main.go --test-list=$ROOT_DIR/testlists/vk-master.txt $@
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 41bc4ca..a9f5889 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -48,6 +48,22 @@
Tests []string
}
+// Load loads the test list file and appends all tests to the Group.
+func (g *Group) Load() error {
+ tests, err := ioutil.ReadFile(g.File)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't read '%s'", tests)
+ }
+ for _, line := range strings.Split(string(tests), "\n") {
+ line = strings.TrimSpace(line)
+ if line != "" && !strings.HasPrefix(line, "#") {
+ g.Tests = append(g.Tests, line)
+ }
+ }
+ sort.Strings(g.Tests)
+ return nil
+}
+
// Filter returns a new Group that contains only tests that match the predicate.
func (g Group) Filter(pred func(string) bool) Group {
out := Group{
@@ -117,27 +133,22 @@
out := make(Lists, len(jsonGroups))
for i, jsonGroup := range jsonGroups {
- path := filepath.Join(dir, jsonGroup.TestFile)
- tests, err := ioutil.ReadFile(path)
- if err != nil {
- return nil, cause.Wrap(err, "Couldn't read '%s'", tests)
- }
- relPath, err := filepath.Rel(root, path)
- if err != nil {
- return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path)
- }
group := Group{
Name: jsonGroup.Name,
- File: relPath,
+ File: filepath.Join(dir, jsonGroup.TestFile),
API: API(jsonGroup.API),
}
- for _, line := range strings.Split(string(tests), "\n") {
- line = strings.TrimSpace(line)
- if line != "" && !strings.HasPrefix(line, "#") {
- group.Tests = append(group.Tests, line)
- }
+ if err := group.Load(); err != nil {
+ return nil, err
}
- sort.Strings(group.Tests)
+
+ // Make the path relative before displaying it to the world.
+ relPath, err := filepath.Rel(root, group.File)
+ if err != nil {
+ return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
+ }
+ group.File = relPath
+
out[i] = group
}
