Regres: Simplify running of local tests

Add `filter` flag to run_testlist so that tests can be filtered by wildcard or regex.
Update `run_testlist.sh` and `run_testlist.bat` to automatically use the vk-master.txt test list.

Change-Id: I94ef1d2e9220f18bdf50555db8c291b3736ec3f3
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/42089
Reviewed-by: Paul Thomson <paulthomson@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Tested-by: Ben Clayton <bclayton@google.com>

tests/regres/cmd/run_testlist/main.go

diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 212aef8..520695c 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -24,14 +24,14 @@
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"log"
 	"os"
+	"path/filepath"
+	"regexp"
 	"runtime"
-	"sort"
 	"strings"
 	"time"
 
-	"../../cause"
 	"../../deqp"
 	"../../shell"
 	"../../testlist"
@@ -43,27 +43,32 @@
 	numThreads    = flag.Int("num-threads", runtime.NumCPU(), "number of parallel test runner processes")
 	maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
 	output        = flag.String("output", "results.json", "path to an output JSON results file")
+	filter        = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
 )
 
 const testTimeout = time.Minute * 2
 
-func runTests() error {
-	tests, err := ioutil.ReadFile(*testList)
-	if err != nil {
-		return cause.Wrap(err, "Couldn't read '%s'", *testList)
-	}
+func run() error {
 	group := testlist.Group{
 		Name: "",
-		File: "",
+		File: *testList,
 		API:  testlist.Vulkan,
 	}
-	for _, line := range strings.Split(string(tests), "\n") {
-		line = strings.TrimSpace(line)
-		if line != "" && !strings.HasPrefix(line, "#") {
-			group.Tests = append(group.Tests, line)
+	if err := group.Load(); err != nil {
+		return err
+	}
+
+	if *filter != "" {
+		if strings.HasPrefix(*filter, "/") {
+			re := regexp.MustCompile((*filter)[1:])
+			group = group.Filter(re.MatchString)
+		} else {
+			group = group.Filter(func(name string) bool {
+				ok, _ := filepath.Match(*filter, name)
+				return ok
+			})
 		}
 	}
-	sort.Strings(group.Tests)
 
 	testLists := testlist.Lists{group}
 
@@ -80,11 +85,23 @@
 		TestTimeout:      testTimeout,
 	}
 
+	log.Printf("Running %d tests...\n", len(group.Tests))
+
 	res, err := config.Run()
 	if err != nil {
 		return err
 	}
 
+	counts := map[testlist.Status]int{}
+	for _, r := range res.Tests {
+		counts[r.Status] = counts[r.Status] + 1
+	}
+	for _, s := range testlist.Statuses {
+		if count := counts[s]; count > 0 {
+			log.Printf("%s: %d\n", string(s), count)
+		}
+	}
+
 	err = res.Save(*output)
 	if err != nil {
 		return err
@@ -96,7 +113,7 @@
 func main() {
 	flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
 	flag.Parse()
-	if err := runTests(); err != nil {
+	if err := run(); err != nil {
 		_, _ = fmt.Fprintln(os.Stderr, err)
 		os.Exit(-1)
 	}

tests/regres/run_testlist.bat

diff --git a/tests/regres/run_testlist.bat b/tests/regres/run_testlist.bat
index ae9aaf1..a83b16f 100644
--- a/tests/regres/run_testlist.bat
+++ b/tests/regres/run_testlist.bat
@@ -1 +1 @@
-go run %~dp0cmd\run_testlist\main.go %*
\ No newline at end of file
+go run %~dp0cmd\run_testlist\main.go --test-list=%~dp0testlists\vk-master.txt %*

tests/regres/run_testlist.sh

diff --git a/tests/regres/run_testlist.sh b/tests/regres/run_testlist.sh
new file mode 100755
index 0000000..04d4d10
--- /dev/null
+++ b/tests/regres/run_testlist.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+
+go run $ROOT_DIR/cmd/run_testlist/main.go --test-list=$ROOT_DIR/testlists/vk-master.txt $@

tests/regres/testlist/testlist.go

diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 41bc4ca..a9f5889 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -48,6 +48,22 @@
 	Tests []string
 }
 
+// Load loads the test list file and appends all tests to the Group.
+func (g *Group) Load() error {
+	tests, err := ioutil.ReadFile(g.File)
+	if err != nil {
+		return cause.Wrap(err, "Couldn't read '%s'", tests)
+	}
+	for _, line := range strings.Split(string(tests), "\n") {
+		line = strings.TrimSpace(line)
+		if line != "" && !strings.HasPrefix(line, "#") {
+			g.Tests = append(g.Tests, line)
+		}
+	}
+	sort.Strings(g.Tests)
+	return nil
+}
+
 // Filter returns a new Group that contains only tests that match the predicate.
 func (g Group) Filter(pred func(string) bool) Group {
 	out := Group{
@@ -117,27 +133,22 @@
 
 	out := make(Lists, len(jsonGroups))
 	for i, jsonGroup := range jsonGroups {
-		path := filepath.Join(dir, jsonGroup.TestFile)
-		tests, err := ioutil.ReadFile(path)
-		if err != nil {
-			return nil, cause.Wrap(err, "Couldn't read '%s'", tests)
-		}
-		relPath, err := filepath.Rel(root, path)
-		if err != nil {
-			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", path)
-		}
 		group := Group{
 			Name: jsonGroup.Name,
-			File: relPath,
+			File: filepath.Join(dir, jsonGroup.TestFile),
 			API:  API(jsonGroup.API),
 		}
-		for _, line := range strings.Split(string(tests), "\n") {
-			line = strings.TrimSpace(line)
-			if line != "" && !strings.HasPrefix(line, "#") {
-				group.Tests = append(group.Tests, line)
-			}
+		if err := group.Load(); err != nil {
+			return nil, err
 		}
-		sort.Strings(group.Tests)
+
+		// Make the path relative before displaying it to the world.
+		relPath, err := filepath.Rel(root, group.File)
+		if err != nil {
+			return nil, cause.Wrap(err, "Couldn't get relative path for '%s'", group.File)
+		}
+		group.File = relPath
+
 		out[i] = group
 	}