Regres: add cmd/run_testlist
This uses regres' test runner, without all the Gerrit functionality.
A faster, less frustrating version of dEQP's --deqp-caselist-file.
Kudos to paulthomson@ who did all the work here.
Change-Id: I1de26e731b197af64aeb60c79da0b75fe13cb338
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/40529
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
new file mode 100644
index 0000000..f489adb
--- /dev/null
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -0,0 +1,108 @@
+// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run_testlist is a tool runs a dEQP test list, using multiple sand-boxed
+// processes.
+//
+// Unlike simply running deqp with its --deqp-caselist-file flag, run_testlist
+// uses multiple sand-boxed processes, which greatly reduces testing time, and
+// gracefully handles crashing processes.
+package main
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "runtime"
+ "sort"
+ "strings"
+ "time"
+
+ "../../cause"
+ "../../deqp"
+ "../../shell"
+ "../../testlist"
+)
+
+var (
+ deqpVkBinary = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
+ testList = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
+ numThreads = flag.Int("num-threads", runtime.NumCPU(), "number of parallel test runner processes")
+ maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+ output = flag.String("output", "results.json", "path to an output JSON results file")
+)
+
+const testTimeout = time.Minute * 2
+
+func runTests() error {
+ tests, err := ioutil.ReadFile(*testList)
+ if err != nil {
+ return cause.Wrap(err, "Couldn't read '%s'", *testList)
+ }
+ group := testlist.Group{
+ Name: "",
+ File: "",
+ API: testlist.Vulkan,
+ }
+ for _, line := range strings.Split(string(tests), "\n") {
+ line = strings.TrimSpace(line)
+ if line != "" && !strings.HasPrefix(line, "#") {
+ group.Tests = append(group.Tests, line)
+ }
+ }
+ sort.Strings(group.Tests)
+
+ testLists := testlist.Lists{group}
+
+ shell.MaxProcMemory = *maxProcMemory
+
+ config := deqp.Config{
+ ExeEgl: "",
+ ExeGles2: "",
+ ExeGles3: "",
+ ExeVulkan: *deqpVkBinary,
+ Env: os.Environ(),
+ NumParallelTests: *numThreads,
+ TestLists: testLists,
+ TestTimeout: testTimeout,
+ }
+
+ res, err := config.Run()
+ if err != nil {
+ return err
+ }
+
+ err = res.Save(*output)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func main() {
+ if runtime.GOOS != "linux" {
+ log.Fatal("regres only currently runs on linux")
+ }
+
+ flag.ErrHelp = errors.New("regres is a tool to detect regressions between versions of SwiftShader")
+ flag.Parse()
+ if err := runTests(); err != nil {
+ _, _ = fmt.Fprintln(os.Stderr, err)
+ os.Exit(-1)
+ }
+}
