Regres: Add new run_testlist flags
`shuffle` will randomize the order in which the tests are run.
`limit` will run (post-shuffle) at most this number of tests.
`no-results` will disable emission of the `results.json` file.
These are useful for testing a random subset of the filtered test list.
Change-Id: Ied54b4fd57c9e02fa6a9e63b808417fdc8f73a43
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/42710
Tested-by: Ben Clayton <bclayton@google.com>
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 520695c..75a0f8e 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -25,6 +25,7 @@
"flag"
"fmt"
"log"
+ "math/rand"
"os"
"path/filepath"
"regexp"
@@ -44,6 +45,9 @@
maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
output = flag.String("output", "results.json", "path to an output JSON results file")
filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
+ limit = flag.Int("limit", 0, "only run a maximum of this number of tests")
+ shuffle = flag.Bool("shuffle", false, "shuffle tests")
+ noResults = flag.Bool("no-results", false, "disable generation of results.json file")
)
const testTimeout = time.Minute * 2
@@ -70,10 +74,19 @@
}
}
- testLists := testlist.Lists{group}
-
shell.MaxProcMemory = *maxProcMemory
+ if *shuffle {
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+ rnd.Shuffle(len(group.Tests), func(i, j int) { group.Tests[i], group.Tests[j] = group.Tests[j], group.Tests[i] })
+ }
+
+ if *limit != 0 && len(group.Tests) > *limit {
+ group.Tests = group.Tests[:*limit]
+ }
+
+ log.Printf("Running %d tests...\n", len(group.Tests))
+
config := deqp.Config{
ExeEgl: "",
ExeGles2: "",
@@ -81,12 +94,10 @@
ExeVulkan: *deqpVkBinary,
Env: os.Environ(),
NumParallelTests: *numThreads,
- TestLists: testLists,
+ TestLists: testlist.Lists{group},
TestTimeout: testTimeout,
}
- log.Printf("Running %d tests...\n", len(group.Tests))
-
res, err := config.Run()
if err != nil {
return err
@@ -102,9 +113,11 @@
}
}
- err = res.Save(*output)
- if err != nil {
- return err
+ if !*noResults {
+ err = res.Save(*output)
+ if err != nil {
+ return err
+ }
}
return nil
