Add support for internal error status checking in regres
This CL adds support for checking if the status is InternalError
on deqp when run on regres. This CL also adds support for running
deqp-vk with --deqp-validation=enabled if needed. The status for
validation errors is InternalError.
Bug: b/181718332
Change-Id: I63196f6f6fb244551be8ed03cce7ed79be95ea38
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/53268
Tested-by: Venni Ihanakangas <venni.ihanakangas@siru.fi>
Reviewed-by: Paul Thomson <paulthomson@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>
Commit-Queue: Venni Ihanakangas <venni.ihanakangas@siru.fi>
diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 10eccbe..8674502 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -53,16 +53,17 @@
}
var (
- deqpVkBinary = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
- testList = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
- numThreads = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
- maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
- output = flag.String("output", "results.json", "path to an output JSON results file")
- filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
- limit = flag.Int("limit", 0, "only run a maximum of this number of tests")
- shuffle = flag.Bool("shuffle", false, "shuffle tests")
- noResults = flag.Bool("no-results", false, "disable generation of results.json file")
- genCoverage = flag.Bool("coverage", false, "generate test coverage")
+ deqpVkBinary = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
+ testList = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
+ numThreads = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
+ maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+ output = flag.String("output", "results.json", "path to an output JSON results file")
+ filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
+ limit = flag.Int("limit", 0, "only run a maximum of this number of tests")
+ shuffle = flag.Bool("shuffle", false, "shuffle tests")
+ noResults = flag.Bool("no-results", false, "disable generation of results.json file")
+ genCoverage = flag.Bool("coverage", false, "generate test coverage")
+ enableValidation = flag.Bool("validation", false, "run deqp-vk with Vulkan validation layers")
)
const testTimeout = time.Minute * 2
@@ -111,6 +112,7 @@
NumParallelTests: *numThreads,
TestLists: testlist.Lists{group},
TestTimeout: testTimeout,
+ ValidationLayer: *enableValidation,
}
if *genCoverage {
diff --git a/tests/regres/deqp/deqp.go b/tests/regres/deqp/deqp.go
index 818a6dc..138ca17 100644
--- a/tests/regres/deqp/deqp.go
+++ b/tests/regres/deqp/deqp.go
@@ -42,7 +42,7 @@
var (
// Regular expression to parse the output of a dEQP test.
- deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning) \(([^\)]*)\)`)
+ deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
// Regular expression to parse a test that failed due to UNIMPLEMENTED()
unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
// Regular expression to parse a test that failed due to UNSUPPORTED()
@@ -68,6 +68,7 @@
NumParallelTests int
CoverageEnv *cov.Env
TestTimeout time.Duration
+ ValidationLayer bool
}
// Results holds the results of tests across all APIs.
@@ -316,7 +317,14 @@
// log.Printf("Running test '%s'\n", name)
start := time.Now()
+ // Set validation layer according to flag.
+ validation := "disable"
+ if c.ValidationLayer {
+ validation = "enable"
+ }
+
outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), env,
+ "--deqp-validation="+validation,
"--deqp-surface-type=pbuffer",
"--deqp-shadercache=disable",
"--deqp-log-images=disable",
@@ -411,6 +419,12 @@
err = toks[2]
}
results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
+ case "InternalError":
+ var err string
+ if toks[2] != "InternalError" {
+ err = toks[2]
+ }
+ results <- TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
default:
err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
log.Println("Warning: ", err)
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 22e72c4..04c8eee 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -198,6 +198,8 @@
CompatibilityWarning = Status("COMPATIBILITY_WARNING")
// QualityWarning is the status passing test with a warning.
QualityWarning = Status("QUALITY_WARNING")
+ // InternalError is the status of a test that failed on an API usage error.
+ InternalError = Status("INTERNAL_ERROR")
)
// Statuses is the full list of status types
@@ -214,12 +216,13 @@
NotSupported,
CompatibilityWarning,
QualityWarning,
+ InternalError,
}
// Failing returns true if the task status requires fixing.
func (s Status) Failing() bool {
switch s {
- case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort:
+ case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
return true
case Unsupported:
// This may seem surprising that this should be a failure, however these
