regres: Various bug fixes
* For the daily update, roll dEQP *before* doing the test run. I don't know why this was done after, as this gives the illusion that the results are for the wrong version of dEQP.
* The vast majority of tests are unsupported with SwiftShader, and produce no test output. This would produce excessive log spam, and would be treated as a failure. Bin these under a new 'Unknown' status.
* Call os.MkdirAll() before attempting to call os.WriteFile(). If the directory didn't exist, then the write would fail.
* Remove the coverage generation logic. It was disabled, and is extremely unlikely to ever come back.
Change-Id: Id65a0fc3c6917040b7081ed0d2149292f0262372
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/73489
Tested-by: Ben Clayton <bclayton@google.com>
Commit-Queue: Shahbaz Youssefi <syoussefi@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Shahbaz Youssefi <syoussefi@google.com>
Presubmit-Ready: Ben Clayton <bclayton@google.com>
Tested-by: Shahbaz Youssefi <syoussefi@google.com>
diff --git a/tests/regres/cmd/regres/main.go b/tests/regres/cmd/regres/main.go
index f51b63e..a3d2e4f 100644
--- a/tests/regres/cmd/regres/main.go
+++ b/tests/regres/cmd/regres/main.go
@@ -300,10 +300,10 @@
for {
if now := time.Now(); toDate(now) != lastUpdatedTestLists {
lastUpdatedTestLists = toDate(now)
- if err := r.runDaily(client, backendLLVM, false); err != nil {
+ if err := r.runDaily(client, backendLLVM); err != nil {
log.Println(err.Error())
}
- if err := r.runDaily(client, backendSubzero, true); err != nil {
+ if err := r.runDaily(client, backendSubzero); err != nil {
log.Println(err.Error())
}
}
@@ -636,26 +636,11 @@
}
// runDaily runs a full deqp run on the HEAD change, posting the results to a
-// new or existing gerrit change. If genCov is true, then coverage
-// information will be generated for the run, and commiteed to the
-// coverageBranch.
-func (r *regres) runDaily(client *gerrit.Client, reactorBackend reactorBackend, genCov bool) error {
- // TODO(b/152192800): Generating coverage data is currently broken.
- genCov = false
-
+// new or existing gerrit change.
+func (r *regres) runDaily(client *gerrit.Client, reactorBackend reactorBackend) error {
log.Printf("Updating test lists (Backend: %v)\n", reactorBackend)
- if genCov {
- if r.githubUser == "" {
- log.Println("--gh-user not specified and SS_GITHUB_USER not set. Disabling code coverage generation")
- genCov = false
- } else if r.githubPass == "" {
- log.Println("--gh-pass not specified and SS_GITHUB_PASS not set. Disabling code coverage generation")
- genCov = false
- }
- }
-
- dailyHash := git.Hash{}
+ var dailyHash git.Hash
if r.dailyChange == "" {
headHash, err := git.FetchRefHash(gitDailyBranch, gitURL)
if err != nil {
@@ -666,34 +651,6 @@
dailyHash = git.ParseHash(r.dailyChange)
}
- return r.runDailyTest(dailyHash, reactorBackend, genCov,
- func(test *test, testLists testlist.Lists, results *deqp.Results) error {
- errs := []error{}
-
- if err := r.postDailyResults(client, test, testLists, results, reactorBackend, dailyHash); err != nil {
- errs = append(errs, err)
- }
-
- if genCov {
- if err := r.postCoverageResults(results.Coverage, dailyHash); err != nil {
- errs = append(errs, err)
- }
- }
-
- if len(errs) > 0 {
- msg := strings.Builder{}
- for _, err := range errs {
- msg.WriteString(err.Error() + "\n")
- }
- return fmt.Errorf("%s", msg.String())
- }
- return nil
- })
-}
-
-// runDailyTest performs the full deqp run on the HEAD change, calling
-// withResults with the test results.
-func (r *regres) runDailyTest(dailyHash git.Hash, reactorBackend reactorBackend, genCov bool, withResults func(*test, testlist.Lists, *deqp.Results) error) error {
// Get the full test results.
test := r.newTest(dailyHash).setReactorBackend(reactorBackend)
defer test.cleanup()
@@ -703,6 +660,12 @@
return fmt.Errorf("failed to checkout '%s': %w", dailyHash, err)
}
+ // Update dEQP to latest
+ newPaths, err := r.updateLocalDeqpFiles(test)
+ if err != nil {
+ return fmt.Errorf("failed to update test lists from dEQP: %w", err)
+ }
+
d, err := r.getOrBuildDEQP(test)
if err != nil {
return fmt.Errorf("failed to build deqp for '%s': %w", dailyHash, err)
@@ -714,15 +677,6 @@
return fmt.Errorf("failed to load full test lists for '%s': %w", dailyHash, err)
}
- if genCov {
- test.coverageEnv = &cov.Env{
- LLVM: *r.toolchain,
- RootDir: test.checkoutDir,
- ExePath: filepath.Join(test.buildDir, "libvk_swiftshader.so"),
- TurboCov: filepath.Join(test.buildDir, "turbo-cov"),
- }
- }
-
// Build the change.
if err := test.build(); err != nil {
return fmt.Errorf("failed to build '%s': %w", dailyHash, err)
@@ -734,7 +688,13 @@
return fmt.Errorf("failed to test '%s': %w", dailyHash, err)
}
- return withResults(test, testLists, results)
+ if err := r.postDailyResults(client, test, testLists, results, reactorBackend, dailyHash, newPaths); err != nil {
+ msg := strings.Builder{}
+ msg.WriteString(err.Error() + "\n")
+ return fmt.Errorf("%s", msg.String())
+ }
+
+ return nil
}
// copyFileIfDifferent copies src to dst if src doesn't exist or if there are differences
@@ -755,6 +715,9 @@
}
if !bytes.Equal(srcContents, dstContents) {
+ if err := os.MkdirAll(path.Dir(dst), 0777); err != nil {
+ return err
+ }
if err := os.WriteFile(dst, srcContents, srcFileInfo.Mode()); err != nil {
return err
}
@@ -808,7 +771,6 @@
// Use getOrBuildDEQPFromConfig as it'll prevent us from copying data from a revision of dEQP that has build errors.
deqpBuild, err := r.getOrBuildDEQPFromConfig(test, cfg, test.checkoutDir)
-
if err != nil {
return nil, fmt.Errorf("failed to retrieve dEQP build information: %w", err)
}
@@ -893,7 +855,8 @@
testLists testlist.Lists,
results *deqp.Results,
reactorBackend reactorBackend,
- dailyHash git.Hash) error {
+ dailyHash git.Hash,
+ newPaths []string) error {
// Write out the test list status files.
filePaths, err := test.writeTestListsByStatus(testLists, results)
@@ -901,11 +864,6 @@
return fmt.Errorf("failed to write test lists by status: %w", err)
}
- newPaths, err := r.updateLocalDeqpFiles(test)
- if err != nil {
- return fmt.Errorf("failed to update test lists from dEQP: %w", err)
- }
-
filePaths = append(filePaths, newPaths...)
// Stage all the updated test files.
@@ -1496,6 +1454,7 @@
}
sb := strings.Builder{}
+ sb.WriteString("```\n")
// list prints the list l to sb, truncating after a limit.
list := func(l []string) {
@@ -1547,7 +1506,7 @@
}
if len(broken) == 0 && len(fixed) == 0 && len(removed) == 0 && len(changed) == 0 {
- sb.WriteString(fmt.Sprintf("\n--- No change in test results ---\n"))
+ sb.WriteString("\n--- No change in test results ---\n")
}
sb.WriteString(fmt.Sprintf(" Total tests: %d\n", totalTests))
@@ -1567,6 +1526,7 @@
{" Not Supported", testlist.NotSupported},
{"Compatibility Warning", testlist.CompatibilityWarning},
{" Quality Warning", testlist.QualityWarning},
+ {" Unknown", testlist.Unknown},
} {
old, new := oldStatusCounts[s.status], newStatusCounts[s.status]
if old == 0 && new == 0 {
@@ -1634,6 +1594,8 @@
}
}
+ sb.WriteString("```\n")
+
return sb.String(), alert
}
diff --git a/tests/regres/deqp/deqp.go b/tests/regres/deqp/deqp.go
index 0ac43ae..b40e17d 100644
--- a/tests/regres/deqp/deqp.go
+++ b/tests/regres/deqp/deqp.go
@@ -462,9 +462,7 @@
case nil:
toks := deqpRE.FindStringSubmatch(out)
if len(toks) < 3 {
- err := fmt.Sprintf("Couldn't parse test '%v' output:\n%s", name, out)
- log.Println("Warning: ", err)
- return TestResult{Test: name, Status: testlist.Fail, Err: err, Coverage: coverage}
+ return TestResult{Test: name, Status: testlist.Unknown, TimeTaken: duration, Coverage: coverage}
}
switch toks[1] {
case "Pass":
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 98e385a..3943fbf 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -208,6 +208,8 @@
QualityWarning = Status("QUALITY_WARNING")
// InternalError is the status of a test that failed on an API usage error.
InternalError = Status("INTERNAL_ERROR")
+ // Unknown is the status of a test that had a result that could not be parsed.
+ Unknown = Status("INTERNAL_ERROR")
)
// Statuses is the full list of status types
@@ -225,6 +227,7 @@
CompatibilityWarning,
QualityWarning,
InternalError,
+ Unknown,
}
// Failing returns true if the task status requires fixing.
