regres: Automatically build dEQP based on new config file
Keeps the test lists and version of dEQP in sync.
Also helps reproducibility by removing another set of external environment flags.
Bug: b/140172528
Change-Id: Ib595a1c0c69ffedbfa7848ce8a31c3927bd78980
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/37448
Kokoro-Presubmit: kokoro <noreply+kokoro@google.com>
Reviewed-by: Alexis Hétu <sugoi@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Tested-by: Ben Clayton <bclayton@google.com>
diff --git a/tests/regres/deqp-patches/deqp-x11.patch b/tests/regres/deqp-patches/deqp-x11.patch
new file mode 100644
index 0000000..d9b760c
--- /dev/null
+++ b/tests/regres/deqp-patches/deqp-x11.patch
@@ -0,0 +1,39 @@
+diff --git a/.gitignore b/.gitignore
+index d6c5c502b..d22bff565 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -8,3 +8,4 @@
+ !.editorconfig
+ !.travis.yml
+ !.github
++/build/
+\ No newline at end of file
+diff --git a/framework/platform/lnx/X11/tcuLnxX11.cpp b/framework/platform/lnx/X11/tcuLnxX11.cpp
+index dfe09d060..b4493e431 100644
+--- a/framework/platform/lnx/X11/tcuLnxX11.cpp
++++ b/framework/platform/lnx/X11/tcuLnxX11.cpp
+@@ -24,6 +24,7 @@
+ #include "tcuLnxX11.hpp"
+ #include "gluRenderConfig.hpp"
+ #include "deMemory.h"
++#include "deThread.hpp"
+
+ #include <X11/Xutil.h>
+
+@@ -80,10 +81,14 @@ XlibDisplay::XlibDisplay (EventState& eventState, const char* name)
+ // a multi-threaded program calls, and it must complete before any other
+ // Xlib call is made.
+ DE_CHECK_RUNTIME_ERR(XInitThreads() != 0);
+- m_display = XOpenDisplay((char*)name); // Won't modify argument string.
++ for (int i = 0; i < 10; i++) {
++ m_display = XOpenDisplay((char*)name); // Won't modify argument string.
++ if (m_display)
++ break;
++ deSleep(100*(1+i));
++ }
+ if (!m_display)
+ throw ResourceError("Failed to open display", name, __FILE__, __LINE__);
+-
+ m_deleteAtom = XInternAtom(m_display, "WM_DELETE_WINDOW", False);
+ }
+
diff --git a/tests/regres/deqp.json b/tests/regres/deqp.json
new file mode 100644
index 0000000..b82fbdf
--- /dev/null
+++ b/tests/regres/deqp.json
@@ -0,0 +1,7 @@
+{
+ "remote": "git@gitlab.khronos.org:Tracker/vk-gl-cts.git",
+ "sha": "76a2b81e0ca86a65a927892ee4340bc6685e86b3",
+ "patches": [
+ "tests/regres/deqp-patches/deqp-x11.patch"
+ ]
+}
\ No newline at end of file
diff --git a/tests/regres/git/git.go b/tests/regres/git/git.go
index 787fa7c..fec3d41 100644
--- a/tests/regres/git/git.go
+++ b/tests/regres/git/git.go
@@ -120,7 +120,7 @@
for _, cmds := range [][]string{
{"init"},
{"remote", "add", "origin", url},
- {"fetch", "origin", commit.String()},
+ {"fetch", "origin", "--depth=1", commit.String()},
{"checkout", commit.String()},
} {
if err := shell.Shell(gitTimeout, exe, path, cmds...); err != nil {
@@ -132,6 +132,11 @@
return nil
}
+// Apply applys the patch file to the git repo at dir.
+func Apply(dir, patch string) error {
+ return shell.Shell(gitTimeout, exe, dir, "apply", patch)
+}
+
// FetchRefHash returns the git hash of the given ref.
func FetchRefHash(ref, url string) (Hash, error) {
out, err := shell.Exec(gitTimeout, exe, "", nil, "ls-remote", url, ref)
diff --git a/tests/regres/main.go b/tests/regres/main.go
index 13a2f85..e8fc57a 100644
--- a/tests/regres/main.go
+++ b/tests/regres/main.go
@@ -26,6 +26,8 @@
package main
import (
+ "crypto/sha1"
+ "encoding/hex"
"encoding/json"
"errors"
"flag"
@@ -35,6 +37,7 @@
"math/rand"
"os"
"os/exec"
+ "path"
"path/filepath"
"regexp"
"runtime"
@@ -64,12 +67,12 @@
dailyUpdateTestListHour = 5 // 5am
fullTestListRelPath = "tests/regres/full-tests.json"
ciTestListRelPath = "tests/regres/ci-tests.json"
+ deqpConfigRelPath = "tests/regres/deqp.json"
)
var (
numParallelTests = runtime.NumCPU()
- deqpPath = flag.String("deqp", "", "path to the deqp build directory")
cacheDir = flag.String("cache", "cache", "path to the output cache directory")
gerritEmail = flag.String("email", "$SS_REGRES_EMAIL", "gerrit email address for posting regres results")
gerritUser = flag.String("user", "$SS_REGRES_USER", "gerrit username for posting regres results")
@@ -92,7 +95,6 @@
shell.MaxProcMemory = *maxProcMemory
r := regres{
- deqpBuild: *deqpPath,
cacheRoot: *cacheDir,
gerritEmail: os.ExpandEnv(*gerritEmail),
gerritUser: os.ExpandEnv(*gerritUser),
@@ -110,9 +112,9 @@
}
type regres struct {
- deqpBuild string // path to the build directory of deqp
- cmake string // path to cmake
- make string // path to make
+ cmake string // path to cmake executable
+ make string // path to make executable
+ python string // path to python executable
cacheRoot string // path to the regres cache directory
gerritEmail string // gerrit email address used for posting results
gerritUser string // gerrit username used for posting results
@@ -128,7 +130,6 @@
// expands them to absolute paths.
func (r *regres) resolveDirs() error {
allDirs := []*string{
- &r.deqpBuild,
&r.cacheRoot,
}
@@ -162,6 +163,7 @@
for _, e := range []exe{
{"cmake", &r.cmake},
{"make", &r.make},
+ {"python", &r.python},
} {
path, err := exec.LookPath(e.name)
if err != nil {
@@ -293,35 +295,128 @@
}
func (r *regres) test(change *changeInfo) (string, error) {
+ latest := r.newTest(change.latest)
+ defer latest.cleanup()
+
+ deqp, err := r.getOrBuildDEQP(latest)
+ if err != nil {
+ return "", cause.Wrap(err, "Failed to build dEQP '%v' for change", change.id)
+ }
+
+ if err := latest.checkout(); err != nil {
+ return "", cause.Wrap(err, "Failed to checkout '%s'", change.latest)
+ }
+
log.Printf("Testing latest patchset for change '%s'\n", change.id)
- latest, testlists, err := r.testLatest(change)
+ latestResults, testlists, err := r.testLatest(change, latest, deqp)
if err != nil {
return "", cause.Wrap(err, "Failed to test latest change of '%v'", change.id)
}
log.Printf("Testing parent of change '%s'\n", change.id)
- parent, err := r.testParent(change, testlists)
+ parentResults, err := r.testParent(change, testlists, deqp)
if err != nil {
return "", cause.Wrap(err, "Failed to test parent change of '%v'", change.id)
}
log.Println("Comparing latest patchset's results with parent")
- msg := compare(parent, latest)
+ msg := compare(parentResults, latestResults)
return msg, nil
}
-var additionalTestsRE = regexp.MustCompile(`\n\s*Test[s]?:\s*([^\s]+)[^\n]*`)
+type deqp struct {
+ path string // path to deqp directory
+ hash string // hash of the deqp config
+}
-func (r *regres) testLatest(change *changeInfo) (*CommitTestResults, testlist.Lists, error) {
- // Get the test results for the latest patchset in the change.
- test := r.newTest(change.latest)
- defer test.cleanup()
+func (r *regres) getOrBuildDEQP(test *test) (deqp, error) {
+ srcDir := test.srcDir
+ if !isFile(path.Join(srcDir, deqpConfigRelPath)) {
+ srcDir, _ = os.Getwd()
+ log.Println("Couldn't open dEQP config file from change, falling back to internal version")
+ }
+ file, err := os.Open(path.Join(srcDir, deqpConfigRelPath))
+ if err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't open dEQP config file")
+ }
+ defer file.Close()
- if err := test.checkout(); err != nil {
- return nil, nil, cause.Wrap(err, "Failed to checkout '%s'", change.latest)
+ cfg := struct {
+ Remote string `json:"remote"`
+ SHA string `json:"sha"`
+ Patches []string `json:"patches"`
+ }{}
+ if err := json.NewDecoder(file).Decode(&cfg); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't parse %s", deqpConfigRelPath)
}
+ hasher := sha1.New()
+ if err := json.NewEncoder(hasher).Encode(&cfg); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't re-encode %s", deqpConfigRelPath)
+ }
+ hash := hex.EncodeToString(hasher.Sum(nil))
+ cacheDir := path.Join(r.cacheRoot, "deqp", hash)
+ buildDir := path.Join(cacheDir, "build")
+ if !isDir(cacheDir) {
+ if err := os.MkdirAll(cacheDir, 0777); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't make deqp cache directory '%s'", cacheDir)
+ }
+
+ success := false
+ defer func() {
+ if !success {
+ os.RemoveAll(cacheDir)
+ }
+ }()
+
+ log.Printf("Checking out deqp %s @ %s into %s\n", cfg.Remote, cfg.SHA, cacheDir)
+ if err := git.Checkout(cacheDir, cfg.Remote, git.ParseHash(cfg.SHA)); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't build deqp %s @ %s", cfg.Remote, cfg.SHA)
+ }
+
+ log.Println("Fetching deqp dependencies")
+ if err := shell.Shell(buildTimeout, r.python, cacheDir, "external/fetch_sources.py"); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't fetch deqp sources %s @ %s", cfg.Remote, cfg.SHA)
+ }
+
+ log.Println("Applying deqp patches")
+ for _, patch := range cfg.Patches {
+ fullPath := path.Join(srcDir, patch)
+ if err := git.Apply(cacheDir, fullPath); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't apply deqp patch %s for %s @ %s", patch, cfg.Remote, cfg.SHA)
+ }
+ }
+
+ log.Printf("Building deqp into %s\n", buildDir)
+ if err := os.MkdirAll(buildDir, 0777); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't make deqp cache directory '%s'", cacheDir)
+ }
+
+ if err := shell.Shell(buildTimeout, r.cmake, buildDir,
+ "-DDEQP_TARGET=x11_egl",
+ "-DCMAKE_BUILD_TYPE=Release",
+ ".."); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't generate build rules for deqp %s @ %s", cfg.Remote, cfg.SHA)
+ }
+
+ if err := shell.Shell(buildTimeout, r.make, buildDir, fmt.Sprintf("-j%d", runtime.NumCPU())); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't build deqp %s @ %s", cfg.Remote, cfg.SHA)
+ }
+
+ success = true
+ }
+
+ return deqp{
+ path: cacheDir,
+ hash: hash,
+ }, nil
+}
+
+var additionalTestsRE = regexp.MustCompile(`\n\s*Test[s]?:\s*([^\s]+)[^\n]*`)
+
+func (r *regres) testLatest(change *changeInfo, test *test, d deqp) (*CommitTestResults, testlist.Lists, error) {
+ // Get the test results for the latest patchset in the change.
testlists, err := test.loadTestLists(ciTestListRelPath)
if err != nil {
return nil, nil, cause.Wrap(err, "Failed to load '%s'", change.latest)
@@ -350,14 +445,14 @@
}
}
- cachePath := test.resultsCachePath(testlists)
+ cachePath := test.resultsCachePath(testlists, d)
if results, err := loadCommitTestResults(cachePath); err == nil {
return results, testlists, nil // Use cached results
}
// Build the change and test it.
- results := test.buildAndRun(testlists)
+ results := test.buildAndRun(testlists, d)
// Cache the results for future tests
if err := results.save(cachePath); err != nil {
@@ -367,12 +462,12 @@
return results, testlists, nil
}
-func (r *regres) testParent(change *changeInfo, testlists testlist.Lists) (*CommitTestResults, error) {
+func (r *regres) testParent(change *changeInfo, testlists testlist.Lists, d deqp) (*CommitTestResults, error) {
// Get the test results for the changes's parent changelist.
test := r.newTest(change.parent)
defer test.cleanup()
- cachePath := test.resultsCachePath(testlists)
+ cachePath := test.resultsCachePath(testlists, d)
if results, err := loadCommitTestResults(cachePath); err == nil {
return results, nil // Use cached results
@@ -384,7 +479,7 @@
}
// Build the parent change and test it.
- results := test.buildAndRun(testlists)
+ results := test.buildAndRun(testlists, d)
// Store the results of the parent change to the cache.
if err := results.save(cachePath); err != nil {
@@ -411,6 +506,11 @@
return cause.Wrap(err, "Failed to checkout '%s'", headHash)
}
+ d, err := r.getOrBuildDEQP(test)
+ if err != nil {
+ return cause.Wrap(err, "Failed to build deqp for '%s'", headHash)
+ }
+
// Load the test lists.
testLists, err := test.loadTestLists(fullTestListRelPath)
if err != nil {
@@ -423,7 +523,7 @@
}
// Run the tests on the change.
- results, err := test.run(testLists)
+ results, err := test.run(testLists, d)
if err != nil {
return cause.Wrap(err, "Failed to test '%s'", headHash)
}
@@ -708,7 +808,7 @@
// buildAndRun calls t.build() followed by t.run(). Errors are logged and
// reported in the returned CommitTestResults.Error field.
-func (t *test) buildAndRun(testLists testlist.Lists) *CommitTestResults {
+func (t *test) buildAndRun(testLists testlist.Lists, d deqp) *CommitTestResults {
// Build the parent change.
if err := t.build(); err != nil {
msg := fmt.Sprintf("Failed to build '%s'", t.commit)
@@ -717,7 +817,7 @@
}
// Run the tests on the parent change.
- results, err := t.run(testLists)
+ results, err := t.run(testLists, d)
if err != nil {
msg := fmt.Sprintf("Failed to test change '%s'", t.commit)
log.Println(cause.Wrap(err, msg))
@@ -752,7 +852,7 @@
}
// run runs all the tests.
-func (t *test) run(testLists testlist.Lists) (*CommitTestResults, error) {
+func (t *test) run(testLists testlist.Lists, d deqp) (*CommitTestResults, error) {
log.Printf("Running tests for '%s'\n", t.commit)
outDir := filepath.Join(t.srcDir, "out")
@@ -778,13 +878,13 @@
var exe string
switch list.API {
case testlist.EGL:
- exe = filepath.Join(t.r.deqpBuild, "modules", "egl", "deqp-egl")
+ exe = filepath.Join(d.path, "build", "modules", "egl", "deqp-egl")
case testlist.GLES2:
- exe = filepath.Join(t.r.deqpBuild, "modules", "gles2", "deqp-gles2")
+ exe = filepath.Join(d.path, "build", "modules", "gles2", "deqp-gles2")
case testlist.GLES3:
- exe = filepath.Join(t.r.deqpBuild, "modules", "gles3", "deqp-gles3")
+ exe = filepath.Join(d.path, "build", "modules", "gles3", "deqp-gles3")
case testlist.Vulkan:
- exe = filepath.Join(t.r.deqpBuild, "external", "vulkancts", "modules", "vulkan", "deqp-vk")
+ exe = filepath.Join(d.path, "build", "external", "vulkancts", "modules", "vulkan", "deqp-vk")
default:
return nil, fmt.Errorf("Unknown API '%v'", list.API)
}
@@ -887,9 +987,9 @@
}
// resultsCachePath returns the path to the cache results file for the given
-// test and testlists.
-func (t *test) resultsCachePath(testLists testlist.Lists) string {
- return filepath.Join(t.resDir, testLists.Hash())
+// test, testlists and path to deqp.
+func (t *test) resultsCachePath(testLists testlist.Lists, d deqp) string {
+ return filepath.Join(t.resDir, testLists.Hash(), d.hash)
}
// CommitTestResults holds the results the tests across all APIs for a given
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 59e8b98..41bc4ca 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -84,9 +84,7 @@
if err := gob.NewEncoder(h).Encode(l); err != nil {
panic(cause.Wrap(err, "Could not encode testlist to produce hash"))
}
- var hash [20]byte
- copy(hash[:], h.Sum(nil))
- return hex.EncodeToString(hash[:])
+ return hex.EncodeToString(h.Sum(nil))
}
// Load loads the test list json file and returns the full set of tests.
