Regres: Add workaround for Khronos Gitlab issues
It seems that once a change is no longer in the top N most recent changes, the server will start responding `server does not allow request for unadvertised object` when attempting to fetch the specific change by SHA.
To work around this, we need to fetch the branch by name, then checkout the specific change locally. As the Khronos seems to be particularly slow to respond today, I've limited the branch depth to 99, for now. This hopefully will not be a problem as we should typically only be updating to newer changes.
Also bumped the git command timeout to 15 mintutes as this repo can be _really_ slow.
Change-Id: Ie08165a6ca92e79d56bb4d2d5376410f2850a471
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/38129
Tested-by: Ben Clayton <bclayton@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
diff --git a/tests/regres/deqp.json b/tests/regres/deqp.json
index 9a4ca29..46168c0 100644
--- a/tests/regres/deqp.json
+++ b/tests/regres/deqp.json
@@ -1,5 +1,6 @@
{
"remote": "git@gitlab.khronos.org:Tracker/vk-gl-cts.git",
+ "branch": "vulkan-cts-1.1.6",
"sha": "a7d268a6a3ee9655298d61cdb8b5c42306a4e335",
"patches": [
"tests/regres/deqp-patches/deqp-x11.patch"
diff --git a/tests/regres/git/git.go b/tests/regres/git/git.go
index fec3d41..6598bd4 100644
--- a/tests/regres/git/git.go
+++ b/tests/regres/git/git.go
@@ -30,7 +30,7 @@
)
const (
- gitTimeout = time.Minute * 5 // timeout for a git operation
+ gitTimeout = time.Minute * 15 // timeout for a git operation
)
var exe string
@@ -111,8 +111,31 @@
return shell.Shell(gitTimeout, exe, project, args...)
}
-// Checkout performs a git checkout of the given commit into path.
-func Checkout(path, url string, commit Hash) error {
+// CheckoutRemoteBranch performs a git fetch and checkout of the given branch into path.
+func CheckoutRemoteBranch(path, url string, branch string) error {
+ if err := os.MkdirAll(path, 0777); err != nil {
+ return cause.Wrap(err, "mkdir '"+path+"' failed")
+ }
+
+ for _, cmds := range [][]string{
+ {"init"},
+ {"remote", "add", "origin", url},
+ // Note: this depth is here to prevent massive dEQP checkouts that can
+ // take all day. If the commit cannot be found in the checked out branch
+ // then this limit may need to be increased.
+ {"fetch", "origin", "--depth=99", branch},
+ } {
+ if err := shell.Shell(gitTimeout, exe, path, cmds...); err != nil {
+ os.RemoveAll(path)
+ return err
+ }
+ }
+
+ return nil
+}
+
+// CheckoutRemoteCommit performs a git fetch and checkout of the given commit into path.
+func CheckoutRemoteCommit(path, url string, commit Hash) error {
if err := os.MkdirAll(path, 0777); err != nil {
return cause.Wrap(err, "mkdir '"+path+"' failed")
}
@@ -132,6 +155,11 @@
return nil
}
+// CheckoutCommit performs a git checkout of the given commit.
+func CheckoutCommit(path string, commit Hash) error {
+ return shell.Shell(gitTimeout, exe, path, "checkout", commit.String())
+}
+
// Apply applys the patch file to the git repo at dir.
func Apply(dir, patch string) error {
return shell.Shell(gitTimeout, exe, dir, "apply", patch)
diff --git a/tests/regres/main.go b/tests/regres/main.go
index b5ac659..fbd87bf 100644
--- a/tests/regres/main.go
+++ b/tests/regres/main.go
@@ -346,6 +346,7 @@
cfg := struct {
Remote string `json:"remote"`
+ Branch string `json:"branch"`
SHA string `json:"sha"`
Patches []string `json:"patches"`
}{}
@@ -372,38 +373,52 @@
}
}()
- log.Printf("Checking out deqp %s @ %s into %s\n", cfg.Remote, cfg.SHA, cacheDir)
- if err := git.Checkout(cacheDir, cfg.Remote, git.ParseHash(cfg.SHA)); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't build deqp %s @ %s", cfg.Remote, cfg.SHA)
+ if cfg.Branch != "" {
+ // If a branch is specified, then fetch the branch then checkout the
+ // commit by SHA. This is a workaround for git repos that error when
+ // attempting to directly checkout a remote commit.
+ log.Printf("Checking out deqp %v branch %v into %v\n", cfg.Remote, cfg.Branch, cacheDir)
+ if err := git.CheckoutRemoteBranch(cacheDir, cfg.Remote, cfg.Branch); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't checkout deqp branch %v @ %v", cfg.Remote, cfg.Branch)
+ }
+ log.Printf("Checking out deqp %v commit %v \n", cfg.Remote, cfg.SHA)
+ if err := git.CheckoutCommit(cacheDir, git.ParseHash(cfg.SHA)); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't checkout deqp commit %v @ %v", cfg.Remote, cfg.SHA)
+ }
+ } else {
+ log.Printf("Checking out deqp %v @ %v into %v\n", cfg.Remote, cfg.SHA, cacheDir)
+ if err := git.CheckoutRemoteCommit(cacheDir, cfg.Remote, git.ParseHash(cfg.SHA)); err != nil {
+ return deqp{}, cause.Wrap(err, "Couldn't checkout deqp commit %v @ %v", cfg.Remote, cfg.SHA)
+ }
}
log.Println("Fetching deqp dependencies")
if err := shell.Shell(buildTimeout, r.python, cacheDir, "external/fetch_sources.py"); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't fetch deqp sources %s @ %s", cfg.Remote, cfg.SHA)
+ return deqp{}, cause.Wrap(err, "Couldn't fetch deqp sources %v @ %v", cfg.Remote, cfg.SHA)
}
log.Println("Applying deqp patches")
for _, patch := range cfg.Patches {
fullPath := path.Join(srcDir, patch)
if err := git.Apply(cacheDir, fullPath); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't apply deqp patch %s for %s @ %s", patch, cfg.Remote, cfg.SHA)
+ return deqp{}, cause.Wrap(err, "Couldn't apply deqp patch %v for %v @ %v", patch, cfg.Remote, cfg.SHA)
}
}
- log.Printf("Building deqp into %s\n", buildDir)
+ log.Printf("Building deqp into %v\n", buildDir)
if err := os.MkdirAll(buildDir, 0777); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't make deqp cache directory '%s'", cacheDir)
+ return deqp{}, cause.Wrap(err, "Couldn't make deqp build directory '%v'", buildDir)
}
if err := shell.Shell(buildTimeout, r.cmake, buildDir,
"-DDEQP_TARGET=x11_egl",
"-DCMAKE_BUILD_TYPE=Release",
".."); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't generate build rules for deqp %s @ %s", cfg.Remote, cfg.SHA)
+ return deqp{}, cause.Wrap(err, "Couldn't generate build rules for deqp %v @ %v", cfg.Remote, cfg.SHA)
}
if err := shell.Shell(buildTimeout, r.make, buildDir, fmt.Sprintf("-j%d", runtime.NumCPU())); err != nil {
- return deqp{}, cause.Wrap(err, "Couldn't build deqp %s @ %s", cfg.Remote, cfg.SHA)
+ return deqp{}, cause.Wrap(err, "Couldn't build deqp %v @ %v", cfg.Remote, cfg.SHA)
}
success = true
@@ -801,7 +816,7 @@
}
log.Printf("Checking out '%s'\n", t.commit)
os.RemoveAll(t.srcDir)
- if err := git.Checkout(t.srcDir, gitURL, t.commit); err != nil {
+ if err := git.CheckoutRemoteCommit(t.srcDir, gitURL, t.commit); err != nil {
return cause.Wrap(err, "Checking out commit '%s'", t.commit)
}
log.Printf("Checked out commit '%s'\n", t.commit)
