From 473700fc8bc323099514b3b417f4ae29beafae2d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 12:10:45 -0600 Subject: [PATCH 01/11] test(e2e): stabilize podman integration tests --- .github/workflows/pr-ci.yml | 48 ++++--- e2e/framework/command.go | 35 ++++- e2e/framework/retry.go | 127 +++++++++++------- e2e/framework/retry_test.go | 8 +- e2e/tests/up/helper.go | 16 ++- .../up/provider_podman_rootful_lifecycle.go | 6 +- .../up/provider_podman_rootful_lifecycle_2.go | 4 +- .../up/provider_podman_rootless_lifecycle.go | 10 +- 8 files changed, 169 insertions(+), 85 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index efe227cb7..84a9c5ff5 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -430,7 +430,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-exec runner: ubuntu-24.04 @@ -440,7 +440,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-lifecycle runner: ubuntu-24.04 @@ -450,7 +450,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-config runner: ubuntu-24.04 @@ -460,7 +460,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-features runner: ubuntu-24.04 @@ -470,7 +470,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-basic runner: ubuntu-24.04 @@ -480,7 +480,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-lifecycle runner: ubuntu-24.04 @@ -490,7 +490,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-lifecycle-2 runner: ubuntu-24.04 @@ -500,7 +500,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-config runner: ubuntu-24.04 @@ -510,7 +510,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-features runner: ubuntu-24.04 @@ -520,7 +520,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-docker runner: ubuntu-latest @@ -863,14 +863,28 @@ jobs: FLAKE_ATTEMPTS: ${{ matrix.flake-attempts || '1' }} run: | if [ "${{ runner.os }}" == "Linux" ]; then - sudo \ + if [ "${{ matrix.install-podman || '' }}" = "rootless" ]; then + echo "Podman rootless test identity:" + id + printf 'HOME=%s\n' "${HOME}" + printf 'XDG_RUNTIME_DIR=%s\n' "${XDG_RUNTIME_DIR:-}" + podman info GH_USERNAME="${GH_USERNAME}" \ - GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ - GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ - KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ - ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ - PATH="${PATH}" \ - ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ + GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ + KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ + PATH="${PATH}" \ + ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + else + sudo \ + GH_USERNAME="${GH_USERNAME}" \ + GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ + GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ + KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ + ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ + PATH="${PATH}" \ + ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + fi else GH_USERNAME="${GH_USERNAME}" \ GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 318ab1558..4efe9a7bf 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/flags/names" @@ -214,6 +215,23 @@ func (f *Framework) DevsySSH( return out, nil } +// DevsySSHOnce performs a single SSH attempt. It is intended for readiness +// polling, where retrying inside the polling callback can outlive the polling +// window and consume the enclosing spec's deadline. +func (f *Framework) DevsySSHOnce( + ctx context.Context, + workspace string, + command string, +) (string, error) { + out, _, err := f.ExecCommandCapture(ctx, []string{ + cmdWorkspace, cmdSSH, workspace, flagCommand, command, flagDebug, + }) + if err != nil { + return "", fmt.Errorf("devsy ssh failed: %w", err) + } + return out, nil +} + func (f *Framework) DevsySSHEchoTestString(ctx context.Context, workspace string) error { err := f.ExecCommand( ctx, @@ -317,13 +335,7 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error baseArgs = append(baseArgs, args...) _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) if err != nil { - // Skip "already exists" errors to make this idempotent - // This occurs when another test begins before ginkgo.DeferCleanup - // is called to delete the workspace. The workspace is linked to the - // provider and the provider cannot be deleted until the workspace is deleted. - if !strings.Contains(stderr, "already exists") { - return fmt.Errorf("devsy provider add failed: %s", stderr) - } + return fmt.Errorf("devsy provider add failed: %s", stderr) } return nil } @@ -436,6 +448,15 @@ func (f *Framework) DevsyWorkspaceDelete( return f.ExecCommand(ctx, false, true, fmt.Sprintf("deleted workspace %s", workspace), baseArgs) } +// CleanupWorkspace gives cleanup a fresh, bounded context. Ginkgo cleanup may +// run after the spec context has expired, so inheriting its cancellation can +// silently skip the workspace deletion. +func (f *Framework) CleanupWorkspace(ctx context.Context, workspace string) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancel() + return f.DevsyWorkspaceDelete(cleanupCtx, workspace) +} + func (f *Framework) SetupGPG(tmpDir string) error { if _, err := exec.LookPath("gpg"); err != nil { if installErr := exec.Command("sudo", "apt-get", "install", "gnupg2", "-y"). diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index b5944c457..23a69f0a1 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -13,10 +13,11 @@ import ( ) // dockerPullBackoff defines retry timing for transient Docker registry errors. -// 4 total attempts (1 initial + 3 retries) with waits of ~30s, ~60s, ~120s. +// The waits are deliberately short so retries consume only a minority of a +// short spec's deadline. var dockerPullBackoff = wait.Backoff{ Steps: 4, - Duration: 30 * time.Second, + Duration: 5 * time.Second, Factor: 2.0, Jitter: 0.1, } @@ -100,30 +101,33 @@ func execWithDockerRetry( var lastErr error attempt := 0 - err := wait.ExponentialBackoffWithContext(ctx, dockerPullBackoff, - func(ctx context.Context) (bool, error) { - attempt++ - lastStdout, lastStderr, lastErr = fn(ctx) - if lastErr == nil { - return true, nil // success - } - if isRetryableDockerError(lastStderr) { - ginkgo.GinkgoWriter.Printf( - "[retry] attempt %d failed with transient Docker error, retrying: %s\n", - attempt, lastErr, - ) - return false, nil // retry - } - return false, lastErr // non-retryable, stop immediately - }, - ) - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return lastStdout, lastStderr, err - } - if err != nil && lastErr != nil { - return lastStdout, lastStderr, fmt.Errorf("after %d attempts: %w", attempt, lastErr) + for attempt = 1; attempt <= dockerPullBackoff.Steps; attempt++ { + if err := ctx.Err(); err != nil { + return lastStdout, lastStderr, err + } + lastStdout, lastStderr, lastErr = fn(ctx) + if lastErr == nil { + return lastStdout, lastStderr, nil + } + if !isRetryableDockerError(lastStderr) || attempt == dockerPullBackoff.Steps { + break + } + delay := nextBackoffDelay(dockerPullBackoff, attempt) + if !retryFitsBudget(ctx, delay) { + return lastStdout, lastStderr, fmt.Errorf( + "after %d attempts: retryable Docker error; retry not attempted because remaining deadline budget was insufficient (next retry delay: %s): %w", + attempt, delay, lastErr, + ) + } + ginkgo.GinkgoWriter.Printf( + "[retry] attempt %d failed with transient Docker error, retrying after %s: %s\n", + attempt, delay, lastErr, + ) + if err := waitForRetry(ctx, delay); err != nil { + return lastStdout, lastStderr, err + } } - return lastStdout, lastStderr, err + return lastStdout, lastStderr, fmt.Errorf("after %d attempts: %w", attempt, lastErr) } // execWithSSHRetry runs fn and retries if the error indicates a transient SSH @@ -139,27 +143,33 @@ func execWithSSHRetry( var lastErr error attempt := 0 - err := wait.ExponentialBackoffWithContext(ctx, sshBackoff, - func(ctx context.Context) (bool, error) { - attempt++ - lastOut, lastStderr, lastErr = fn(ctx) - if lastErr == nil { - return true, nil // success - } - if isRetryableSSHError(lastErr, lastStderr) { - ginkgo.GinkgoWriter.Printf( - "[retry] ssh %s: attempt %d failed with transient error, retrying: %s\n", - workspace, attempt, lastErr, - ) - return false, nil // retry - } - return false, lastErr // non-retryable, stop immediately - }, - ) - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return lastOut, err + for attempt = 1; attempt <= sshBackoff.Steps; attempt++ { + if err := ctx.Err(); err != nil { + return lastOut, err + } + lastOut, lastStderr, lastErr = fn(ctx) + if lastErr == nil { + return lastOut, nil + } + if !isRetryableSSHError(lastErr, lastStderr) || attempt == sshBackoff.Steps { + break + } + delay := nextBackoffDelay(sshBackoff, attempt) + if !retryFitsBudget(ctx, delay) { + return lastOut, fmt.Errorf( + "after %d attempts: retryable SSH error; retry not attempted because remaining deadline budget was insufficient (next retry delay: %s): %w", + attempt, delay, lastErr, + ) + } + ginkgo.GinkgoWriter.Printf( + "[retry] ssh %s: attempt %d failed with transient error, retrying after %s: %s\n", + workspace, attempt, delay, lastErr, + ) + if err := waitForRetry(ctx, delay); err != nil { + return lastOut, err + } } - if err != nil && lastErr != nil { + if lastErr != nil { if lastStderr != "" { return lastOut, fmt.Errorf( "after %d attempts: %w (stderr: %s)", attempt, lastErr, lastStderr, @@ -167,5 +177,30 @@ func execWithSSHRetry( } return lastOut, fmt.Errorf("after %d attempts: %w", attempt, lastErr) } - return lastOut, err + return lastOut, lastErr +} + +func retryFitsBudget(ctx context.Context, delay time.Duration) bool { + deadline, ok := ctx.Deadline() + return !ok || time.Until(deadline) > delay +} + +func nextBackoffDelay(backoff wait.Backoff, retryNumber int) time.Duration { + delay := backoff.DelayFunc() + var next time.Duration + for i := 0; i < retryNumber; i++ { + next = delay() + } + return next +} + +func waitForRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } } diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index b88ece04d..3872b9d6f 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -305,8 +305,8 @@ func TestExecWithDockerRetry_ContextDeadlineExceeded(t *testing.T) { }, ) require.Error(t, err) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.NotContains(t, err.Error(), "after") + assert.Contains(t, err.Error(), "remaining deadline budget was insufficient") + assert.Contains(t, err.Error(), "next retry delay") } func TestExecWithSSHRetry_SuccessFirstTry(t *testing.T) { @@ -402,6 +402,6 @@ func TestExecWithSSHRetry_ContextDeadlineExceeded(t *testing.T) { }, ) require.Error(t, err) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.NotContains(t, err.Error(), "after") + assert.Contains(t, err.Error(), "remaining deadline budget was insufficient") + assert.Contains(t, err.Error(), "next retry delay") } diff --git a/e2e/tests/up/helper.go b/e2e/tests/up/helper.go index 38b9dc2a6..90e8c1091 100644 --- a/e2e/tests/up/helper.go +++ b/e2e/tests/up/helper.go @@ -7,6 +7,7 @@ import ( "io" "os" "strings" + "time" "github.com/devsy-org/devsy/e2e/framework" pkgconfig "github.com/devsy-org/devsy/pkg/config" @@ -89,6 +90,19 @@ type dockerTestContext struct { baseTestContext } +// probeSSH bounds one readiness attempt. The polling window owns the overall +// wait; a probe must not start the framework's multi-attempt SSH retry loop. +func probeSSH( + f *framework.Framework, + ctx context.Context, + workspace string, + command string, +) (string, error) { + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return f.DevsySSHOnce(probeCtx, workspace, command) +} + func (dtc *dockerTestContext) setupAndUp( ctx context.Context, testDataPath string, @@ -174,7 +188,7 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st return "", err } ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) - ginkgo.DeferCleanup(f.DevsyWorkspaceDelete, tempDir) + ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) return tempDir, nil } diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index 0412b75e8..5465f54ba 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -144,7 +144,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("updateContentDone")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/deferred.marker 2>/dev/null", ) if err != nil { @@ -165,7 +165,7 @@ var _ = ginkgo.Describe( gomega.Expect(envPath).NotTo(gomega.ContainSubstring("${containerEnv:")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-start-deferred.out 2>/dev/null", @@ -206,7 +206,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-attach.out 2>/dev/null", ) if err != nil { diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go index 7fc5ea256..f401a2a47 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go @@ -83,7 +83,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { @@ -98,7 +98,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { diff --git a/e2e/tests/up/provider_podman_rootless_lifecycle.go b/e2e/tests/up/provider_podman_rootless_lifecycle.go index 1f483d560..0f4793dba 100644 --- a/e2e/tests/up/provider_podman_rootless_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootless_lifecycle.go @@ -110,7 +110,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("updateContentDone")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/deferred.marker 2>/dev/null", ) if err != nil { @@ -131,7 +131,7 @@ var _ = ginkgo.Describe( gomega.Expect(envPath).NotTo(gomega.ContainSubstring("${containerEnv:")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-start-deferred.out 2>/dev/null", @@ -172,7 +172,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-attach.out 2>/dev/null", ) if err != nil { @@ -200,7 +200,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { @@ -215,7 +215,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { From 8fa1a349f504d0ea2fff12787b0f0f54c0993b91 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 12:25:47 -0600 Subject: [PATCH 02/11] fix(e2e): satisfy lint for retry budget --- e2e/framework/retry.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index 23a69f0a1..62db435bf 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -115,7 +115,8 @@ func execWithDockerRetry( delay := nextBackoffDelay(dockerPullBackoff, attempt) if !retryFitsBudget(ctx, delay) { return lastStdout, lastStderr, fmt.Errorf( - "after %d attempts: retryable Docker error; retry not attempted because remaining deadline budget was insufficient (next retry delay: %s): %w", + "after %d attempts: retryable Docker error; retry not attempted because "+ + "remaining deadline budget was insufficient (next retry delay: %s): %w", attempt, delay, lastErr, ) } @@ -157,7 +158,8 @@ func execWithSSHRetry( delay := nextBackoffDelay(sshBackoff, attempt) if !retryFitsBudget(ctx, delay) { return lastOut, fmt.Errorf( - "after %d attempts: retryable SSH error; retry not attempted because remaining deadline budget was insufficient (next retry delay: %s): %w", + "after %d attempts: retryable SSH error; retry not attempted because "+ + "remaining deadline budget was insufficient (next retry delay: %s): %w", attempt, delay, lastErr, ) } @@ -188,7 +190,7 @@ func retryFitsBudget(ctx context.Context, delay time.Duration) bool { func nextBackoffDelay(backoff wait.Backoff, retryNumber int) time.Duration { delay := backoff.DelayFunc() var next time.Duration - for i := 0; i < retryNumber; i++ { + for range retryNumber { next = delay() } return next From ca2cf1bdc0e716cff3f1cf64b2bb5e6712514617 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 17:11:46 -0600 Subject: [PATCH 03/11] fix(e2e): recreate stale test providers safely --- e2e/framework/command.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 4efe9a7bf..d5384c1ee 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -334,12 +334,39 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error baseArgs := []string{cmdProvider, "add"} baseArgs = append(baseArgs, args...) _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) + if err != nil { + if strings.Contains(stderr, "already exists") { + providerName := providerNameFromAddArgs(args) + if deleteErr := f.DevsyProviderDelete(ctx, providerName); deleteErr != nil { + return fmt.Errorf( + "devsy provider add failed: existing provider %q could not be removed: %w", + providerName, + deleteErr, + ) + } + + _, stderr, err = f.ExecCommandCapture(ctx, baseArgs) + } + } if err != nil { return fmt.Errorf("devsy provider add failed: %s", stderr) } return nil } +func providerNameFromAddArgs(args []string) string { + if len(args) == 0 { + return "" + } + providerName := args[0] + for i := 0; i+1 < len(args); i++ { + if args[i] == "--name" { + return args[i+1] + } + } + return providerName +} + func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) error { baseArgs := []string{cmdProvider, cmdDelete} baseArgs = append(baseArgs, args...) From 02b314d0c11c862a3c20d235a2d708c462329234 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 21:50:42 -0600 Subject: [PATCH 04/11] fix(e2e): handle shared docker provider in integration tests --- e2e/framework/command.go | 12 ++++++++++++ e2e/tests/up/provider_podman_rootful_config.go | 10 +++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index d5384c1ee..3a1a1759d 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -3,6 +3,7 @@ package framework import ( "context" "encoding/json" + "errors" "fmt" "os/exec" "path/filepath" @@ -22,6 +23,8 @@ var ( flagCommand = names.Flag(names.Command) ) +var errProviderInUse = errors.New("provider is in use by a workspace") + const ( formatJSON = "json" cmdList = "list" @@ -372,6 +375,9 @@ func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) err baseArgs = append(baseArgs, args...) err := f.ExecCommand(ctx, false, false, "", baseArgs) if err != nil { + if strings.Contains(err.Error(), "because workspace") { + return fmt.Errorf("%w: %v", errProviderInUse, err) + } return err } @@ -647,6 +653,12 @@ func SetupDockerProvider(binDir, dockerPath string) (*Framework, error) { "-o", "DOCKER_PATH="+dockerPath, ); err != nil { + // The shared default Docker provider may be referenced by an existing + // workspace. It is safe to reuse only for the standard Docker setup; + // custom runtime setups must still fail rather than inherit stale state. + if dockerPath == "docker" && errors.Is(err, errProviderInUse) { + return f, f.DevsyProviderUse(context.Background(), "docker") + } return nil, fmt.Errorf("failed to add docker provider: %w", err) } return f, f.DevsyProviderUse(context.Background(), "docker") diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index b497a1220..e0ee42829 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -140,7 +140,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Expect(strings.TrimSpace(customImage)). To(gomega.Equal("ghcr.io/devsy-org/test-images/base:alpine")) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It("should substitute variables with defaults", func(ctx context.Context) { tempDir, err := setupWorkspaceAndUp( @@ -164,7 +164,7 @@ var _ = ginkgo.Describe( setVar, err := f.DevsySSH(ctx, tempDir, "cat $HOME/set-var.out") framework.ExpectNoError(err) gomega.Expect(strings.TrimSpace(setVar)).To(gomega.Equal(os.Getenv("HOME"))) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It("should merge extra devcontainer config", func(ctx context.Context) { tempDir, err := setupWorkspace( @@ -193,7 +193,7 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It( "should override with extra devcontainer config", @@ -221,7 +221,7 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It("should select from multiple devcontainers", func(ctx context.Context) { @@ -260,7 +260,7 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) }) }) }, From 56b14e4d13cf090e7dc1e0cd9ed579bc661df4b2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 23:05:01 -0600 Subject: [PATCH 05/11] fix(e2e): stabilize rootful lifecycle failures --- e2e/framework/command.go | 6 +++--- e2e/tests/up/provider_podman_rootful_lifecycle.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 3a1a1759d..37cc8e8ec 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -373,10 +373,10 @@ func providerNameFromAddArgs(args []string) string { func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) error { baseArgs := []string{cmdProvider, cmdDelete} baseArgs = append(baseArgs, args...) - err := f.ExecCommand(ctx, false, false, "", baseArgs) + _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) if err != nil { - if strings.Contains(err.Error(), "because workspace") { - return fmt.Errorf("%w: %v", errProviderInUse, err) + if strings.Contains(stderr, "because workspace") { + return fmt.Errorf("%w: %s", errProviderInUse, stderr) } return err } diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index 5465f54ba..0e6c545c7 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -89,7 +89,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Expect(strings.TrimSpace(two)).To(gomega.Equal("postCreateTwo")) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It("should run postStartCommand after restart", func(ctx context.Context) { @@ -178,7 +178,7 @@ var _ = ginkgo.Describe( gomega.Equal("postStartDone"), ) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It( @@ -217,7 +217,7 @@ var _ = ginkgo.Describe( gomega.Equal("postAttachDone"), ) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) }) }) From 4e47ebf615afe79cbeeafccdbce4aef5a2ed636d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 12 Sep 2026 23:54:32 -0600 Subject: [PATCH 06/11] fix(e2e): wait for rootful podman ssh readiness --- .../up/provider_podman_rootful_config.go | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index e0ee42829..dc545d08f 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/e2e/framework" "github.com/devsy-org/devsy/pkg/docker" @@ -183,13 +184,29 @@ var _ = ginkgo.Describe( ) framework.ExpectNoError(err) - out, err := f.DevsySSH(ctx, tempDir, "bash -l -c 'echo -n $BASE_VAR'") - framework.ExpectNoError(err) - framework.ExpectEqual(out, "base_value") + gomega.Eventually(func() string { + out, err := probeSSH( + f, ctx, tempDir, "bash -l -c 'echo -n $BASE_VAR'", + ) + if err != nil { + return "" + } + return strings.TrimSpace(out) + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should( + gomega.Equal("base_value"), + ) - out, err = f.DevsySSH(ctx, tempDir, "bash -l -c 'echo -n $EXTRA_VAR'") - framework.ExpectNoError(err) - framework.ExpectEqual(out, "extra_value") + gomega.Eventually(func() string { + out, err := probeSSH( + f, ctx, tempDir, "bash -l -c 'echo -n $EXTRA_VAR'", + ) + if err != nil { + return "" + } + return strings.TrimSpace(out) + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should( + gomega.Equal("extra_value"), + ) err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) From 80178306120bcb147c893c3fec04b84441e21075 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 13 Sep 2026 12:25:49 -0600 Subject: [PATCH 07/11] fix(e2e): extend rootful lifecycle readiness timeout --- e2e/tests/up/provider_podman_rootful_lifecycle_2.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go index f401a2a47..e3e2fc813 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go @@ -90,7 +90,7 @@ var _ = ginkgo.Describe( return "" } return strings.TrimSpace(out) - }).WithTimeout(15 * time.Second).WithPolling(1 * time.Second).Should( + }).WithTimeout(60 * time.Second).WithPolling(1 * time.Second).Should( gomega.Equal("1"), ) @@ -105,7 +105,7 @@ var _ = ginkgo.Describe( return "" } return strings.TrimSpace(out) - }).WithTimeout(15 * time.Second).WithPolling(1 * time.Second).Should( + }).WithTimeout(60 * time.Second).WithPolling(1 * time.Second).Should( gomega.Equal("2"), ) }, @@ -179,7 +179,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)). To(gomega.Equal("second-secret-42")) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) }) }) From 156d05990a533e80cc37c9970669384f446fe8d1 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 13 Sep 2026 13:15:56 -0600 Subject: [PATCH 08/11] fix(e2e): bound rootful config ssh probes --- .../up/provider_podman_rootful_config.go | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index dc545d08f..76e658914 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -82,64 +82,53 @@ var _ = ginkgo.Describe( ) framework.ExpectNoError(err) - devContainerID, err := f.DevsySSH( - ctx, - tempDir, - "cat $HOME/dev-container-id.out", + devContainerID := eventuallySSH( + f, ctx, tempDir, "cat $HOME/dev-container-id.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(devContainerID)).NotTo(gomega.BeEmpty()) + gomega.Expect(devContainerID).NotTo(gomega.BeEmpty()) - containerEnvPath, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-env-path.out", + containerEnvPath := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-env-path.out", ) - framework.ExpectNoError(err) gomega.Expect(containerEnvPath).To(gomega.ContainSubstring("/usr/local/bin")) - localEnvHome, err := f.DevsySSH(ctx, tempDir, "cat $HOME/local-env-home.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(localEnvHome)). + localEnvHome := eventuallySSH(f, ctx, tempDir, "cat $HOME/local-env-home.out") + gomega.Expect(localEnvHome). To(gomega.Equal(os.Getenv("HOME"))) - localWorkspaceFolder, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/local-workspace-folder.out", + localWorkspaceFolder := eventuallySSH( + f, ctx, tempDir, "cat $HOME/local-workspace-folder.out", ) - framework.ExpectNoError(err) gomega.Expect( - framework.CleanString(strings.TrimSpace(localWorkspaceFolder)), + framework.CleanString(localWorkspaceFolder), ).To(gomega.Equal(framework.CleanString(tempDir))) - localWorkspaceFolderBasename, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/local-workspace-folder-basename.out", + localWorkspaceFolderBasename := eventuallySSH( + f, ctx, tempDir, "cat $HOME/local-workspace-folder-basename.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(localWorkspaceFolderBasename)). + gomega.Expect(localWorkspaceFolderBasename). To(gomega.Equal(filepath.Base(tempDir))) - containerWorkspaceFolder, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-workspace-folder.out", + containerWorkspaceFolder := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-workspace-folder.out", ) - framework.ExpectNoError(err) gomega.Expect( - framework.CleanString(strings.TrimSpace(containerWorkspaceFolder)), + framework.CleanString(containerWorkspaceFolder), ).To(gomega.Equal( framework.CleanString(path.Join("/workspaces", filepath.Base(tempDir))), )) - containerWorkspaceFolderBasename, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-workspace-folder-basename.out", + containerWorkspaceFolderBasename := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-workspace-folder-basename.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(containerWorkspaceFolderBasename)). + gomega.Expect(containerWorkspaceFolderBasename). To(gomega.Equal(filepath.Base(tempDir))) - customVar, err := f.DevsySSH(ctx, tempDir, "cat $HOME/custom-var.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(customVar)).To(gomega.Equal("custom_value")) + customVar := eventuallySSH(f, ctx, tempDir, "cat $HOME/custom-var.out") + gomega.Expect(customVar).To(gomega.Equal("custom_value")) - customImage, err := f.DevsySSH(ctx, tempDir, "cat $HOME/custom-image.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(customImage)). + customImage := eventuallySSH(f, ctx, tempDir, "cat $HOME/custom-image.out") + gomega.Expect(customImage). To(gomega.Equal("ghcr.io/devsy-org/test-images/base:alpine")) }, ginkgo.SpecTimeout(framework.TimeoutModerate())) @@ -282,3 +271,16 @@ var _ = ginkgo.Describe( }) }, ) + +func eventuallySSH(f *framework.Framework, ctx context.Context, workspace, command string) string { + var output string + gomega.Eventually(func() bool { + out, err := probeSSH(f, ctx, workspace, command) + if err != nil { + return false + } + output = strings.TrimSpace(out) + return true + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should(gomega.BeTrue()) + return output +} From aad849a0adf7b5247c3e0f3247f1a6a18ed5b25a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 13 Sep 2026 14:05:50 -0600 Subject: [PATCH 09/11] fix(e2e): allow rootful config ssh setup time --- e2e/tests/up/provider_podman_rootful_config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index 76e658914..5082fcbb8 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -275,7 +275,9 @@ var _ = ginkgo.Describe( func eventuallySSH(f *framework.Framework, ctx context.Context, workspace, command string) string { var output string gomega.Eventually(func() bool { - out, err := probeSSH(f, ctx, workspace, command) + probeCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + out, err := f.DevsySSHOnce(probeCtx, workspace, command) if err != nil { return false } From 0d2332f52ad988e2d81d582ba246b9eeb3fab544 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 13 Sep 2026 21:00:07 -0600 Subject: [PATCH 10/11] fix(e2e): recover provider names from config files --- e2e/framework/command.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 37cc8e8ec..7e19b2f8e 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -1,10 +1,12 @@ package framework import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -361,13 +363,24 @@ func providerNameFromAddArgs(args []string) string { if len(args) == 0 { return "" } - providerName := args[0] for i := 0; i+1 < len(args); i++ { if args[i] == "--name" { return args[i+1] } } - return providerName + + providerSource := args[0] + providerConfigRaw, err := os.ReadFile( // #nosec G304 -- test provider source path + providerSource, + ) + if err == nil { + providerConfig, parseErr := provider2.ParseProvider(bytes.NewReader(providerConfigRaw)) + if parseErr == nil { + return providerConfig.Name + } + } + + return providerSource } func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) error { @@ -646,9 +659,12 @@ func (f *Framework) DevsyIDEList(ctx context.Context, extraArgs ...string) (stri // adds a fresh one with the given docker path, and sets it as the active provider. func SetupDockerProvider(binDir, dockerPath string) (*Framework, error) { f := NewDefaultFramework(binDir) - _ = f.DevsyProviderDelete(context.Background(), "docker") + setupCtx, cancel := context.WithTimeout(context.Background(), TimeoutModerate()) + defer cancel() + + _ = f.DevsyProviderDelete(setupCtx, "docker") if err := f.DevsyProviderAdd( - context.Background(), + setupCtx, "docker", "-o", "DOCKER_PATH="+dockerPath, @@ -657,9 +673,9 @@ func SetupDockerProvider(binDir, dockerPath string) (*Framework, error) { // workspace. It is safe to reuse only for the standard Docker setup; // custom runtime setups must still fail rather than inherit stale state. if dockerPath == "docker" && errors.Is(err, errProviderInUse) { - return f, f.DevsyProviderUse(context.Background(), "docker") + return f, f.DevsyProviderUse(setupCtx, "docker") } return nil, fmt.Errorf("failed to add docker provider: %w", err) } - return f, f.DevsyProviderUse(context.Background(), "docker") + return f, f.DevsyProviderUse(setupCtx, "docker") } From 1cd46e2462970f00c55ad592e0d582ae2488a1cc Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 13 Sep 2026 21:55:05 -0600 Subject: [PATCH 11/11] fix(e2e): wait for rootful post-start lifecycle --- .../up/provider_podman_rootful_lifecycle.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index 0e6c545c7..5f202146a 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -115,10 +115,20 @@ var _ = ginkgo.Describe( err = f.DevsyUp(ctx, tempDir) framework.ExpectNoError(err) - out, err = f.DevsySSH(ctx, tempDir, "cat $HOME/post-start-count.log") - framework.ExpectNoError(err) - lines = strings.Count(strings.TrimSpace(out), "\n") + 1 - gomega.Expect(lines).To(gomega.Equal(2), + gomega.Eventually(func() int { + out, err := probeSSH( + f, ctx, tempDir, "cat $HOME/post-start-count.log", + ) + if err != nil { + return 0 + } + trimmed := strings.TrimSpace(out) + if trimmed == "" { + return 0 + } + return strings.Count(trimmed, "\n") + 1 + }).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should( + gomega.Equal(2), "postStartCommand should have run again after restart") }, ginkgo.SpecTimeout(framework.TimeoutModerate()))