diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index efe227cb7..84a9c5ff5 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -430,7 +430,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-exec runner: ubuntu-24.04 @@ -440,7 +440,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-lifecycle runner: ubuntu-24.04 @@ -450,7 +450,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-config runner: ubuntu-24.04 @@ -460,7 +460,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootless-features runner: ubuntu-24.04 @@ -470,7 +470,7 @@ jobs: install-podman: rootless test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-basic runner: ubuntu-24.04 @@ -480,7 +480,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-lifecycle runner: ubuntu-24.04 @@ -490,7 +490,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-lifecycle-2 runner: ubuntu-24.04 @@ -500,7 +500,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-config runner: ubuntu-24.04 @@ -510,7 +510,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-podman-rootful-features runner: ubuntu-24.04 @@ -520,7 +520,7 @@ jobs: install-podman: rootful test-timeout: 600s job-timeout-minutes: 20 - flake-attempts: 2 + flake-attempts: 1 - label: up-provider-docker runner: ubuntu-latest @@ -863,14 +863,28 @@ jobs: FLAKE_ATTEMPTS: ${{ matrix.flake-attempts || '1' }} run: | if [ "${{ runner.os }}" == "Linux" ]; then - sudo \ + if [ "${{ matrix.install-podman || '' }}" = "rootless" ]; then + echo "Podman rootless test identity:" + id + printf 'HOME=%s\n' "${HOME}" + printf 'XDG_RUNTIME_DIR=%s\n' "${XDG_RUNTIME_DIR:-}" + podman info GH_USERNAME="${GH_USERNAME}" \ - GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ - GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ - KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ - ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ - PATH="${PATH}" \ - ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ + GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ + KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ + PATH="${PATH}" \ + ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + else + sudo \ + GH_USERNAME="${GH_USERNAME}" \ + GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ + GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ + KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ + ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ + PATH="${PATH}" \ + ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" + fi else GH_USERNAME="${GH_USERNAME}" \ GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ diff --git a/e2e/framework/command.go b/e2e/framework/command.go index 318ab1558..7e19b2f8e 100644 --- a/e2e/framework/command.go +++ b/e2e/framework/command.go @@ -1,12 +1,16 @@ package framework import ( + "bytes" "context" "encoding/json" + "errors" "fmt" + "os" "os/exec" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/pkg/client" "github.com/devsy-org/devsy/pkg/flags/names" @@ -21,6 +25,8 @@ var ( flagCommand = names.Flag(names.Command) ) +var errProviderInUse = errors.New("provider is in use by a workspace") + const ( formatJSON = "json" cmdList = "list" @@ -214,6 +220,23 @@ func (f *Framework) DevsySSH( return out, nil } +// DevsySSHOnce performs a single SSH attempt. It is intended for readiness +// polling, where retrying inside the polling callback can outlive the polling +// window and consume the enclosing spec's deadline. +func (f *Framework) DevsySSHOnce( + ctx context.Context, + workspace string, + command string, +) (string, error) { + out, _, err := f.ExecCommandCapture(ctx, []string{ + cmdWorkspace, cmdSSH, workspace, flagCommand, command, flagDebug, + }) + if err != nil { + return "", fmt.Errorf("devsy ssh failed: %w", err) + } + return out, nil +} + func (f *Framework) DevsySSHEchoTestString(ctx context.Context, workspace string) error { err := f.ExecCommand( ctx, @@ -317,22 +340,57 @@ func (f *Framework) DevsyProviderAdd(ctx context.Context, args ...string) error baseArgs = append(baseArgs, args...) _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) if err != nil { - // Skip "already exists" errors to make this idempotent - // This occurs when another test begins before ginkgo.DeferCleanup - // is called to delete the workspace. The workspace is linked to the - // provider and the provider cannot be deleted until the workspace is deleted. - if !strings.Contains(stderr, "already exists") { - return fmt.Errorf("devsy provider add failed: %s", stderr) + if strings.Contains(stderr, "already exists") { + providerName := providerNameFromAddArgs(args) + if deleteErr := f.DevsyProviderDelete(ctx, providerName); deleteErr != nil { + return fmt.Errorf( + "devsy provider add failed: existing provider %q could not be removed: %w", + providerName, + deleteErr, + ) + } + + _, stderr, err = f.ExecCommandCapture(ctx, baseArgs) } } + if err != nil { + return fmt.Errorf("devsy provider add failed: %s", stderr) + } return nil } +func providerNameFromAddArgs(args []string) string { + if len(args) == 0 { + return "" + } + for i := 0; i+1 < len(args); i++ { + if args[i] == "--name" { + return args[i+1] + } + } + + providerSource := args[0] + providerConfigRaw, err := os.ReadFile( // #nosec G304 -- test provider source path + providerSource, + ) + if err == nil { + providerConfig, parseErr := provider2.ParseProvider(bytes.NewReader(providerConfigRaw)) + if parseErr == nil { + return providerConfig.Name + } + } + + return providerSource +} + func (f *Framework) DevsyProviderDelete(ctx context.Context, args ...string) error { baseArgs := []string{cmdProvider, cmdDelete} baseArgs = append(baseArgs, args...) - err := f.ExecCommand(ctx, false, false, "", baseArgs) + _, stderr, err := f.ExecCommandCapture(ctx, baseArgs) if err != nil { + if strings.Contains(stderr, "because workspace") { + return fmt.Errorf("%w: %s", errProviderInUse, stderr) + } return err } @@ -436,6 +494,15 @@ func (f *Framework) DevsyWorkspaceDelete( return f.ExecCommand(ctx, false, true, fmt.Sprintf("deleted workspace %s", workspace), baseArgs) } +// CleanupWorkspace gives cleanup a fresh, bounded context. Ginkgo cleanup may +// run after the spec context has expired, so inheriting its cancellation can +// silently skip the workspace deletion. +func (f *Framework) CleanupWorkspace(ctx context.Context, workspace string) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancel() + return f.DevsyWorkspaceDelete(cleanupCtx, workspace) +} + func (f *Framework) SetupGPG(tmpDir string) error { if _, err := exec.LookPath("gpg"); err != nil { if installErr := exec.Command("sudo", "apt-get", "install", "gnupg2", "-y"). @@ -592,14 +659,23 @@ func (f *Framework) DevsyIDEList(ctx context.Context, extraArgs ...string) (stri // adds a fresh one with the given docker path, and sets it as the active provider. func SetupDockerProvider(binDir, dockerPath string) (*Framework, error) { f := NewDefaultFramework(binDir) - _ = f.DevsyProviderDelete(context.Background(), "docker") + setupCtx, cancel := context.WithTimeout(context.Background(), TimeoutModerate()) + defer cancel() + + _ = f.DevsyProviderDelete(setupCtx, "docker") if err := f.DevsyProviderAdd( - context.Background(), + setupCtx, "docker", "-o", "DOCKER_PATH="+dockerPath, ); err != nil { + // The shared default Docker provider may be referenced by an existing + // workspace. It is safe to reuse only for the standard Docker setup; + // custom runtime setups must still fail rather than inherit stale state. + if dockerPath == "docker" && errors.Is(err, errProviderInUse) { + return f, f.DevsyProviderUse(setupCtx, "docker") + } return nil, fmt.Errorf("failed to add docker provider: %w", err) } - return f, f.DevsyProviderUse(context.Background(), "docker") + return f, f.DevsyProviderUse(setupCtx, "docker") } diff --git a/e2e/framework/retry.go b/e2e/framework/retry.go index b5944c457..62db435bf 100644 --- a/e2e/framework/retry.go +++ b/e2e/framework/retry.go @@ -13,10 +13,11 @@ import ( ) // dockerPullBackoff defines retry timing for transient Docker registry errors. -// 4 total attempts (1 initial + 3 retries) with waits of ~30s, ~60s, ~120s. +// The waits are deliberately short so retries consume only a minority of a +// short spec's deadline. var dockerPullBackoff = wait.Backoff{ Steps: 4, - Duration: 30 * time.Second, + Duration: 5 * time.Second, Factor: 2.0, Jitter: 0.1, } @@ -100,30 +101,34 @@ func execWithDockerRetry( var lastErr error attempt := 0 - err := wait.ExponentialBackoffWithContext(ctx, dockerPullBackoff, - func(ctx context.Context) (bool, error) { - attempt++ - lastStdout, lastStderr, lastErr = fn(ctx) - if lastErr == nil { - return true, nil // success - } - if isRetryableDockerError(lastStderr) { - ginkgo.GinkgoWriter.Printf( - "[retry] attempt %d failed with transient Docker error, retrying: %s\n", - attempt, lastErr, - ) - return false, nil // retry - } - return false, lastErr // non-retryable, stop immediately - }, - ) - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return lastStdout, lastStderr, err - } - if err != nil && lastErr != nil { - return lastStdout, lastStderr, fmt.Errorf("after %d attempts: %w", attempt, lastErr) + for attempt = 1; attempt <= dockerPullBackoff.Steps; attempt++ { + if err := ctx.Err(); err != nil { + return lastStdout, lastStderr, err + } + lastStdout, lastStderr, lastErr = fn(ctx) + if lastErr == nil { + return lastStdout, lastStderr, nil + } + if !isRetryableDockerError(lastStderr) || attempt == dockerPullBackoff.Steps { + break + } + delay := nextBackoffDelay(dockerPullBackoff, attempt) + if !retryFitsBudget(ctx, delay) { + return lastStdout, lastStderr, fmt.Errorf( + "after %d attempts: retryable Docker error; retry not attempted because "+ + "remaining deadline budget was insufficient (next retry delay: %s): %w", + attempt, delay, lastErr, + ) + } + ginkgo.GinkgoWriter.Printf( + "[retry] attempt %d failed with transient Docker error, retrying after %s: %s\n", + attempt, delay, lastErr, + ) + if err := waitForRetry(ctx, delay); err != nil { + return lastStdout, lastStderr, err + } } - return lastStdout, lastStderr, err + return lastStdout, lastStderr, fmt.Errorf("after %d attempts: %w", attempt, lastErr) } // execWithSSHRetry runs fn and retries if the error indicates a transient SSH @@ -139,27 +144,34 @@ func execWithSSHRetry( var lastErr error attempt := 0 - err := wait.ExponentialBackoffWithContext(ctx, sshBackoff, - func(ctx context.Context) (bool, error) { - attempt++ - lastOut, lastStderr, lastErr = fn(ctx) - if lastErr == nil { - return true, nil // success - } - if isRetryableSSHError(lastErr, lastStderr) { - ginkgo.GinkgoWriter.Printf( - "[retry] ssh %s: attempt %d failed with transient error, retrying: %s\n", - workspace, attempt, lastErr, - ) - return false, nil // retry - } - return false, lastErr // non-retryable, stop immediately - }, - ) - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return lastOut, err + for attempt = 1; attempt <= sshBackoff.Steps; attempt++ { + if err := ctx.Err(); err != nil { + return lastOut, err + } + lastOut, lastStderr, lastErr = fn(ctx) + if lastErr == nil { + return lastOut, nil + } + if !isRetryableSSHError(lastErr, lastStderr) || attempt == sshBackoff.Steps { + break + } + delay := nextBackoffDelay(sshBackoff, attempt) + if !retryFitsBudget(ctx, delay) { + return lastOut, fmt.Errorf( + "after %d attempts: retryable SSH error; retry not attempted because "+ + "remaining deadline budget was insufficient (next retry delay: %s): %w", + attempt, delay, lastErr, + ) + } + ginkgo.GinkgoWriter.Printf( + "[retry] ssh %s: attempt %d failed with transient error, retrying after %s: %s\n", + workspace, attempt, delay, lastErr, + ) + if err := waitForRetry(ctx, delay); err != nil { + return lastOut, err + } } - if err != nil && lastErr != nil { + if lastErr != nil { if lastStderr != "" { return lastOut, fmt.Errorf( "after %d attempts: %w (stderr: %s)", attempt, lastErr, lastStderr, @@ -167,5 +179,30 @@ func execWithSSHRetry( } return lastOut, fmt.Errorf("after %d attempts: %w", attempt, lastErr) } - return lastOut, err + return lastOut, lastErr +} + +func retryFitsBudget(ctx context.Context, delay time.Duration) bool { + deadline, ok := ctx.Deadline() + return !ok || time.Until(deadline) > delay +} + +func nextBackoffDelay(backoff wait.Backoff, retryNumber int) time.Duration { + delay := backoff.DelayFunc() + var next time.Duration + for range retryNumber { + next = delay() + } + return next +} + +func waitForRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } } diff --git a/e2e/framework/retry_test.go b/e2e/framework/retry_test.go index b88ece04d..3872b9d6f 100644 --- a/e2e/framework/retry_test.go +++ b/e2e/framework/retry_test.go @@ -305,8 +305,8 @@ func TestExecWithDockerRetry_ContextDeadlineExceeded(t *testing.T) { }, ) require.Error(t, err) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.NotContains(t, err.Error(), "after") + assert.Contains(t, err.Error(), "remaining deadline budget was insufficient") + assert.Contains(t, err.Error(), "next retry delay") } func TestExecWithSSHRetry_SuccessFirstTry(t *testing.T) { @@ -402,6 +402,6 @@ func TestExecWithSSHRetry_ContextDeadlineExceeded(t *testing.T) { }, ) require.Error(t, err) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.NotContains(t, err.Error(), "after") + assert.Contains(t, err.Error(), "remaining deadline budget was insufficient") + assert.Contains(t, err.Error(), "next retry delay") } diff --git a/e2e/tests/up/helper.go b/e2e/tests/up/helper.go index 38b9dc2a6..90e8c1091 100644 --- a/e2e/tests/up/helper.go +++ b/e2e/tests/up/helper.go @@ -7,6 +7,7 @@ import ( "io" "os" "strings" + "time" "github.com/devsy-org/devsy/e2e/framework" pkgconfig "github.com/devsy-org/devsy/pkg/config" @@ -89,6 +90,19 @@ type dockerTestContext struct { baseTestContext } +// probeSSH bounds one readiness attempt. The polling window owns the overall +// wait; a probe must not start the framework's multi-attempt SSH retry loop. +func probeSSH( + f *framework.Framework, + ctx context.Context, + workspace string, + command string, +) (string, error) { + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return f.DevsySSHOnce(probeCtx, workspace, command) +} + func (dtc *dockerTestContext) setupAndUp( ctx context.Context, testDataPath string, @@ -174,7 +188,7 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st return "", err } ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) - ginkgo.DeferCleanup(f.DevsyWorkspaceDelete, tempDir) + ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) return tempDir, nil } diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index b497a1220..5082fcbb8 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -7,6 +7,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/e2e/framework" "github.com/devsy-org/devsy/pkg/docker" @@ -81,66 +82,55 @@ var _ = ginkgo.Describe( ) framework.ExpectNoError(err) - devContainerID, err := f.DevsySSH( - ctx, - tempDir, - "cat $HOME/dev-container-id.out", + devContainerID := eventuallySSH( + f, ctx, tempDir, "cat $HOME/dev-container-id.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(devContainerID)).NotTo(gomega.BeEmpty()) + gomega.Expect(devContainerID).NotTo(gomega.BeEmpty()) - containerEnvPath, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-env-path.out", + containerEnvPath := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-env-path.out", ) - framework.ExpectNoError(err) gomega.Expect(containerEnvPath).To(gomega.ContainSubstring("/usr/local/bin")) - localEnvHome, err := f.DevsySSH(ctx, tempDir, "cat $HOME/local-env-home.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(localEnvHome)). + localEnvHome := eventuallySSH(f, ctx, tempDir, "cat $HOME/local-env-home.out") + gomega.Expect(localEnvHome). To(gomega.Equal(os.Getenv("HOME"))) - localWorkspaceFolder, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/local-workspace-folder.out", + localWorkspaceFolder := eventuallySSH( + f, ctx, tempDir, "cat $HOME/local-workspace-folder.out", ) - framework.ExpectNoError(err) gomega.Expect( - framework.CleanString(strings.TrimSpace(localWorkspaceFolder)), + framework.CleanString(localWorkspaceFolder), ).To(gomega.Equal(framework.CleanString(tempDir))) - localWorkspaceFolderBasename, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/local-workspace-folder-basename.out", + localWorkspaceFolderBasename := eventuallySSH( + f, ctx, tempDir, "cat $HOME/local-workspace-folder-basename.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(localWorkspaceFolderBasename)). + gomega.Expect(localWorkspaceFolderBasename). To(gomega.Equal(filepath.Base(tempDir))) - containerWorkspaceFolder, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-workspace-folder.out", + containerWorkspaceFolder := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-workspace-folder.out", ) - framework.ExpectNoError(err) gomega.Expect( - framework.CleanString(strings.TrimSpace(containerWorkspaceFolder)), + framework.CleanString(containerWorkspaceFolder), ).To(gomega.Equal( framework.CleanString(path.Join("/workspaces", filepath.Base(tempDir))), )) - containerWorkspaceFolderBasename, err := f.DevsySSH( - ctx, tempDir, "cat $HOME/container-workspace-folder-basename.out", + containerWorkspaceFolderBasename := eventuallySSH( + f, ctx, tempDir, "cat $HOME/container-workspace-folder-basename.out", ) - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(containerWorkspaceFolderBasename)). + gomega.Expect(containerWorkspaceFolderBasename). To(gomega.Equal(filepath.Base(tempDir))) - customVar, err := f.DevsySSH(ctx, tempDir, "cat $HOME/custom-var.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(customVar)).To(gomega.Equal("custom_value")) + customVar := eventuallySSH(f, ctx, tempDir, "cat $HOME/custom-var.out") + gomega.Expect(customVar).To(gomega.Equal("custom_value")) - customImage, err := f.DevsySSH(ctx, tempDir, "cat $HOME/custom-image.out") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(customImage)). + customImage := eventuallySSH(f, ctx, tempDir, "cat $HOME/custom-image.out") + gomega.Expect(customImage). To(gomega.Equal("ghcr.io/devsy-org/test-images/base:alpine")) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It("should substitute variables with defaults", func(ctx context.Context) { tempDir, err := setupWorkspaceAndUp( @@ -164,7 +154,7 @@ var _ = ginkgo.Describe( setVar, err := f.DevsySSH(ctx, tempDir, "cat $HOME/set-var.out") framework.ExpectNoError(err) gomega.Expect(strings.TrimSpace(setVar)).To(gomega.Equal(os.Getenv("HOME"))) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It("should merge extra devcontainer config", func(ctx context.Context) { tempDir, err := setupWorkspace( @@ -183,17 +173,33 @@ var _ = ginkgo.Describe( ) framework.ExpectNoError(err) - out, err := f.DevsySSH(ctx, tempDir, "bash -l -c 'echo -n $BASE_VAR'") - framework.ExpectNoError(err) - framework.ExpectEqual(out, "base_value") + gomega.Eventually(func() string { + out, err := probeSSH( + f, ctx, tempDir, "bash -l -c 'echo -n $BASE_VAR'", + ) + if err != nil { + return "" + } + return strings.TrimSpace(out) + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should( + gomega.Equal("base_value"), + ) - out, err = f.DevsySSH(ctx, tempDir, "bash -l -c 'echo -n $EXTRA_VAR'") - framework.ExpectNoError(err) - framework.ExpectEqual(out, "extra_value") + gomega.Eventually(func() string { + out, err := probeSSH( + f, ctx, tempDir, "bash -l -c 'echo -n $EXTRA_VAR'", + ) + if err != nil { + return "" + } + return strings.TrimSpace(out) + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should( + gomega.Equal("extra_value"), + ) err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) ginkgo.It( "should override with extra devcontainer config", @@ -221,7 +227,7 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It("should select from multiple devcontainers", func(ctx context.Context) { @@ -260,8 +266,23 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) - }, ginkgo.SpecTimeout(framework.TimeoutShort())) + }, ginkgo.SpecTimeout(framework.TimeoutModerate())) }) }) }, ) + +func eventuallySSH(f *framework.Framework, ctx context.Context, workspace, command string) string { + var output string + gomega.Eventually(func() bool { + probeCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + out, err := f.DevsySSHOnce(probeCtx, workspace, command) + if err != nil { + return false + } + output = strings.TrimSpace(out) + return true + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should(gomega.BeTrue()) + return output +} diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index 0412b75e8..5f202146a 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -89,7 +89,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Expect(strings.TrimSpace(two)).To(gomega.Equal("postCreateTwo")) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It("should run postStartCommand after restart", func(ctx context.Context) { @@ -115,10 +115,20 @@ var _ = ginkgo.Describe( err = f.DevsyUp(ctx, tempDir) framework.ExpectNoError(err) - out, err = f.DevsySSH(ctx, tempDir, "cat $HOME/post-start-count.log") - framework.ExpectNoError(err) - lines = strings.Count(strings.TrimSpace(out), "\n") + 1 - gomega.Expect(lines).To(gomega.Equal(2), + gomega.Eventually(func() int { + out, err := probeSSH( + f, ctx, tempDir, "cat $HOME/post-start-count.log", + ) + if err != nil { + return 0 + } + trimmed := strings.TrimSpace(out) + if trimmed == "" { + return 0 + } + return strings.Count(trimmed, "\n") + 1 + }).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should( + gomega.Equal(2), "postStartCommand should have run again after restart") }, ginkgo.SpecTimeout(framework.TimeoutModerate())) @@ -144,7 +154,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("updateContentDone")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/deferred.marker 2>/dev/null", ) if err != nil { @@ -165,7 +175,7 @@ var _ = ginkgo.Describe( gomega.Expect(envPath).NotTo(gomega.ContainSubstring("${containerEnv:")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-start-deferred.out 2>/dev/null", @@ -178,7 +188,7 @@ var _ = ginkgo.Describe( gomega.Equal("postStartDone"), ) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) ginkgo.It( @@ -206,7 +216,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-attach.out 2>/dev/null", ) if err != nil { @@ -217,7 +227,7 @@ var _ = ginkgo.Describe( gomega.Equal("postAttachDone"), ) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) }) }) diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go index 7fc5ea256..e3e2fc813 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go @@ -83,14 +83,14 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { return "" } return strings.TrimSpace(out) - }).WithTimeout(15 * time.Second).WithPolling(1 * time.Second).Should( + }).WithTimeout(60 * time.Second).WithPolling(1 * time.Second).Should( gomega.Equal("1"), ) @@ -98,14 +98,14 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { return "" } return strings.TrimSpace(out) - }).WithTimeout(15 * time.Second).WithPolling(1 * time.Second).Should( + }).WithTimeout(60 * time.Second).WithPolling(1 * time.Second).Should( gomega.Equal("2"), ) }, @@ -179,7 +179,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)). To(gomega.Equal("second-secret-42")) }, - ginkgo.SpecTimeout(framework.TimeoutShort()), + ginkgo.SpecTimeout(framework.TimeoutModerate()), ) }) }) diff --git a/e2e/tests/up/provider_podman_rootless_lifecycle.go b/e2e/tests/up/provider_podman_rootless_lifecycle.go index 1f483d560..0f4793dba 100644 --- a/e2e/tests/up/provider_podman_rootless_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootless_lifecycle.go @@ -110,7 +110,7 @@ var _ = ginkgo.Describe( gomega.Expect(strings.TrimSpace(out)).To(gomega.Equal("updateContentDone")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/deferred.marker 2>/dev/null", ) if err != nil { @@ -131,7 +131,7 @@ var _ = ginkgo.Describe( gomega.Expect(envPath).NotTo(gomega.ContainSubstring("${containerEnv:")) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-start-deferred.out 2>/dev/null", @@ -172,7 +172,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/post-attach.out 2>/dev/null", ) if err != nil { @@ -200,7 +200,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil { @@ -215,7 +215,7 @@ var _ = ginkgo.Describe( framework.ExpectNoError(err) gomega.Eventually(func() string { - out, err := f.DevsySSH( + out, err := probeSSH(f, ctx, tempDir, "cat $HOME/attach-count.out 2>/dev/null", ) if err != nil {