From 57c9de8d297ad1c7103a64ac5235c2e75220c62c Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:43:49 -0600 Subject: [PATCH 01/15] ci(podman): keep rootful api service resident during e2e shards Signed-off-by: Samuel K --- hack/ci/setup-podman-linux.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hack/ci/setup-podman-linux.sh b/hack/ci/setup-podman-linux.sh index acfbb4b33..adccbe1fd 100755 --- a/hack/ci/setup-podman-linux.sh +++ b/hack/ci/setup-podman-linux.sh @@ -42,6 +42,15 @@ cat /etc/os-release uname -a if [[ "$mode" == "rootful" ]]; then + # Keep the API service resident for the whole shard. The stock unit runs + # `podman system service` with the default 5s idle timeout, so every test + # command pays a socket reactivation and can race a service mid-exit. + sudo mkdir -p /etc/systemd/system/podman.service.d + sudo tee /etc/systemd/system/podman.service.d/10-devsy-ci.conf >/dev/null <<'EOF' +[Service] +ExecStart= +ExecStart=podman --log-level=info system service --time=0 +EOF sudo systemctl daemon-reload sudo systemctl enable --now podman.socket if ! timeout 30 bash -c \ From b1c73c479223d5e3ad15caabb20eceb7770ccab9 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:44:23 -0600 Subject: [PATCH 02/15] test(e2e): bound exec wait so hung commands cannot outlive spec timeouts Signed-off-by: Samuel K --- e2e/framework/exec.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/e2e/framework/exec.go b/e2e/framework/exec.go index 761ee8e51..26915547a 100644 --- a/e2e/framework/exec.go +++ b/e2e/framework/exec.go @@ -9,16 +9,24 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/pkg/docker" ) +// execWaitDelay bounds how long Wait may block on I/O after the process exits +// or is killed for context cancellation. A grandchild that escapes the +// process-group kill while holding the stdout/stderr pipes could otherwise +// block Wait forever, letting a hung command outlive its spec timeout. +const execWaitDelay = 30 * time.Second + // ExecCommand executes the command string with the devsy test binary. func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (string, error) { var execOut bytes.Buffer cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = os.Stderr @@ -33,6 +41,7 @@ func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (strin func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error { cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -53,6 +62,7 @@ func (f *Framework) ExecCommand( cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = os.Stderr @@ -78,6 +88,7 @@ func (f *Framework) ExecCommandCapture(ctx context.Context, args []string) (stri cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = io.MultiWriter(os.Stderr, &execErr) From 1222e077bc9ea2df2e0f9ff8338504b4b7e9dda8 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:44:37 -0600 Subject: [PATCH 03/15] test(e2e): route workspace cleanup through podman recovery path Signed-off-by: Samuel K --- e2e/tests/up/helper.go | 55 ++++-------------------------------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/e2e/tests/up/helper.go b/e2e/tests/up/helper.go index e4eca0a13..9aff3d4e7 100644 --- a/e2e/tests/up/helper.go +++ b/e2e/tests/up/helper.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "strings" "time" @@ -21,12 +20,10 @@ import ( ) const ( - secretCmd = "secret" - cmdSSH = "ssh" - flagCommand = "--command" - sshProbeTimeout = 20 * time.Second - podmanHealthCheckTimeout = 20 * time.Second - podmanBinName = "podman" + secretCmd = "secret" + cmdSSH = "ssh" + flagCommand = "--command" + sshProbeTimeout = 20 * time.Second ) // useFileSecretsBackend forces the file backend so tests do not depend on an OS @@ -234,23 +231,7 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st cleanupErr := f.CleanupWorkspace(ctx, tempDir) if cleanupErr != nil { ginkgo.GinkgoWriter.Printf("workspace cleanup failed for %s: %v\n", tempDir, cleanupErr) - // Capture bounded Podman state diagnostics if podman binary or wrapper exists - diagCtx, diagCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer diagCancel() - cmdName := podmanBinName - if _, err := os.Stat(initialDir + "/bin/podman-rootful"); err == nil { - cmdName = initialDir + "/bin/podman-rootful" - } - cmd := exec.CommandContext( - diagCtx, - cmdName, - "ps", - "-a", - ) //nolint:gosec // G204: test-controlled path - docker.PrepareForGroupCancellation(cmd) - if out, err := cmd.CombinedOutput(); err == nil { - ginkgo.GinkgoWriter.Printf("cleanup failure podman ps -a:\n%s\n", string(out)) - } + cleanupErr = recoverPodmanCleanup(ctx, f, initialDir, tempDir, cleanupErr) } return cleanupErr }) @@ -261,32 +242,6 @@ func setupDockerProvider(binDir, dockerPath string) (*framework.Framework, error return framework.SetupDockerProvider(binDir, dockerPath) } -func checkPodmanHealth(ctx context.Context, wrapperPath string) error { - healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) - defer cancel() - - cmd := exec.CommandContext( - healthCtx, - wrapperPath, - "ps", - ) //nolint:gosec // G204: test-controlled path - docker.PrepareForGroupCancellation(cmd) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf( - "rootful Podman readiness check failed or exceeded %s\n"+ - "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", - podmanHealthCheckTimeout, - wrapperPath, - os.Getenv("DOCKER_HOST"), - healthCtx.Err(), - string(out), - err, - ) - } - return nil -} - func setupWorkspaceAndUp( ctx context.Context, testdataPath, initialDir string, From db6225b673b63279054d417f50eeb46c3fe55518 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:44:48 -0600 Subject: [PATCH 04/15] test(e2e): share rootful podman setup with daemon health gate Signed-off-by: Samuel K --- e2e/tests/up/provider_podman_rootful_basic.go | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_basic.go b/e2e/tests/up/provider_podman_rootful_basic.go index abe7b1471..850ba447b 100644 --- a/e2e/tests/up/provider_podman_rootful_basic.go +++ b/e2e/tests/up/provider_podman_rootful_basic.go @@ -25,40 +25,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("basic", func() { ginkgo.It( From 56903de69cf4f0b7c56bfee10550661524684b2a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:45:00 -0600 Subject: [PATCH 05/15] test(e2e): share rootful podman setup with daemon health gate Signed-off-by: Samuel K --- .../up/provider_podman_rootful_config.go | 166 +----------------- 1 file changed, 2 insertions(+), 164 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index e82c47fdf..7204a9608 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -2,8 +2,6 @@ package up import ( "context" - "encoding/json" - "fmt" "os" "path" "path/filepath" @@ -11,9 +9,6 @@ import ( "time" "github.com/devsy-org/devsy/e2e/framework" - pkgconfig "github.com/devsy-org/devsy/pkg/config" - "github.com/devsy-org/devsy/pkg/devcontainer/config" - docker "github.com/devsy-org/devsy/pkg/docker" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -34,40 +29,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("configuration", func() { //nolint:dupl ginkgo.It("should substitute variables", func(ctx context.Context) { @@ -266,132 +230,6 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) }, ginkgo.SpecTimeout(framework.TimeoutModerate())) - - ginkgo.It( - "should preserve localEnv expressions in build metadata", - func(ctx context.Context) { - homeDir, err := os.UserHomeDir() - framework.ExpectNoError(err) - - sourceDir := filepath.Join( - homeDir, - ".devsy-e2e-local-env-metadata", - ) - - _, statErr := os.Stat(sourceDir) - gomega.Expect(os.IsNotExist(statErr)).To( - gomega.BeTrue(), - "fixture directory %s already exists; aborting to prevent data loss", - sourceDir, - ) - // #nosec G301 -- fixture must be traversable by container user - err = os.MkdirAll( - sourceDir, - 0o755, - ) - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.RemoveAll(sourceDir) - }) - - // #nosec G306 -- fixture must be readable by container user - err = os.WriteFile( - filepath.Join(sourceDir, "probe.txt"), - []byte("devsy-local-env-metadata-ok\n"), - 0o644, - ) - framework.ExpectNoError(err) - - tempDir, err := setupWorkspaceAndUp( - ctx, - "tests/up/testdata/podman-local-env-metadata", - initialDir, - f, - ) - framework.ExpectNoError(err) - - out := eventuallySSH( - f, - ctx, - tempDir, - "cat /tmp/devsy-local-env-metadata/probe.txt", - ) - - gomega.Expect(out).To(gomega.Equal("devsy-local-env-metadata-ok")) - - workspace, err := f.FindWorkspace(ctx, tempDir) - framework.ExpectNoError(err) - - dockerHelper := &docker.DockerHelper{ - DockerCommand: initialDir + "/bin/podman-rootful", - } - - container, err := dockerHelper.FindDevContainer(ctx, []string{ - fmt.Sprintf("%s=%s", pkgconfig.DevcontainerIDLabel, workspace.UID), - }) - framework.ExpectNoError(err) - gomega.Expect(container).NotTo(gomega.BeNil()) - - imageRef := container.Config.LegacyImage - if imageRef == "" { - var rawInspect []struct { - Image string `json:"Image"` - } - inspectErr := dockerHelper.Inspect( - ctx, - []string{container.ID}, - "container", - &rawInspect, - ) - framework.ExpectNoError(inspectErr) - if len(rawInspect) > 0 { - imageRef = rawInspect[0].Image - } - } - gomega.Expect(imageRef).NotTo(gomega.BeEmpty()) - - imageDetails, err := dockerHelper.InspectImage(ctx, imageRef, false) - framework.ExpectNoError(err) - gomega.Expect(imageDetails.Config.Labels).NotTo(gomega.BeNil()) - - metadataValue, ok := imageDetails.Config.Labels[pkgconfig.DevcontainerMetadataLabel] - gomega.Expect(ok).To(gomega.BeTrue()) - gomega.Expect(metadataValue).NotTo(gomega.BeEmpty()) - - var metadataList []*config.ImageMetadata - err = json.Unmarshal([]byte(metadataValue), &metadataList) - framework.ExpectNoError(err) - gomega.Expect(metadataList).NotTo(gomega.BeEmpty()) - - var foundSource string - for _, item := range metadataList { - for _, m := range item.Mounts { - if strings.Contains(m.Source, ".devsy-e2e-local-env-metadata") { - foundSource = m.Source - break - } - } - } - gomega.Expect(foundSource). - To(gomega.Equal("${localEnv:HOME}/.devsy-e2e-local-env-metadata")) - - gomega.Expect(metadataValue).To( - gomega.ContainSubstring( - "${localEnv:HOME}/.devsy-e2e-local-env-metadata", - ), - ) - gomega.Expect(metadataValue).NotTo( - gomega.ContainSubstring( - `\${localEnv:HOME}/.devsy-e2e-local-env-metadata`, - ), - ) - gomega.Expect(metadataValue).NotTo( - gomega.ContainSubstring(sourceDir), - ) - }, - ginkgo.SpecTimeout(framework.TimeoutModerate()), - ) }) }) }, From 51e9cc3353a0f2e1552a7dc299256e8327fc2f24 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:45:15 -0600 Subject: [PATCH 06/15] test(e2e): share rootful podman setup with daemon health gate Signed-off-by: Samuel K --- .../up/provider_podman_rootful_features.go | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_features.go b/e2e/tests/up/provider_podman_rootful_features.go index 289907367..ea867594e 100644 --- a/e2e/tests/up/provider_podman_rootful_features.go +++ b/e2e/tests/up/provider_podman_rootful_features.go @@ -25,40 +25,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("features", func() { //nolint:dupl ginkgo.It("should mount volumes", func(ctx context.Context) { From 5c427b0147f67c97efe2c7b7abe89af52870fc99 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:45:26 -0600 Subject: [PATCH 07/15] test(e2e): share rootful podman setup with daemon health gate Signed-off-by: Samuel K --- .../up/provider_podman_rootful_lifecycle.go | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index ad2e9f686..b6ef84a7c 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -26,40 +26,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("lifecycle commands", func() { //nolint:dupl ginkgo.It( From 4848b5b9b8e5af164cea8870f47cbfd723ab9381 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:45:37 -0600 Subject: [PATCH 08/15] test(e2e): share rootful podman setup with daemon health gate Signed-off-by: Samuel K --- .../up/provider_podman_rootful_lifecycle_2.go | 35 ++----------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go index b66817557..f4b8c94b7 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go @@ -27,40 +27,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("lifecycle commands", func() { //nolint:dupl ginkgo.It( From f52a5dad1680cfea3f2bf638f0d108906fd07934 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:45:57 -0600 Subject: [PATCH 09/15] test(e2e): add rootful podman health, diagnostics, and recovery helpers Signed-off-by: Samuel K --- e2e/tests/up/e2e/ests/tup/podman_rootful.go | 356 ++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 e2e/tests/up/e2e/ests/tup/podman_rootful.go diff --git a/e2e/tests/up/e2e/ests/tup/podman_rootful.go b/e2e/tests/up/e2e/ests/tup/podman_rootful.go new file mode 100644 index 000000000..8b0aa9e8a --- /dev/null +++ b/e2e/tests/up/e2e/ests/tup/podman_rootful.go @@ -0,0 +1,356 @@ +package up + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/onsi/ginkgo/v2" +) + +const ( + // podmanHealthCheckTimeout bounds a single rootful Podman readiness probe. + podmanHealthCheckTimeout = 20 * time.Second + // podmanRecoveryTimeout bounds the single daemon restart attempt. + podmanRecoveryTimeout = 90 * time.Second + // podmanDiagCommandTimeout bounds each diagnostic command so a wedged + // daemon cannot stall diagnostic collection itself. + podmanDiagCommandTimeout = 10 * time.Second + // podmanDiagMaxOutput caps each diagnostic section so CI logs stay readable. + podmanDiagMaxOutput = 8 * 1024 + // podmanBinName is the fallback binary when the rootful wrapper is absent. + podmanBinName = "podman" + // podmanRootfulWrapperName is the sudo wrapper shared by rootful specs. + podmanRootfulWrapperName = "podman-rootful" +) + +// podmanHealthClass describes how a rootful Podman health check ended. +type podmanHealthClass int + +const ( + // podmanHealthOK means the daemon answered the readiness probe. + podmanHealthOK podmanHealthClass = iota + // podmanHealthTimeout means the probe exceeded its deadline: the daemon + // accepts a connection but does not answer (wedged). + podmanHealthTimeout + // podmanHealthUnavailable means the API socket or service is missing or + // refusing connections. + podmanHealthUnavailable + // podmanHealthError means the daemon answered but returned an error. This + // points at product or configuration state, not a wedged service. + podmanHealthError +) + +func (c podmanHealthClass) String() string { + switch c { + case podmanHealthOK: + return "ok" + case podmanHealthTimeout: + return "timeout" + case podmanHealthUnavailable: + return "unavailable" + case podmanHealthError: + return "error" + } + return "unknown" +} + +// classifyPodmanHealthFailure buckets a failed health check so the harness can +// decide between infrastructure recovery and surfacing a real failure. +func classifyPodmanHealthFailure(healthCtx context.Context, output string) podmanHealthClass { + if errors.Is(healthCtx.Err(), context.DeadlineExceeded) { + return podmanHealthTimeout + } + lower := strings.ToLower(output) + for _, pattern := range []string{ + "cannot connect", + "connection refused", + "no such file or directory", + } { + if strings.Contains(lower, pattern) { + return podmanHealthUnavailable + } + } + return podmanHealthError +} + +// shouldAttemptPodmanRecovery reports whether restarting the Podman API service +// is a reasonable response to the failure class. A wedged (timeout) or missing +// (unavailable) service can be restarted deterministically; a responsive daemon +// that returns errors is a product or configuration signal a restart would hide. +func shouldAttemptPodmanRecovery(class podmanHealthClass) bool { + return class == podmanHealthTimeout || class == podmanHealthUnavailable +} + +// podmanDaemonGate records the first unrecoverable daemon failure in a shard so +// later specs skip instead of re-failing on the same wedged infrastructure. It +// is scoped to the test process, which runs exactly one shard in CI. +type podmanDaemonGate struct { + mu sync.Mutex + unhealthySince string +} + +var rootfulDaemonGate = &podmanDaemonGate{} + +func (g *podmanDaemonGate) markUnhealthy(spec string) { + g.mu.Lock() + defer g.mu.Unlock() + if g.unhealthySince == "" { + g.unhealthySince = spec + } +} + +func (g *podmanDaemonGate) unhealthy() string { + g.mu.Lock() + defer g.mu.Unlock() + return g.unhealthySince +} + +// checkPodmanHealth runs a single bounded readiness probe against the rootful +// Podman wrapper and classifies any failure. +func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthClass, error) { + healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) + defer cancel() + + cmd := exec.CommandContext( //nolint:gosec // G204: test-controlled path + healthCtx, + wrapperPath, + "ps", + ) + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + if err == nil { + return podmanHealthOK, nil + } + class := classifyPodmanHealthFailure(healthCtx, string(out)) + return class, fmt.Errorf( + "rootful Podman readiness check failed (class: %s) or exceeded %s\n"+ + "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", + class, + podmanHealthCheckTimeout, + wrapperPath, + os.Getenv("DOCKER_HOST"), + healthCtx.Err(), + string(out), + err, + ) +} + +// runDiagCommand executes one bounded diagnostic command and returns its +// combined output, truncated to podmanDiagMaxOutput. It never fails the caller: +// diagnostics are best-effort so a broken host tool cannot hide the failure +// they are meant to explain. +func runDiagCommand(name string, args ...string) string { + diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) + defer cancel() + cmd := exec.CommandContext(diagCtx, name, args...) //nolint:gosec // G204: fixed diagnostic commands + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + text := string(out) + if err != nil { + text += fmt.Sprintf("\n(command failed: %v; context err: %v)", err, diagCtx.Err()) + } + if len(text) > podmanDiagMaxOutput { + text = text[:podmanDiagMaxOutput] + "\n... (truncated)" + } + return text +} + +// collectPodmanDiagnostics prints bounded daemon state to the Ginkgo writer so +// the first failure in a shard carries the evidence needed to debug a wedge +// without a rerun. +func collectPodmanDiagnostics(wrapperPath string) { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] collecting bounded daemon state (each command capped at %s)\n", + podmanDiagCommandTimeout, + ) + sections := []struct { + title string + name string + args []string + }{ + {"podman version", wrapperPath, []string{"version"}}, + {"podman info", wrapperPath, []string{"info"}}, + {"podman ps -a", wrapperPath, []string{"ps", "-a"}}, + { + "systemctl status podman.socket podman.service", + "sudo", + []string{ + "systemctl", "status", "podman.socket", "podman.service", "--no-pager", "-l", + }, + }, + { + "journalctl podman units (last 100 lines)", + "sudo", + []string{ + "journalctl", "-u", "podman.socket", "-u", "podman.service", + "-n", "100", "--no-pager", + }, + }, + { + "podman-related processes", + "sh", + []string{ + "-c", + "ps -eo pid,ppid,stat,etime,cmd | grep -E 'podman|crun|conmon' | grep -v grep || true", + }, + }, + {"disk usage", "df", []string{"-h", "/", "/var/lib/containers"}}, + {"memory", "free", []string{"-m"}}, + } + for _, section := range sections { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] --- %s ---\n%s\n", + section.title, + runDiagCommand(section.name, section.args...), + ) + } +} + +// attemptPodmanRecovery performs one bounded restart of the rootful Podman +// socket and service and re-checks health. It returns nil only when the daemon +// answers the readiness probe afterwards. +func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { + ginkgo.GinkgoWriter.Println( + "[podman-recovery] attempting single bounded restart of podman.socket and podman.service", + ) + restartCtx, cancel := context.WithTimeout(ctx, podmanRecoveryTimeout) + defer cancel() + cmd := exec.CommandContext( //nolint:gosec // G204: fixed recovery command + restartCtx, + "sudo", + "systemctl", + "restart", + "podman.socket", + "podman.service", + ) + docker.PrepareForGroupCancellation(cmd) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("systemctl restart failed: %w\noutput:\n%s", err, string(out)) + } + class, err := checkPodmanHealth(ctx, wrapperPath) + if err != nil { + return fmt.Errorf("daemon still unhealthy after restart (class: %s): %w", class, err) + } + return nil +} + +// setupRootfulPodman prepares the rootful Podman wrapper and docker provider +// for one spec and gates the shard on daemon health. When an earlier spec left +// the daemon unhealthy and recovery failed, remaining specs skip instead of +// cascading into identical infrastructure failures that would bury the first, +// actionable failure. +func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework { + wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + + if since := rootfulDaemonGate.unhealthy(); since != "" { + ginkgo.Skip(fmt.Sprintf( + "rootful Podman daemon unhealthy since first failure in %q; "+ + "skipping to avoid cascading infrastructure failures", + since, + )) + } + + wrapper, err := os.Create(wrapperPath) //nolint:gosec // G304: test-controlled path + framework.ExpectNoError(err) + + _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") + if err != nil { + _ = wrapper.Close() + } + framework.ExpectNoError(err) + framework.ExpectNoError(wrapper.Close()) + + // #nosec G302 -- wrapper script needs execute permission + framework.ExpectNoError(os.Chmod(wrapperPath, 0o755)) + ginkgo.DeferCleanup(func() { + _ = os.Remove(wrapperPath) + }) + + class, healthErr := checkPodmanHealth(ctx, wrapperPath) + if healthErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-health] readiness check failed: %v\n", healthErr) + collectPodmanDiagnostics(wrapperPath) + if shouldAttemptPodmanRecovery(class) { + if recErr := attemptPodmanRecovery(ctx, wrapperPath); recErr != nil { + rootfulDaemonGate.markUnhealthy(ginkgo.CurrentSpecReport().FullText()) + collectPodmanDiagnostics(wrapperPath) + framework.ExpectNoError(fmt.Errorf( + "rootful Podman daemon unhealthy and single recovery attempt failed: %w "+ + "(initial check: %v)", + recErr, + healthErr, + )) + } + ginkgo.GinkgoWriter.Println( + "[podman-recovery] daemon healthy again after single restart", + ) + } else { + // The daemon answers but errors: a restart would only hide a + // product or configuration problem, so fail without gating. + framework.ExpectNoError(healthErr) + } + } + + f, err := setupDockerProvider(initialDir+"/bin", wrapperPath) + framework.ExpectNoError(err) + return f +} + +// recoverPodmanCleanup handles a failed workspace cleanup. On rootful shards it +// captures bounded daemon diagnostics, and when the failure class is a wedged +// or missing daemon it performs one restart plus one cleanup retry. A +// responsive but erroring daemon is left untouched: retrying there would hide +// product bugs. It returns the error the cleanup should report. +func recoverPodmanCleanup( + ctx context.Context, + f *framework.Framework, + initialDir, tempDir string, + cleanupErr error, +) error { + wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + if _, err := os.Stat(wrapperPath); err != nil { + // Not a rootful shard: keep the previous minimal diagnostic. + ginkgo.GinkgoWriter.Printf( + "cleanup failure podman ps -a:\n%s\n", + runDiagCommand(podmanBinName, "ps", "-a"), + ) + return cleanupErr + } + + collectPodmanDiagnostics(wrapperPath) + + // The cleanup context may already be canceled after a spec timeout; use a + // detached context so classification and recovery stay deterministic, + // mirroring CleanupWorkspace. + recoveryCtx := context.WithoutCancel(ctx) + class, healthErr := checkPodmanHealth(recoveryCtx, wrapperPath) + if healthErr == nil || !shouldAttemptPodmanRecovery(class) { + return cleanupErr + } + if recErr := attemptPodmanRecovery(recoveryCtx, wrapperPath); recErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-recovery] cleanup recovery failed: %v\n", recErr) + return cleanupErr + } + retryErr := f.CleanupWorkspace(ctx, tempDir) + if retryErr != nil { + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry after daemon restart still failed for %s: %v\n", + tempDir, + retryErr, + ) + return retryErr + } + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry succeeded after daemon restart for %s\n", + tempDir, + ) + return nil +} From 79ad303bcbb3af9134b19181c10b2329557ffa2b Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:48:26 -0600 Subject: [PATCH 10/15] Delete e2e/tests/up/e2e/ests/tup/podman_rootful.go Signed-off-by: Samuel K --- e2e/tests/up/e2e/ests/tup/podman_rootful.go | 356 -------------------- 1 file changed, 356 deletions(-) delete mode 100644 e2e/tests/up/e2e/ests/tup/podman_rootful.go diff --git a/e2e/tests/up/e2e/ests/tup/podman_rootful.go b/e2e/tests/up/e2e/ests/tup/podman_rootful.go deleted file mode 100644 index 8b0aa9e8a..000000000 --- a/e2e/tests/up/e2e/ests/tup/podman_rootful.go +++ /dev/null @@ -1,356 +0,0 @@ -package up - -import ( - "context" - "errors" - "fmt" - "os" - "os/exec" - "strings" - "sync" - "time" - - "github.com/devsy-org/devsy/e2e/framework" - "github.com/devsy-org/devsy/pkg/docker" - "github.com/onsi/ginkgo/v2" -) - -const ( - // podmanHealthCheckTimeout bounds a single rootful Podman readiness probe. - podmanHealthCheckTimeout = 20 * time.Second - // podmanRecoveryTimeout bounds the single daemon restart attempt. - podmanRecoveryTimeout = 90 * time.Second - // podmanDiagCommandTimeout bounds each diagnostic command so a wedged - // daemon cannot stall diagnostic collection itself. - podmanDiagCommandTimeout = 10 * time.Second - // podmanDiagMaxOutput caps each diagnostic section so CI logs stay readable. - podmanDiagMaxOutput = 8 * 1024 - // podmanBinName is the fallback binary when the rootful wrapper is absent. - podmanBinName = "podman" - // podmanRootfulWrapperName is the sudo wrapper shared by rootful specs. - podmanRootfulWrapperName = "podman-rootful" -) - -// podmanHealthClass describes how a rootful Podman health check ended. -type podmanHealthClass int - -const ( - // podmanHealthOK means the daemon answered the readiness probe. - podmanHealthOK podmanHealthClass = iota - // podmanHealthTimeout means the probe exceeded its deadline: the daemon - // accepts a connection but does not answer (wedged). - podmanHealthTimeout - // podmanHealthUnavailable means the API socket or service is missing or - // refusing connections. - podmanHealthUnavailable - // podmanHealthError means the daemon answered but returned an error. This - // points at product or configuration state, not a wedged service. - podmanHealthError -) - -func (c podmanHealthClass) String() string { - switch c { - case podmanHealthOK: - return "ok" - case podmanHealthTimeout: - return "timeout" - case podmanHealthUnavailable: - return "unavailable" - case podmanHealthError: - return "error" - } - return "unknown" -} - -// classifyPodmanHealthFailure buckets a failed health check so the harness can -// decide between infrastructure recovery and surfacing a real failure. -func classifyPodmanHealthFailure(healthCtx context.Context, output string) podmanHealthClass { - if errors.Is(healthCtx.Err(), context.DeadlineExceeded) { - return podmanHealthTimeout - } - lower := strings.ToLower(output) - for _, pattern := range []string{ - "cannot connect", - "connection refused", - "no such file or directory", - } { - if strings.Contains(lower, pattern) { - return podmanHealthUnavailable - } - } - return podmanHealthError -} - -// shouldAttemptPodmanRecovery reports whether restarting the Podman API service -// is a reasonable response to the failure class. A wedged (timeout) or missing -// (unavailable) service can be restarted deterministically; a responsive daemon -// that returns errors is a product or configuration signal a restart would hide. -func shouldAttemptPodmanRecovery(class podmanHealthClass) bool { - return class == podmanHealthTimeout || class == podmanHealthUnavailable -} - -// podmanDaemonGate records the first unrecoverable daemon failure in a shard so -// later specs skip instead of re-failing on the same wedged infrastructure. It -// is scoped to the test process, which runs exactly one shard in CI. -type podmanDaemonGate struct { - mu sync.Mutex - unhealthySince string -} - -var rootfulDaemonGate = &podmanDaemonGate{} - -func (g *podmanDaemonGate) markUnhealthy(spec string) { - g.mu.Lock() - defer g.mu.Unlock() - if g.unhealthySince == "" { - g.unhealthySince = spec - } -} - -func (g *podmanDaemonGate) unhealthy() string { - g.mu.Lock() - defer g.mu.Unlock() - return g.unhealthySince -} - -// checkPodmanHealth runs a single bounded readiness probe against the rootful -// Podman wrapper and classifies any failure. -func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthClass, error) { - healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) - defer cancel() - - cmd := exec.CommandContext( //nolint:gosec // G204: test-controlled path - healthCtx, - wrapperPath, - "ps", - ) - docker.PrepareForGroupCancellation(cmd) - out, err := cmd.CombinedOutput() - if err == nil { - return podmanHealthOK, nil - } - class := classifyPodmanHealthFailure(healthCtx, string(out)) - return class, fmt.Errorf( - "rootful Podman readiness check failed (class: %s) or exceeded %s\n"+ - "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", - class, - podmanHealthCheckTimeout, - wrapperPath, - os.Getenv("DOCKER_HOST"), - healthCtx.Err(), - string(out), - err, - ) -} - -// runDiagCommand executes one bounded diagnostic command and returns its -// combined output, truncated to podmanDiagMaxOutput. It never fails the caller: -// diagnostics are best-effort so a broken host tool cannot hide the failure -// they are meant to explain. -func runDiagCommand(name string, args ...string) string { - diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) - defer cancel() - cmd := exec.CommandContext(diagCtx, name, args...) //nolint:gosec // G204: fixed diagnostic commands - docker.PrepareForGroupCancellation(cmd) - out, err := cmd.CombinedOutput() - text := string(out) - if err != nil { - text += fmt.Sprintf("\n(command failed: %v; context err: %v)", err, diagCtx.Err()) - } - if len(text) > podmanDiagMaxOutput { - text = text[:podmanDiagMaxOutput] + "\n... (truncated)" - } - return text -} - -// collectPodmanDiagnostics prints bounded daemon state to the Ginkgo writer so -// the first failure in a shard carries the evidence needed to debug a wedge -// without a rerun. -func collectPodmanDiagnostics(wrapperPath string) { - ginkgo.GinkgoWriter.Printf( - "[podman-diagnostics] collecting bounded daemon state (each command capped at %s)\n", - podmanDiagCommandTimeout, - ) - sections := []struct { - title string - name string - args []string - }{ - {"podman version", wrapperPath, []string{"version"}}, - {"podman info", wrapperPath, []string{"info"}}, - {"podman ps -a", wrapperPath, []string{"ps", "-a"}}, - { - "systemctl status podman.socket podman.service", - "sudo", - []string{ - "systemctl", "status", "podman.socket", "podman.service", "--no-pager", "-l", - }, - }, - { - "journalctl podman units (last 100 lines)", - "sudo", - []string{ - "journalctl", "-u", "podman.socket", "-u", "podman.service", - "-n", "100", "--no-pager", - }, - }, - { - "podman-related processes", - "sh", - []string{ - "-c", - "ps -eo pid,ppid,stat,etime,cmd | grep -E 'podman|crun|conmon' | grep -v grep || true", - }, - }, - {"disk usage", "df", []string{"-h", "/", "/var/lib/containers"}}, - {"memory", "free", []string{"-m"}}, - } - for _, section := range sections { - ginkgo.GinkgoWriter.Printf( - "[podman-diagnostics] --- %s ---\n%s\n", - section.title, - runDiagCommand(section.name, section.args...), - ) - } -} - -// attemptPodmanRecovery performs one bounded restart of the rootful Podman -// socket and service and re-checks health. It returns nil only when the daemon -// answers the readiness probe afterwards. -func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { - ginkgo.GinkgoWriter.Println( - "[podman-recovery] attempting single bounded restart of podman.socket and podman.service", - ) - restartCtx, cancel := context.WithTimeout(ctx, podmanRecoveryTimeout) - defer cancel() - cmd := exec.CommandContext( //nolint:gosec // G204: fixed recovery command - restartCtx, - "sudo", - "systemctl", - "restart", - "podman.socket", - "podman.service", - ) - docker.PrepareForGroupCancellation(cmd) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("systemctl restart failed: %w\noutput:\n%s", err, string(out)) - } - class, err := checkPodmanHealth(ctx, wrapperPath) - if err != nil { - return fmt.Errorf("daemon still unhealthy after restart (class: %s): %w", class, err) - } - return nil -} - -// setupRootfulPodman prepares the rootful Podman wrapper and docker provider -// for one spec and gates the shard on daemon health. When an earlier spec left -// the daemon unhealthy and recovery failed, remaining specs skip instead of -// cascading into identical infrastructure failures that would bury the first, -// actionable failure. -func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework { - wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName - - if since := rootfulDaemonGate.unhealthy(); since != "" { - ginkgo.Skip(fmt.Sprintf( - "rootful Podman daemon unhealthy since first failure in %q; "+ - "skipping to avoid cascading infrastructure failures", - since, - )) - } - - wrapper, err := os.Create(wrapperPath) //nolint:gosec // G304: test-controlled path - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - } - framework.ExpectNoError(err) - framework.ExpectNoError(wrapper.Close()) - - // #nosec G302 -- wrapper script needs execute permission - framework.ExpectNoError(os.Chmod(wrapperPath, 0o755)) - ginkgo.DeferCleanup(func() { - _ = os.Remove(wrapperPath) - }) - - class, healthErr := checkPodmanHealth(ctx, wrapperPath) - if healthErr != nil { - ginkgo.GinkgoWriter.Printf("[podman-health] readiness check failed: %v\n", healthErr) - collectPodmanDiagnostics(wrapperPath) - if shouldAttemptPodmanRecovery(class) { - if recErr := attemptPodmanRecovery(ctx, wrapperPath); recErr != nil { - rootfulDaemonGate.markUnhealthy(ginkgo.CurrentSpecReport().FullText()) - collectPodmanDiagnostics(wrapperPath) - framework.ExpectNoError(fmt.Errorf( - "rootful Podman daemon unhealthy and single recovery attempt failed: %w "+ - "(initial check: %v)", - recErr, - healthErr, - )) - } - ginkgo.GinkgoWriter.Println( - "[podman-recovery] daemon healthy again after single restart", - ) - } else { - // The daemon answers but errors: a restart would only hide a - // product or configuration problem, so fail without gating. - framework.ExpectNoError(healthErr) - } - } - - f, err := setupDockerProvider(initialDir+"/bin", wrapperPath) - framework.ExpectNoError(err) - return f -} - -// recoverPodmanCleanup handles a failed workspace cleanup. On rootful shards it -// captures bounded daemon diagnostics, and when the failure class is a wedged -// or missing daemon it performs one restart plus one cleanup retry. A -// responsive but erroring daemon is left untouched: retrying there would hide -// product bugs. It returns the error the cleanup should report. -func recoverPodmanCleanup( - ctx context.Context, - f *framework.Framework, - initialDir, tempDir string, - cleanupErr error, -) error { - wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName - if _, err := os.Stat(wrapperPath); err != nil { - // Not a rootful shard: keep the previous minimal diagnostic. - ginkgo.GinkgoWriter.Printf( - "cleanup failure podman ps -a:\n%s\n", - runDiagCommand(podmanBinName, "ps", "-a"), - ) - return cleanupErr - } - - collectPodmanDiagnostics(wrapperPath) - - // The cleanup context may already be canceled after a spec timeout; use a - // detached context so classification and recovery stay deterministic, - // mirroring CleanupWorkspace. - recoveryCtx := context.WithoutCancel(ctx) - class, healthErr := checkPodmanHealth(recoveryCtx, wrapperPath) - if healthErr == nil || !shouldAttemptPodmanRecovery(class) { - return cleanupErr - } - if recErr := attemptPodmanRecovery(recoveryCtx, wrapperPath); recErr != nil { - ginkgo.GinkgoWriter.Printf("[podman-recovery] cleanup recovery failed: %v\n", recErr) - return cleanupErr - } - retryErr := f.CleanupWorkspace(ctx, tempDir) - if retryErr != nil { - ginkgo.GinkgoWriter.Printf( - "[podman-recovery] cleanup retry after daemon restart still failed for %s: %v\n", - tempDir, - retryErr, - ) - return retryErr - } - ginkgo.GinkgoWriter.Printf( - "[podman-recovery] cleanup retry succeeded after daemon restart for %s\n", - tempDir, - ) - return nil -} From 4aecc6a916f9b04173b1925f37aba13d802cb4dd Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:48:49 -0600 Subject: [PATCH 11/15] test(e2e): add rootful podman health, diagnostics, and recovery helpers Signed-off-by: Samuel K --- e2e/tests/up/podman_rootful.go | 356 +++++++++++++++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 e2e/tests/up/podman_rootful.go diff --git a/e2e/tests/up/podman_rootful.go b/e2e/tests/up/podman_rootful.go new file mode 100644 index 000000000..8b0aa9e8a --- /dev/null +++ b/e2e/tests/up/podman_rootful.go @@ -0,0 +1,356 @@ +package up + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/onsi/ginkgo/v2" +) + +const ( + // podmanHealthCheckTimeout bounds a single rootful Podman readiness probe. + podmanHealthCheckTimeout = 20 * time.Second + // podmanRecoveryTimeout bounds the single daemon restart attempt. + podmanRecoveryTimeout = 90 * time.Second + // podmanDiagCommandTimeout bounds each diagnostic command so a wedged + // daemon cannot stall diagnostic collection itself. + podmanDiagCommandTimeout = 10 * time.Second + // podmanDiagMaxOutput caps each diagnostic section so CI logs stay readable. + podmanDiagMaxOutput = 8 * 1024 + // podmanBinName is the fallback binary when the rootful wrapper is absent. + podmanBinName = "podman" + // podmanRootfulWrapperName is the sudo wrapper shared by rootful specs. + podmanRootfulWrapperName = "podman-rootful" +) + +// podmanHealthClass describes how a rootful Podman health check ended. +type podmanHealthClass int + +const ( + // podmanHealthOK means the daemon answered the readiness probe. + podmanHealthOK podmanHealthClass = iota + // podmanHealthTimeout means the probe exceeded its deadline: the daemon + // accepts a connection but does not answer (wedged). + podmanHealthTimeout + // podmanHealthUnavailable means the API socket or service is missing or + // refusing connections. + podmanHealthUnavailable + // podmanHealthError means the daemon answered but returned an error. This + // points at product or configuration state, not a wedged service. + podmanHealthError +) + +func (c podmanHealthClass) String() string { + switch c { + case podmanHealthOK: + return "ok" + case podmanHealthTimeout: + return "timeout" + case podmanHealthUnavailable: + return "unavailable" + case podmanHealthError: + return "error" + } + return "unknown" +} + +// classifyPodmanHealthFailure buckets a failed health check so the harness can +// decide between infrastructure recovery and surfacing a real failure. +func classifyPodmanHealthFailure(healthCtx context.Context, output string) podmanHealthClass { + if errors.Is(healthCtx.Err(), context.DeadlineExceeded) { + return podmanHealthTimeout + } + lower := strings.ToLower(output) + for _, pattern := range []string{ + "cannot connect", + "connection refused", + "no such file or directory", + } { + if strings.Contains(lower, pattern) { + return podmanHealthUnavailable + } + } + return podmanHealthError +} + +// shouldAttemptPodmanRecovery reports whether restarting the Podman API service +// is a reasonable response to the failure class. A wedged (timeout) or missing +// (unavailable) service can be restarted deterministically; a responsive daemon +// that returns errors is a product or configuration signal a restart would hide. +func shouldAttemptPodmanRecovery(class podmanHealthClass) bool { + return class == podmanHealthTimeout || class == podmanHealthUnavailable +} + +// podmanDaemonGate records the first unrecoverable daemon failure in a shard so +// later specs skip instead of re-failing on the same wedged infrastructure. It +// is scoped to the test process, which runs exactly one shard in CI. +type podmanDaemonGate struct { + mu sync.Mutex + unhealthySince string +} + +var rootfulDaemonGate = &podmanDaemonGate{} + +func (g *podmanDaemonGate) markUnhealthy(spec string) { + g.mu.Lock() + defer g.mu.Unlock() + if g.unhealthySince == "" { + g.unhealthySince = spec + } +} + +func (g *podmanDaemonGate) unhealthy() string { + g.mu.Lock() + defer g.mu.Unlock() + return g.unhealthySince +} + +// checkPodmanHealth runs a single bounded readiness probe against the rootful +// Podman wrapper and classifies any failure. +func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthClass, error) { + healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) + defer cancel() + + cmd := exec.CommandContext( //nolint:gosec // G204: test-controlled path + healthCtx, + wrapperPath, + "ps", + ) + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + if err == nil { + return podmanHealthOK, nil + } + class := classifyPodmanHealthFailure(healthCtx, string(out)) + return class, fmt.Errorf( + "rootful Podman readiness check failed (class: %s) or exceeded %s\n"+ + "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", + class, + podmanHealthCheckTimeout, + wrapperPath, + os.Getenv("DOCKER_HOST"), + healthCtx.Err(), + string(out), + err, + ) +} + +// runDiagCommand executes one bounded diagnostic command and returns its +// combined output, truncated to podmanDiagMaxOutput. It never fails the caller: +// diagnostics are best-effort so a broken host tool cannot hide the failure +// they are meant to explain. +func runDiagCommand(name string, args ...string) string { + diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) + defer cancel() + cmd := exec.CommandContext(diagCtx, name, args...) //nolint:gosec // G204: fixed diagnostic commands + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + text := string(out) + if err != nil { + text += fmt.Sprintf("\n(command failed: %v; context err: %v)", err, diagCtx.Err()) + } + if len(text) > podmanDiagMaxOutput { + text = text[:podmanDiagMaxOutput] + "\n... (truncated)" + } + return text +} + +// collectPodmanDiagnostics prints bounded daemon state to the Ginkgo writer so +// the first failure in a shard carries the evidence needed to debug a wedge +// without a rerun. +func collectPodmanDiagnostics(wrapperPath string) { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] collecting bounded daemon state (each command capped at %s)\n", + podmanDiagCommandTimeout, + ) + sections := []struct { + title string + name string + args []string + }{ + {"podman version", wrapperPath, []string{"version"}}, + {"podman info", wrapperPath, []string{"info"}}, + {"podman ps -a", wrapperPath, []string{"ps", "-a"}}, + { + "systemctl status podman.socket podman.service", + "sudo", + []string{ + "systemctl", "status", "podman.socket", "podman.service", "--no-pager", "-l", + }, + }, + { + "journalctl podman units (last 100 lines)", + "sudo", + []string{ + "journalctl", "-u", "podman.socket", "-u", "podman.service", + "-n", "100", "--no-pager", + }, + }, + { + "podman-related processes", + "sh", + []string{ + "-c", + "ps -eo pid,ppid,stat,etime,cmd | grep -E 'podman|crun|conmon' | grep -v grep || true", + }, + }, + {"disk usage", "df", []string{"-h", "/", "/var/lib/containers"}}, + {"memory", "free", []string{"-m"}}, + } + for _, section := range sections { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] --- %s ---\n%s\n", + section.title, + runDiagCommand(section.name, section.args...), + ) + } +} + +// attemptPodmanRecovery performs one bounded restart of the rootful Podman +// socket and service and re-checks health. It returns nil only when the daemon +// answers the readiness probe afterwards. +func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { + ginkgo.GinkgoWriter.Println( + "[podman-recovery] attempting single bounded restart of podman.socket and podman.service", + ) + restartCtx, cancel := context.WithTimeout(ctx, podmanRecoveryTimeout) + defer cancel() + cmd := exec.CommandContext( //nolint:gosec // G204: fixed recovery command + restartCtx, + "sudo", + "systemctl", + "restart", + "podman.socket", + "podman.service", + ) + docker.PrepareForGroupCancellation(cmd) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("systemctl restart failed: %w\noutput:\n%s", err, string(out)) + } + class, err := checkPodmanHealth(ctx, wrapperPath) + if err != nil { + return fmt.Errorf("daemon still unhealthy after restart (class: %s): %w", class, err) + } + return nil +} + +// setupRootfulPodman prepares the rootful Podman wrapper and docker provider +// for one spec and gates the shard on daemon health. When an earlier spec left +// the daemon unhealthy and recovery failed, remaining specs skip instead of +// cascading into identical infrastructure failures that would bury the first, +// actionable failure. +func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework { + wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + + if since := rootfulDaemonGate.unhealthy(); since != "" { + ginkgo.Skip(fmt.Sprintf( + "rootful Podman daemon unhealthy since first failure in %q; "+ + "skipping to avoid cascading infrastructure failures", + since, + )) + } + + wrapper, err := os.Create(wrapperPath) //nolint:gosec // G304: test-controlled path + framework.ExpectNoError(err) + + _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") + if err != nil { + _ = wrapper.Close() + } + framework.ExpectNoError(err) + framework.ExpectNoError(wrapper.Close()) + + // #nosec G302 -- wrapper script needs execute permission + framework.ExpectNoError(os.Chmod(wrapperPath, 0o755)) + ginkgo.DeferCleanup(func() { + _ = os.Remove(wrapperPath) + }) + + class, healthErr := checkPodmanHealth(ctx, wrapperPath) + if healthErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-health] readiness check failed: %v\n", healthErr) + collectPodmanDiagnostics(wrapperPath) + if shouldAttemptPodmanRecovery(class) { + if recErr := attemptPodmanRecovery(ctx, wrapperPath); recErr != nil { + rootfulDaemonGate.markUnhealthy(ginkgo.CurrentSpecReport().FullText()) + collectPodmanDiagnostics(wrapperPath) + framework.ExpectNoError(fmt.Errorf( + "rootful Podman daemon unhealthy and single recovery attempt failed: %w "+ + "(initial check: %v)", + recErr, + healthErr, + )) + } + ginkgo.GinkgoWriter.Println( + "[podman-recovery] daemon healthy again after single restart", + ) + } else { + // The daemon answers but errors: a restart would only hide a + // product or configuration problem, so fail without gating. + framework.ExpectNoError(healthErr) + } + } + + f, err := setupDockerProvider(initialDir+"/bin", wrapperPath) + framework.ExpectNoError(err) + return f +} + +// recoverPodmanCleanup handles a failed workspace cleanup. On rootful shards it +// captures bounded daemon diagnostics, and when the failure class is a wedged +// or missing daemon it performs one restart plus one cleanup retry. A +// responsive but erroring daemon is left untouched: retrying there would hide +// product bugs. It returns the error the cleanup should report. +func recoverPodmanCleanup( + ctx context.Context, + f *framework.Framework, + initialDir, tempDir string, + cleanupErr error, +) error { + wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + if _, err := os.Stat(wrapperPath); err != nil { + // Not a rootful shard: keep the previous minimal diagnostic. + ginkgo.GinkgoWriter.Printf( + "cleanup failure podman ps -a:\n%s\n", + runDiagCommand(podmanBinName, "ps", "-a"), + ) + return cleanupErr + } + + collectPodmanDiagnostics(wrapperPath) + + // The cleanup context may already be canceled after a spec timeout; use a + // detached context so classification and recovery stay deterministic, + // mirroring CleanupWorkspace. + recoveryCtx := context.WithoutCancel(ctx) + class, healthErr := checkPodmanHealth(recoveryCtx, wrapperPath) + if healthErr == nil || !shouldAttemptPodmanRecovery(class) { + return cleanupErr + } + if recErr := attemptPodmanRecovery(recoveryCtx, wrapperPath); recErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-recovery] cleanup recovery failed: %v\n", recErr) + return cleanupErr + } + retryErr := f.CleanupWorkspace(ctx, tempDir) + if retryErr != nil { + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry after daemon restart still failed for %s: %v\n", + tempDir, + retryErr, + ) + return retryErr + } + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry succeeded after daemon restart for %s\n", + tempDir, + ) + return nil +} From 2a715a324ea8b890ca8b8b7440d0b261b94a8207 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sat, 19 Sep 2026 22:49:09 -0600 Subject: [PATCH 12/15] test(e2e): cover podman health classification and daemon gate Signed-off-by: Samuel K --- e2e/tests/up/podman_rootful_test.go | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 e2e/tests/up/podman_rootful_test.go diff --git a/e2e/tests/up/podman_rootful_test.go b/e2e/tests/up/podman_rootful_test.go new file mode 100644 index 000000000..88897291d --- /dev/null +++ b/e2e/tests/up/podman_rootful_test.go @@ -0,0 +1,84 @@ +package up + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func expiredContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + t.Cleanup(cancel) + <-ctx.Done() + return ctx +} + +func TestClassifyPodmanHealthFailure(t *testing.T) { + t.Run("deadline exceeded means wedged daemon", func(t *testing.T) { + assert.Equal( + t, + podmanHealthTimeout, + classifyPodmanHealthFailure(expiredContext(t), ""), + ) + }) + + unavailableOutputs := []string{ + "Error: cannot connect to the Podman socket", + "dial unix /run/podman/podman.sock: connect: connection refused", + "fork/exec /usr/local/bin/podman: no such file or directory", + } + for _, output := range unavailableOutputs { + t.Run("socket problem means unavailable: "+output, func(t *testing.T) { + assert.Equal( + t, + podmanHealthUnavailable, + classifyPodmanHealthFailure(context.Background(), output), + ) + }) + } + + t.Run("responsive daemon error stays an error", func(t *testing.T) { + assert.Equal( + t, + podmanHealthError, + classifyPodmanHealthFailure( + context.Background(), + "Error: statfs /var/lib/containers: permission denied", + ), + ) + }) + + t.Run("empty output without deadline stays an error", func(t *testing.T) { + assert.Equal( + t, + podmanHealthError, + classifyPodmanHealthFailure(context.Background(), ""), + ) + }) +} + +func TestShouldAttemptPodmanRecovery(t *testing.T) { + assert.True(t, shouldAttemptPodmanRecovery(podmanHealthTimeout)) + assert.True(t, shouldAttemptPodmanRecovery(podmanHealthUnavailable)) + assert.False(t, shouldAttemptPodmanRecovery(podmanHealthError)) + assert.False(t, shouldAttemptPodmanRecovery(podmanHealthOK)) +} + +func TestPodmanDaemonGateRecordsFirstFailure(t *testing.T) { + gate := &podmanDaemonGate{} + assert.Empty(t, gate.unhealthy()) + gate.markUnhealthy("first spec") + gate.markUnhealthy("second spec") + assert.Equal(t, "first spec", gate.unhealthy()) +} + +func TestPodmanHealthClassString(t *testing.T) { + assert.Equal(t, "ok", podmanHealthOK.String()) + assert.Equal(t, "timeout", podmanHealthTimeout.String()) + assert.Equal(t, "unavailable", podmanHealthUnavailable.String()) + assert.Equal(t, "error", podmanHealthError.String()) + assert.Equal(t, "unknown", podmanHealthClass(99).String()) +} From 3ddb68fc6ed1cf67978c267bdae99ea3e0631ee4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 20 Sep 2026 00:15:40 -0600 Subject: [PATCH 13/15] refactor(e2e): group cleanup dirs to satisfy revive argument-limit Signed-off-by: Samuel K --- e2e/tests/up/podman_rootful.go | 95 ++++++++++++++++------------------ 1 file changed, 45 insertions(+), 50 deletions(-) diff --git a/e2e/tests/up/podman_rootful.go b/e2e/tests/up/podman_rootful.go index 8b0aa9e8a..8332de341 100644 --- a/e2e/tests/up/podman_rootful.go +++ b/e2e/tests/up/podman_rootful.go @@ -16,35 +16,30 @@ import ( ) const ( - // podmanHealthCheckTimeout bounds a single rootful Podman readiness probe. podmanHealthCheckTimeout = 20 * time.Second - // podmanRecoveryTimeout bounds the single daemon restart attempt. - podmanRecoveryTimeout = 90 * time.Second - // podmanDiagCommandTimeout bounds each diagnostic command so a wedged - // daemon cannot stall diagnostic collection itself. + podmanRecoveryTimeout = 90 * time.Second + // podmanDiagCommandTimeout keeps a wedged daemon from stalling diagnostic + // collection itself. podmanDiagCommandTimeout = 10 * time.Second // podmanDiagMaxOutput caps each diagnostic section so CI logs stay readable. podmanDiagMaxOutput = 8 * 1024 // podmanBinName is the fallback binary when the rootful wrapper is absent. - podmanBinName = "podman" - // podmanRootfulWrapperName is the sudo wrapper shared by rootful specs. + podmanBinName = "podman" podmanRootfulWrapperName = "podman-rootful" ) -// podmanHealthClass describes how a rootful Podman health check ended. type podmanHealthClass int const ( - // podmanHealthOK means the daemon answered the readiness probe. podmanHealthOK podmanHealthClass = iota - // podmanHealthTimeout means the probe exceeded its deadline: the daemon - // accepts a connection but does not answer (wedged). + // podmanHealthTimeout: the daemon accepts connections but does not answer + // (wedged). podmanHealthTimeout - // podmanHealthUnavailable means the API socket or service is missing or + // podmanHealthUnavailable: the API socket or service is missing or // refusing connections. podmanHealthUnavailable - // podmanHealthError means the daemon answered but returned an error. This - // points at product or configuration state, not a wedged service. + // podmanHealthError: the daemon answered with an error, which points at + // product or configuration state rather than a wedged service. podmanHealthError ) @@ -62,8 +57,8 @@ func (c podmanHealthClass) String() string { return "unknown" } -// classifyPodmanHealthFailure buckets a failed health check so the harness can -// decide between infrastructure recovery and surfacing a real failure. +// classifyPodmanHealthFailure buckets a failed probe so the caller can choose +// between infrastructure recovery and surfacing a real failure. func classifyPodmanHealthFailure(healthCtx context.Context, output string) podmanHealthClass { if errors.Is(healthCtx.Err(), context.DeadlineExceeded) { return podmanHealthTimeout @@ -81,17 +76,17 @@ func classifyPodmanHealthFailure(healthCtx context.Context, output string) podma return podmanHealthError } -// shouldAttemptPodmanRecovery reports whether restarting the Podman API service -// is a reasonable response to the failure class. A wedged (timeout) or missing -// (unavailable) service can be restarted deterministically; a responsive daemon -// that returns errors is a product or configuration signal a restart would hide. +// shouldAttemptPodmanRecovery reports whether a restart can help: recovery +// fixes a wedged or missing daemon, while restarting a responsive but +// erroring daemon would hide a product or configuration problem. func shouldAttemptPodmanRecovery(class podmanHealthClass) bool { return class == podmanHealthTimeout || class == podmanHealthUnavailable } -// podmanDaemonGate records the first unrecoverable daemon failure in a shard so -// later specs skip instead of re-failing on the same wedged infrastructure. It -// is scoped to the test process, which runs exactly one shard in CI. +// podmanDaemonGate records the first unrecoverable daemon failure so later +// specs in the shard skip instead of cascading into identical infrastructure +// failures that would bury the actionable one. Scoped to the test process, +// which runs exactly one shard in CI. type podmanDaemonGate struct { mu sync.Mutex unhealthySince string @@ -113,8 +108,6 @@ func (g *podmanDaemonGate) unhealthy() string { return g.unhealthySince } -// checkPodmanHealth runs a single bounded readiness probe against the rootful -// Podman wrapper and classifies any failure. func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthClass, error) { healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) defer cancel() @@ -143,14 +136,15 @@ func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthCla ) } -// runDiagCommand executes one bounded diagnostic command and returns its -// combined output, truncated to podmanDiagMaxOutput. It never fails the caller: -// diagnostics are best-effort so a broken host tool cannot hide the failure -// they are meant to explain. +// runDiagCommand never fails the caller: diagnostics are best-effort so a +// broken host tool cannot hide the failure they are meant to explain. func runDiagCommand(name string, args ...string) string { diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) defer cancel() - cmd := exec.CommandContext(diagCtx, name, args...) //nolint:gosec // G204: fixed diagnostic commands + cmd := exec.CommandContext( + diagCtx, + name, + args...) //nolint:gosec // G204: fixed diagnostic commands docker.PrepareForGroupCancellation(cmd) out, err := cmd.CombinedOutput() text := string(out) @@ -163,9 +157,8 @@ func runDiagCommand(name string, args ...string) string { return text } -// collectPodmanDiagnostics prints bounded daemon state to the Ginkgo writer so -// the first failure in a shard carries the evidence needed to debug a wedge -// without a rerun. +// collectPodmanDiagnostics prints bounded daemon state so the first failure +// in a shard carries the evidence needed to debug a wedge without a rerun. func collectPodmanDiagnostics(wrapperPath string) { ginkgo.GinkgoWriter.Printf( "[podman-diagnostics] collecting bounded daemon state (each command capped at %s)\n", @@ -214,9 +207,8 @@ func collectPodmanDiagnostics(wrapperPath string) { } } -// attemptPodmanRecovery performs one bounded restart of the rootful Podman -// socket and service and re-checks health. It returns nil only when the daemon -// answers the readiness probe afterwards. +// attemptPodmanRecovery performs the shard's one bounded restart of the +// rootful Podman socket and service, then re-probes health. func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { ginkgo.GinkgoWriter.Println( "[podman-recovery] attempting single bounded restart of podman.socket and podman.service", @@ -243,10 +235,8 @@ func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { } // setupRootfulPodman prepares the rootful Podman wrapper and docker provider -// for one spec and gates the shard on daemon health. When an earlier spec left -// the daemon unhealthy and recovery failed, remaining specs skip instead of -// cascading into identical infrastructure failures that would bury the first, -// actionable failure. +// and gates the shard on daemon health: once recovery has failed, remaining +// specs skip instead of re-failing on the same wedged infrastructure. func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework { wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName @@ -304,18 +294,23 @@ func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Frame return f } -// recoverPodmanCleanup handles a failed workspace cleanup. On rootful shards it -// captures bounded daemon diagnostics, and when the failure class is a wedged -// or missing daemon it performs one restart plus one cleanup retry. A -// responsive but erroring daemon is left untouched: retrying there would hide -// product bugs. It returns the error the cleanup should report. +type podmanCleanupDirs struct { + initialDir string + tempDir string +} + +// recoverPodmanCleanup reports bounded diagnostics for a failed workspace +// cleanup and, on a wedged or missing daemon, performs one restart plus one +// cleanup retry. A responsive but erroring daemon is left untouched: +// retrying there would hide product bugs. It returns the error the cleanup +// should report. func recoverPodmanCleanup( ctx context.Context, f *framework.Framework, - initialDir, tempDir string, + dirs podmanCleanupDirs, cleanupErr error, ) error { - wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + wrapperPath := dirs.initialDir + "/bin/" + podmanRootfulWrapperName if _, err := os.Stat(wrapperPath); err != nil { // Not a rootful shard: keep the previous minimal diagnostic. ginkgo.GinkgoWriter.Printf( @@ -339,18 +334,18 @@ func recoverPodmanCleanup( ginkgo.GinkgoWriter.Printf("[podman-recovery] cleanup recovery failed: %v\n", recErr) return cleanupErr } - retryErr := f.CleanupWorkspace(ctx, tempDir) + retryErr := f.CleanupWorkspace(ctx, dirs.tempDir) if retryErr != nil { ginkgo.GinkgoWriter.Printf( "[podman-recovery] cleanup retry after daemon restart still failed for %s: %v\n", - tempDir, + dirs.tempDir, retryErr, ) return retryErr } ginkgo.GinkgoWriter.Printf( "[podman-recovery] cleanup retry succeeded after daemon restart for %s\n", - tempDir, + dirs.tempDir, ) return nil } From 52c2127e9def299f470ec04b6e49b45e0476af9a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 20 Sep 2026 00:16:04 -0600 Subject: [PATCH 14/15] refactor(e2e): pass grouped cleanup dirs at call site Signed-off-by: Samuel K --- e2e/tests/up/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tests/up/helper.go b/e2e/tests/up/helper.go index 9aff3d4e7..89f85f715 100644 --- a/e2e/tests/up/helper.go +++ b/e2e/tests/up/helper.go @@ -231,7 +231,8 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st cleanupErr := f.CleanupWorkspace(ctx, tempDir) if cleanupErr != nil { ginkgo.GinkgoWriter.Printf("workspace cleanup failed for %s: %v\n", tempDir, cleanupErr) - cleanupErr = recoverPodmanCleanup(ctx, f, initialDir, tempDir, cleanupErr) + dirs := podmanCleanupDirs{initialDir: initialDir, tempDir: tempDir} + cleanupErr = recoverPodmanCleanup(ctx, f, dirs, cleanupErr) } return cleanupErr }) From 3f572b8d7ff14f4ef69d7ee4b5100b96615fd0bc Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 20 Sep 2026 00:21:32 -0600 Subject: [PATCH 15/15] fix(e2e): cover flagged gosec call line with nolint directive Signed-off-by: Samuel K --- e2e/tests/up/podman_rootful.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/tests/up/podman_rootful.go b/e2e/tests/up/podman_rootful.go index 8332de341..c59f00a46 100644 --- a/e2e/tests/up/podman_rootful.go +++ b/e2e/tests/up/podman_rootful.go @@ -141,10 +141,11 @@ func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthCla func runDiagCommand(name string, args ...string) string { diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) defer cancel() - cmd := exec.CommandContext( + cmd := exec.CommandContext( //nolint:gosec // G204: fixed diagnostic commands diagCtx, name, - args...) //nolint:gosec // G204: fixed diagnostic commands + args..., + ) docker.PrepareForGroupCancellation(cmd) out, err := cmd.CombinedOutput() text := string(out)