test(e2e): harden rootful Podman shards against daemon stalls

e2e/framework/exec.go

@@ -9,16 +9,24 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/devsy-org/devsy/pkg/docker"
 )
 
+// execWaitDelay bounds how long Wait may block on I/O after the process exits
+// or is killed for context cancellation. A grandchild that escapes the
+// process-group kill while holding the stdout/stderr pipes could otherwise
+// block Wait forever, letting a hung command outlive its spec timeout.
+const execWaitDelay = 30 * time.Second
+
 // ExecCommand executes the command string with the devsy test binary.
 func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (string, error) {
 	var execOut bytes.Buffer
 
 	cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...)
 	docker.PrepareForGroupCancellation(cmd)
+	cmd.WaitDelay = execWaitDelay
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = os.Stderr
 
@@ -33,6 +41,7 @@ func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (strin
 func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error {
 	cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...)
 	docker.PrepareForGroupCancellation(cmd)
+	cmd.WaitDelay = execWaitDelay
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
@@ -53,6 +62,7 @@ func (f *Framework) ExecCommand(
 
 	cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...)
 	docker.PrepareForGroupCancellation(cmd)
+	cmd.WaitDelay = execWaitDelay
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = os.Stderr
 
@@ -78,6 +88,7 @@ func (f *Framework) ExecCommandCapture(ctx context.Context, args []string) (stri
 
 	cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...)
 	docker.PrepareForGroupCancellation(cmd)
+	cmd.WaitDelay = execWaitDelay
 	cmd.Stdout = io.MultiWriter(os.Stdout, &execOut)
 	cmd.Stderr = io.MultiWriter(os.Stderr, &execErr)
 

e2e/tests/up/helper.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -21,12 +20,10 @@ import (
 )
 
 const (
-	secretCmd                = "secret"
-	cmdSSH                   = "ssh"
-	flagCommand              = "--command"
-	sshProbeTimeout          = 20 * time.Second
-	podmanHealthCheckTimeout = 20 * time.Second
-	podmanBinName            = "podman"
+	secretCmd       = "secret"
+	cmdSSH          = "ssh"
+	flagCommand     = "--command"
+	sshProbeTimeout = 20 * time.Second
 )
 
 // useFileSecretsBackend forces the file backend so tests do not depend on an OS
@@ -234,23 +231,8 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st
 		cleanupErr := f.CleanupWorkspace(ctx, tempDir)
 		if cleanupErr != nil {
 			ginkgo.GinkgoWriter.Printf("workspace cleanup failed for %s: %v\n", tempDir, cleanupErr)
-			// Capture bounded Podman state diagnostics if podman binary or wrapper exists
-			diagCtx, diagCancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer diagCancel()
-			cmdName := podmanBinName
-			if _, err := os.Stat(initialDir + "/bin/podman-rootful"); err == nil {
-				cmdName = initialDir + "/bin/podman-rootful"
-			}
-			cmd := exec.CommandContext(
-				diagCtx,
-				cmdName,
-				"ps",
-				"-a",
-			) //nolint:gosec // G204: test-controlled path
-			docker.PrepareForGroupCancellation(cmd)
-			if out, err := cmd.CombinedOutput(); err == nil {
-				ginkgo.GinkgoWriter.Printf("cleanup failure podman ps -a:\n%s\n", string(out))
-			}
+			dirs := podmanCleanupDirs{initialDir: initialDir, tempDir: tempDir}
+			cleanupErr = recoverPodmanCleanup(ctx, f, dirs, cleanupErr)
 		}
 		return cleanupErr
 	})
@@ -261,32 +243,6 @@ func setupDockerProvider(binDir, dockerPath string) (*framework.Framework, error
 	return framework.SetupDockerProvider(binDir, dockerPath)
 }
 
-func checkPodmanHealth(ctx context.Context, wrapperPath string) error {
-	healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout)
-	defer cancel()
-
-	cmd := exec.CommandContext(
-		healthCtx,
-		wrapperPath,
-		"ps",
-	) //nolint:gosec // G204: test-controlled path
-	docker.PrepareForGroupCancellation(cmd)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf(
-			"rootful Podman readiness check failed or exceeded %s\n"+
-				"command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w",
-			podmanHealthCheckTimeout,
-			wrapperPath,
-			os.Getenv("DOCKER_HOST"),
-			healthCtx.Err(),
-			string(out),
-			err,
-		)
-	}
-	return nil
-}
-
 func setupWorkspaceAndUp(
 	ctx context.Context,
 	testdataPath, initialDir string,