diff --git a/e2e/framework/exec.go b/e2e/framework/exec.go index 761ee8e51..26915547a 100644 --- a/e2e/framework/exec.go +++ b/e2e/framework/exec.go @@ -9,16 +9,24 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/devsy-org/devsy/pkg/docker" ) +// execWaitDelay bounds how long Wait may block on I/O after the process exits +// or is killed for context cancellation. A grandchild that escapes the +// process-group kill while holding the stdout/stderr pipes could otherwise +// block Wait forever, letting a hung command outlive its spec timeout. +const execWaitDelay = 30 * time.Second + // ExecCommand executes the command string with the devsy test binary. func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (string, error) { var execOut bytes.Buffer cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = os.Stderr @@ -33,6 +41,7 @@ func (f *Framework) ExecCommandOutput(ctx context.Context, args []string) (strin func (f *Framework) ExecCommandStdout(ctx context.Context, args []string) error { cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -53,6 +62,7 @@ func (f *Framework) ExecCommand( cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = os.Stderr @@ -78,6 +88,7 @@ func (f *Framework) ExecCommandCapture(ctx context.Context, args []string) (stri cmd := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), args...) docker.PrepareForGroupCancellation(cmd) + cmd.WaitDelay = execWaitDelay cmd.Stdout = io.MultiWriter(os.Stdout, &execOut) cmd.Stderr = io.MultiWriter(os.Stderr, &execErr) diff --git a/e2e/tests/up/helper.go b/e2e/tests/up/helper.go index e4eca0a13..89f85f715 100644 --- a/e2e/tests/up/helper.go +++ b/e2e/tests/up/helper.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "strings" "time" @@ -21,12 +20,10 @@ import ( ) const ( - secretCmd = "secret" - cmdSSH = "ssh" - flagCommand = "--command" - sshProbeTimeout = 20 * time.Second - podmanHealthCheckTimeout = 20 * time.Second - podmanBinName = "podman" + secretCmd = "secret" + cmdSSH = "ssh" + flagCommand = "--command" + sshProbeTimeout = 20 * time.Second ) // useFileSecretsBackend forces the file backend so tests do not depend on an OS @@ -234,23 +231,8 @@ func setupWorkspace(testdataPath, initialDir string, f *framework.Framework) (st cleanupErr := f.CleanupWorkspace(ctx, tempDir) if cleanupErr != nil { ginkgo.GinkgoWriter.Printf("workspace cleanup failed for %s: %v\n", tempDir, cleanupErr) - // Capture bounded Podman state diagnostics if podman binary or wrapper exists - diagCtx, diagCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer diagCancel() - cmdName := podmanBinName - if _, err := os.Stat(initialDir + "/bin/podman-rootful"); err == nil { - cmdName = initialDir + "/bin/podman-rootful" - } - cmd := exec.CommandContext( - diagCtx, - cmdName, - "ps", - "-a", - ) //nolint:gosec // G204: test-controlled path - docker.PrepareForGroupCancellation(cmd) - if out, err := cmd.CombinedOutput(); err == nil { - ginkgo.GinkgoWriter.Printf("cleanup failure podman ps -a:\n%s\n", string(out)) - } + dirs := podmanCleanupDirs{initialDir: initialDir, tempDir: tempDir} + cleanupErr = recoverPodmanCleanup(ctx, f, dirs, cleanupErr) } return cleanupErr }) @@ -261,32 +243,6 @@ func setupDockerProvider(binDir, dockerPath string) (*framework.Framework, error return framework.SetupDockerProvider(binDir, dockerPath) } -func checkPodmanHealth(ctx context.Context, wrapperPath string) error { - healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) - defer cancel() - - cmd := exec.CommandContext( - healthCtx, - wrapperPath, - "ps", - ) //nolint:gosec // G204: test-controlled path - docker.PrepareForGroupCancellation(cmd) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf( - "rootful Podman readiness check failed or exceeded %s\n"+ - "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", - podmanHealthCheckTimeout, - wrapperPath, - os.Getenv("DOCKER_HOST"), - healthCtx.Err(), - string(out), - err, - ) - } - return nil -} - func setupWorkspaceAndUp( ctx context.Context, testdataPath, initialDir string, diff --git a/e2e/tests/up/podman_rootful.go b/e2e/tests/up/podman_rootful.go new file mode 100644 index 000000000..c59f00a46 --- /dev/null +++ b/e2e/tests/up/podman_rootful.go @@ -0,0 +1,352 @@ +package up + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/devsy-org/devsy/e2e/framework" + "github.com/devsy-org/devsy/pkg/docker" + "github.com/onsi/ginkgo/v2" +) + +const ( + podmanHealthCheckTimeout = 20 * time.Second + podmanRecoveryTimeout = 90 * time.Second + // podmanDiagCommandTimeout keeps a wedged daemon from stalling diagnostic + // collection itself. + podmanDiagCommandTimeout = 10 * time.Second + // podmanDiagMaxOutput caps each diagnostic section so CI logs stay readable. + podmanDiagMaxOutput = 8 * 1024 + // podmanBinName is the fallback binary when the rootful wrapper is absent. + podmanBinName = "podman" + podmanRootfulWrapperName = "podman-rootful" +) + +type podmanHealthClass int + +const ( + podmanHealthOK podmanHealthClass = iota + // podmanHealthTimeout: the daemon accepts connections but does not answer + // (wedged). + podmanHealthTimeout + // podmanHealthUnavailable: the API socket or service is missing or + // refusing connections. + podmanHealthUnavailable + // podmanHealthError: the daemon answered with an error, which points at + // product or configuration state rather than a wedged service. + podmanHealthError +) + +func (c podmanHealthClass) String() string { + switch c { + case podmanHealthOK: + return "ok" + case podmanHealthTimeout: + return "timeout" + case podmanHealthUnavailable: + return "unavailable" + case podmanHealthError: + return "error" + } + return "unknown" +} + +// classifyPodmanHealthFailure buckets a failed probe so the caller can choose +// between infrastructure recovery and surfacing a real failure. +func classifyPodmanHealthFailure(healthCtx context.Context, output string) podmanHealthClass { + if errors.Is(healthCtx.Err(), context.DeadlineExceeded) { + return podmanHealthTimeout + } + lower := strings.ToLower(output) + for _, pattern := range []string{ + "cannot connect", + "connection refused", + "no such file or directory", + } { + if strings.Contains(lower, pattern) { + return podmanHealthUnavailable + } + } + return podmanHealthError +} + +// shouldAttemptPodmanRecovery reports whether a restart can help: recovery +// fixes a wedged or missing daemon, while restarting a responsive but +// erroring daemon would hide a product or configuration problem. +func shouldAttemptPodmanRecovery(class podmanHealthClass) bool { + return class == podmanHealthTimeout || class == podmanHealthUnavailable +} + +// podmanDaemonGate records the first unrecoverable daemon failure so later +// specs in the shard skip instead of cascading into identical infrastructure +// failures that would bury the actionable one. Scoped to the test process, +// which runs exactly one shard in CI. +type podmanDaemonGate struct { + mu sync.Mutex + unhealthySince string +} + +var rootfulDaemonGate = &podmanDaemonGate{} + +func (g *podmanDaemonGate) markUnhealthy(spec string) { + g.mu.Lock() + defer g.mu.Unlock() + if g.unhealthySince == "" { + g.unhealthySince = spec + } +} + +func (g *podmanDaemonGate) unhealthy() string { + g.mu.Lock() + defer g.mu.Unlock() + return g.unhealthySince +} + +func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthClass, error) { + healthCtx, cancel := context.WithTimeout(ctx, podmanHealthCheckTimeout) + defer cancel() + + cmd := exec.CommandContext( //nolint:gosec // G204: test-controlled path + healthCtx, + wrapperPath, + "ps", + ) + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + if err == nil { + return podmanHealthOK, nil + } + class := classifyPodmanHealthFailure(healthCtx, string(out)) + return class, fmt.Errorf( + "rootful Podman readiness check failed (class: %s) or exceeded %s\n"+ + "command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w", + class, + podmanHealthCheckTimeout, + wrapperPath, + os.Getenv("DOCKER_HOST"), + healthCtx.Err(), + string(out), + err, + ) +} + +// runDiagCommand never fails the caller: diagnostics are best-effort so a +// broken host tool cannot hide the failure they are meant to explain. +func runDiagCommand(name string, args ...string) string { + diagCtx, cancel := context.WithTimeout(context.Background(), podmanDiagCommandTimeout) + defer cancel() + cmd := exec.CommandContext( //nolint:gosec // G204: fixed diagnostic commands + diagCtx, + name, + args..., + ) + docker.PrepareForGroupCancellation(cmd) + out, err := cmd.CombinedOutput() + text := string(out) + if err != nil { + text += fmt.Sprintf("\n(command failed: %v; context err: %v)", err, diagCtx.Err()) + } + if len(text) > podmanDiagMaxOutput { + text = text[:podmanDiagMaxOutput] + "\n... (truncated)" + } + return text +} + +// collectPodmanDiagnostics prints bounded daemon state so the first failure +// in a shard carries the evidence needed to debug a wedge without a rerun. +func collectPodmanDiagnostics(wrapperPath string) { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] collecting bounded daemon state (each command capped at %s)\n", + podmanDiagCommandTimeout, + ) + sections := []struct { + title string + name string + args []string + }{ + {"podman version", wrapperPath, []string{"version"}}, + {"podman info", wrapperPath, []string{"info"}}, + {"podman ps -a", wrapperPath, []string{"ps", "-a"}}, + { + "systemctl status podman.socket podman.service", + "sudo", + []string{ + "systemctl", "status", "podman.socket", "podman.service", "--no-pager", "-l", + }, + }, + { + "journalctl podman units (last 100 lines)", + "sudo", + []string{ + "journalctl", "-u", "podman.socket", "-u", "podman.service", + "-n", "100", "--no-pager", + }, + }, + { + "podman-related processes", + "sh", + []string{ + "-c", + "ps -eo pid,ppid,stat,etime,cmd | grep -E 'podman|crun|conmon' | grep -v grep || true", + }, + }, + {"disk usage", "df", []string{"-h", "/", "/var/lib/containers"}}, + {"memory", "free", []string{"-m"}}, + } + for _, section := range sections { + ginkgo.GinkgoWriter.Printf( + "[podman-diagnostics] --- %s ---\n%s\n", + section.title, + runDiagCommand(section.name, section.args...), + ) + } +} + +// attemptPodmanRecovery performs the shard's one bounded restart of the +// rootful Podman socket and service, then re-probes health. +func attemptPodmanRecovery(ctx context.Context, wrapperPath string) error { + ginkgo.GinkgoWriter.Println( + "[podman-recovery] attempting single bounded restart of podman.socket and podman.service", + ) + restartCtx, cancel := context.WithTimeout(ctx, podmanRecoveryTimeout) + defer cancel() + cmd := exec.CommandContext( //nolint:gosec // G204: fixed recovery command + restartCtx, + "sudo", + "systemctl", + "restart", + "podman.socket", + "podman.service", + ) + docker.PrepareForGroupCancellation(cmd) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("systemctl restart failed: %w\noutput:\n%s", err, string(out)) + } + class, err := checkPodmanHealth(ctx, wrapperPath) + if err != nil { + return fmt.Errorf("daemon still unhealthy after restart (class: %s): %w", class, err) + } + return nil +} + +// setupRootfulPodman prepares the rootful Podman wrapper and docker provider +// and gates the shard on daemon health: once recovery has failed, remaining +// specs skip instead of re-failing on the same wedged infrastructure. +func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework { + wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName + + if since := rootfulDaemonGate.unhealthy(); since != "" { + ginkgo.Skip(fmt.Sprintf( + "rootful Podman daemon unhealthy since first failure in %q; "+ + "skipping to avoid cascading infrastructure failures", + since, + )) + } + + wrapper, err := os.Create(wrapperPath) //nolint:gosec // G304: test-controlled path + framework.ExpectNoError(err) + + _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") + if err != nil { + _ = wrapper.Close() + } + framework.ExpectNoError(err) + framework.ExpectNoError(wrapper.Close()) + + // #nosec G302 -- wrapper script needs execute permission + framework.ExpectNoError(os.Chmod(wrapperPath, 0o755)) + ginkgo.DeferCleanup(func() { + _ = os.Remove(wrapperPath) + }) + + class, healthErr := checkPodmanHealth(ctx, wrapperPath) + if healthErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-health] readiness check failed: %v\n", healthErr) + collectPodmanDiagnostics(wrapperPath) + if shouldAttemptPodmanRecovery(class) { + if recErr := attemptPodmanRecovery(ctx, wrapperPath); recErr != nil { + rootfulDaemonGate.markUnhealthy(ginkgo.CurrentSpecReport().FullText()) + collectPodmanDiagnostics(wrapperPath) + framework.ExpectNoError(fmt.Errorf( + "rootful Podman daemon unhealthy and single recovery attempt failed: %w "+ + "(initial check: %v)", + recErr, + healthErr, + )) + } + ginkgo.GinkgoWriter.Println( + "[podman-recovery] daemon healthy again after single restart", + ) + } else { + // The daemon answers but errors: a restart would only hide a + // product or configuration problem, so fail without gating. + framework.ExpectNoError(healthErr) + } + } + + f, err := setupDockerProvider(initialDir+"/bin", wrapperPath) + framework.ExpectNoError(err) + return f +} + +type podmanCleanupDirs struct { + initialDir string + tempDir string +} + +// recoverPodmanCleanup reports bounded diagnostics for a failed workspace +// cleanup and, on a wedged or missing daemon, performs one restart plus one +// cleanup retry. A responsive but erroring daemon is left untouched: +// retrying there would hide product bugs. It returns the error the cleanup +// should report. +func recoverPodmanCleanup( + ctx context.Context, + f *framework.Framework, + dirs podmanCleanupDirs, + cleanupErr error, +) error { + wrapperPath := dirs.initialDir + "/bin/" + podmanRootfulWrapperName + if _, err := os.Stat(wrapperPath); err != nil { + // Not a rootful shard: keep the previous minimal diagnostic. + ginkgo.GinkgoWriter.Printf( + "cleanup failure podman ps -a:\n%s\n", + runDiagCommand(podmanBinName, "ps", "-a"), + ) + return cleanupErr + } + + collectPodmanDiagnostics(wrapperPath) + + // The cleanup context may already be canceled after a spec timeout; use a + // detached context so classification and recovery stay deterministic, + // mirroring CleanupWorkspace. + recoveryCtx := context.WithoutCancel(ctx) + class, healthErr := checkPodmanHealth(recoveryCtx, wrapperPath) + if healthErr == nil || !shouldAttemptPodmanRecovery(class) { + return cleanupErr + } + if recErr := attemptPodmanRecovery(recoveryCtx, wrapperPath); recErr != nil { + ginkgo.GinkgoWriter.Printf("[podman-recovery] cleanup recovery failed: %v\n", recErr) + return cleanupErr + } + retryErr := f.CleanupWorkspace(ctx, dirs.tempDir) + if retryErr != nil { + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry after daemon restart still failed for %s: %v\n", + dirs.tempDir, + retryErr, + ) + return retryErr + } + ginkgo.GinkgoWriter.Printf( + "[podman-recovery] cleanup retry succeeded after daemon restart for %s\n", + dirs.tempDir, + ) + return nil +} diff --git a/e2e/tests/up/podman_rootful_test.go b/e2e/tests/up/podman_rootful_test.go new file mode 100644 index 000000000..88897291d --- /dev/null +++ b/e2e/tests/up/podman_rootful_test.go @@ -0,0 +1,84 @@ +package up + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func expiredContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + t.Cleanup(cancel) + <-ctx.Done() + return ctx +} + +func TestClassifyPodmanHealthFailure(t *testing.T) { + t.Run("deadline exceeded means wedged daemon", func(t *testing.T) { + assert.Equal( + t, + podmanHealthTimeout, + classifyPodmanHealthFailure(expiredContext(t), ""), + ) + }) + + unavailableOutputs := []string{ + "Error: cannot connect to the Podman socket", + "dial unix /run/podman/podman.sock: connect: connection refused", + "fork/exec /usr/local/bin/podman: no such file or directory", + } + for _, output := range unavailableOutputs { + t.Run("socket problem means unavailable: "+output, func(t *testing.T) { + assert.Equal( + t, + podmanHealthUnavailable, + classifyPodmanHealthFailure(context.Background(), output), + ) + }) + } + + t.Run("responsive daemon error stays an error", func(t *testing.T) { + assert.Equal( + t, + podmanHealthError, + classifyPodmanHealthFailure( + context.Background(), + "Error: statfs /var/lib/containers: permission denied", + ), + ) + }) + + t.Run("empty output without deadline stays an error", func(t *testing.T) { + assert.Equal( + t, + podmanHealthError, + classifyPodmanHealthFailure(context.Background(), ""), + ) + }) +} + +func TestShouldAttemptPodmanRecovery(t *testing.T) { + assert.True(t, shouldAttemptPodmanRecovery(podmanHealthTimeout)) + assert.True(t, shouldAttemptPodmanRecovery(podmanHealthUnavailable)) + assert.False(t, shouldAttemptPodmanRecovery(podmanHealthError)) + assert.False(t, shouldAttemptPodmanRecovery(podmanHealthOK)) +} + +func TestPodmanDaemonGateRecordsFirstFailure(t *testing.T) { + gate := &podmanDaemonGate{} + assert.Empty(t, gate.unhealthy()) + gate.markUnhealthy("first spec") + gate.markUnhealthy("second spec") + assert.Equal(t, "first spec", gate.unhealthy()) +} + +func TestPodmanHealthClassString(t *testing.T) { + assert.Equal(t, "ok", podmanHealthOK.String()) + assert.Equal(t, "timeout", podmanHealthTimeout.String()) + assert.Equal(t, "unavailable", podmanHealthUnavailable.String()) + assert.Equal(t, "error", podmanHealthError.String()) + assert.Equal(t, "unknown", podmanHealthClass(99).String()) +} diff --git a/e2e/tests/up/provider_podman_rootful_basic.go b/e2e/tests/up/provider_podman_rootful_basic.go index abe7b1471..850ba447b 100644 --- a/e2e/tests/up/provider_podman_rootful_basic.go +++ b/e2e/tests/up/provider_podman_rootful_basic.go @@ -25,40 +25,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("basic", func() { ginkgo.It( diff --git a/e2e/tests/up/provider_podman_rootful_config.go b/e2e/tests/up/provider_podman_rootful_config.go index e82c47fdf..7204a9608 100644 --- a/e2e/tests/up/provider_podman_rootful_config.go +++ b/e2e/tests/up/provider_podman_rootful_config.go @@ -2,8 +2,6 @@ package up import ( "context" - "encoding/json" - "fmt" "os" "path" "path/filepath" @@ -11,9 +9,6 @@ import ( "time" "github.com/devsy-org/devsy/e2e/framework" - pkgconfig "github.com/devsy-org/devsy/pkg/config" - "github.com/devsy-org/devsy/pkg/devcontainer/config" - docker "github.com/devsy-org/devsy/pkg/docker" "github.com/devsy-org/devsy/pkg/flags/names" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -34,40 +29,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("configuration", func() { //nolint:dupl ginkgo.It("should substitute variables", func(ctx context.Context) { @@ -266,132 +230,6 @@ var _ = ginkgo.Describe( err = f.DevsyWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) }, ginkgo.SpecTimeout(framework.TimeoutModerate())) - - ginkgo.It( - "should preserve localEnv expressions in build metadata", - func(ctx context.Context) { - homeDir, err := os.UserHomeDir() - framework.ExpectNoError(err) - - sourceDir := filepath.Join( - homeDir, - ".devsy-e2e-local-env-metadata", - ) - - _, statErr := os.Stat(sourceDir) - gomega.Expect(os.IsNotExist(statErr)).To( - gomega.BeTrue(), - "fixture directory %s already exists; aborting to prevent data loss", - sourceDir, - ) - // #nosec G301 -- fixture must be traversable by container user - err = os.MkdirAll( - sourceDir, - 0o755, - ) - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.RemoveAll(sourceDir) - }) - - // #nosec G306 -- fixture must be readable by container user - err = os.WriteFile( - filepath.Join(sourceDir, "probe.txt"), - []byte("devsy-local-env-metadata-ok\n"), - 0o644, - ) - framework.ExpectNoError(err) - - tempDir, err := setupWorkspaceAndUp( - ctx, - "tests/up/testdata/podman-local-env-metadata", - initialDir, - f, - ) - framework.ExpectNoError(err) - - out := eventuallySSH( - f, - ctx, - tempDir, - "cat /tmp/devsy-local-env-metadata/probe.txt", - ) - - gomega.Expect(out).To(gomega.Equal("devsy-local-env-metadata-ok")) - - workspace, err := f.FindWorkspace(ctx, tempDir) - framework.ExpectNoError(err) - - dockerHelper := &docker.DockerHelper{ - DockerCommand: initialDir + "/bin/podman-rootful", - } - - container, err := dockerHelper.FindDevContainer(ctx, []string{ - fmt.Sprintf("%s=%s", pkgconfig.DevcontainerIDLabel, workspace.UID), - }) - framework.ExpectNoError(err) - gomega.Expect(container).NotTo(gomega.BeNil()) - - imageRef := container.Config.LegacyImage - if imageRef == "" { - var rawInspect []struct { - Image string `json:"Image"` - } - inspectErr := dockerHelper.Inspect( - ctx, - []string{container.ID}, - "container", - &rawInspect, - ) - framework.ExpectNoError(inspectErr) - if len(rawInspect) > 0 { - imageRef = rawInspect[0].Image - } - } - gomega.Expect(imageRef).NotTo(gomega.BeEmpty()) - - imageDetails, err := dockerHelper.InspectImage(ctx, imageRef, false) - framework.ExpectNoError(err) - gomega.Expect(imageDetails.Config.Labels).NotTo(gomega.BeNil()) - - metadataValue, ok := imageDetails.Config.Labels[pkgconfig.DevcontainerMetadataLabel] - gomega.Expect(ok).To(gomega.BeTrue()) - gomega.Expect(metadataValue).NotTo(gomega.BeEmpty()) - - var metadataList []*config.ImageMetadata - err = json.Unmarshal([]byte(metadataValue), &metadataList) - framework.ExpectNoError(err) - gomega.Expect(metadataList).NotTo(gomega.BeEmpty()) - - var foundSource string - for _, item := range metadataList { - for _, m := range item.Mounts { - if strings.Contains(m.Source, ".devsy-e2e-local-env-metadata") { - foundSource = m.Source - break - } - } - } - gomega.Expect(foundSource). - To(gomega.Equal("${localEnv:HOME}/.devsy-e2e-local-env-metadata")) - - gomega.Expect(metadataValue).To( - gomega.ContainSubstring( - "${localEnv:HOME}/.devsy-e2e-local-env-metadata", - ), - ) - gomega.Expect(metadataValue).NotTo( - gomega.ContainSubstring( - `\${localEnv:HOME}/.devsy-e2e-local-env-metadata`, - ), - ) - gomega.Expect(metadataValue).NotTo( - gomega.ContainSubstring(sourceDir), - ) - }, - ginkgo.SpecTimeout(framework.TimeoutModerate()), - ) }) }) }, diff --git a/e2e/tests/up/provider_podman_rootful_features.go b/e2e/tests/up/provider_podman_rootful_features.go index 289907367..ea867594e 100644 --- a/e2e/tests/up/provider_podman_rootful_features.go +++ b/e2e/tests/up/provider_podman_rootful_features.go @@ -25,40 +25,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("features", func() { //nolint:dupl ginkgo.It("should mount volumes", func(ctx context.Context) { diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle.go b/e2e/tests/up/provider_podman_rootful_lifecycle.go index ad2e9f686..b6ef84a7c 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle.go @@ -26,40 +26,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("lifecycle commands", func() { //nolint:dupl ginkgo.It( diff --git a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go index b66817557..f4b8c94b7 100644 --- a/e2e/tests/up/provider_podman_rootful_lifecycle_2.go +++ b/e2e/tests/up/provider_podman_rootful_lifecycle_2.go @@ -27,40 +27,9 @@ var _ = ginkgo.Describe( ginkgo.Context("with rootful podman", func() { var f *framework.Framework - //nolint:dupl // shared rootful podman wrapper setup across split files ginkgo.BeforeEach(func(ctx context.Context) { - wrapper, err := os.Create( //nolint:gosec // G304: test-controlled path - initialDir + "/bin/podman-rootful", - ) - framework.ExpectNoError(err) - - _, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n") - if err != nil { - _ = wrapper.Close() - framework.ExpectNoError(err) - } - - err = wrapper.Close() - framework.ExpectNoError(err) - - // #nosec G302 -- wrapper script needs execute permission - err = os.Chmod(initialDir+"/bin/podman-rootful", 0o755) - framework.ExpectNoError(err) - - err = checkPodmanHealth(ctx, initialDir+"/bin/podman-rootful") - framework.ExpectNoError(err) - - ginkgo.DeferCleanup(func() { - _ = os.Remove(initialDir + "/bin/podman-rootful") - }) - - f, err = setupDockerProvider( - initialDir+"/bin", - initialDir+"/bin/podman-rootful", - ) - framework.ExpectNoError(err) - }, - ) + f = setupRootfulPodman(ctx, initialDir) + }) ginkgo.Context("lifecycle commands", func() { //nolint:dupl ginkgo.It( diff --git a/hack/ci/setup-podman-linux.sh b/hack/ci/setup-podman-linux.sh index acfbb4b33..adccbe1fd 100755 --- a/hack/ci/setup-podman-linux.sh +++ b/hack/ci/setup-podman-linux.sh @@ -42,6 +42,15 @@ cat /etc/os-release uname -a if [[ "$mode" == "rootful" ]]; then + # Keep the API service resident for the whole shard. The stock unit runs + # `podman system service` with the default 5s idle timeout, so every test + # command pays a socket reactivation and can race a service mid-exit. + sudo mkdir -p /etc/systemd/system/podman.service.d + sudo tee /etc/systemd/system/podman.service.d/10-devsy-ci.conf >/dev/null <<'EOF' +[Service] +ExecStart= +ExecStart=podman --log-level=info system service --time=0 +EOF sudo systemctl daemon-reload sudo systemctl enable --now podman.socket if ! timeout 30 bash -c \