Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 83 additions & 12 deletions e2e/tests/up/podman_rootful.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"net/url"
"os"
"os/exec"
"strings"
Expand All @@ -12,6 +13,7 @@ import (

"github.com/devsy-org/devsy/e2e/framework"
"github.com/devsy-org/devsy/pkg/docker"
"github.com/devsy-org/devsy/pkg/secrets"
"github.com/onsi/ginkgo/v2"
)

Expand Down Expand Up @@ -88,6 +90,26 @@ func shouldAttemptPodmanRecovery(class podmanHealthClass) bool {
return class == podmanHealthTimeout || class == podmanHealthUnavailable
}

func rootfulPodmanUsesManagedDaemon(endpoint string) bool {
if endpoint == "" {
return true
}
return endpoint == "unix:///run/podman/podman.sock" ||
endpoint == "unix:///var/run/podman/podman.sock"
}

func rootfulPodmanWrapperScript() string {
return `#!/bin/sh
set -eu

if [ -n "${DOCKER_HOST:-}" ]; then
exec sudo podman --remote --url "$DOCKER_HOST" "$@"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
fi

exec sudo podman "$@"
`
}

// podmanDaemonGate records the first unrecoverable daemon failure so later
// specs in the shard skip instead of cascading into identical infrastructure
// failures that would bury the actionable one. Scoped to the test process,
Expand Down Expand Up @@ -139,17 +161,17 @@ func checkPodmanHealth(ctx context.Context, wrapperPath string) (podmanHealthCla
return podmanHealthOK, nil
}
class := classifyPodmanHealthFailure(healthCtx, string(out))
return class, fmt.Errorf(
return class, redactPodmanError(err, fmt.Sprintf(
"rootful Podman readiness check failed (class: %s) or exceeded %s\n"+
"command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %w",
"command: %s ps\nDOCKER_HOST: %s\ncontext err: %v\noutput:\n%s\nerror: %v",
class,
podmanHealthCheckTimeout,
wrapperPath,
os.Getenv("DOCKER_HOST"),
healthCtx.Err(),
string(out),
err,
)
))
}

// runDiagCommand never fails the caller: diagnostics are best-effort so a
Expand All @@ -168,6 +190,7 @@ func runDiagCommand(ctx context.Context, name string, args ...string) string {
if err != nil {
text += fmt.Sprintf("\n(command failed: %v; context err: %v)", err, diagCtx.Err())
}
text = redactPodmanDiagnostics(text)
limit := podmanDiagMaxOutput
if name == "ps" || name == "lslocks" || name == "sh" ||
strings.Contains(strings.Join(args, " "), "lslocks") {
Expand Down Expand Up @@ -274,12 +297,12 @@ func attemptPodmanRecovery(ctx context.Context, wrapperPath string) (podmanHealt
docker.PrepareForGroupCancellation(cmd)
started := time.Now()
if out, err := cmd.CombinedOutput(); err != nil {
return podmanHealthPoisoned, fmt.Errorf(
"PODMAN_ROOTFUL_RECOVERY_FAILED: systemctl restart failed after %s: %w\noutput:\n%s",
return podmanHealthPoisoned, redactPodmanError(err, fmt.Sprintf(
"PODMAN_ROOTFUL_RECOVERY_FAILED: systemctl restart failed after %s: %v\noutput:\n%s",
time.Since(started),
err,
string(out),
)
))
}
ginkgo.GinkgoWriter.Printf("[podman-recovery] restart elapsed=%s\n", time.Since(started))
class, err := checkPodmanHealth(ctx, wrapperPath)
Expand All @@ -300,7 +323,8 @@ func attemptPodmanRecovery(ctx context.Context, wrapperPath string) (podmanHealt
func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Framework {
wrapperPath := initialDir + "/bin/" + podmanRootfulWrapperName

if since := rootfulDaemonGate.unhealthy(); since != "" {
managedDaemon := rootfulPodmanUsesManagedDaemon(os.Getenv("DOCKER_HOST"))
if since := rootfulDaemonGate.unhealthy(); managedDaemon && since != "" {
ginkgo.Skip(fmt.Sprintf(
"rootful Podman daemon unhealthy since first failure in %q; "+
"skipping to avoid cascading infrastructure failures",
Expand All @@ -311,7 +335,7 @@ func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Frame
wrapper, err := os.Create(wrapperPath) //nolint:gosec // G304: test-controlled path
framework.ExpectNoError(err)

_, err = wrapper.WriteString("#!/bin/sh\nsudo podman \"$@\"\n")
_, err = wrapper.WriteString(rootfulPodmanWrapperScript())
if err != nil {
_ = wrapper.Close()
}
Expand All @@ -335,7 +359,7 @@ func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Frame
healthErr,
)
collectPodmanDiagnostics(wrapperPath)
if shouldAttemptPodmanRecovery(class) {
if managedDaemon && shouldAttemptPodmanRecovery(class) {
if !rootfulDaemonGate.claimRecovery() {
rootfulDaemonGate.markUnhealthy(ginkgo.CurrentSpecReport().FullText())
framework.ExpectNoError(
Expand All @@ -360,8 +384,8 @@ func setupRootfulPodman(ctx context.Context, initialDir string) *framework.Frame
"[podman-recovery] daemon healthy again after single restart",
)
} else {
// The daemon answers but errors: a restart would only hide a
// product or configuration problem, so fail without gating.
// The managed service cannot repair another endpoint or a
// responsive daemon error, so fail without gating.
framework.ExpectNoError(healthErr)
}
}
Expand Down Expand Up @@ -404,7 +428,8 @@ func recoverPodmanCleanup(
// mirroring CleanupWorkspace.
recoveryCtx := context.WithoutCancel(ctx)
class, healthErr := checkPodmanHealth(recoveryCtx, wrapperPath)
if healthErr == nil || !shouldAttemptPodmanRecovery(class) {
if healthErr == nil || !rootfulPodmanUsesManagedDaemon(os.Getenv("DOCKER_HOST")) ||
!shouldAttemptPodmanRecovery(class) {
return cleanupErr
}
if !rootfulDaemonGate.claimRecovery() {
Expand Down Expand Up @@ -446,3 +471,49 @@ func recoverPodmanCleanup(
)
return nil
}

// Redact before truncation so a clipped URL cannot expose partial credentials.
func redactPodmanDiagnostics(text string) string {
return secrets.Combine(secrets.NewRedactor(podmanEndpointSecrets(os.Getenv("DOCKER_HOST"))),
secrets.NewEnvironmentRedactor(os.Environ())).Redact(text)
}

func podmanEndpointSecrets(value string) []string {
endpoint, err := url.Parse(value)
if err != nil {
return []string{"endpoint=" + value}
}
if endpoint.User == nil {
return nil
}
entries := podmanUserinfoSecrets(endpoint.User.String())
entries = append(entries, "username="+endpoint.User.Username())
if password, ok := endpoint.User.Password(); ok {
entries = append(entries, "password="+password)
}
start, end := strings.Index(value, "://"), strings.LastIndex(value, "@")
if start >= 0 && end > start+3 {
entries = append(entries, podmanUserinfoSecrets(value[start+3:end])...)
}
return entries
}

func podmanUserinfoSecrets(userinfo string) []string {
entries := []string{"userinfo=" + userinfo}
if _, encoded, ok := strings.Cut(userinfo, ":"); ok {
entries = append(entries, "encoded="+encoded)
}
return entries
}

type podmanDiagnosticError struct {
cause error
message string
}

func (e podmanDiagnosticError) Error() string { return e.message }
func (e podmanDiagnosticError) Unwrap() error { return e.cause }

func redactPodmanError(cause error, message string) error {
return podmanDiagnosticError{cause: cause, message: redactPodmanDiagnostics(message)}
}
Loading
Loading