Sync release-1.0 with upstream backports

.ko.yaml

@@ -19,6 +19,15 @@ defaultPlatforms:
   - linux/arm64
 
 baseImageOverrides:
+  # The actor images built here write at the root of their filesystem and into
+  # volumes atelet creates as root. A container runs as its image's USER, so
+  # they stay on the root base.
+  go.ate.dev/substrate/demos/counter: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
+  go.ate.dev/substrate/demos/egress: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
+  go.ate.dev/substrate/demos/multi-template/fspersist: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
+  go.ate.dev/substrate/internal/e2e/fixtures/probe: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
+  go.ate.dev/substrate/internal/e2e/fixtures/testserver: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
+  go.ate.dev/substrate/cmd/benchmarking/glutton: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78
   go.ate.dev/substrate/demos/sandbox: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a
   # atelet needs glibc to run the node's kubelet credential provider, which
   # some platforms ship dynamically linked (e.g. AKS, GKE Ubuntu).

charts/substrate/README.md

@@ -148,6 +148,11 @@ This replaces the previous `v5` config. Existing `v5` memory snapshots need a
 compatible runtime; falling back to a durable filesystem snapshot does not
 restore the actor's memory.
 
+The [image cache](../../internal/imagecache/README.md) now uses layout version 3
+to preserve image ownership and permissions. Atelet refuses version 2 caches.
+Stop the actors on each node before deleting its old image cache
+(`/var/lib/ate/image-cache` by default). Atelet recreates it on startup.
+
 When upgrading an installation with RustFS enabled, delete its completed bucket
 initializer Job before running `helm upgrade`. The default Job name is
 `rustfs-bucket-init` in the release namespace. This update changes the AWS CLI

charts/substrate/templates/atelet.yaml

@@ -215,7 +215,10 @@ spec:
             drop:
             - ALL
             add:
+            - CHOWN
             - DAC_OVERRIDE
+            - FOWNER
+            - FSETID
         resources:
 {{ toYaml .Values.atelet.resources | indent 10 }}
         env:

charts/substrate/tests/control_plane_test.yaml

@@ -114,6 +114,21 @@ tests:
         key: cloud.google.com/compute-class
         operator: Exists
         effect: NoSchedule
+- it: lets atelet preserve image ownership and permissions
+  template: atelet.yaml
+  documentSelector:
+    path: kind
+    value: DaemonSet
+  asserts:
+  - equal:
+      path: spec.template.spec.containers[0].securityContext
+      value:
+        allowPrivilegeEscalation: false
+        runAsUser: 0
+        runAsGroup: 0
+        capabilities:
+          drop: [ALL]
+          add: [CHOWN, DAC_OVERRIDE, FOWNER, FSETID]
 - it: probes the credential provider on its health listener
   template: k8s-credential-provider.yaml
   documentSelector:

cmd/ate-setup/internal/ko/ko.go

@@ -33,6 +33,9 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
+
 	"go.ate.dev/substrate/cmd/ate-setup/internal/log"
 )
 
@@ -162,6 +165,37 @@ func (r *Runner) BuildTagged(ctx context.Context, tag string, labels []string, i
 	return refs, nil
 }
 
+// buildBackoff paces the retries of an image that fails to build or push: 3
+// attempts, 10s then 20s apart. ghcr.io can briefly deny pushes to a package it
+// has just created.
+var buildBackoff = wait.Backoff{Steps: 3, Duration: 10 * time.Second, Factor: 2}
+
+// BuildEach runs BuildTagged once per import path, retrying each failure per
+// buildBackoff. It returns the refs in importPaths order.
+func (r *Runner) BuildEach(ctx context.Context, tag string, labels []string, importPaths ...string) ([]string, error) {
+	refs := make([]string, 0, len(importPaths))
+	for _, path := range importPaths {
+		var ref string
+		logAndRetry := func(err error) bool {
+			log.Warnf("%s: %v", path, err)
+			return true
+		}
+		err := retry.OnError(buildBackoff, logAndRetry, func() error {
+			built, err := r.BuildTagged(ctx, tag, labels, path)
+			if err != nil {
+				return err
+			}
+			ref = built[0]
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		refs = append(refs, ref)
+	}
+	return refs, nil
+}
+
 // ResolveBytes resolves an in-memory manifest, such as kustomize output.
 func (r *Runner) ResolveBytes(ctx context.Context, manifest []byte) ([]byte, error) {
 	return r.Resolve(ctx, "-", manifest)

cmd/ate-setup/internal/ko/ko_test.go

@@ -15,12 +15,17 @@
 package ko
 
 import (
+	"errors"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // Without --base-import-paths ko publishes cmd/atelet as "atelet-<md5>", and
@@ -174,3 +179,80 @@ func TestBuildTaggedRejectsMissingRefs(t *testing.T) {
 		t.Error("BuildTagged() error = nil, want one for 1 ref from 2 packages")
 	}
 }
+
+// flakyKo writes a ko stand-in that fails the first failures builds of each
+// package, printing stderr, then prints a ref for it. It returns the
+// directory where it counts builds per package.
+func flakyKo(t *testing.T, failures int, stderr string) (binary, dir string) {
+	t.Helper()
+	dir = t.TempDir()
+	binary = filepath.Join(dir, "ko")
+	script := "#!/bin/sh\n" +
+		"pkg=$(basename \"$2\")\n" +
+		"echo run >> " + dir + "/runs-$pkg\n" +
+		"if [ $(wc -l < " + dir + "/runs-$pkg) -le " + strconv.Itoa(failures) + " ]; then echo '" + stderr + "' >&2; exit 1; fi\n" +
+		"echo repo/$pkg:v1@sha256:a\n"
+	if err := os.WriteFile(binary, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return binary, dir
+}
+
+// koRuns returns how many times flakyKo built pkg.
+func koRuns(t *testing.T, dir, pkg string) int {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join(dir, "runs-"+pkg))
+	if errors.Is(err, os.ErrNotExist) {
+		return 0
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.Count(string(raw), "run")
+}
+
+// fastBackoff shortens buildBackoff for the test.
+func fastBackoff(t *testing.T) {
+	t.Helper()
+	original := buildBackoff
+	buildBackoff = wait.Backoff{Steps: original.Steps, Duration: time.Millisecond, Factor: 1}
+	t.Cleanup(func() { buildBackoff = original })
+}
+
+const deniedStderr = "PATCH https://ghcr.io/v2/agent-substrate/substrate/ateapi/blobs/upload/1: DENIED: permission_denied: write_package"
+
+func TestBuildEachRetriesEachImage(t *testing.T) {
+	fastBackoff(t)
+	binary, dir := flakyKo(t, 2, deniedStderr)
+	r := &Runner{Root: t.TempDir(), binary: binary}
+
+	refs, err := r.BuildEach(t.Context(), "v1", nil, "./cmd/ateapi", "./cmd/atelet")
+	if err != nil {
+		t.Fatalf("BuildEach() error = %v", err)
+	}
+	if want := []string{"repo/ateapi:v1@sha256:a", "repo/atelet:v1@sha256:a"}; !slices.Equal(refs, want) {
+		t.Errorf("BuildEach() = %q, want %q", refs, want)
+	}
+	for _, pkg := range []string{"ateapi", "atelet"} {
+		if got := koRuns(t, dir, pkg); got != 3 {
+			t.Errorf("ko built %s %d times, want 3", pkg, got)
+		}
+	}
+}
+
+func TestBuildEachStopsAfterAttempts(t *testing.T) {
+	fastBackoff(t)
+	binary, dir := flakyKo(t, 100, deniedStderr)
+	r := &Runner{Root: t.TempDir(), binary: binary}
+
+	_, err := r.BuildEach(t.Context(), "v1", nil, "./cmd/ateapi", "./cmd/atelet")
+	if err == nil {
+		t.Error("BuildEach() error = nil, want one after 3 failed attempts")
+	}
+	if got := koRuns(t, dir, "ateapi"); got != 3 {
+		t.Errorf("ko built ateapi %d times, want 3", got)
+	}
+	if got := koRuns(t, dir, "atelet"); got != 0 {
+		t.Errorf("ko built atelet %d times, want 0", got)
+	}
+}

cmd/ate-setup/internal/steps/publish.go

@@ -254,7 +254,7 @@ func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer, opts Publis
 		}
 		refs = append(refs, ref)
 	}
-	koRefs, err := runner.BuildTagged(ctx, tag, labels, pkgs...)
+	koRefs, err := runner.BuildEach(ctx, tag, labels, pkgs...)
 	if err != nil {
 		return err
 	}

cmd/atelet/imagevolume_test.go

@@ -75,11 +75,26 @@ func singleFileLayer(t *testing.T, path, body string) v1.Layer {
 // pushTestImage pushes the image and returns its digest-pinned reference,
 // which is what validated workload specs carry.
 func pushTestImage(t *testing.T, ref string, layers ...v1.Layer) string {
+	t.Helper()
+	return pushTestImageWithConfig(t, ref, v1.Config{}, layers...)
+}
+
+// pushTestImageWithConfig is pushTestImage with the image config set.
+func pushTestImageWithConfig(t *testing.T, ref string, cfg v1.Config, layers ...v1.Layer) string {
 	t.Helper()
 	img, err := mutate.AppendLayers(empty.Image, layers...)
 	if err != nil {
 		t.Fatalf("mutate.AppendLayers: %v", err)
 	}
+	cf, err := img.ConfigFile()
+	if err != nil {
+		t.Fatalf("img.ConfigFile: %v", err)
+	}
+	cf = cf.DeepCopy()
+	cf.Config = cfg
+	if img, err = mutate.ConfigFile(img, cf); err != nil {
+		t.Fatalf("mutate.ConfigFile: %v", err)
+	}
 	tag, err := name.ParseReference(ref, name.Insecure)
 	if err != nil {
 		t.Fatalf("name.ParseReference(%q): %v", ref, err)

cmd/atelet/main.go

@@ -2281,19 +2281,25 @@ func (s *Atelet) prepareOCIBundles(
 			envs = append(envs, fmt.Sprintf("%s=%s", env.GetName(), env.GetValue()))
 		}
 		g.Go(func() error {
-			imageConfig, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts())
+			img, cwd, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts())
 			if err != nil {
 				return wrapFileSystemErr(fmt.Sprintf("while creating %q OCI bundle", ctr.GetName()), err)
 			}
-			args, err := resolveProcessArgs(imageConfig, ctr.GetCommand(), ctr.GetArgs())
+			args, err := resolveProcessArgs(&img.Config, ctr.GetCommand(), ctr.GetArgs())
 			if err != nil {
 				return fmt.Errorf("while resolving process args for container %q: %w", ctr.GetName(), err)
 			}
+			user, err := resolveImageUser(img)
+			if err != nil {
+				return fmt.Errorf("while resolving user for container %q: %w", ctr.GetName(), err)
+			}
 			containerSpecs[i] = &ateworkerpb.ContainerSpec{
 				Args:         args,
-				Env:          resolveActorEnv(imageConfig, envs),
+				Env:          resolveActorEnv(&img.Config, envs),
 				Capabilities: resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()),
 				Resources:    toAteworkerResources(ctr.GetResources()),
+				User:         user,
+				Cwd:          cwd,
 			}
 			return nil
 		})

cmd/atelet/main_test.go

@@ -34,6 +34,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/spf13/pflag"
 	"go.ate.dev/substrate/cmd/atelet/internal/actordirs"
 	"go.ate.dev/substrate/cmd/atelet/internal/ateletpath"
@@ -42,6 +43,7 @@ import (
 	"go.ate.dev/substrate/internal/ateattr"
 	"go.ate.dev/substrate/internal/ateinterceptors"
 	"go.ate.dev/substrate/internal/atelet"
+	"go.ate.dev/substrate/internal/imagecache"
 	"go.ate.dev/substrate/internal/proto/ateletpb"
 	"go.ate.dev/substrate/internal/proto/ateworkerpb"
 	"go.ate.dev/substrate/internal/resources"
@@ -265,12 +267,69 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) {
 		Env:          []string{"GREETING=hello", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
 		Capabilities: []string{"CAP_AUDIT_WRITE", "CAP_NET_ADMIN", "CAP_NET_BIND_SERVICE"},
 		Resources:    &ateworkerpb.ResourceRequirements{Limits: &ateworkerpb.Resources{MemoryBytes: 1 << 30, CpuMillicores: 500}},
+		Cwd:          "/",
 	}}
 	if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" {
 		t.Errorf("prepareOCIBundles container specs mismatch (-want +got):\n%s", diff)
 	}
 }
 
+// TestPrepareOCIBundlesResolvesUsers pins that a container runs as its image's
+// USER, resolved against the image's own passwd and group files; a USER the
+// image cannot resolve fails the run.
+func TestPrepareOCIBundlesResolvesUsers(t *testing.T) {
+	useTempNodeDirs(t)
+	host := imageVolumeTestRegistry(t)
+	etc := []v1.Layer{
+		singleFileLayer(t, "etc/passwd", "root:x:0:0:root:/root:/bin/sh\nnonroot:x:65532:65532::/home/nonroot:/sbin/nologin\n"),
+		singleFileLayer(t, "etc/group", "root:x:0:\nnonroot:x:65532:\nvideo:x:44:nonroot\n"),
+	}
+	app := pushTestImageWithConfig(t, host+"/app:v1", v1.Config{User: "nonroot", Cmd: []string{"/bin/app"}}, etc...)
+	s := &Atelet{imageCache: newImageVolumeStore(t)}
+
+	spec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: app}}}
+	got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec)
+	if err != nil {
+		t.Fatalf("prepareOCIBundles: %v", err)
+	}
+	want := &ateworkerpb.ProcessUser{Uid: 65532, Gid: 65532, AdditionalGids: []uint32{44}}
+	if diff := cmp.Diff(want, got[0].GetUser(), protocmp.Transform()); diff != "" {
+		t.Errorf("app user mismatch (-want +got):\n%s", diff)
+	}
+
+	unresolvable := pushTestImageWithConfig(t, host+"/nobody:v1", v1.Config{User: "nobody", Cmd: []string{"/bin/app"}})
+	spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: unresolvable}}}
+	if _, err := s.prepareOCIBundles(t.Context(), "actor-uid-2", resources.ActorRef{}, spec); err == nil {
+		t.Error("prepareOCIBundles succeeded with a USER the image has no passwd entry for, want an error")
+	}
+}
+
+// TestPrepareOCIBundlesStartsInTheImageWorkdir pins that a container starts in
+// its image's WORKDIR, created through the overlay when the image lacks it.
+func TestPrepareOCIBundlesStartsInTheImageWorkdir(t *testing.T) {
+	useTempNodeDirs(t)
+	host := imageVolumeTestRegistry(t)
+	image := pushTestImageWithConfig(t, host+"/app:v1", v1.Config{Cmd: []string{"/bin/app"}, WorkingDir: "/home/nonroot"})
+	spec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: image}}}
+
+	const actorUID = "actor-uid-1"
+	s := &Atelet{imageCache: newImageVolumeStore(t)}
+	got, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec)
+	if err != nil {
+		t.Fatalf("prepareOCIBundles: %v", err)
+	}
+	if cwd := got[0].GetCwd(); cwd != "/home/nonroot" {
+		t.Errorf("Cwd = %q, want /home/nonroot", cwd)
+	}
+	overlay, err := imagecache.ReadSpec(ateletpath.OCIBundlePath(actorUID, "app"))
+	if err != nil {
+		t.Fatalf("imagecache.ReadSpec: %v", err)
+	}
+	if !slices.Contains(overlay.ExtraDirs, "/home/nonroot") {
+		t.Errorf("ExtraDirs = %v, want the working directory", overlay.ExtraDirs)
+	}
+}
+
 func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) {
 	manifest, err := json.Marshal(sandboxAssetsRecord{
 		SandboxClass:  "gvisor",