From f86fde9dfb5424c8d16f613de80469bdbef06541 Mon Sep 17 00:00:00 2001 From: Anna Pendleton Date: Thu, 29 Oct 2026 14:05:23 -0700 Subject: [PATCH 1/2] [release-1.0] Cherry-pick #3446: ate-setup: retry each ko release image with backoff (#3459) Cherry-pick of #3446 onto `release-1.0`, with no changes (`git cherry-pick -x` of both commits). Needed for the 1.0 release: the v1.0.0-rc.1 release run (https://github.com/agent-substrate/substrate/actions/runs/40925903340) failed in **Publish images** with `DENIED: permission_denied`. ghcr.io can briefly deny pushes to a package it has just created, and this is the first release that creates them. `ate-setup publish release-images` now builds each ko image in its own `ko build`. If an image fails, it logs the error and retries that image, for up to 3 attempts, waiting 10s then 20s (`retry.OnError` with a `wait.Backoff`). AI use: AI-assisted. - [x] Tests pass --------- Signed-off-by: Anna Pendleton <13793293+annapendleton@users.noreply.github.com> Co-authored-by: Anna Pendleton <13793293+annapendleton@users.noreply.github.com> --- cmd/ate-setup/internal/ko/ko.go | 34 ++++++++++ cmd/ate-setup/internal/ko/ko_test.go | 82 +++++++++++++++++++++++++ cmd/ate-setup/internal/steps/publish.go | 2 +- 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/cmd/ate-setup/internal/ko/ko.go b/cmd/ate-setup/internal/ko/ko.go index 139fa6bd63..c2b719c5c3 100644 --- a/cmd/ate-setup/internal/ko/ko.go +++ b/cmd/ate-setup/internal/ko/ko.go @@ -33,6 +33,9 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" ) @@ -162,6 +165,37 @@ func (r *Runner) BuildTagged(ctx context.Context, tag string, labels []string, i return refs, nil } +// buildBackoff paces the retries of an image that fails to build or push: 3 +// attempts, 10s then 20s apart. ghcr.io can briefly deny pushes to a package it +// has just created. +var buildBackoff = wait.Backoff{Steps: 3, Duration: 10 * time.Second, Factor: 2} + +// BuildEach runs BuildTagged once per import path, retrying each failure per +// buildBackoff. It returns the refs in importPaths order. +func (r *Runner) BuildEach(ctx context.Context, tag string, labels []string, importPaths ...string) ([]string, error) { + refs := make([]string, 0, len(importPaths)) + for _, path := range importPaths { + var ref string + logAndRetry := func(err error) bool { + log.Warnf("%s: %v", path, err) + return true + } + err := retry.OnError(buildBackoff, logAndRetry, func() error { + built, err := r.BuildTagged(ctx, tag, labels, path) + if err != nil { + return err + } + ref = built[0] + return nil + }) + if err != nil { + return nil, err + } + refs = append(refs, ref) + } + return refs, nil +} + // ResolveBytes resolves an in-memory manifest, such as kustomize output. func (r *Runner) ResolveBytes(ctx context.Context, manifest []byte) ([]byte, error) { return r.Resolve(ctx, "-", manifest) diff --git a/cmd/ate-setup/internal/ko/ko_test.go b/cmd/ate-setup/internal/ko/ko_test.go index 8c5377a60b..415676b4be 100644 --- a/cmd/ate-setup/internal/ko/ko_test.go +++ b/cmd/ate-setup/internal/ko/ko_test.go @@ -15,12 +15,17 @@ package ko import ( + "errors" "os" "os/exec" "path/filepath" "slices" + "strconv" "strings" "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" ) // Without --base-import-paths ko publishes cmd/atelet as "atelet-", and @@ -174,3 +179,80 @@ func TestBuildTaggedRejectsMissingRefs(t *testing.T) { t.Error("BuildTagged() error = nil, want one for 1 ref from 2 packages") } } + +// flakyKo writes a ko stand-in that fails the first failures builds of each +// package, printing stderr, then prints a ref for it. It returns the +// directory where it counts builds per package. +func flakyKo(t *testing.T, failures int, stderr string) (binary, dir string) { + t.Helper() + dir = t.TempDir() + binary = filepath.Join(dir, "ko") + script := "#!/bin/sh\n" + + "pkg=$(basename \"$2\")\n" + + "echo run >> " + dir + "/runs-$pkg\n" + + "if [ $(wc -l < " + dir + "/runs-$pkg) -le " + strconv.Itoa(failures) + " ]; then echo '" + stderr + "' >&2; exit 1; fi\n" + + "echo repo/$pkg:v1@sha256:a\n" + if err := os.WriteFile(binary, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return binary, dir +} + +// koRuns returns how many times flakyKo built pkg. +func koRuns(t *testing.T, dir, pkg string) int { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, "runs-"+pkg)) + if errors.Is(err, os.ErrNotExist) { + return 0 + } + if err != nil { + t.Fatal(err) + } + return strings.Count(string(raw), "run") +} + +// fastBackoff shortens buildBackoff for the test. +func fastBackoff(t *testing.T) { + t.Helper() + original := buildBackoff + buildBackoff = wait.Backoff{Steps: original.Steps, Duration: time.Millisecond, Factor: 1} + t.Cleanup(func() { buildBackoff = original }) +} + +const deniedStderr = "PATCH https://ghcr.io/v2/agent-substrate/substrate/ateapi/blobs/upload/1: DENIED: permission_denied: write_package" + +func TestBuildEachRetriesEachImage(t *testing.T) { + fastBackoff(t) + binary, dir := flakyKo(t, 2, deniedStderr) + r := &Runner{Root: t.TempDir(), binary: binary} + + refs, err := r.BuildEach(t.Context(), "v1", nil, "./cmd/ateapi", "./cmd/atelet") + if err != nil { + t.Fatalf("BuildEach() error = %v", err) + } + if want := []string{"repo/ateapi:v1@sha256:a", "repo/atelet:v1@sha256:a"}; !slices.Equal(refs, want) { + t.Errorf("BuildEach() = %q, want %q", refs, want) + } + for _, pkg := range []string{"ateapi", "atelet"} { + if got := koRuns(t, dir, pkg); got != 3 { + t.Errorf("ko built %s %d times, want 3", pkg, got) + } + } +} + +func TestBuildEachStopsAfterAttempts(t *testing.T) { + fastBackoff(t) + binary, dir := flakyKo(t, 100, deniedStderr) + r := &Runner{Root: t.TempDir(), binary: binary} + + _, err := r.BuildEach(t.Context(), "v1", nil, "./cmd/ateapi", "./cmd/atelet") + if err == nil { + t.Error("BuildEach() error = nil, want one after 3 failed attempts") + } + if got := koRuns(t, dir, "ateapi"); got != 3 { + t.Errorf("ko built ateapi %d times, want 3", got) + } + if got := koRuns(t, dir, "atelet"); got != 0 { + t.Errorf("ko built atelet %d times, want 0", got) + } +} diff --git a/cmd/ate-setup/internal/steps/publish.go b/cmd/ate-setup/internal/steps/publish.go index f98326e733..cbeac86ad0 100644 --- a/cmd/ate-setup/internal/steps/publish.go +++ b/cmd/ate-setup/internal/steps/publish.go @@ -254,7 +254,7 @@ func (e *Env) PublishReleaseImages(ctx context.Context, w io.Writer, opts Publis } refs = append(refs, ref) } - koRefs, err := runner.BuildTagged(ctx, tag, labels, pkgs...) + koRefs, err := runner.BuildEach(ctx, tag, labels, pkgs...) if err != nil { return err } From 0c662eba7191f5bc4357d5bbd1bde44dd4ec6861 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Thu, 29 Oct 2026 19:51:27 -0700 Subject: [PATCH 2/2] [release-1.0] Cherry-pick #2392: Run containers as the image's USER and WORKDIR, with layer file owners kept (#3473) Cherry-pick of #2392 onto `release-1.0` (`git cherry-pick -x` of the squash commit), it applied cleanly with no changes. Containers run as the image's `USER` and start in its `WORKDIR`, and atelet keeps the file owners recorded in image layers. Fixes #2036 for 1.0. AI use: AI-assisted. Clean pick. This carries the same breaking changes as #2392: - After an upgrade or rollback, atelet refuses the existing image cache. Delete `/var/lib/ate/image-cache` on each node once no actor runs on it, as atelet's error says. After an upgrade, a node reboot also works. - A custom atelet securityContext needs `CHOWN`, `DAC_OVERRIDE`, `FOWNER` and `FSETID`. - Images with a `USER` stop running as root, and images with a `WORKDIR` stop starting in `/`. A non-root `USER` has no effective capabilities, so `securityContext.capabilities.add` no longer helps it. - gVisor snapshots of such images don't restore, because `runsc restore` checks each container's user and working directory. Recreate those templates. MicroVM snapshots resume as they were. - Files an earlier run wrote as root into a durable dir stay root-owned. - [x] Tests pass - [x] Docs updated Signed-off-by: Anish Ramasekar Signed-off-by: Davanum Srinivas Signed-off-by: Benjamin Elder Co-authored-by: Anish Ramasekar Co-authored-by: Davanum Srinivas --- .ko.yaml | 9 + cmd/atelet/imagevolume_test.go | 15 + cmd/atelet/main.go | 12 +- cmd/atelet/main_test.go | 59 ++ cmd/atelet/oci.go | 99 ++- cmd/atelet/oci_test.go | 71 ++ .../internal/kata/specconv_linux_test.go | 20 + demos/jupyter/jupyter-template.yaml.tmpl | 1 - docs/reference/api-guide.md | 7 + go.mod | 2 +- .../zz_generated.validation.go | 2 + internal/e2e/fixture.go | 13 +- internal/e2e/fixtures/probe/main.go | 31 + internal/e2e/fixtures/probe/main_test.go | 45 ++ internal/e2e/fixtures/probe/nonroot/main.go | 91 +++ .../probe/probe-nonroot-template.yaml.tmpl | 38 + internal/e2e/manifest.go | 8 +- internal/e2e/sandbox_test.go | 4 + internal/e2e/suites/imagefs/imagefs_test.go | 243 +++++++ internal/e2e/suites/imagefs/testmain_test.go | 24 + internal/imagecache/README.md | 33 +- internal/imagecache/ateletcaps_test.go | 199 ++++++ internal/imagecache/bundle_linux.go | 9 + internal/imagecache/bundle_linux_test.go | 178 ++++- internal/imagecache/imagecache.go | 20 +- internal/imagecache/imagefile.go | 117 +++ internal/imagecache/imagefile_test.go | 127 ++++ internal/imagecache/implicitdirs.go | 16 +- internal/imagecache/removeall.go | 9 +- internal/imagecache/retire.go | 6 +- internal/imagecache/spec.go | 2 +- internal/imagecache/unpack.go | 263 +++++-- internal/imagecache/unpack_owner_test.go | 673 ++++++++++++++++++ internal/imagecache/unpack_test.go | 161 +++++ internal/ocispec/ocispec.go | 13 +- internal/ocispec/ocispec_test.go | 21 + internal/ocispec/parity_test.go | 39 + internal/proto/ateworkerpb/ateworker.pb.go | 302 +++++--- internal/proto/ateworkerpb/ateworker.proto | 13 + internal/tarutil/owner.go | 58 ++ internal/tarutil/owner_linux.go | 21 - internal/tarutil/tarutil.go | 24 +- manifests/ate-install/atelet.yaml | 29 +- 43 files changed, 2844 insertions(+), 283 deletions(-) create mode 100644 internal/e2e/fixtures/probe/nonroot/main.go create mode 100644 internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl create mode 100644 internal/e2e/suites/imagefs/imagefs_test.go create mode 100644 internal/e2e/suites/imagefs/testmain_test.go create mode 100644 internal/imagecache/ateletcaps_test.go create mode 100644 internal/imagecache/imagefile.go create mode 100644 internal/imagecache/imagefile_test.go create mode 100644 internal/imagecache/unpack_owner_test.go create mode 100644 internal/tarutil/owner.go diff --git a/.ko.yaml b/.ko.yaml index 80d54c7973..6465081548 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -19,6 +19,15 @@ defaultPlatforms: - linux/arm64 baseImageOverrides: + # The actor images built here write at the root of their filesystem and into + # volumes atelet creates as root. A container runs as its image's USER, so + # they stay on the root base. + go.ate.dev/substrate/demos/counter: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 + go.ate.dev/substrate/demos/egress: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 + go.ate.dev/substrate/demos/multi-template/fspersist: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 + go.ate.dev/substrate/internal/e2e/fixtures/probe: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 + go.ate.dev/substrate/internal/e2e/fixtures/testserver: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 + go.ate.dev/substrate/cmd/benchmarking/glutton: gcr.io/distroless/static-debian13:latest@sha256:15f8637509792e05b97d83451b0b6ccfa4abd548f997e406ac1e72c87fe1fa78 go.ate.dev/substrate/demos/sandbox: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a # atelet needs glibc to run the node's kubelet credential provider, which # some platforms ship dynamically linked (e.g. AKS, GKE Ubuntu). diff --git a/cmd/atelet/imagevolume_test.go b/cmd/atelet/imagevolume_test.go index 387688a823..c5a2bd17cb 100644 --- a/cmd/atelet/imagevolume_test.go +++ b/cmd/atelet/imagevolume_test.go @@ -75,11 +75,26 @@ func singleFileLayer(t *testing.T, path, body string) v1.Layer { // pushTestImage pushes the image and returns its digest-pinned reference, // which is what validated workload specs carry. func pushTestImage(t *testing.T, ref string, layers ...v1.Layer) string { + t.Helper() + return pushTestImageWithConfig(t, ref, v1.Config{}, layers...) +} + +// pushTestImageWithConfig is pushTestImage with the image config set. +func pushTestImageWithConfig(t *testing.T, ref string, cfg v1.Config, layers ...v1.Layer) string { t.Helper() img, err := mutate.AppendLayers(empty.Image, layers...) if err != nil { t.Fatalf("mutate.AppendLayers: %v", err) } + cf, err := img.ConfigFile() + if err != nil { + t.Fatalf("img.ConfigFile: %v", err) + } + cf = cf.DeepCopy() + cf.Config = cfg + if img, err = mutate.ConfigFile(img, cf); err != nil { + t.Fatalf("mutate.ConfigFile: %v", err) + } tag, err := name.ParseReference(ref, name.Insecure) if err != nil { t.Fatalf("name.ParseReference(%q): %v", ref, err) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 24f86122fb..1b51210e32 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -2262,19 +2262,25 @@ func (s *Atelet) prepareOCIBundles( envs = append(envs, fmt.Sprintf("%s=%s", env.GetName(), env.GetValue())) } g.Go(func() error { - imageConfig, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts()) + img, cwd, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), spec.GetVolumes(), ctr.GetVolumeMounts()) if err != nil { return wrapFileSystemErr(fmt.Sprintf("while creating %q OCI bundle", ctr.GetName()), err) } - args, err := resolveProcessArgs(imageConfig, ctr.GetCommand(), ctr.GetArgs()) + args, err := resolveProcessArgs(&img.Config, ctr.GetCommand(), ctr.GetArgs()) if err != nil { return fmt.Errorf("while resolving process args for container %q: %w", ctr.GetName(), err) } + user, err := resolveImageUser(img) + if err != nil { + return fmt.Errorf("while resolving user for container %q: %w", ctr.GetName(), err) + } containerSpecs[i] = &ateworkerpb.ContainerSpec{ Args: args, - Env: resolveActorEnv(imageConfig, envs), + Env: resolveActorEnv(&img.Config, envs), Capabilities: resolveCapabilities(ctr.GetSecurityContext().GetCapabilities()), Resources: toAteworkerResources(ctr.GetResources()), + User: user, + Cwd: cwd, } return nil }) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index d3daf4e277..162d1fba5b 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -34,6 +34,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/spf13/pflag" "go.ate.dev/substrate/cmd/atelet/internal/actordirs" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" @@ -42,6 +43,7 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" + "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -265,12 +267,69 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { Env: []string{"GREETING=hello", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, Capabilities: []string{"CAP_AUDIT_WRITE", "CAP_NET_ADMIN", "CAP_NET_BIND_SERVICE"}, Resources: &ateworkerpb.ResourceRequirements{Limits: &ateworkerpb.Resources{MemoryBytes: 1 << 30, CpuMillicores: 500}}, + Cwd: "/", }} if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("prepareOCIBundles container specs mismatch (-want +got):\n%s", diff) } } +// TestPrepareOCIBundlesResolvesUsers pins that a container runs as its image's +// USER, resolved against the image's own passwd and group files; a USER the +// image cannot resolve fails the run. +func TestPrepareOCIBundlesResolvesUsers(t *testing.T) { + useTempNodeDirs(t) + host := imageVolumeTestRegistry(t) + etc := []v1.Layer{ + singleFileLayer(t, "etc/passwd", "root:x:0:0:root:/root:/bin/sh\nnonroot:x:65532:65532::/home/nonroot:/sbin/nologin\n"), + singleFileLayer(t, "etc/group", "root:x:0:\nnonroot:x:65532:\nvideo:x:44:nonroot\n"), + } + app := pushTestImageWithConfig(t, host+"/app:v1", v1.Config{User: "nonroot", Cmd: []string{"/bin/app"}}, etc...) + s := &Atelet{imageCache: newImageVolumeStore(t)} + + spec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: app}}} + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec) + if err != nil { + t.Fatalf("prepareOCIBundles: %v", err) + } + want := &ateworkerpb.ProcessUser{Uid: 65532, Gid: 65532, AdditionalGids: []uint32{44}} + if diff := cmp.Diff(want, got[0].GetUser(), protocmp.Transform()); diff != "" { + t.Errorf("app user mismatch (-want +got):\n%s", diff) + } + + unresolvable := pushTestImageWithConfig(t, host+"/nobody:v1", v1.Config{User: "nobody", Cmd: []string{"/bin/app"}}) + spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: unresolvable}}} + if _, err := s.prepareOCIBundles(t.Context(), "actor-uid-2", resources.ActorRef{}, spec); err == nil { + t.Error("prepareOCIBundles succeeded with a USER the image has no passwd entry for, want an error") + } +} + +// TestPrepareOCIBundlesStartsInTheImageWorkdir pins that a container starts in +// its image's WORKDIR, created through the overlay when the image lacks it. +func TestPrepareOCIBundlesStartsInTheImageWorkdir(t *testing.T) { + useTempNodeDirs(t) + host := imageVolumeTestRegistry(t) + image := pushTestImageWithConfig(t, host+"/app:v1", v1.Config{Cmd: []string{"/bin/app"}, WorkingDir: "/home/nonroot"}) + spec := &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "app", Image: image}}} + + const actorUID = "actor-uid-1" + s := &Atelet{imageCache: newImageVolumeStore(t)} + got, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec) + if err != nil { + t.Fatalf("prepareOCIBundles: %v", err) + } + if cwd := got[0].GetCwd(); cwd != "/home/nonroot" { + t.Errorf("Cwd = %q, want /home/nonroot", cwd) + } + overlay, err := imagecache.ReadSpec(ateletpath.OCIBundlePath(actorUID, "app")) + if err != nil { + t.Fatalf("imagecache.ReadSpec: %v", err) + } + if !slices.Contains(overlay.ExtraDirs, "/home/nonroot") { + t.Errorf("ExtraDirs = %v, want the working directory", overlay.ExtraDirs) + } +} + func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index 3f72633d5b..1cf5a1a9ca 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -15,17 +15,23 @@ package main import ( + "bytes" "context" + "errors" "fmt" + "io" + "io/fs" "os" "path" "sort" "strings" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/moby/sys/user" "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "golang.org/x/sync/errgroup" @@ -74,7 +80,10 @@ func resolveCapabilities(caps *ateletpb.Capabilities) []string { return out } -func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) (*v1.Config, error) { +// prepareOCIDirectory pulls the container's image and writes the overlay spec +// the worker composes its rootfs from. It returns the image and the process's +// working directory, which the worker creates through the overlay. +func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, volumes []*ateletpb.Volume, volumeMounts []*ateletpb.VolumeMount) (*imagecache.Image, string, error) { tracer := otel.Tracer("prepareOCIDirectory") ctx, span := tracer.Start(ctx, "prepareOCIDirectory") @@ -87,18 +96,18 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto // actordirs.ResetActorDirs already wiped the bundle dir on the Run/Restore // path). if err := imagecache.RemoveAllWritable(bundlePath); err != nil { - return nil, fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) + return nil, "", fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) } // The bundle's rootfs is composed by worker as an overlay mount just before // the workload runs: the cached image layers are the read-only lowerdirs, // and the bundle-local upper/work hold this actor's private writes (wiped // between runs, preserving the pristine-rootfs-per-run contract the old - // full re-untar provided). atelet only prepares the (empty) directories — - // it deliberately runs with no capabilities, so it cannot mount. + // full re-untar provided). atelet only prepares the (empty) directories: + // it has no CAP_SYS_ADMIN, so it cannot mount. for _, d := range []string{"rootfs", "upper", "work"} { if err := os.MkdirAll(path.Join(bundlePath, d), 0o700); err != nil { - return nil, fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) + return nil, "", fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) } } @@ -120,26 +129,45 @@ func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, acto return err }) if err := g.Wait(); err != nil { - return nil, err + return nil, "", err + } + cwd, err := resolveCwd(&img.Config) + if err != nil { + return nil, "", fmt.Errorf("while resolving the working directory of container %q: %w", containerName, err) } - // Every bind target must exist in the rootfs for the mount to attach; - // worker creates them through the mounted overlay (they land in the - // actor's upper). + // Every bind target must exist in the rootfs for the mount to attach, and + // the working directory for the process to start; worker creates them + // through the mounted overlay (they land in the actor's upper). var extraDirs []string for _, vm := range volumeMounts { extraDirs = append(extraDirs, vm.GetMountPath()) } + if cwd != "/" { + extraDirs = append(extraDirs, cwd) + } if err := imagecache.WriteSpec(bundlePath, &imagecache.OverlaySpec{ ImageDigest: img.Digest.String(), Layers: img.LayerDirs, ExtraDirs: extraDirs, ImageVolumes: imageVolumes, }); err != nil { - return nil, fmt.Errorf("while writing overlay spec: %w", err) + return nil, "", fmt.Errorf("while writing overlay spec: %w", err) } - return &img.Config, nil + return img, cwd, nil +} + +// resolveCwd reads the image's WORKDIR; none means "/". The OCI spec requires +// an absolute path. +func resolveCwd(imageCfg *v1.Config) (string, error) { + if imageCfg == nil || imageCfg.WorkingDir == "" { + return "/", nil + } + if !path.IsAbs(imageCfg.WorkingDir) { + return "", fmt.Errorf("image WorkingDir %q is not absolute", imageCfg.WorkingDir) + } + return path.Clean(imageCfg.WorkingDir), nil } // resolveImageVolumes pulls the image behind every image-typed volume this @@ -241,3 +269,52 @@ func resolveProcessArgs(imageCfg *v1.Config, command, args []string) ([]string, } return argv, nil } + +// resolveImageUser resolves the image's USER against the image's own +// /etc/passwd and /etc/group, either of which may be absent. No USER is root, +// reported as nil. +func resolveImageUser(img *imagecache.Image) (*ateworkerpb.ProcessUser, error) { + if img.Config.User == "" { + return nil, nil + } + passwd, err := readOptionalImageFile(img, "etc/passwd") + if err != nil { + return nil, err + } + group, err := readOptionalImageFile(img, "etc/group") + if err != nil { + return nil, err + } + return resolveUser(img.Config.User, passwd, group) +} + +// readOptionalImageFile returns a nil reader for a file the image lacks. +func readOptionalImageFile(img *imagecache.Image, name string) (io.Reader, error) { + b, err := img.ReadFile(name) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +// resolveUser follows Docker's rules for USER "user[:group]": numeric ids as +// is, a bare uid with its login group from passwd (gid 0 without an entry), +// names looked up or an error, group memberships as supplementary groups. +// Ids are bounded to int32. A nil reader means the image lacks the file. +func resolveUser(userSpec string, passwd, group io.Reader) (*ateworkerpb.ProcessUser, error) { + if strings.Count(userSpec, ":") > 1 { + return nil, fmt.Errorf("image User %q: want user[:group]", userSpec) + } + u, err := user.GetExecUser(userSpec, nil, passwd, group) + if err != nil { + return nil, fmt.Errorf("image User %q: %w", userSpec, err) + } + identity := &ateworkerpb.ProcessUser{Uid: uint32(u.Uid), Gid: uint32(u.Gid)} + for _, gid := range u.Sgids { + identity.AdditionalGids = append(identity.AdditionalGids, uint32(gid)) + } + return identity, nil +} diff --git a/cmd/atelet/oci_test.go b/cmd/atelet/oci_test.go index c76d2d0d03..1eed244303 100644 --- a/cmd/atelet/oci_test.go +++ b/cmd/atelet/oci_test.go @@ -15,11 +15,16 @@ package main import ( + "io" "slices" + "strings" "testing" + "github.com/google/go-cmp/cmp" v1 "github.com/google/go-containerregistry/pkg/v1" "go.ate.dev/substrate/internal/proto/ateletpb" + "go.ate.dev/substrate/internal/proto/ateworkerpb" + "google.golang.org/protobuf/testing/protocmp" ) func TestResolveActorEnv(t *testing.T) { @@ -176,6 +181,72 @@ func withCaps(in []string, add ...string) []string { return out } +func TestResolveUser(t *testing.T) { + const passwd = "root:x:0:0:root:/root:/bin/sh\n" + + "jovyan:x:1000:100::/home/jovyan:/bin/bash\n" + + "nonroot:x:65532:65532::/home/nonroot:/sbin/nologin\n" + const group = "root:x:0:\n" + + "users:x:100:\n" + + "nonroot:x:65532:\n" + + "video:x:44:nonroot,jovyan\n" + + "audio:x:29:nonroot\n" + tests := []struct { + name string + user string + passwd, group string // empty: the image lacks the file + want *ateworkerpb.ProcessUser + wantErr bool + }{ + {name: "bare uid, no passwd", user: "1000", want: &ateworkerpb.ProcessUser{Uid: 1000}}, + {name: "bare uid takes its login group and memberships", user: "1000", passwd: passwd, group: group, want: &ateworkerpb.ProcessUser{Uid: 1000, Gid: 100, AdditionalGids: []uint32{44}}}, + {name: "bare uid without a passwd entry", user: "1001", passwd: passwd, group: group, want: &ateworkerpb.ProcessUser{Uid: 1001}}, + {name: "uid:gid passes through", user: "1000:2000", passwd: passwd, group: group, want: &ateworkerpb.ProcessUser{Uid: 1000, Gid: 2000}}, + {name: "named user", user: "nonroot", passwd: passwd, group: group, want: &ateworkerpb.ProcessUser{Uid: 65532, Gid: 65532, AdditionalGids: []uint32{44, 29}}}, + {name: "named user and group", user: "jovyan:video", passwd: passwd, group: group, want: &ateworkerpb.ProcessUser{Uid: 1000, Gid: 44}}, + {name: "named user missing from passwd", user: "nobody", passwd: passwd, wantErr: true}, + {name: "named group missing from group", user: "1000:staff", passwd: passwd, group: group, wantErr: true}, + {name: "above int32", user: "2147483648", wantErr: true}, + {name: "third field", user: "1000:1000:1", wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var passwdR, groupR io.Reader + if tc.passwd != "" { + passwdR = strings.NewReader(tc.passwd) + } + if tc.group != "" { + groupR = strings.NewReader(tc.group) + } + got, err := resolveUser(tc.user, passwdR, groupR) + if (err != nil) != tc.wantErr { + t.Fatalf("resolveUser(%q) = %v, %v; wantErr %v", tc.user, got, err, tc.wantErr) + } + if diff := cmp.Diff(tc.want, got, protocmp.Transform()); err == nil && diff != "" { + t.Errorf("resolveUser(%q) mismatch (-want +got):\n%s", tc.user, diff) + } + }) + } +} + +func TestResolveCwd(t *testing.T) { + for _, tc := range []struct { + image *v1.Config + want string + wantErr bool + }{ + {image: nil, want: "/"}, + {image: &v1.Config{}, want: "/"}, + {image: &v1.Config{WorkingDir: "/home/jovyan"}, want: "/home/jovyan"}, + {image: &v1.Config{WorkingDir: "/app/"}, want: "/app"}, + {image: &v1.Config{WorkingDir: "app"}, wantErr: true}, + } { + got, err := resolveCwd(tc.image) + if (err != nil) != tc.wantErr || got != tc.want { + t.Errorf("resolveCwd(%+v) = %q, %v; want %q, wantErr %v", tc.image, got, err, tc.want, tc.wantErr) + } + } +} + func TestResolveCapabilities(t *testing.T) { tests := []struct { name string diff --git a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go index 4b7d3afdc7..b211346f9f 100644 --- a/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go +++ b/cmd/ateworker-microvm/internal/kata/specconv_linux_test.go @@ -17,6 +17,7 @@ package kata import ( + "slices" "testing" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -174,3 +175,22 @@ func TestSpecToAgentPB_ForwardsSysctl(t *testing.T) { t.Fatalf("Linux.Sysctl = %v, want %s=0 carried through", got.GetLinux().GetSysctl(), key) } } + +// The agent starts the process as the user it is sent; a dropped identity +// would run the workload as root in the guest. +func TestSpecToAgentPB_ForwardsProcessUser(t *testing.T) { + got := SpecToAgentPB(&specs.Spec{ + Process: &specs.Process{User: specs.User{UID: 65532, GID: 65534, AdditionalGids: []uint32{44}}}, + }) + u := got.GetProcess().GetUser() + if u == nil || u.UID != 65532 || u.GID != 65534 || !slices.Equal(u.AdditionalGids, []uint32{44}) { + t.Fatalf("Process.User = %v, want 65532:65534 +44", u) + } +} + +func TestSpecToAgentPB_ForwardsCwd(t *testing.T) { + got := SpecToAgentPB(&specs.Spec{Process: &specs.Process{Cwd: "/work"}}) + if got.GetProcess().GetCwd() != "/work" { + t.Fatalf("Process.Cwd = %q, want /work", got.GetProcess().GetCwd()) + } +} diff --git a/demos/jupyter/jupyter-template.yaml.tmpl b/demos/jupyter/jupyter-template.yaml.tmpl index df8cd69396..1877a397a1 100644 --- a/demos/jupyter/jupyter-template.yaml.tmpl +++ b/demos/jupyter/jupyter-template.yaml.tmpl @@ -28,7 +28,6 @@ containers: command: - jupyter - lab - - "--allow-root" - "--ServerApp.token=''" - "--ServerApp.password=''" - "--ServerApp.allow_origin='*'" diff --git a/docs/reference/api-guide.md b/docs/reference/api-guide.md index 68e883c867..9a26af21d4 100644 --- a/docs/reference/api-guide.md +++ b/docs/reference/api-guide.md @@ -309,10 +309,16 @@ Each entry in `containers` describes one process to run in the actor's sandbox. `command` and `args` resolve against the container image's `ENTRYPOINT`/`CMD` the same way [Kubernetes Pod `command`/`args`](https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/) resolve against `ENTRYPOINT`/`CMD`. If the resolved argv is empty — the image sets neither `ENTRYPOINT` nor `CMD`, and the container sets neither `command` nor `args` — `Run`/`Restore` fails. +The process runs as the image's `USER`, resolved as Docker does against the image's own `/etc/passwd` and `/etc/group`: a numeric `uid:gid` as is; a bare uid with its login group, or gid 0 when it has no entry; a name must have an entry or `Run`/`Restore` fails. Group memberships become supplementary groups. Ids are at most 2147483647. No `USER` means root. There is no `runAsUser` field. + +The process starts in the image's `WORKDIR`, or in `/` when the image sets none. A `WORKDIR` the image lacks is created in the actor's writable layer. There is no `workingDir` field. + ### Container Capabilities (`securityContext.capabilities`) Each container runs with a default set of Linux capabilities — `AUDIT_WRITE`, `KILL` and `NET_BIND_SERVICE`. `securityContext.capabilities` adjusts that set, mirroring `securityContext.capabilities` on a Kubernetes Pod container. +A container running as root has that set as its effective capabilities. A container whose image `USER` is not root starts with no effective capabilities, as under Kubernetes, because ambient capabilities are not supported. Its set then only limits what a setuid-root program can gain. + | Field | Type | Description | | :--- | :--- | :--- | | `securityContext.capabilities.add` | `[]string` | Optional. Capabilities to grant on top of the default set. `ALL` is **not** accepted here. | @@ -333,6 +339,7 @@ Each container runs with a default set of Linux capabilities — `AUDIT_WRITE`, - **Ambient capabilities are not supported** ([gvisor#3166](https://github.com/google/gvisor/issues/3166)). - **Low ports.** A container may bind a port below 1024 without `NET_BIND_SERVICE` on both runtimes: gVisor's network stack has no privileged-port limit, and the micro-VM guest sets `net.ipv4.ip_unprivileged_port_start=0` to match. - **Setuid binaries work.** Neither runtime sets `no_new_privileges`, as for a Kubernetes container with `allowPrivilegeEscalation: true`; gVisor runs with `--allow-suid`. +- **Image files owned by other users.** Image files keep the owner their image's layers record, as under containerd. A container running as its image's `USER` can use what the image gave that user, for example after `RUN chown -R app:app /app` and `USER app`. A container running as root, because its image sets no `USER` or `USER root`, has no `DAC_OVERRIDE` or `DAC_READ_SEARCH` by default. On a file or directory another user owns, it gets only what the permission bits grant to others. Add `DAC_OVERRIDE` if it writes to such files, or `DAC_READ_SEARCH` if it only reads them. The sandbox — gVisor or microVM — remains the isolation boundary; capabilities constrain the workload *inside* it. diff --git a/go.mod b/go.mod index deca33e7cd..7e81ca7c74 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/jackc/pgx/v5 v5.10.0 github.com/klauspost/compress v1.19.1 + github.com/moby/sys/user v0.4.1 github.com/myzhan/boomer v1.6.1-0.20250711115830-8a2c7ce4c7b1 github.com/opencontainers/runtime-spec v1.3.0 github.com/openfga/api/proto v0.0.0-20260319214821-f153694bfc20 @@ -191,7 +192,6 @@ require ( github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/sequential v0.7.0 // indirect - github.com/moby/sys/user v0.4.1 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/internal/ateworkervalidation/zz_generated.validation.go b/internal/ateworkervalidation/zz_generated.validation.go index eb633428cf..fc44c8eeed 100644 --- a/internal/ateworkervalidation/zz_generated.validation.go +++ b/internal/ateworkervalidation/zz_generated.validation.go @@ -922,6 +922,8 @@ func Validate_ContainerSpec( errs = append(errs, fn(fldPath.Child("resources"), obj.Resources, oldVal, oldObj != nil)...) } + // field ateworkerpb.ContainerSpec.User has no validation + // field ateworkerpb.ContainerSpec.Cwd has no validation return errs } diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index 5410bb7c0c..1abf83d227 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -92,6 +92,7 @@ type FixtureOption func(*fixtureConfig) type fixtureConfig struct { templateValues map[string]string + imageUser string } // WithTemplateValue replaces the inline placeholder ${name} in the fixture's @@ -105,6 +106,12 @@ func WithTemplateValue(name, value string) FixtureOption { } } +// WithImageUser sets user as the USER of the images ko builds for the +// fixture's ActorTemplates, as a Dockerfile USER line would. +func WithImageUser(user string) FixtureOption { + return func(c *fixtureConfig) { c.imageUser = user } +} + // SubstrateFixtureManifests names the two manifest templates a substrate // fixture is built from (both repo-relative, under internal/e2e/fixtures). type SubstrateFixtureManifests struct { @@ -157,7 +164,11 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, tmplInline, tmplBlocks := substrateTemplateSubstitutions(name, trustBundles) maps.Copy(tmplInline, cfg.templateValues) - rendered := koResolve(t, renderManifest(t, manifests.Template, tmplInline, tmplBlocks)) + var resolveArgs []string + if cfg.imageUser != "" { + resolveArgs = append(resolveArgs, "--image-user", cfg.imageUser) + } + rendered := koResolve(t, renderManifest(t, manifests.Template, tmplInline, tmplBlocks), resolveArgs...) templates := decodeSubstrateTemplates(t, rendered) atespace := templates[0].GetMetadata().GetAtespace() diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 5dedcb1005..80a68179db 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -223,6 +223,36 @@ func readfile(w http.ResponseWriter, r *http.Request) { writeJSON(w, resp) } +// statResponse is the /stat payload: path's numeric owner and mode as the +// actor sees them, without following a final-component symlink. +type statResponse struct { + Path string `json:"path"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + // Mode is the os.FileMode string, e.g. "drwx------". + Mode string `json:"mode"` + Error string `json:"error,omitempty"` +} + +// stat reports a path's owner and mode inside the actor, so a test can assert +// on the ownership the image's layers gave it. +func stat(w http.ResponseWriter, r *http.Request) { + resp := statResponse{Path: r.URL.Query().Get("path")} + fi, err := os.Lstat(resp.Path) + if err != nil { + resp.Error = err.Error() + writeJSON(w, resp) + return + } + resp.Mode = fi.Mode().String() + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + resp.UID, resp.GID = st.Uid, st.Gid + } else { + resp.Error = "no owner information for " + resp.Path + } + writeJSON(w, resp) +} + func writefile(w http.ResponseWriter, r *http.Request) { path := r.URL.Query().Get("path") resp := map[string]string{"path": path} @@ -629,6 +659,7 @@ func main() { mux.HandleFunc("/readfile", readfile) mux.HandleFunc("/writefile", writefile) mux.HandleFunc("/lookup", lookup) + mux.HandleFunc("/stat", stat) mux.HandleFunc("/resources", resources) mux.HandleFunc("/capabilities", capabilities) mux.HandleFunc("/healthz", healthz) diff --git a/internal/e2e/fixtures/probe/main_test.go b/internal/e2e/fixtures/probe/main_test.go index 7d26c6a151..e7ec377308 100644 --- a/internal/e2e/fixtures/probe/main_test.go +++ b/internal/e2e/fixtures/probe/main_test.go @@ -515,3 +515,48 @@ func TestRestoreFetchRetriesOnNextProbe(t *testing.T) { time.Sleep(50 * time.Millisecond) } } + +func doStat(t *testing.T, path string) statResponse { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/stat?"+url.Values{"path": {path}}.Encode(), nil) + rec := httptest.NewRecorder() + stat(rec, req) + var resp statResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding /stat response %q: %v", rec.Body.String(), err) + } + return resp +} + +// The imagefs e2e suite asserts on /stat's owner and mode, so this pins both +// against files whose owner and mode the test sets itself. +func TestStatReportsOwnerAndMode(t *testing.T) { + dir := t.TempDir() + priv := filepath.Join(dir, "private") + if err := os.Mkdir(priv, 0o700); err != nil { + t.Fatal(err) + } + // Mkdir's mode is clipped by the umask; set it exactly. + if err := os.Chmod(priv, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link") + if err := os.Symlink("private", link); err != nil { + t.Fatal(err) + } + + got := doStat(t, priv) + want := statResponse{Path: priv, UID: uint32(os.Getuid()), GID: got.GID, Mode: "drwx------"} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("/stat %s mismatch (-want +got):\n%s", priv, diff) + } + + // The symlink itself, not its target. + if got := doStat(t, link); !strings.HasPrefix(got.Mode, "L") || got.Error != "" { + t.Errorf("/stat %s = %+v, want the symlink's own mode", link, got) + } + + if got := doStat(t, filepath.Join(dir, "missing")); got.Error == "" { + t.Errorf("/stat of a missing path reported no error: %+v", got) + } +} diff --git a/internal/e2e/fixtures/probe/nonroot/main.go b/internal/e2e/fixtures/probe/nonroot/main.go new file mode 100644 index 0000000000..587b69f283 --- /dev/null +++ b/internal/e2e/fixtures/probe/nonroot/main.go @@ -0,0 +1,91 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command nonroot is the probe for an image with a non-root USER and a +// WORKDIR that user owns. It reports the identity, effective capabilities and +// working directory the actor's process runs with. +// +// It is a separate binary from the probe because ko picks the base image per +// import path. The probe stays on the root base, and this one builds on the +// default nonroot distroless base, whose WORKDIR is /home/nonroot. +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "strings" +) + +// identityResponse is the /identity payload. +type identityResponse struct { + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups"` + Cwd string `json:"cwd"` + // CapEff is the CapEff mask from /proc/self/status, in hex. + CapEff string `json:"capEff"` + // WriteError is why creating a file in the working directory failed. + WriteError string `json:"writeError,omitempty"` + Error string `json:"error,omitempty"` +} + +func identity(w http.ResponseWriter, _ *http.Request) { + resp := identityResponse{UID: os.Getuid(), GID: os.Getgid(), Groups: []int{}} + if groups, err := os.Getgroups(); err == nil { + resp.Groups = append(resp.Groups, groups...) + } else { + resp.Error += "getgroups: " + err.Error() + "; " + } + if cwd, err := os.Getwd(); err == nil { + resp.Cwd = cwd + } else { + resp.Error += "getwd: " + err.Error() + "; " + } + if status, err := os.ReadFile("/proc/self/status"); err == nil { + for line := range strings.Lines(string(status)) { + if v, ok := strings.CutPrefix(line, "CapEff:"); ok { + resp.CapEff = strings.TrimSpace(v) + } + } + } else { + resp.Error += "reading /proc/self/status: " + err.Error() + "; " + } + if f, err := os.CreateTemp(".", "write-check-"); err == nil { + _ = f.Close() + if err := os.Remove(f.Name()); err != nil { + resp.WriteError = err.Error() + } + } else { + resp.WriteError = err.Error() + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp); err != nil { + log.Printf("nonroot probe: writing response: %v", err) + } +} + +func main() { + mux := http.NewServeMux() + mux.HandleFunc("/identity", identity) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + + const addr = ":80" + log.Printf("nonroot probe listening on %s", addr) + if err := http.ListenAndServe(addr, mux); err != nil { + log.Fatalf("nonroot probe server: %v", err) + } +} diff --git a/internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl new file mode 100644 index 0000000000..6ada59c5b1 --- /dev/null +++ b/internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl @@ -0,0 +1,38 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# An ActorTemplate on the probe pool (probe.yaml.tmpl) whose image runs as a +# named non-root USER in a WORKDIR that user owns. The imagefs suite resolves +# it with `ko resolve --image-user nonroot:nonroot` (e2e.WithImageUser); the +# nonroot distroless base sets WORKDIR /home/nonroot. + +metadata: + atespace: ate-e2e-probe${FIXTURE_SUFFIX} + name: probe-nonroot +workerSelector: + matchLabels: + workload: probe${FIXTURE_SUFFIX} +containers: +- name: probe + image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe/nonroot + command: ["/ko-app/nonroot"] + wakeupProbe: + httpGet: + path: /healthz + port: 80 + timeoutSeconds: 60 +resources: + limits: + cpuMillicores: 1000 + memoryBytes: 536870912 # 512Mi diff --git a/internal/e2e/manifest.go b/internal/e2e/manifest.go index 5714973ba6..b7623cd2ea 100644 --- a/internal/e2e/manifest.go +++ b/internal/e2e/manifest.go @@ -120,12 +120,14 @@ func koApply(t *testing.T, manifest string) { // the manifest with those references replaced by pushed digests. Same pinned // ko and KO_CONFIG_PATH rules as koApply; resolve is how a manifest that is // not destined for kubectl (a protojson ActorTemplate document) still gets -// its image built. -func koResolve(t *testing.T, manifest string) []byte { +// its image built. args go to ko resolve before the manifest. +func koResolve(t *testing.T, manifest string, args ...string) []byte { t.Helper() root, err := FindRepoRoot() if err != nil { t.Fatalf("FindRepoRoot: %v", err) } - return RunCmdOutput(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), "ko", "resolve", "-f", manifest) + resolveArgs := append([]string{"ko", "resolve"}, args...) + resolveArgs = append(resolveArgs, "-f", manifest) + return RunCmdOutput(t, []string{"KO_CONFIG_PATH=" + root}, filepath.Join(root, "hack/run-tool.sh"), resolveArgs...) } diff --git a/internal/e2e/sandbox_test.go b/internal/e2e/sandbox_test.go index 427bb5d823..d60ce2aec0 100644 --- a/internal/e2e/sandbox_test.go +++ b/internal/e2e/sandbox_test.go @@ -40,6 +40,10 @@ var substrateFixtures = []struct { Pool: "internal/e2e/fixtures/probe/probe.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-template.yaml.tmpl", }, 1, nil}, + {SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/probe/probe.yaml.tmpl", + Template: "internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl", + }, 1, nil}, {SubstrateFixtureManifests{ Pool: "internal/e2e/fixtures/probe/probe.yaml.tmpl", Template: "internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl", diff --git a/internal/e2e/suites/imagefs/imagefs_test.go b/internal/e2e/suites/imagefs/imagefs_test.go new file mode 100644 index 0000000000..e354d420f8 --- /dev/null +++ b/internal/e2e/suites/imagefs/imagefs_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imagefs checks the actor's view of its image's filesystem and +// process identity against what the image records. +package imagefs + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strconv" + "testing" + + "go.ate.dev/substrate/api/ateapi/v1" + "go.ate.dev/substrate/internal/e2e" + "go.ate.dev/substrate/internal/resources" +) + +// statResponse mirrors the probe's /stat payload. +type statResponse struct { + Path string `json:"path"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Mode string `json:"mode"` + Error string `json:"error"` +} + +// layerOwners are paths whose owner and mode the probe image's layers record. +// .ko.yaml builds the probe on the root distroless static base, whose base +// layer has /home/nonroot as drwx------ owned by 65532:65532 (distroless's +// nonroot user), /var/tmp as root-owned 1777, and root-owned system files. +// /var/tmp, not /tmp, checks the sticky bit: gVisor mounts its own 1777 tmpfs +// over an empty /tmp. +var layerOwners = []statResponse{ + {Path: "/home/nonroot", UID: 65532, GID: 65532, Mode: "drwx------"}, + {Path: "/var/tmp", UID: 0, GID: 0, Mode: "dtrwxrwxrwx"}, + {Path: "/etc/passwd", UID: 0, GID: 0, Mode: "-rw-r--r--"}, + {Path: "/home", UID: 0, GID: 0, Mode: "drwxr-xr-x"}, +} + +// TestImageLayerOwnership asserts that files keep the owners the image's +// layer tars record, as they do under containerd: a private home directory +// given to a non-root user must not show up as root-owned inside the actor. +// It checks once on the first start and again after a suspend and resume, +// which restores the actor over the same cached layers. +// +// It runs against whichever sandbox class E2E_SANDBOX_CLASS selects, so CI +// covers both gVisor and microVM from one suite. +func TestImageLayerOwnership(t *testing.T) { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + atespace, tmpl := e2e.DeployProbe(t, "imagefs") + ref := &ateapi.ObjectRef{Atespace: atespace, Name: "imagefs-actor"} + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + ActorTemplate: &ateapi.ObjectRef{Atespace: atespace, Name: tmpl.GetMetadata().GetName()}, + }}); err != nil { + t.Fatalf("CreateActor %q: %v", ref.Name, err) + } + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapi.DeleteActorRequest{Actor: ref}) + }) + + rc, err := e2e.NewRouterClient(t, ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %q: %v", ref.Name, err) + } + t.Run("first start", func(t *testing.T) { checkOwners(t, ctx, rc, ref) }) + + suspended, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapi.SuspendActorRequest{Actor: ref}) + if err != nil { + t.Fatalf("SuspendActor %q: %v", ref.Name, err) + } + if got := suspended.GetActor().GetStatus().GetState(); got != ateapi.ActorState_ACTOR_STATE_SUSPENDED { + t.Fatalf("SuspendActor %q: state %v, want %v", ref.Name, got, ateapi.ActorState_ACTOR_STATE_SUSPENDED) + } + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %q after suspend: %v", ref.Name, err) + } + t.Run("after suspend and resume", func(t *testing.T) { checkOwners(t, ctx, rc, ref) }) +} + +// identityResponse mirrors the nonroot probe's /identity payload. +type identityResponse struct { + UID int `json:"uid"` + GID int `json:"gid"` + Groups []int `json:"groups"` + Cwd string `json:"cwd"` + CapEff string `json:"capEff"` + WriteError string `json:"writeError"` + Error string `json:"error"` +} + +// The nonroot probe image's USER is nonroot:nonroot. The distroless base's +// /etc/passwd and /etc/group give that user and group id 65532 and list the +// user in no other group. Its WORKDIR, /home/nonroot, is 65532:65532 and +// drwx------. +const ( + nonrootID = 65532 + nonrootWorkdir = "/home/nonroot" +) + +// TestImageUserAndWorkdir asserts that the actor's process runs as its image's +// USER, resolved by name against the image's own /etc/passwd and /etc/group, +// with no effective capabilities, and starts in the image's WORKDIR, where +// that user can create files. The WORKDIR is private to the user, so the +// write also needs the layer owners TestImageLayerOwnership checks. +// +// The golden bake runs the probe as that user and binds :80, and the actor +// resumes from that golden snapshot. It runs against whichever sandbox class +// E2E_SANDBOX_CLASS selects, so CI covers both gVisor and microVM. +func TestImageUserAndWorkdir(t *testing.T) { + if _, err := e2e.CheckEnv("KO_DOCKER_REPO"); err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + + atespace, templates := e2e.DeploySubstrateFixture(t, ctx, clients, e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/probe/probe.yaml.tmpl", + Template: "internal/e2e/fixtures/probe/probe-nonroot-template.yaml.tmpl", + }, "imagefs-user", nil, e2e.WithImageUser("nonroot:nonroot")) + ref := &ateapi.ObjectRef{Atespace: atespace, Name: "imagefs-user-actor"} + if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapi.CreateActorRequest{Actor: &ateapi.Actor{ + Metadata: &ateapi.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + ActorTemplate: &ateapi.ObjectRef{Atespace: atespace, Name: templates[0].GetMetadata().GetName()}, + }}); err != nil { + t.Fatalf("CreateActor %q: %v", ref.Name, err) + } + t.Cleanup(func() { + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapi.DeleteActorRequest{Actor: ref}) + }) + + rc, err := e2e.NewRouterClient(t, ctx) + if err != nil { + t.Fatalf("NewRouterClient: %v", err) + } + defer rc.Close() + + if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapi.ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor %q: %v", ref.Name, err) + } + + got := probeIdentity(t, ctx, rc, ref) + if got.Error != "" { + t.Errorf("probe reported an error reading its identity: %s", got.Error) + } + if got.UID != nonrootID || got.GID != nonrootID { + t.Errorf("process runs as %d:%d, want %d:%d from the image's USER nonroot:nonroot", got.UID, got.GID, nonrootID, nonrootID) + } + // Linux may list the primary group among the supplementary ones. Any + // other group, such as root's 0, was not granted by the image. + for _, g := range got.Groups { + if g != nonrootID { + t.Errorf("supplementary groups = %v, want none besides %d", got.Groups, nonrootID) + break + } + } + // Without ambient capabilities, a non-root process has none effective. + if capEff, err := strconv.ParseUint(got.CapEff, 16, 64); err != nil || capEff != 0 { + t.Errorf("CapEff = %q, want no effective capabilities for a non-root USER", got.CapEff) + } + if got.Cwd != nonrootWorkdir { + t.Errorf("working directory = %q, want the image's WORKDIR %q", got.Cwd, nonrootWorkdir) + } + if got.WriteError != "" { + t.Errorf("creating a file in the WORKDIR as the image's USER: %s", got.WriteError) + } +} + +func probeIdentity(t *testing.T, ctx context.Context, rc *e2e.RouterClient, ref *ateapi.ObjectRef) identityResponse { + t.Helper() + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: ref.Atespace, Name: ref.Name}, "/identity") + if err != nil { + t.Fatalf("GET /identity: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET /identity: status %d, body %q", resp.StatusCode, body) + } + var out identityResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding /identity: %v", err) + } + return out +} + +func checkOwners(t *testing.T, ctx context.Context, rc *e2e.RouterClient, ref *ateapi.ObjectRef) { + t.Helper() + for _, want := range layerOwners { + got := probeStat(t, ctx, rc, ref, want.Path) + if got.Error != "" { + t.Errorf("stat %s inside the actor: %s", want.Path, got.Error) + continue + } + if got.UID != want.UID || got.GID != want.GID || got.Mode != want.Mode { + t.Errorf("%s inside the actor is %s %d:%d, want %s %d:%d as the image's layer records it", + want.Path, got.Mode, got.UID, got.GID, want.Mode, want.UID, want.GID) + } + } +} + +func probeStat(t *testing.T, ctx context.Context, rc *e2e.RouterClient, ref *ateapi.ObjectRef, path string) statResponse { + t.Helper() + resp, err := rc.Get(ctx, resources.ActorRef{Atespace: ref.Atespace, Name: ref.Name}, "/stat?"+url.Values{"path": {path}}.Encode()) + if err != nil { + t.Fatalf("GET /stat %s: %v", path, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("GET /stat %s: status %d, body %q", path, resp.StatusCode, body) + } + var out statResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatalf("decoding /stat %s: %v", path, err) + } + return out +} diff --git a/internal/e2e/suites/imagefs/testmain_test.go b/internal/e2e/suites/imagefs/testmain_test.go new file mode 100644 index 0000000000..e22b83e12d --- /dev/null +++ b/internal/e2e/suites/imagefs/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagefs + +import ( + "os" + "testing" + + "go.ate.dev/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index db37c4c2e3..126a00deec 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -37,13 +37,15 @@ What it buys, concretely: ## The privilege split The design is shaped by an existing substrate boundary: **atelet runs as -plain root with every Linux capability dropped** ("atelet does no mounts" — -see `manifests/ate-install/atelet.yaml`), while the **worker pods are -privileged** and own all mounts on the node. The module is split accordingly: +root with every Linux capability dropped but a few for files other users +own** ("atelet does no mounts" — see +[`manifests/ate-install/atelet.yaml`](../../manifests/ate-install/atelet.yaml), +which gives the reason for each), while the **worker pods hold the mount +capabilities** and own all mounts on the node. The module is split accordingly: | Half | Runs in | Files | Needs | |---|---|---|---| -| Store: pull, parse, unpack, record | atelet | `imagecache.go`, `unpack.go`, `spec.go` (portable) | nothing but file I/O | +| Store: pull, parse, unpack, record | atelet | `imagecache.go`, `unpack.go`, `spec.go` (portable) | file I/O; `CAP_CHOWN`, `CAP_DAC_OVERRIDE`, `CAP_FOWNER`, `CAP_FSETID` | | Consumer: finalize, mount, unmount | ateworker-gvisor / ateworker-microvm | `bundle_linux.go` (`//go:build linux`) | `CAP_MKNOD`, `CAP_SYS_ADMIN` | The two halves communicate through the filesystem only: the shared cache @@ -59,7 +61,7 @@ anywhere. ``` / default: /var/lib/ate/image-cache - version layout version marker ("2") + version layout version marker ("3") layers/sha256// fs/ the unpacked layer tree (an overlay lowerdir) whiteouts.json whiteout state recorded at unpack time @@ -101,12 +103,18 @@ layer diffIDs in order — layers shared by N images exist once. confinement (path traversal and symlink/hardlink escapes are refused), "later entry wins" within a layer, read-only-dir handling that works without `CAP_DAC_OVERRIDE`, and creation of parent directories that the - layer tar omits (they may exist only in lower layers). Character and - block device nodes are skipped (atelet runs without `CAP_MKNOD` and the - runtime mounts `/dev`). Whiteout entries (`.wh.*`) are **not** written - into the tree — overlayfs whiteouts are char devices atelet cannot - create — they are recorded in `whiteouts.json` for the consumer to - materialize. + layer tar omits (they may exist only in lower layers). Every entry keeps + the numeric owner and the exact mode its tar header records, setuid, + setgid and sticky bits included, as containerd and moby apply them + (see `unpackLayer` for how a failed chown or chmod is handled). A root + atelet inside a user namespace (rootless or userns-remapped nodes) can + only give files to uids and gids the namespace maps, so a layer with an + owner outside that range fails to unpack, as it does under rootless + Docker. Character and block device nodes are skipped (atelet runs + without `CAP_MKNOD` and the runtime mounts `/dev`). Whiteout entries + (`.wh.*`) are **not** written into the tree — overlayfs whiteouts are + char devices atelet cannot create — they are recorded in + `whiteouts.json` for the consumer to materialize. 5. **Verify** before the layer enters the pool. Images share layers by diffID, but each image's author chooses the diffIDs in its config. So the uncompressed stream is hashed as it unpacks and must match the @@ -117,7 +125,8 @@ layer diffIDs in order — layers shared by N images exist once. A layer already in the pool is reused without re-hashing, because it was checked when it was written. The reusing image's own blob is never downloaded, so it is never checked. atelet refuses a layout version 1 - pool, whose layers may not have been checked. + pool, whose layers may not have been checked, and a version 2 pool, + whose layers are root-owned without setuid, setgid or sticky bits. 6. **Record**: the image config + diffID list is written under the requested digest (and the per-platform child digest for multi-arch refs). diff --git a/internal/imagecache/ateletcaps_test.go b/internal/imagecache/ateletcaps_test.go new file mode 100644 index 0000000000..26894ef135 --- /dev/null +++ b/internal/imagecache/ateletcaps_test.go @@ -0,0 +1,199 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "bytes" + "os" + "os/exec" + "regexp" + "runtime" + "slices" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" +) + +// Capability bits, from linux/capability.h. +const ( + capChown = 0 + capDacOverride = 1 + capFowner = 3 + capFsetid = 4 + capSysAdmin = 21 + capMknod = 27 +) + +// capSet is a capability set a root-gated test runs under, as root with +// every other capability dropped. +type capSet struct { + // bounding is the set in setpriv(1)'s --bounding-set syntax. + bounding string + // eff is the effective set the child process must report. + eff uint64 +} + +var ( + // ateletCaps is the set manifests/ate-install/atelet.yaml grants atelet. + ateletCaps = capSet{ + bounding: "-all,+chown,+dac_override,+fowner,+fsetid", + eff: 1<