feat(atelet): pull actor images with dockerconfigjson Secrets

charts/substrate/README.md

@@ -53,6 +53,7 @@ See `values.yaml` for the full set; the important keys:
 | `atelet.imageCredentialProviderConfig` | `""` | Host path to the kubelet credential provider config; set together with the bin directory |
 | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path |
 | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only |
+| `atelet.imagePullSecrets` | `[]` | `kubernetes.io/dockerconfigjson` Secrets (`[{name: ...}]`) in the release namespace whose credentials atelet pulls actor images with, before the credential provider's; mounted read-only, rotation needs no restart |
 | `atelet.imageCache.{gcPeriod,highPercent,lowPercent,minAge,maxBytes}` | `null` (atelet's defaults: `5m`, `85`, `80`, `2m`, no cap) | atelet's image-cache eviction pass: `--image-cache-gc-period`, `--image-cache-high-percent`, `--image-cache-low-percent`, `--image-cache-min-age`, `--image-cache-max-bytes` |
 | `atelet.imageCache.pinnedImages` | `[]` | Image references every pass pulls and never evicts (the ConfigMap `atelet-pinned-images`, mounted for `--image-cache-pinned-images-file`): the pool's ActorTemplate images on nodes whose cache volume stays above the watermark. atelet reads the list at every pass, so a change restarts no atelet pod |
 | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider. A policy with an `ownNamespaceSelector` (a label selector) in place of `atespace` and `allowedNamespaces` lets each namespace it matches be read by the atespace of the same name only, matched by the namespace's current labels; an `allowedNamespaceSelector` beside `atespace` grants that atespace every namespace it matches (an explicit opt-in, the only way `ate-golden` reads a namespace by label); with either, the chart adds list/watch RBAC on namespaces |

charts/substrate/templates/atelet.yaml

@@ -193,6 +193,9 @@ spec:
 {{- if $credentialProviderConfig }}
         - --image-credential-provider-config=/run/image-credential-provider/config.yaml
         - --image-credential-provider-bin-dir=/run/image-credential-provider/bin
+{{- end }}
+{{- range .Values.atelet.imagePullSecrets }}
+        - --image-pull-secret-file=/run/image-pull-secrets/{{ required "atelet.imagePullSecrets entries need a name" .name }}/.dockerconfigjson
 {{- end }}
         - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem
         - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem
@@ -310,6 +313,11 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
+{{- end }}
+{{- range $i, $secret := .Values.atelet.imagePullSecrets }}
+        - name: image-pull-secret-{{ $i }}
+          mountPath: /run/image-pull-secrets/{{ $secret.name }}
+          readOnly: true
 {{- end }}
         # Mounted whole, never through subPath: kubelet then swaps in a
         # changed list in place.
@@ -347,6 +355,14 @@ spec:
           path: {{ .Values.atelet.imageCredentialProviderConfig | quote }}
           type: File
 {{- end }}
+{{- end }}
+{{- range $i, $secret := .Values.atelet.imagePullSecrets }}
+      - name: image-pull-secret-{{ $i }}
+        secret:
+          secretName: {{ $secret.name | quote }}
+          items:
+          - key: .dockerconfigjson
+            path: .dockerconfigjson
 {{- end }}
       - name: pinned-images
         configMap:

charts/substrate/values.yaml

@@ -97,6 +97,12 @@ atelet:
   # either one still needs the bin directory.
   imageCredentialProviderConfigYAML: ""
   imageCredentialProviderBinDir: ""
+  # kubernetes.io/dockerconfigjson Secrets in the release namespace whose
+  # credentials atelet uses to pull actor images, tried before the credential
+  # provider's: [{name: my-registry}]. Each is mounted read-only, so a rotated
+  # Secret takes effect without a restart. Unlike imagePullSecrets, which
+  # pull the chart's own pods, these pull the images the actors run.
+  imagePullSecrets: []
   storageBackend: s3
   extraArgs: []
   extraEnv: []

cmd/atelet/helm_test.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"os/exec"
+	"reflect"
 	"slices"
 	"strings"
 	"testing"
@@ -136,3 +137,61 @@ providers: []`
 		})
 	}
 }
+
+func TestHelmImagePullSecrets(t *testing.T) {
+	if _, err := exec.LookPath("helm"); err != nil {
+		t.Skip("helm is not installed")
+	}
+	out, err := exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate",
+		"--show-only", "templates/atelet.yaml",
+		"--set", "atelet.imagePullSecrets[0].name=registry-a",
+		"--set", "atelet.imagePullSecrets[1].name=registry-b").CombinedOutput()
+	if err != nil {
+		t.Fatalf("render: %v\n%s", err, out)
+	}
+	var pod *corev1.PodSpec
+	for _, doc := range strings.Split(string(out), "\n---\n") {
+		var ds appsv1.DaemonSet
+		if err := yaml.Unmarshal([]byte(doc), &ds); err != nil {
+			t.Fatal(err)
+		}
+		if ds.Kind == "DaemonSet" {
+			pod = &ds.Spec.Template.Spec
+		}
+	}
+	if pod == nil || len(pod.Containers) != 1 {
+		t.Fatal("expected atelet DaemonSet with one container")
+	}
+	container := pod.Containers[0]
+	var flagged []string
+	for _, arg := range container.Args {
+		if file, ok := strings.CutPrefix(arg, "--image-pull-secret-file="); ok {
+			flagged = append(flagged, file)
+		}
+	}
+	if want := []string{"/run/image-pull-secrets/registry-a/.dockerconfigjson", "/run/image-pull-secrets/registry-b/.dockerconfigjson"}; !slices.Equal(flagged, want) {
+		t.Fatalf("--image-pull-secret-file flags = %v, want %v in order", flagged, want)
+	}
+	for i, secret := range []string{"registry-a", "registry-b"} {
+		mountIndex := slices.IndexFunc(container.VolumeMounts, func(m corev1.VolumeMount) bool {
+			return m.MountPath == "/run/image-pull-secrets/"+secret
+		})
+		if mountIndex < 0 || !container.VolumeMounts[mountIndex].ReadOnly || container.VolumeMounts[mountIndex].SubPath != "" {
+			t.Fatalf("Secret %s must mount read-only without subPath, so rotation reaches atelet", secret)
+		}
+		volumeIndex := slices.IndexFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == container.VolumeMounts[mountIndex].Name })
+		if volumeIndex < 0 || pod.Volumes[volumeIndex].Secret == nil {
+			t.Fatalf("mount %d has no Secret volume", i)
+		}
+		src := pod.Volumes[volumeIndex].Secret
+		if src.SecretName != secret || !reflect.DeepEqual(src.Items, []corev1.KeyToPath{{Key: ".dockerconfigjson", Path: ".dockerconfigjson"}}) {
+			t.Errorf("volume for %s projects %+v", secret, src)
+		}
+	}
+
+	out, err = exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate",
+		"--show-only", "templates/atelet.yaml", "--set", "atelet.imagePullSecrets[0].other=x").CombinedOutput()
+	if err == nil || !strings.Contains(string(out), "need a name") {
+		t.Errorf("render accepted a pull secret without a name: %v\n%s", err, out)
+	}
+}

cmd/atelet/internal/credentialprovider/credentialprovider.go

@@ -72,14 +72,7 @@ func (k *Keychain) Resolve(target authn.Resource) (authn.Authenticator, error) {
 // ResolveContext implements authn.ContextKeychain. It returns the first
 // candidate, or authn.Anonymous if there are none.
 func (k *Keychain) ResolveContext(ctx context.Context, target authn.Resource) (authn.Authenticator, error) {
-	candidates, err := k.Candidates(ctx, target)
-	if err != nil {
-		return nil, err
-	}
-	if len(candidates) == 0 {
-		return authn.Anonymous, nil
-	}
-	return candidates[0], nil
+	return firstCandidate(k.Candidates(ctx, target))
 }
 
 // Candidates returns the credentials of all providers matching target, in the

cmd/atelet/internal/credentialprovider/dockerconfig.go

@@ -0,0 +1,199 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package credentialprovider
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"os"
+	"slices"
+	"strings"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+)
+
+// DockerConfigKeychain resolves registry credentials from .dockerconfigjson
+// files, the content of a kubernetes.io/dockerconfigjson Secret, matched to an
+// image like the kubelet matches a Pod's imagePullSecrets. Every lookup reads
+// the files again, so a rotated Secret mounted into the pod takes effect
+// without a restart. It is safe for concurrent use.
+type DockerConfigKeychain struct {
+	paths []string
+}
+
+var (
+	_ authn.Keychain        = (*DockerConfigKeychain)(nil)
+	_ authn.ContextKeychain = (*DockerConfigKeychain)(nil)
+)
+
+// NewDockerConfig returns a keychain over the .dockerconfigjson files at
+// paths, tried in order. Each file must parse now, so a misconfigured mount
+// fails at start-up instead of at the first private pull.
+func NewDockerConfig(paths []string) (*DockerConfigKeychain, error) {
+	for _, path := range paths {
+		if _, err := readDockerConfig(path); err != nil {
+			return nil, err
+		}
+	}
+	slog.Info("Loaded image pull secrets", slog.Any("files", paths))
+	return &DockerConfigKeychain{paths: paths}, nil
+}
+
+// Resolve implements authn.Keychain.
+func (k *DockerConfigKeychain) Resolve(target authn.Resource) (authn.Authenticator, error) {
+	return k.ResolveContext(context.Background(), target)
+}
+
+// ResolveContext implements authn.ContextKeychain. It returns the first
+// candidate, or authn.Anonymous if there are none.
+func (k *DockerConfigKeychain) ResolveContext(ctx context.Context, target authn.Resource) (authn.Authenticator, error) {
+	return firstCandidate(k.Candidates(ctx, target))
+}
+
+// Candidates returns the credentials of every file entry matching target, in
+// the order to try them: most specific registry key first, then file order.
+// A file that cannot be read any more is logged and skipped.
+func (k *DockerConfigKeychain) Candidates(ctx context.Context, target authn.Resource) ([]authn.Authenticator, error) {
+	image := target.String()
+	byKey := make(map[string][]authn.AuthConfig)
+	for _, path := range k.paths {
+		auths, err := readDockerConfig(path)
+		if err != nil {
+			slog.WarnContext(ctx, "Image pull secret unreadable; skipping it",
+				slog.String("file", path), slog.Any("err", err))
+			continue
+		}
+		keys, err := matchingAuthKeys(auths, image)
+		if err != nil {
+			slog.WarnContext(ctx, "Image pull secret has an invalid registry key; skipping it",
+				slog.String("file", path), slog.String("image", image), slog.Any("err", err))
+			continue
+		}
+		if len(keys) > 0 {
+			slog.InfoContext(ctx, "Resolved image credentials from image pull secret",
+				slog.String("file", path), slog.String("image", image))
+		}
+		for _, key := range keys {
+			byKey[key] = append(byKey[key], auths[key]...)
+		}
+	}
+
+	keys := make([]string, 0, len(byKey))
+	for key := range byKey {
+		keys = append(keys, key)
+	}
+	sortAuthKeys(keys)
+	var candidates []authn.Authenticator
+	seen := make(map[authn.AuthConfig]struct{})
+	for _, key := range keys {
+		for _, auth := range byKey[key] {
+			if _, dup := seen[auth]; dup {
+				continue
+			}
+			seen[auth] = struct{}{}
+			candidates = append(candidates, authn.FromConfig(auth))
+		}
+	}
+	return candidates, nil
+}
+
+// readDockerConfig returns the auths of the .dockerconfigjson file at path,
+// keyed by registry in the scheme-less form matchesImage compares. Keys that
+// differ only in scheme or trailing slash share an entry, in key order.
+func readDockerConfig(path string) (map[string][]authn.AuthConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("while reading image pull secret: %w", err)
+	}
+	var config struct {
+		Auths map[string]authn.AuthConfig `json:"auths"`
+	}
+	if err := json.Unmarshal(data, &config); err != nil {
+		return nil, fmt.Errorf("while parsing image pull secret %s: %w", path, err)
+	}
+	auths := make(map[string][]authn.AuthConfig, len(config.Auths))
+	for _, key := range slices.Sorted(maps.Keys(config.Auths)) {
+		auth := config.Auths[key]
+		// The encoded form is derived from Username and Password; dropping it
+		// keeps equal credentials equal for deduplication.
+		auth.Auth = ""
+		normalized := normalizeRegistryKey(key)
+		auths[normalized] = append(auths[normalized], auth)
+	}
+	return auths, nil
+}
+
+// normalizeRegistryKey strips a registry key's scheme and trailing slash and
+// maps Docker Hub's aliases (the legacy "https://index.docker.io/v1/" that
+// docker login writes among them) onto the name go-containerregistry gives
+// its repositories, "index.docker.io".
+func normalizeRegistryKey(key string) string {
+	key = strings.TrimPrefix(strings.TrimPrefix(key, "https://"), "http://")
+	key = strings.TrimSuffix(key, "/")
+	switch key {
+	case "docker.io", "index.docker.io/v1", "registry-1.docker.io":
+		return "index.docker.io"
+	}
+	return key
+}
+
+// CandidateSource is a keychain that returns several credentials to try.
+type CandidateSource interface {
+	Candidates(ctx context.Context, target authn.Resource) ([]authn.Authenticator, error)
+}
+
+// Chain is a keychain that tries the candidates of its keychains in order.
+type Chain []CandidateSource
+
+var (
+	_ authn.Keychain        = Chain(nil)
+	_ authn.ContextKeychain = Chain(nil)
+)
+
+// Resolve implements authn.Keychain.
+func (c Chain) Resolve(target authn.Resource) (authn.Authenticator, error) {
+	return c.ResolveContext(context.Background(), target)
+}
+
+// ResolveContext implements authn.ContextKeychain.
+func (c Chain) ResolveContext(ctx context.Context, target authn.Resource) (authn.Authenticator, error) {
+	return firstCandidate(c.Candidates(ctx, target))
+}
+
+// Candidates returns every keychain's candidates, the first keychain's first.
+func (c Chain) Candidates(ctx context.Context, target authn.Resource) ([]authn.Authenticator, error) {
+	var candidates []authn.Authenticator
+	for _, kc := range c {
+		got, err := kc.Candidates(ctx, target)
+		if err != nil {
+			return nil, err
+		}
+		candidates = append(candidates, got...)
+	}
+	return candidates, nil
+}
+
+func firstCandidate(candidates []authn.Authenticator, err error) (authn.Authenticator, error) {
+	if err != nil {
+		return nil, err
+	}
+	if len(candidates) == 0 {
+		return authn.Anonymous, nil
+	}
+	return candidates[0], nil
+}