Mirror container images with proxy mirror

README.md

@@ -1002,6 +1002,9 @@ proxy mirror --dry-run pkg:npm/lodash
 
 # Control parallelism
 proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
+
+# Mirror a container image for one platform
+proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
 ```
 
 The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped.

cmd/proxy/main.go

@@ -113,11 +113,9 @@ import (
 	"github.com/git-pkgs/proxy/internal/config"
 	"github.com/git-pkgs/proxy/internal/database"
 	"github.com/git-pkgs/proxy/internal/denylist"
-	"github.com/git-pkgs/proxy/internal/handler"
 	"github.com/git-pkgs/proxy/internal/mirror"
 	"github.com/git-pkgs/proxy/internal/server"
 	"github.com/git-pkgs/proxy/internal/storage"
-	"github.com/git-pkgs/registries/fetch"
 )
 
 const defaultTopN = 10
@@ -421,20 +419,30 @@ func runMirror() {
 	sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
 	concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
 	dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
+	platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
 
 	fs.Usage = func() {
 		fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
 		fmt.Fprintf(os.Stderr, "Usage: proxy mirror [flags] [purl...]\n\n")
 		fmt.Fprintf(os.Stderr, "Examples:\n")
 		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash@4.17.21\n")
 		fmt.Fprintf(os.Stderr, "  proxy mirror --sbom sbom.cdx.json\n")
-		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash  # all versions\n\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash  # all versions\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
 		fmt.Fprintf(os.Stderr, "Flags:\n")
 		fs.PrintDefaults()
 	}
 
 	_ = fs.Parse(os.Args[1:])
 	purls := fs.Args()
+	var platformList []string
+	if *platforms != "" {
+		platformList = []string{*platforms}
+	}
+	if _, err := mirror.ParsePlatforms(platformList); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
 
 	// Determine source
 	var source mirror.Source
@@ -445,9 +453,9 @@ func runMirror() {
 			fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
 			os.Exit(1)
 		}
-		source = &mirror.SBOMSource{Data: data, Name: *sbomPath}
+		source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList}
 	case len(purls) > 0:
-		source = &mirror.PURLSource{PURLs: purls}
+		source = &mirror.PURLSource{PURLs: purls, Platforms: platformList}
 	default:
 		fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
 		fs.Usage()
@@ -518,10 +526,8 @@ func runMirror() {
 		os.Exit(1) //nolint:gocritic // db closed above
 	}
 
-	// Build proxy (reuses same pipeline as serve)
-	fetcher := fetch.NewFetcher()
-	resolver := fetch.NewResolver()
-	proxy := handler.NewProxy(db, store, handler.LimitArtifactSize(fetcher, cfg.ParseArtifactMaxSize()), resolver, logger)
+	// Build proxy with the same upstream clients and authentication as serve
+	proxy, images := server.NewMirrorProxy(cfg, db, store, logger)
 	proxy.Denylist, err = denylist.New(cfg.Denylist.Packages)
 	if err != nil {
 		_ = db.Close()
@@ -533,6 +539,12 @@ func runMirror() {
 	proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
 
 	m := mirror.New(proxy, db, store, logger, *concurrency)
+	m.Images = images
+	if cfg.Scanning.Enabled {
+		// Scanners fetch staged layers from the running server, which this
+		// command does not start.
+		m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"}
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {

docs/architecture.md

@@ -327,7 +327,7 @@ Package metadata enrichment. Fetches license, description, homepage, repository
 
 ### `internal/mirror`
 
-Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work.
+Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads.
 
 The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
 

docs/configuration.md

@@ -1002,6 +1002,14 @@ curl -X POST http://localhost:8080/api/mirror \
   -d '{"sbom":{"bomFormat":"CycloneDX","components":[{"purl":"pkg:npm/lodash@4.17.21"}]}}'
 ```
 
+`platforms` limits which platforms of container images are mirrored, like the command's `--platform` flag:
+
+```bash
+curl -X POST http://localhost:8080/api/mirror \
+  -H "Content-Type: application/json" \
+  -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}'
+```
+
 ## Mirror Command
 
 The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`.
@@ -1011,6 +1019,7 @@ The `proxy mirror` command pre-populates the cache from various sources. It acce
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |
 | `--concurrency` | `4` | Number of parallel downloads |
 | `--dry-run` | `false` | Show what would be mirrored without downloading |
+| `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored |
 | `--config` | | Path to configuration file |
 | `--storage-url` | | Storage URL |
 | `--database-driver` | | Database driver |
@@ -1023,6 +1032,24 @@ Positional arguments are treated as PURLs:
 proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
 ```
 
+### Container images
+
+`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.
+
+```bash
+proxy mirror --platform linux/amd64,linux/arm64 pkg:docker/library/alpine@3.20
+proxy mirror 'pkg:docker/containerd/alpine@3.14.0?repository_url=ghcr.io'
+proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian&tag=bookworm'
+```
+
+- The version is a tag or a digest. Unversioned image PURLs fail; mirroring all tags is not supported.
+- Without `repository_url` the name is routed like a pull without containerd's `ns` parameter: the default OCI registry, repository prefix routes and `upstream/{name}/`. Single-segment Docker Hub names get the `library/` prefix.
+- `repository_url` is resolved like containerd's `ns` parameter, so it has to name the default registry, Docker Hub, a configured `upstream.oci` host or a host listed in `upstream.oci_mirrors`. Other hosts are refused.
+- `pkg:oci` needs `repository_url`. With `tag=` the tag is mirrored as well; if it now points to another digest, both are mirrored and a warning is logged. `arch=` limits the platforms when `--platform` is not given.
+- Without `--platform` every platform of an index is mirrored, including Windows images, which can be large. Attestation manifests of the selected platforms are mirrored too, because Docker fetches them on every pull.
+- Non-distributable (foreign) layers are skipped.
+- While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead.
+
 ## Docker
 
 ### SQLite with Local Storage

internal/handler/container_manifest.go

@@ -8,6 +8,7 @@ import (
 	"mime"
 	"net/http"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -23,6 +24,14 @@ const (
 	containerAcceptExactSpecificity
 )
 
+const (
+	// containerManifestIndexCacheClass replaces the Accept header in tag cache
+	// keys for clients that accept an index type.
+	containerManifestIndexCacheClass = "index"
+	ociImageIndexMediaType           = "application/vnd.oci.image.index.v1+json"
+	dockerManifestListMediaType      = "application/vnd.docker.distribution.manifest.list.v2+json"
+)
+
 var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`)
 
 type cachedContainerManifest struct {
@@ -38,8 +47,7 @@ type cachedContainerManifest struct {
 func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) {
 	accept := containerManifestAccept(r)
 	cacheAccept := normalizeContainerManifestAccept(accept)
-	cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
-	cached := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheKey)
+	cached, fromFallback := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept)
 
 	immutable := manifestDigestReferencePattern.MatchString(reference)
 	if cached != nil && (immutable || h.containerManifestFresh(cached)) {
@@ -67,7 +75,13 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request,
 
 	if resp.StatusCode == http.StatusNotModified && cached != nil {
 		cached.fetchedAt = time.Now()
-		h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
+		if fromFallback {
+			// The shared entry holds another representation for other
+			// clients; only this client's own entries are refreshed.
+			h.storeContainerManifestLegacyEntries(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
+		} else {
+			h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
+		}
 		writeContainerManifest(w, r, cached, false)
 		return
 	}
@@ -140,49 +154,95 @@ func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, referenc
 	return hex.EncodeToString(sum[:])
 }
 
-func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheKey string) *cachedContainerManifest {
-	cached, err := h.loadContainerManifest(ctx, cacheKey)
-	if err != nil {
-		h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
-		return nil
-	}
-	if cached != nil {
-		if containerManifestCacheCompatible(accept, cached) {
-			return cached
+// containerManifestSharedCacheKey is the key clients with different Accept
+// headers share. A digest names exactly one document. A tag is stored once for
+// all clients that accept an index type, whatever upstream returned for it, so
+// Docker, containerd and Podman read the same entry. Clients without index
+// support keep a key per normalized Accept header, because registries may
+// convert an index into a single platform manifest for them.
+func (h *ContainerHandler) containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept string) string {
+	if manifestDigestReferencePattern.MatchString(reference) {
+		identity := strings.Join([]string{registryURL, name, reference}, "\x00")
+		sum := sha256.Sum256([]byte(identity))
+		return hex.EncodeToString(sum[:])
+	}
+	if containerManifestAcceptsIndex(cacheAccept) {
+		return h.containerManifestCacheKey(registryURL, name, reference, containerManifestIndexCacheClass)
+	}
+	return h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
+}
+
+func containerManifestAcceptsIndex(accept string) bool {
+	return containerManifestAccepts(accept, ociImageIndexMediaType) || containerManifestAccepts(accept, dockerManifestListMediaType)
+}
+
+// containerManifestLegacyCacheKeys are the per-Accept keys older versions
+// used. They are still written so a rollback finds the cache, and read when
+// the shared key has no entry yet.
+func (h *ContainerHandler) containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, sharedKey string) []string {
+	var keys []string
+	for _, variant := range []string{cacheAccept, accept} {
+		key := h.containerManifestCacheKey(registryURL, name, reference, variant)
+		if key != sharedKey && !slices.Contains(keys, key) {
+			keys = append(keys, key)
 		}
-		return nil
 	}
+	return keys
+}
 
-	legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
-	if legacyCacheKey == cacheKey {
-		return nil
-	}
-	cached, err = h.loadContainerManifest(ctx, legacyCacheKey)
+// loadContainerManifestForAccept returns the cached manifest for the client and
+// whether it came from a per-Accept fallback key while the shared key holds a
+// representation the client does not accept.
+func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) (*cachedContainerManifest, bool) {
+	cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
+	cached, err := h.loadContainerManifest(ctx, cacheKey)
 	if err != nil {
-		h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err)
-		return nil
+		h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
+		return nil, false
 	}
-	if cached == nil || !containerManifestCacheCompatible(accept, cached) {
-		return nil
+	if cached != nil && containerManifestCacheCompatible(accept, cached) {
+		return cached, false
 	}
-	if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil {
-		h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err)
+	// The shared entry may hold a representation this client does not accept,
+	// for example an OCI index for a client that only accepts Docker manifest
+	// lists. Its own per-Accept entry is still written, so look there.
+	sharedEntryExists := cached != nil
+
+	for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) {
+		cached, err = h.loadContainerManifest(ctx, legacyCacheKey)
+		if err != nil {
+			h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err)
+			return nil, false
+		}
+		if cached == nil || !containerManifestCacheCompatible(accept, cached) {
+			continue
+		}
+		if !sharedEntryExists {
+			if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil {
+				h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err)
+			}
+		}
+		return cached, sharedEntryExists
 	}
-	return cached
+	return nil, false
 }
 
 func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) {
-	cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
+	cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
 	if err := h.storeContainerManifest(ctx, cacheKey, manifest); err != nil {
 		h.proxy.Logger.Warn("failed to cache container manifest", "error", err)
 	}
 
-	legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
-	if legacyCacheKey == cacheKey {
-		return
-	}
-	if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil {
-		h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err)
+	h.storeContainerManifestLegacyEntries(ctx, registryURL, name, reference, accept, cacheAccept, manifest)
+}
+
+// storeContainerManifestLegacyEntries writes only the per-Accept keys.
+func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) {
+	cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
+	for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) {
+		if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil {
+			h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err)
+		}
 	}
 }