Skip to content

Commit ba81c1d

Browse files
authored
Merge pull request #438 from pinguinfuss/issue-411-mirror-images
Mirror container images with proxy mirror
2 parents 784402e + 02ae49d commit ba81c1d

16 files changed

Lines changed: 1803 additions & 75 deletions

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,9 @@ proxy mirror --dry-run pkg:npm/lodash
10021002
10031003
# Control parallelism
10041004
proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
1005+
1006+
# Mirror a container image for one platform
1007+
proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
10051008
```
10061009

10071010
The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped.

‎cmd/proxy/main.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,9 @@ import (
113113
"github.com/git-pkgs/proxy/internal/config"
114114
"github.com/git-pkgs/proxy/internal/database"
115115
"github.com/git-pkgs/proxy/internal/denylist"
116-
"github.com/git-pkgs/proxy/internal/handler"
117116
"github.com/git-pkgs/proxy/internal/mirror"
118117
"github.com/git-pkgs/proxy/internal/server"
119118
"github.com/git-pkgs/proxy/internal/storage"
120-
"github.com/git-pkgs/registries/fetch"
121119
)
122120

123121
const defaultTopN = 10
@@ -421,20 +419,30 @@ func runMirror() {
421419
sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
422420
concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
423421
dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
422+
platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
424423

425424
fs.Usage = func() {
426425
fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
427426
fmt.Fprintf(os.Stderr, "Usage: proxy mirror [flags] [purl...]\n\n")
428427
fmt.Fprintf(os.Stderr, "Examples:\n")
429428
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n")
430429
fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n")
431-
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n\n")
430+
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n")
431+
fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
432432
fmt.Fprintf(os.Stderr, "Flags:\n")
433433
fs.PrintDefaults()
434434
}
435435

436436
_ = fs.Parse(os.Args[1:])
437437
purls := fs.Args()
438+
var platformList []string
439+
if *platforms != "" {
440+
platformList = []string{*platforms}
441+
}
442+
if _, err := mirror.ParsePlatforms(platformList); err != nil {
443+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
444+
os.Exit(1)
445+
}
438446

439447
// Determine source
440448
var source mirror.Source
@@ -445,9 +453,9 @@ func runMirror() {
445453
fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
446454
os.Exit(1)
447455
}
448-
source = &mirror.SBOMSource{Data: data, Name: *sbomPath}
456+
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList}
449457
case len(purls) > 0:
450-
source = &mirror.PURLSource{PURLs: purls}
458+
source = &mirror.PURLSource{PURLs: purls, Platforms: platformList}
451459
default:
452460
fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
453461
fs.Usage()
@@ -518,10 +526,8 @@ func runMirror() {
518526
os.Exit(1) //nolint:gocritic // db closed above
519527
}
520528

521-
// Build proxy (reuses same pipeline as serve)
522-
fetcher := fetch.NewFetcher()
523-
resolver := fetch.NewResolver()
524-
proxy := handler.NewProxy(db, store, handler.LimitArtifactSize(fetcher, cfg.ParseArtifactMaxSize()), resolver, logger)
529+
// Build proxy with the same upstream clients and authentication as serve
530+
proxy, images := server.NewMirrorProxy(cfg, db, store, logger)
525531
proxy.Denylist, err = denylist.New(cfg.Denylist.Packages)
526532
if err != nil {
527533
_ = db.Close()
@@ -533,6 +539,12 @@ func runMirror() {
533539
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
534540

535541
m := mirror.New(proxy, db, store, logger, *concurrency)
542+
m.Images = images
543+
if cfg.Scanning.Enabled {
544+
// Scanners fetch staged layers from the running server, which this
545+
// command does not start.
546+
m.Images = mirror.UnavailableImages{Reason: "container images cannot be mirrored by this command while scanning is enabled; use the mirror API of the running server"}
547+
}
536548

537549
ctx, cancel := context.WithCancel(context.Background())
538550
go func() {

‎docs/architecture.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ Package metadata enrichment. Fetches license, description, homepage, repository
327327

328328
### `internal/mirror`
329329

330-
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work.
330+
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads.
331331

332332
The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
333333

‎docs/configuration.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,14 @@ curl -X POST http://localhost:8080/api/mirror \
10021002
-d '{"sbom":{"bomFormat":"CycloneDX","components":[{"purl":"pkg:npm/lodash@4.17.21"}]}}'
10031003
```
10041004

1005+
`platforms` limits which platforms of container images are mirrored, like the command's `--platform` flag:
1006+
1007+
```bash
1008+
curl -X POST http://localhost:8080/api/mirror \
1009+
-H "Content-Type: application/json" \
1010+
-d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}'
1011+
```
1012+
10051013
## Mirror Command
10061014

10071015
The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`.
@@ -1011,6 +1019,7 @@ The `proxy mirror` command pre-populates the cache from various sources. It acce
10111019
| `--sbom` | | Path to CycloneDX or SPDX SBOM file |
10121020
| `--concurrency` | `4` | Number of parallel downloads |
10131021
| `--dry-run` | `false` | Show what would be mirrored without downloading |
1022+
| `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored |
10141023
| `--config` | | Path to configuration file |
10151024
| `--storage-url` | | Storage URL |
10161025
| `--database-driver` | | Database driver |
@@ -1023,6 +1032,24 @@ Positional arguments are treated as PURLs:
10231032
proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
10241033
```
10251034

1035+
### Container images
1036+
1037+
`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.
1038+
1039+
```bash
1040+
proxy mirror --platform linux/amd64,linux/arm64 pkg:docker/library/alpine@3.20
1041+
proxy mirror 'pkg:docker/containerd/alpine@3.14.0?repository_url=ghcr.io'
1042+
proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian&tag=bookworm'
1043+
```
1044+
1045+
- The version is a tag or a digest. Unversioned image PURLs fail; mirroring all tags is not supported.
1046+
- Without `repository_url` the name is routed like a pull without containerd's `ns` parameter: the default OCI registry, repository prefix routes and `upstream/{name}/`. Single-segment Docker Hub names get the `library/` prefix.
1047+
- `repository_url` is resolved like containerd's `ns` parameter, so it has to name the default registry, Docker Hub, a configured `upstream.oci` host or a host listed in `upstream.oci_mirrors`. Other hosts are refused.
1048+
- `pkg:oci` needs `repository_url`. With `tag=` the tag is mirrored as well; if it now points to another digest, both are mirrored and a warning is logged. `arch=` limits the platforms when `--platform` is not given.
1049+
- Without `--platform` every platform of an index is mirrored, including Windows images, which can be large. Attestation manifests of the selected platforms are mirrored too, because Docker fetches them on every pull.
1050+
- Non-distributable (foreign) layers are skipped.
1051+
- While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead.
1052+
10261053
## Docker
10271054

10281055
### SQLite with Local Storage

‎internal/handler/container_manifest.go‎

Lines changed: 92 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"mime"
99
"net/http"
1010
"regexp"
11+
"slices"
1112
"sort"
1213
"strconv"
1314
"strings"
@@ -23,6 +24,14 @@ const (
2324
containerAcceptExactSpecificity
2425
)
2526

27+
const (
28+
// containerManifestIndexCacheClass replaces the Accept header in tag cache
29+
// keys for clients that accept an index type.
30+
containerManifestIndexCacheClass = "index"
31+
ociImageIndexMediaType = "application/vnd.oci.image.index.v1+json"
32+
dockerManifestListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json"
33+
)
34+
2635
var manifestDigestReferencePattern = regexp.MustCompile(`^[a-z0-9]+:[a-f0-9]+$`)
2736

2837
type cachedContainerManifest struct {
@@ -38,8 +47,7 @@ type cachedContainerManifest struct {
3847
func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) {
3948
accept := containerManifestAccept(r)
4049
cacheAccept := normalizeContainerManifestAccept(accept)
41-
cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
42-
cached := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheKey)
50+
cached, fromFallback := h.loadContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept)
4351

4452
immutable := manifestDigestReferencePattern.MatchString(reference)
4553
if cached != nil && (immutable || h.containerManifestFresh(cached)) {
@@ -67,7 +75,13 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request,
6775

6876
if resp.StatusCode == http.StatusNotModified && cached != nil {
6977
cached.fetchedAt = time.Now()
70-
h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
78+
if fromFallback {
79+
// The shared entry holds another representation for other
80+
// clients; only this client's own entries are refreshed.
81+
h.storeContainerManifestLegacyEntries(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
82+
} else {
83+
h.storeContainerManifestForAccept(r.Context(), registryURL, name, reference, accept, cacheAccept, cached)
84+
}
7185
writeContainerManifest(w, r, cached, false)
7286
return
7387
}
@@ -140,49 +154,95 @@ func (h *ContainerHandler) containerManifestCacheKey(registryURL, name, referenc
140154
return hex.EncodeToString(sum[:])
141155
}
142156

143-
func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheKey string) *cachedContainerManifest {
144-
cached, err := h.loadContainerManifest(ctx, cacheKey)
145-
if err != nil {
146-
h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
147-
return nil
148-
}
149-
if cached != nil {
150-
if containerManifestCacheCompatible(accept, cached) {
151-
return cached
157+
// containerManifestSharedCacheKey is the key clients with different Accept
158+
// headers share. A digest names exactly one document. A tag is stored once for
159+
// all clients that accept an index type, whatever upstream returned for it, so
160+
// Docker, containerd and Podman read the same entry. Clients without index
161+
// support keep a key per normalized Accept header, because registries may
162+
// convert an index into a single platform manifest for them.
163+
func (h *ContainerHandler) containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept string) string {
164+
if manifestDigestReferencePattern.MatchString(reference) {
165+
identity := strings.Join([]string{registryURL, name, reference}, "\x00")
166+
sum := sha256.Sum256([]byte(identity))
167+
return hex.EncodeToString(sum[:])
168+
}
169+
if containerManifestAcceptsIndex(cacheAccept) {
170+
return h.containerManifestCacheKey(registryURL, name, reference, containerManifestIndexCacheClass)
171+
}
172+
return h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
173+
}
174+
175+
func containerManifestAcceptsIndex(accept string) bool {
176+
return containerManifestAccepts(accept, ociImageIndexMediaType) || containerManifestAccepts(accept, dockerManifestListMediaType)
177+
}
178+
179+
// containerManifestLegacyCacheKeys are the per-Accept keys older versions
180+
// used. They are still written so a rollback finds the cache, and read when
181+
// the shared key has no entry yet.
182+
func (h *ContainerHandler) containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, sharedKey string) []string {
183+
var keys []string
184+
for _, variant := range []string{cacheAccept, accept} {
185+
key := h.containerManifestCacheKey(registryURL, name, reference, variant)
186+
if key != sharedKey && !slices.Contains(keys, key) {
187+
keys = append(keys, key)
152188
}
153-
return nil
154189
}
190+
return keys
191+
}
155192

156-
legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
157-
if legacyCacheKey == cacheKey {
158-
return nil
159-
}
160-
cached, err = h.loadContainerManifest(ctx, legacyCacheKey)
193+
// loadContainerManifestForAccept returns the cached manifest for the client and
194+
// whether it came from a per-Accept fallback key while the shared key holds a
195+
// representation the client does not accept.
196+
func (h *ContainerHandler) loadContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string) (*cachedContainerManifest, bool) {
197+
cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
198+
cached, err := h.loadContainerManifest(ctx, cacheKey)
161199
if err != nil {
162-
h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err)
163-
return nil
200+
h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
201+
return nil, false
164202
}
165-
if cached == nil || !containerManifestCacheCompatible(accept, cached) {
166-
return nil
203+
if cached != nil && containerManifestCacheCompatible(accept, cached) {
204+
return cached, false
167205
}
168-
if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil {
169-
h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err)
206+
// The shared entry may hold a representation this client does not accept,
207+
// for example an OCI index for a client that only accepts Docker manifest
208+
// lists. Its own per-Accept entry is still written, so look there.
209+
sharedEntryExists := cached != nil
210+
211+
for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) {
212+
cached, err = h.loadContainerManifest(ctx, legacyCacheKey)
213+
if err != nil {
214+
h.proxy.Logger.Warn("failed to read legacy cached container manifest", "error", err)
215+
return nil, false
216+
}
217+
if cached == nil || !containerManifestCacheCompatible(accept, cached) {
218+
continue
219+
}
220+
if !sharedEntryExists {
221+
if err := h.storeContainerManifest(ctx, cacheKey, cached); err != nil {
222+
h.proxy.Logger.Warn("failed to migrate cached container manifest", "error", err)
223+
}
224+
}
225+
return cached, sharedEntryExists
170226
}
171-
return cached
227+
return nil, false
172228
}
173229

174230
func (h *ContainerHandler) storeContainerManifestForAccept(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) {
175-
cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
231+
cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
176232
if err := h.storeContainerManifest(ctx, cacheKey, manifest); err != nil {
177233
h.proxy.Logger.Warn("failed to cache container manifest", "error", err)
178234
}
179235

180-
legacyCacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
181-
if legacyCacheKey == cacheKey {
182-
return
183-
}
184-
if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil {
185-
h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err)
236+
h.storeContainerManifestLegacyEntries(ctx, registryURL, name, reference, accept, cacheAccept, manifest)
237+
}
238+
239+
// storeContainerManifestLegacyEntries writes only the per-Accept keys.
240+
func (h *ContainerHandler) storeContainerManifestLegacyEntries(ctx context.Context, registryURL, name, reference, accept, cacheAccept string, manifest *cachedContainerManifest) {
241+
cacheKey := h.containerManifestSharedCacheKey(registryURL, name, reference, cacheAccept)
242+
for _, legacyCacheKey := range h.containerManifestLegacyCacheKeys(registryURL, name, reference, accept, cacheAccept, cacheKey) {
243+
if err := h.storeContainerManifest(ctx, legacyCacheKey, manifest); err != nil {
244+
h.proxy.Logger.Warn("failed to cache legacy container manifest", "error", err)
245+
}
186246
}
187247
}
188248

0 commit comments

Comments
 (0)