From 1d88be53abcbbc22e285c50e2aa9a2bc5600e9c9 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 11 Oct 2026 01:07:07 +0200 Subject: [PATCH] Enable storage retention for alpine Packages are stored as pkg:apk/alpine/{name} whichever configured repository they came from, so a package rule covers the package in all of them. Rules may leave out the alpine namespace (pkg:apk/curl); another namespace could never match and is refused. The docs row warns that Alpine mirrors only keep the current build, so an evicted older one can't come back. The mirror can't fetch apk packages, so there's no mirror test. Refs #306 --- docs/configuration.md | 2 +- internal/handler/apk.go | 21 +++ internal/handler/apk_test.go | 30 ++++ internal/server/retention_alpine_test.go | 195 +++++++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 internal/server/retention_alpine_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 6b174521..d4c2148d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,7 +88,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | Ecosystem key | Retention supported | Package rule form | |---------------|---------------------|-------------------| -| `alpine` | no | - | +| `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. | | `cargo` | no | - | | `composer` | no | - | | `conan` | no | - | diff --git a/internal/handler/apk.go b/internal/handler/apk.go index e0b4b31b..6e18e742 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -6,6 +6,9 @@ import ( "net/http" "regexp" "strings" + + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -18,6 +21,24 @@ const ( apkMatchCount = 3 // full match + name + version ) +// Packages are stored as pkg:apk/alpine/{name}, the namespace purl gives +// every apk package, whichever configured repository they came from. Package +// rules may leave the namespace out. +func init() { + retention.Default.Register(retention.Spec{Key: apkEcosystem, CanonicalPackage: apkRetentionKey}) +} + +// apkRetentionKey maps pkg:apk/{name} and pkg:apk/alpine/{name} to the stored +// package PURL. Another namespace, or a name with a slash, can never match a +// stored package and is refused. +func apkRetentionKey(p *purl.PURL) (string, bool) { + if p == nil || p.Type != "apk" || (p.Namespace != "" && p.Namespace != "alpine") || strings.Contains(p.Name, "/") { + return "", false + } + canonical := purl.MakePURLString(apkEcosystem, p.Name, "") + return canonical, canonical != "" +} + // APKHandler handles Alpine APK repository protocol requests. Each configured // upstream repository is mounted at /apk/{repository}/ and the remaining path // mirrors the upstream layout ({release}/{repo}/{arch}/{file}). diff --git a/internal/handler/apk_test.go b/internal/handler/apk_test.go index 77e1da59..61c874ee 100644 --- a/internal/handler/apk_test.go +++ b/internal/handler/apk_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -360,3 +361,32 @@ func serveAPKRequest(h *APKHandler, target string) *httptest.ResponseRecorder { h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) return w } + +func TestAPKRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:apk/curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/alpine/curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/Alpine/Curl", "pkg:apk/alpine/curl", true}, + {"pkg:apk/alpine/libstdc++", "pkg:apk/alpine/libstdc%2B%2B", true}, + {"pkg:apk/wolfi/curl", "", false}, + {"pkg:apk/alpine/main%2Fcurl", "", false}, + {"pkg:deb/alpine/curl", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := apkRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("apkRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := apkRetentionKey(nil); ok { + t.Error("apkRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_alpine_test.go b/internal/server/retention_alpine_test.go new file mode 100644 index 00000000..4118b443 --- /dev/null +++ b/internal/server/retention_alpine_test.go @@ -0,0 +1,195 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +const ( + alpineCurlPath = "/v3.22/main/x86_64/curl-8.14.1-r1.apk" + alpineBusyboxPath = "/v3.22/main/x86_64/busybox-1.37.0-r12.apk" +) + +// alpineRetentionUpstream serves package files the way an Alpine mirror +// does, counting downloads. +func alpineRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case alpineCurlPath, alpineBusyboxPath: + downloads.Add(1) + _, _ = io.WriteString(w, "apk bytes") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +// newAlpineRetentionProxy mounts the upstream twice, as the default "alpine" +// repository and as a named "edge" repository. +func newAlpineRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewAPKHandler(proxy, "http://proxy.test", map[string]string{"alpine": upstream.URL, "edge": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func alpineGet(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// alpineArtifact returns the record of the file cached under filename, +// which for apk is the repository-qualified request path. +func alpineArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func alpineBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestAlpineRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("alpine"); !ok { + t.Fatal("alpine is not registered for retention") + } + for _, value := range []string{"alpine", "apk"} { + if key, ok := retention.Default.KeyForDBEcosystem(value); !ok || key != "alpine" { + t.Errorf("KeyForDBEcosystem(%s) = %q, %v, want alpine, true", value, key, ok) + } + } +} + +// A package served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestAlpineRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := alpineRetentionUpstream(t) + srv, db := newAlpineRetentionProxy(t, upstream) + const file = "alpine" + alpineCurlPath + + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !alpineArtifact(t, db, file).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"alpine": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !alpineArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package used today was evicted") + } + + old := now.Add(-2 * retentionDay) + alpineBackdate(t, db, file, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if alpineArtifact(t, db, file).StoragePath.Valid { + t.Fatal("package unused for two days kept under a one-day rule") + } + + alpineGet(t, srv.URL+"/alpine"+alpineCurlPath) + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := alpineArtifact(t, db, file) + alpineBackdate(t, db, file, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !alpineArtifact(t, db, file).StoragePath.Valid { + t.Error("refetched package evicted again by the next sweep") + } +} + +// A key in another namespace parses as an apk PURL but can never match a +// stored package, so it is refused at validation. Both documented forms are +// accepted. +func TestAlpineRetentionRefusesOtherNamespaces(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:apk/curl": "0", "pkg:apk/alpine/busybox": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules: %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:apk/wolfi/curl": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how alpine packages are stored") { + t.Errorf("RetentionRules(pkg:apk/wolfi/curl) error = %v, want the key refused", err) + } +} + +// A package rule reaches the package in every configured repository, in +// either documented form. +func TestAlpineRetentionPackageRules(t *testing.T) { + upstream, _ := alpineRetentionUpstream(t) + srv, db := newAlpineRetentionProxy(t, upstream) + files := map[string]bool{ + "alpine" + alpineCurlPath: true, + "edge" + alpineCurlPath: true, + "alpine" + alpineBusyboxPath: false, + } + + for _, key := range []string{"pkg:apk/curl", "pkg:apk/alpine/curl"} { + old := time.Now().Add(-30 * retentionDay) + for file := range files { + alpineGet(t, srv.URL+"/"+file) + alpineBackdate(t, db, file, old, old) + } + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"alpine": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + for file, want := range files { + if got := alpineArtifact(t, db, file).StoragePath.Valid; got != want { + t.Errorf("with %s: %s cached = %v, want %v", key, file, got, want) + } + } + } +}