Enable storage retention for swift

docs/configuration.md

@@ -107,7 +107,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r
 | `pub` | yes | `pkg:pub/http`, the package name as published. |
 | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. |
 | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. |
-| `swift` | no | - |
+| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. |
 
 An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind.
 

internal/handler/swift.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/git-pkgs/proxy/internal/config"
 	"github.com/git-pkgs/proxy/internal/packageurl"
+	"github.com/git-pkgs/proxy/internal/retention"
+	"github.com/git-pkgs/purl"
 )
 
 const (
@@ -26,6 +28,41 @@ const (
 	swiftMaxNameLength  = 100
 )
 
+// Source archives are stored under the registry identity,
+// pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write
+// the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored
+// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio
+// doesn't name a registry identity and is refused.
+func init() {
+	retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey})
+}
+
+// swiftRetentionKey maps a package rule to the stored package PURL, refusing
+// keys that name no valid registry scope and package. MakeCacheStrings
+// lowercases the identity the way the handler stores it.
+func swiftRetentionKey(p *purl.PURL) (string, bool) {
+	if p == nil {
+		return "", false
+	}
+	var scope, name string
+	switch {
+	case p.Type == "swift":
+		scope, name = p.Namespace, p.Name
+	case p.Type == "generic" && strings.EqualFold(p.Namespace, "swift-registry"):
+		var found bool
+		if scope, name, found = strings.Cut(p.Name, "."); !found {
+			return "", false
+		}
+	default:
+		return "", false
+	}
+	if !validSwiftScope(scope) || !validSwiftPackageName(name) {
+		return "", false
+	}
+	packagePURL, _ := packageurl.MakeCacheStrings("swift", scope+"/"+name, "")
+	return packagePURL, packagePURL != ""
+}
+
 // SwiftHandler handles the read-only Swift Package Registry v1 protocol.
 type SwiftHandler struct {
 	proxy       *Proxy

internal/handler/swift_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 
 	"github.com/git-pkgs/proxy/internal/packageurl"
+	"github.com/git-pkgs/purl"
 	"github.com/git-pkgs/registries/fetch"
 )
 
@@ -548,3 +549,37 @@ func TestSwiftIdentifierValidation(t *testing.T) {
 		})
 	}
 }
+
+func TestSwiftRetentionKey(t *testing.T) {
+	tests := []struct {
+		key  string
+		want string
+		ok   bool
+	}{
+		{"pkg:swift/apple/swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true},
+		{"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true},
+		{"pkg:generic/swift-registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true},
+		{"pkg:generic/swift-registry/Apple.Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true},
+		{"pkg:generic/Swift-Registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true},
+		{"pkg:swift/github.com/apple/swift-nio", "", false},
+		{"pkg:generic/swift-registry/swift-nio", "", false},
+		{"pkg:generic/swift-registry/apple.swift.nio", "", false},
+		{"pkg:swift/apple/-nio", "", false},
+		{"pkg:generic/apple.swift-nio", "", false},
+		{"pkg:generic/other/apple.swift-nio", "", false},
+		{"pkg:npm/apple/swift-nio", "", false},
+	}
+	for _, tt := range tests {
+		p, err := purl.Parse(tt.key)
+		if err != nil {
+			t.Fatalf("Parse(%s): %v", tt.key, err)
+		}
+		got, ok := swiftRetentionKey(p)
+		if ok != tt.ok || (ok && got != tt.want) {
+			t.Errorf("swiftRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok)
+		}
+	}
+	if _, ok := swiftRetentionKey(nil); ok {
+		t.Error("swiftRetentionKey(nil) accepted")
+	}
+}

internal/server/retention_swift_test.go

@@ -0,0 +1,212 @@
+package server
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/config"
+	"github.com/git-pkgs/proxy/internal/database"
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/proxy/internal/retention"
+	"github.com/git-pkgs/registries/fetch"
+)
+
+// swiftRetentionUpstream serves release metadata and source archives for
+// apple/swift-nio 2.65.0 and apple/swift-log 1.5.4 the way a Swift package
+// registry does, counting archive downloads.
+func swiftRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) {
+	t.Helper()
+	var downloads atomic.Int64
+	archives := map[string]string{
+		"/apple/swift-nio/2.65.0": "swift-nio archive",
+		"/apple/swift-log/1.5.4":  "swift-log archive",
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if archive, ok := archives[strings.TrimSuffix(r.URL.Path, ".zip")]; ok {
+			if strings.HasSuffix(r.URL.Path, ".zip") {
+				downloads.Add(1)
+				w.Header().Set("Content-Type", "application/zip")
+				_, _ = io.WriteString(w, archive)
+				return
+			}
+			sum := sha256.Sum256([]byte(archive))
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Content-Version", "1")
+			_, _ = fmt.Fprintf(w, `{"resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:]))
+			return
+		}
+		http.NotFound(w, r)
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &downloads
+}
+
+func newSwiftRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) {
+	t.Helper()
+	db, store := setupEvictionTest(t)
+	// The upstream is a plain-HTTP loopback server; the default fetcher
+	// refuses loopback, a stock client reaches it.
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
+	t.Cleanup(func() { _ = fetcher.Close() })
+	proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger())
+	proxy.HTTPClient = upstream.Client()
+	h := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL)
+	srv := httptest.NewServer(h.Routes())
+	t.Cleanup(srv.Close)
+	return srv, db
+}
+
+func swiftGet(t *testing.T, url string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Accept", "application/vnd.swift.registry.v1+zip")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("GET %s: %v", url, err)
+	}
+	body, _ := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body)
+	}
+}
+
+// swiftArtifact returns the record of the archive named filename.
+func swiftArtifact(t *testing.T, db *database.DB, filename string) database.Artifact {
+	t.Helper()
+	var art database.Artifact
+	if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil {
+		t.Fatalf("reading artifact %s: %v", filename, err)
+	}
+	return art
+}
+
+func swiftBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) {
+	t.Helper()
+	query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`)
+	if _, err := db.Exec(query, fetched, accessed, filename); err != nil {
+		t.Fatalf("backdating %s: %v", filename, err)
+	}
+}
+
+func TestSwiftRegistersForRetention(t *testing.T) {
+	if _, ok := retention.Default.Lookup("swift"); !ok {
+		t.Fatal("swift is not registered for retention")
+	}
+	if key, ok := retention.Default.KeyForDBEcosystem("swift"); !ok || key != "swift" {
+		t.Errorf("KeyForDBEcosystem(swift) = %q, %v, want swift, true", key, ok)
+	}
+}
+
+// An archive served from the cache stays while it is used, is evicted once
+// its rule runs out, comes back from upstream on the next download, and
+// stays after that even with the evicted copy's old access time on the
+// record.
+func TestSwiftRetentionEvictsAndRefetches(t *testing.T) {
+	upstream, downloads := swiftRetentionUpstream(t)
+	srv, db := newSwiftRetentionProxy(t, upstream)
+	const archive = "swift-nio-2.65.0.zip"
+
+	swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip")
+	swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip")
+	if got := downloads.Load(); got != 1 {
+		t.Fatalf("upstream downloads = %d, want 1", got)
+	}
+	if !swiftArtifact(t, db, archive).LastAccessedAt.Valid {
+		t.Fatal("cache hit did not record an access")
+	}
+
+	rules := retention.Rules{Ecosystems: map[string]time.Duration{"swift": retentionDay}}
+	now := time.Now()
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now)
+	if !swiftArtifact(t, db, archive).StoragePath.Valid {
+		t.Fatal("archive used today was evicted")
+	}
+
+	old := now.Add(-2 * retentionDay)
+	swiftBackdate(t, db, archive, old, old)
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now)
+	if swiftArtifact(t, db, archive).StoragePath.Valid {
+		t.Fatal("archive unused for two days kept under a one-day rule")
+	}
+
+	swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip")
+	if got := downloads.Load(); got != 2 {
+		t.Errorf("upstream downloads = %d, want 2 after the eviction", got)
+	}
+	art := swiftArtifact(t, db, archive)
+	swiftBackdate(t, db, archive, art.FetchedAt.Time, now.Add(-30*retentionDay))
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now())
+	if !swiftArtifact(t, db, archive).StoragePath.Valid {
+		t.Error("refetched archive evicted again by the next sweep")
+	}
+}
+
+// A source repository key parses as a swift PURL but names no registry
+// identity, so it is refused at validation. A scope/name key is accepted.
+func TestSwiftRetentionRefusesRepositoryKeys(t *testing.T) {
+	cfg := config.Default()
+	cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/apple/swift-nio": "0"}
+	if _, err := cfg.RetentionRules(retention.Default); err != nil {
+		t.Fatalf("RetentionRules(pkg:swift/apple/swift-nio): %v", err)
+	}
+	cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"}
+	_, err := cfg.RetentionRules(retention.Default)
+	if err == nil || !strings.Contains(err.Error(), "does not match how swift packages are stored") {
+		t.Errorf("RetentionRules(pkg:swift/github.com/apple/swift-nio) error = %v, want the key refused", err)
+	}
+}
+
+// A package rule reaches the stored archive in either documented form and
+// whatever case it is written in, even when the client asked in another case.
+func TestSwiftRetentionPackageRules(t *testing.T) {
+	upstream, _ := swiftRetentionUpstream(t)
+	srv, db := newSwiftRetentionProxy(t, upstream)
+	const nio, log = "swift-nio-2.65.0.zip", "swift-log-1.5.4.zip"
+
+	swiftGet(t, srv.URL+"/Apple/Swift-NIO/2.65.0.zip")
+	var stored string
+	art := swiftArtifact(t, db, nio)
+	if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil {
+		t.Fatalf("reading stored package PURL: %v", err)
+	}
+	if stored != "pkg:generic/swift-registry/apple.swift-nio" {
+		t.Errorf("stored package PURL = %q, want pkg:generic/swift-registry/apple.swift-nio", stored)
+	}
+
+	for _, key := range []string{"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio"} {
+		swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip")
+		swiftGet(t, srv.URL+"/apple/swift-log/1.5.4.zip")
+		old := time.Now().Add(-30 * retentionDay)
+		swiftBackdate(t, db, nio, old, old)
+		swiftBackdate(t, db, log, old, old)
+
+		cfg := config.Default()
+		cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"}
+		cfg.Storage.Retention.Packages = map[string]string{key: "0"}
+		rules, err := cfg.RetentionRules(retention.Default)
+		if err != nil {
+			t.Fatalf("RetentionRules(%s): %v", key, err)
+		}
+		sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now())
+
+		if !swiftArtifact(t, db, nio).StoragePath.Valid {
+			t.Errorf("with %s: swift-nio evicted despite its never-evict rule", key)
+		}
+		if swiftArtifact(t, db, log).StoragePath.Valid {
+			t.Errorf("with %s: swift-log kept past the swift rule", key)
+		}
+	}
+}