diff --git a/docs/configuration.md b/docs/configuration.md index fa557345..c4cd0e68 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,7 +107,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r | `pub` | yes | `pkg:pub/http`, the package name as published. | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | -| `swift` | no | - | +| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. | An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 6cc7d78a..bd46fab2 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -15,6 +15,8 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) const ( @@ -26,6 +28,41 @@ const ( swiftMaxNameLength = 100 ) +// Source archives are stored under the registry identity, +// pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write +// the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored +// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio +// doesn't name a registry identity and is refused. +func init() { + retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey}) +} + +// swiftRetentionKey maps a package rule to the stored package PURL, refusing +// keys that name no valid registry scope and package. MakeCacheStrings +// lowercases the identity the way the handler stores it. +func swiftRetentionKey(p *purl.PURL) (string, bool) { + if p == nil { + return "", false + } + var scope, name string + switch { + case p.Type == "swift": + scope, name = p.Namespace, p.Name + case p.Type == "generic" && strings.EqualFold(p.Namespace, "swift-registry"): + var found bool + if scope, name, found = strings.Cut(p.Name, "."); !found { + return "", false + } + default: + return "", false + } + if !validSwiftScope(scope) || !validSwiftPackageName(name) { + return "", false + } + packagePURL, _ := packageurl.MakeCacheStrings("swift", scope+"/"+name, "") + return packagePURL, packagePURL != "" +} + // SwiftHandler handles the read-only Swift Package Registry v1 protocol. type SwiftHandler struct { proxy *Proxy diff --git a/internal/handler/swift_test.go b/internal/handler/swift_test.go index c9df35a5..199c7122 100644 --- a/internal/handler/swift_test.go +++ b/internal/handler/swift_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" ) @@ -548,3 +549,37 @@ func TestSwiftIdentifierValidation(t *testing.T) { }) } } + +func TestSwiftRetentionKey(t *testing.T) { + tests := []struct { + key string + want string + ok bool + }{ + {"pkg:swift/apple/swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/swift-registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/swift-registry/Apple.Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:generic/Swift-Registry/apple.swift-nio", "pkg:generic/swift-registry/apple.swift-nio", true}, + {"pkg:swift/github.com/apple/swift-nio", "", false}, + {"pkg:generic/swift-registry/swift-nio", "", false}, + {"pkg:generic/swift-registry/apple.swift.nio", "", false}, + {"pkg:swift/apple/-nio", "", false}, + {"pkg:generic/apple.swift-nio", "", false}, + {"pkg:generic/other/apple.swift-nio", "", false}, + {"pkg:npm/apple/swift-nio", "", false}, + } + for _, tt := range tests { + p, err := purl.Parse(tt.key) + if err != nil { + t.Fatalf("Parse(%s): %v", tt.key, err) + } + got, ok := swiftRetentionKey(p) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("swiftRetentionKey(%s) = %q, %v, want %q, %v", tt.key, got, ok, tt.want, tt.ok) + } + } + if _, ok := swiftRetentionKey(nil); ok { + t.Error("swiftRetentionKey(nil) accepted") + } +} diff --git a/internal/server/retention_swift_test.go b/internal/server/retention_swift_test.go new file mode 100644 index 00000000..441320a4 --- /dev/null +++ b/internal/server/retention_swift_test.go @@ -0,0 +1,212 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/registries/fetch" +) + +// swiftRetentionUpstream serves release metadata and source archives for +// apple/swift-nio 2.65.0 and apple/swift-log 1.5.4 the way a Swift package +// registry does, counting archive downloads. +func swiftRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + var downloads atomic.Int64 + archives := map[string]string{ + "/apple/swift-nio/2.65.0": "swift-nio archive", + "/apple/swift-log/1.5.4": "swift-log archive", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if archive, ok := archives[strings.TrimSuffix(r.URL.Path, ".zip")]; ok { + if strings.HasSuffix(r.URL.Path, ".zip") { + downloads.Add(1) + w.Header().Set("Content-Type", "application/zip") + _, _ = io.WriteString(w, archive) + return + } + sum := sha256.Sum256([]byte(archive)) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Version", "1") + _, _ = fmt.Fprintf(w, `{"resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:])) + return + } + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + return srv, &downloads +} + +func newSwiftRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) { + t.Helper() + db, store := setupEvictionTest(t) + // The upstream is a plain-HTTP loopback server; the default fetcher + // refuses loopback, a stock client reaches it. + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = upstream.Client() + h := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL) + srv := httptest.NewServer(h.Routes()) + t.Cleanup(srv.Close) + return srv, db +} + +func swiftGet(t *testing.T, url string) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Accept", "application/vnd.swift.registry.v1+zip") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s = %d %s, want 200", url, resp.StatusCode, body) + } +} + +// swiftArtifact returns the record of the archive named filename. +func swiftArtifact(t *testing.T, db *database.DB, filename string) database.Artifact { + t.Helper() + var art database.Artifact + if err := db.Get(&art, db.Rebind(`SELECT * FROM artifacts WHERE filename = ?`), filename); err != nil { + t.Fatalf("reading artifact %s: %v", filename, err) + } + return art +} + +func swiftBackdate(t *testing.T, db *database.DB, filename string, fetched, accessed time.Time) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ? WHERE filename = ?`) + if _, err := db.Exec(query, fetched, accessed, filename); err != nil { + t.Fatalf("backdating %s: %v", filename, err) + } +} + +func TestSwiftRegistersForRetention(t *testing.T) { + if _, ok := retention.Default.Lookup("swift"); !ok { + t.Fatal("swift is not registered for retention") + } + if key, ok := retention.Default.KeyForDBEcosystem("swift"); !ok || key != "swift" { + t.Errorf("KeyForDBEcosystem(swift) = %q, %v, want swift, true", key, ok) + } +} + +// An archive served from the cache stays while it is used, is evicted once +// its rule runs out, comes back from upstream on the next download, and +// stays after that even with the evicted copy's old access time on the +// record. +func TestSwiftRetentionEvictsAndRefetches(t *testing.T) { + upstream, downloads := swiftRetentionUpstream(t) + srv, db := newSwiftRetentionProxy(t, upstream) + const archive = "swift-nio-2.65.0.zip" + + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + if got := downloads.Load(); got != 1 { + t.Fatalf("upstream downloads = %d, want 1", got) + } + if !swiftArtifact(t, db, archive).LastAccessedAt.Valid { + t.Fatal("cache hit did not record an access") + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"swift": retentionDay}} + now := time.Now() + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if !swiftArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive used today was evicted") + } + + old := now.Add(-2 * retentionDay) + swiftBackdate(t, db, archive, old, old) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now) + if swiftArtifact(t, db, archive).StoragePath.Valid { + t.Fatal("archive unused for two days kept under a one-day rule") + } + + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + if got := downloads.Load(); got != 2 { + t.Errorf("upstream downloads = %d, want 2 after the eviction", got) + } + art := swiftArtifact(t, db, archive) + swiftBackdate(t, db, archive, art.FetchedAt.Time, now.Add(-30*retentionDay)) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, archive).StoragePath.Valid { + t.Error("refetched archive evicted again by the next sweep") + } +} + +// A source repository key parses as a swift PURL but names no registry +// identity, so it is refused at validation. A scope/name key is accepted. +func TestSwiftRetentionRefusesRepositoryKeys(t *testing.T) { + cfg := config.Default() + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/apple/swift-nio": "0"} + if _, err := cfg.RetentionRules(retention.Default); err != nil { + t.Fatalf("RetentionRules(pkg:swift/apple/swift-nio): %v", err) + } + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} + _, err := cfg.RetentionRules(retention.Default) + if err == nil || !strings.Contains(err.Error(), "does not match how swift packages are stored") { + t.Errorf("RetentionRules(pkg:swift/github.com/apple/swift-nio) error = %v, want the key refused", err) + } +} + +// A package rule reaches the stored archive in either documented form and +// whatever case it is written in, even when the client asked in another case. +func TestSwiftRetentionPackageRules(t *testing.T) { + upstream, _ := swiftRetentionUpstream(t) + srv, db := newSwiftRetentionProxy(t, upstream) + const nio, log = "swift-nio-2.65.0.zip", "swift-log-1.5.4.zip" + + swiftGet(t, srv.URL+"/Apple/Swift-NIO/2.65.0.zip") + var stored string + art := swiftArtifact(t, db, nio) + if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), art.VersionPURL); err != nil { + t.Fatalf("reading stored package PURL: %v", err) + } + if stored != "pkg:generic/swift-registry/apple.swift-nio" { + t.Errorf("stored package PURL = %q, want pkg:generic/swift-registry/apple.swift-nio", stored) + } + + for _, key := range []string{"pkg:swift/Apple/Swift-NIO", "pkg:generic/swift-registry/apple.swift-nio"} { + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-log/1.5.4.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + swiftBackdate(t, db, log, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} + cfg.Storage.Retention.Packages = map[string]string{key: "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules(%s): %v", key, err) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Errorf("with %s: swift-nio evicted despite its never-evict rule", key) + } + if swiftArtifact(t, db, log).StoragePath.Valid { + t.Errorf("with %s: swift-log kept past the swift rule", key) + } + } +}