From e74e5abc79523de0b1660fb434523fa690b91992 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Thu, 15 Oct 2026 12:46:55 +0200 Subject: [PATCH] Serve cached npm and Composer metadata gzip-compressed - Keep a gzip copy of each rewrite cache entry when it is smaller, made once per rewrite and counted against metadata_rewrite_cache_size - Serve it with Content-Encoding: gzip to clients that accept gzip, with an ETag of its own ("-gzip") and Vary: Accept-Encoding - Rewrites made outside the cache (cache disabled, cooldown on) stay uncompressed - rewriteCache.get and rewrite return rewrittenMetadata --- config.example.yaml | 4 +- docs/configuration.md | 2 + internal/handler/metadata_gzip_test.go | 227 +++++++++++++++++++ internal/handler/metadata_validators_test.go | 16 +- internal/handler/npm_rewrite_test.go | 2 +- internal/handler/rewrite_cache.go | 174 ++++++++++---- internal/handler/rewrite_cache_test.go | 40 ++-- 7 files changed, 384 insertions(+), 81 deletions(-) create mode 100644 internal/handler/metadata_gzip_test.go diff --git a/config.example.yaml b/config.example.yaml index c41f30f3..accae89a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -19,8 +19,8 @@ base_url: "http://localhost:8080" # artifact_max_size: "" # Memory for rewritten npm and Composer metadata, so each upstream document is -# rewritten once rather than on every request. Set to "0" to rewrite on every -# request. Default: "256MB". +# rewritten and gzip-compressed once rather than on every request. Set to "0" +# to rewrite on every request. Default: "256MB". # metadata_rewrite_cache_size: "256MB" # Public URL where the web UI is reached. Defaults to base_url when unset. diff --git a/docs/configuration.md b/docs/configuration.md index c735c34b..ffd57fe2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -966,6 +966,8 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time. +Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed. + ```yaml metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request ``` diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go new file mode 100644 index 00000000..5826441d --- /dev/null +++ b/internal/handler/metadata_gzip_test.go @@ -0,0 +1,227 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// compressibleDocuments sets upstream documents with enough versions that +// gzip makes them smaller. +func compressibleDocuments(u *validatorUpstream) { + var npm, composer []string + for i := range 50 { + v := fmt.Sprintf("1.0.%d", i) + npm = append(npm, `"`+v+`":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-`+v+`.tgz"}}`) + composer = append(composer, `{"version":"`+v+`","dist":{"type":"zip","url":"https://example.com/pkg-`+v+`.zip"}}`) + } + u.set( + `{"name":"left-pad","versions":{`+strings.Join(npm, ",")+`}}`, + `{"packages":{"vendor/pkg":[`+strings.Join(composer, ",")+`]}}`, + ) +} + +func gunzip(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(rec.Body.Bytes())) + if err != nil { + t.Fatalf("body is not gzip: %v", err) + } + body, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("reading gzip body: %v", err) + } + return string(body) +} + +var acceptGzip = http.Header{"Accept-Encoding": {"gzip, deflate, br"}} + +// TestRewrittenMetadataGzip checks that a client accepting gzip gets the +// cached compressed copy of rewritten metadata, on the first fetch and on +// the rewrite fast path, with an ETag of its own so it never matches the +// uncompressed representation. +func TestRewrittenMetadataGzip(t *testing.T) { + for _, c := range validatorCases { + for _, ttl := range []time.Duration{time.Hour, 0} { + t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, ttl), u) + + identity := serve(handler, c.path, nil) + for range 2 { + expectGzipOf(t, serve(handler, c.path, acceptGzip), identity) + } + }) + } + } +} + +// expectGzipOf checks that rec is the gzip representation of the +// uncompressed response identity. +func expectGzipOf(t *testing.T, rec, identity *httptest.ResponseRecorder) { + t.Helper() + if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" { + t.Fatalf("status %d, Content-Encoding %q; want 200 with gzip", rec.Code, rec.Header().Get("Content-Encoding")) + } + if rec.Body.Len() >= identity.Body.Len() { + t.Errorf("gzip body is %d bytes, uncompressed %d", rec.Body.Len(), identity.Body.Len()) + } + if got := gunzip(t, rec); got != identity.Body.String() { + t.Errorf("decompressed body differs from the uncompressed response") + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q, want Accept-Encoding", got) + } + if etag := rec.Header().Get("ETag"); etag == "" || etag == identity.Header().Get("ETag") { + t.Errorf("gzip ETag %q must differ from the uncompressed ETag %q", etag, identity.Header().Get("ETag")) + } + if rec.Header().Get("Last-Modified") != identity.Header().Get("Last-Modified") { + t.Errorf("Last-Modified %q, want %q", rec.Header().Get("Last-Modified"), identity.Header().Get("Last-Modified")) + } +} + +// TestRewrittenMetadataGzipRevalidation checks that each representation +// revalidates with its own ETag, also when a reverse proxy weakened it. +func TestRewrittenMetadataGzipRevalidation(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, time.Hour), u) + gzipETag := serve(handler, c.path, acceptGzip).Header().Get("ETag") + identityETag := serve(handler, c.path, nil).Header().Get("ETag") + + for _, ifNoneMatch := range []string{gzipETag, "W/" + gzipETag} { + rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {ifNoneMatch}}) + expectNotModified(t, rec, gzipETag) + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("304 Vary = %q, want Accept-Encoding", got) + } + } + if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {identityETag}}); rec.Code != http.StatusOK { + t.Errorf("gzip request revalidating the uncompressed ETag: status %d, want 200", rec.Code) + } + if rec := serve(handler, c.path, http.Header{"If-None-Match": {gzipETag}}); rec.Code != http.StatusOK { + t.Errorf("uncompressed request revalidating the gzip ETag: status %d, want 200", rec.Code) + } + }) + } +} + +// TestRewrittenMetadataWithoutGzip checks that clients that do not accept +// gzip get the uncompressed document, which still varies on Accept-Encoding. +func TestRewrittenMetadataWithoutGzip(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + handler := c.handler(validatorProxy(t, u, time.Hour), u) + + for _, acceptEncoding := range []string{"", "identity", "gzip;q=0", "br"} { + header := http.Header{} + if acceptEncoding != "" { + header.Set("Accept-Encoding", acceptEncoding) + } + rec := serve(handler, c.path, header) + if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { + t.Errorf("Accept-Encoding %q: status %d, Content-Encoding %q; want 200 uncompressed", acceptEncoding, rec.Code, rec.Header().Get("Content-Encoding")) + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", acceptEncoding, got) + } + } + }) + } +} + +// TestRewrittenMetadataGzipOnlyFromRewriteCache checks that rewrites made +// outside the cache, with it disabled or with cooldown on, stay uncompressed: +// compressing them on every request is better left to a reverse proxy. +func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) { + setups := map[string]func(*Proxy){ + "cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) }, + "cooldown": func(p *Proxy) { p.Cooldown = &cooldown.Config{Default: "3d"} }, + } + for _, c := range validatorCases { + for name, setup := range setups { + t.Run(c.ecosystem+"/"+name, func(t *testing.T) { + u := newValidatorUpstream(t) + compressibleDocuments(u) + proxy := validatorProxy(t, u, time.Hour) + setup(proxy) + handler := c.handler(proxy, u) + + for range 2 { + if rec := serve(handler, c.path, acceptGzip); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" { + t.Errorf("status %d, Content-Encoding %q; want 200 uncompressed", rec.Code, rec.Header().Get("Content-Encoding")) + } + } + }) + } + } +} + +// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps a +// compressed copy of a rewrite, counted against its size, unless compressing +// does not make the rewrite smaller. +func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { + c := newRewriteCache(1 << 20) + identity := func(b []byte) ([]byte, error) { return b, nil } + large := []byte(strings.Repeat(`{"version":"1.0.0"},`, 100)) + + for _, doc := range [][]byte{large, []byte("abc")} { + key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) + if _, err := c.rewrite(t.Context(), key, doc, identity); err != nil { + t.Fatal(err) + } + } + + m, ok := c.get(rewriteCacheKey("npm", "http://proxy", string(large[:3]), large)) + if !ok || m.gzipped == nil { + t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped)) + } + if c.size != int64(len(large)+len(m.gzipped)+3) { + t.Errorf("cache size = %d, want both rewrites and the gzip copy (%d)", c.size, len(large)+len(m.gzipped)+3) + } + if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil { + t.Errorf("small rewrite: cached %v, gzip copy %d bytes; want none", ok, len(small.gzipped)) + } +} + +func TestAcceptsGzip(t *testing.T) { + cases := map[string]bool{ + "": false, + "identity": false, + "br, deflate": false, + "gzip": true, + "GZIP": true, + "x-gzip": true, + "deflate, gzip;q=1.0": true, + "gzip; q=0.5": true, + "gzip;q=0": false, + "gzip;q=0.000": false, + "*": true, + "*;q=0": false, + "br;q=1, *;q=0.5": true, + "gzip;q=0, *": false, + "gzip;q=invalid, deflate": false, + } + for header, want := range cases { + r := httptest.NewRequest(http.MethodGet, "/", nil) + if header != "" { + r.Header.Set("Accept-Encoding", header) + } + if got := acceptsGzip(r); got != want { + t.Errorf("acceptsGzip(%q) = %v, want %v", header, got, want) + } + } +} diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index a057e5eb..e34395d1 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -481,17 +481,17 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - out, etag, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) - if err != nil || string(out) != "doc!" { - t.Fatalf("rewrite = %q, %v", out, err) + m, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim) + if err != nil || string(m.body) != "doc!" { + t.Fatalf("rewrite = %q, %v", m.body, err) } - if want := metadataETag([]byte("doc!")); etag.etag != want { - t.Errorf("rewrite ETag = %q, want %q", etag.etag, want) + if want := metadataETag([]byte("doc!")); m.etag != want { + t.Errorf("rewrite ETag = %q, want %q", m.etag, want) } - if _, cached, ok := c.get(key); !ok || cached != etag { - t.Errorf("get validators = %+v, %v; want %+v", cached, ok, etag) + if cached, ok := c.get(key); !ok || cached.metadataValidators != m.metadataValidators { + t.Errorf("get validators = %+v, %v; want %+v", cached.metadataValidators, ok, m.metadataValidators) } - if metadataETag([]byte("other")) == etag.etag { + if metadataETag([]byte("other")) == m.etag { t.Error("different output has the same ETag") } } diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index d9a787eb..6503ae1e 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -247,7 +247,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 42273c3b..fb3114a6 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -1,6 +1,8 @@ package handler import ( + "bytes" + "compress/gzip" "container/list" "context" "crypto/rand" @@ -11,6 +13,7 @@ import ( "io" "net/http" "slices" + "strconv" "strings" "sync" "time" @@ -36,6 +39,10 @@ import ( // copy without the output being hashed again on every request. Dates are not // the cache's to give: they must agree across replicas, so the database holds // them (see metadataDate). +// +// Each entry also keeps its output gzip-compressed, when that is smaller, so +// clients accepting gzip get it without the document being compressed again +// on every request, by the proxy or by a reverse proxy in front of it. type rewriteCache struct { maxBytes int64 @@ -47,18 +54,21 @@ type rewriteCache struct { } type rewriteEntry struct { - key string - out []byte - etag string + key string + rewrittenMetadata +} + +// size is how much of the cache's room the entry takes. +func (e *rewriteEntry) size() int64 { + return int64(len(e.body) + len(e.gzipped)) } -// inflightRewrite is one rewrite that concurrent callers share. out, etag -// and err are written before done closes and read only after, so the close is -// the handoff. +// inflightRewrite is one rewrite that concurrent callers share. m and err are +// written before done closes and read only after, so the close is the +// handoff. type inflightRewrite struct { done chan struct{} - out []byte - etag string + m rewrittenMetadata err error } @@ -70,9 +80,12 @@ type metadataValidators struct { } // rewrittenMetadata is a rewritten metadata document with its validators. +// gzipped is the document gzip-compressed, or nil when there is no +// compressed copy to serve. type rewrittenMetadata struct { body []byte metadataValidators + gzipped []byte } // metadataETag returns the strong ETag of a rewritten document. @@ -139,28 +152,27 @@ func rewriteCacheKeyForDigest(ecosystem, proxyURL, name, digest string) string { return strings.Join([]string{ecosystem, proxyURL, name, digest}, "\x00") } -// get returns the cached rewrite for key and its ETag, if there is one. -func (c *rewriteCache) get(key string) ([]byte, metadataValidators, bool) { +// get returns the cached rewrite for key, if there is one. +func (c *rewriteCache) get(key string) (rewrittenMetadata, bool) { if c == nil { - return nil, metadataValidators{}, false + return rewrittenMetadata{}, false } c.mu.Lock() defer c.mu.Unlock() el, ok := c.entries[key] if !ok { - return nil, metadataValidators{}, false + return rewrittenMetadata{}, false } c.order.MoveToFront(el) - e := el.Value.(*rewriteEntry) - return e.out, metadataValidators{etag: e.etag}, true + return el.Value.(*rewriteEntry).rewrittenMetadata, true } -// rewrite returns rewrite(in) and its ETag, from the cache when it can. -// Callers must treat the returned bytes as read-only: a cached result is -// shared. A caller waiting on another's rewrite leaves when ctx ends; the -// rewrite itself always runs to completion, so the result is cached for the -// next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (out []byte, v metadataValidators, err error) { +// rewrite returns rewrite(in) with its ETag and gzip copy, from the cache +// when it can. Callers must treat the returned bytes as read-only: a cached +// result is shared. A caller waiting on another's rewrite leaves when ctx +// ends; the rewrite itself always runs to completion, so the result is cached +// for the next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -168,17 +180,16 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() if el, ok := c.entries[key]; ok { c.order.MoveToFront(el) - e := el.Value.(*rewriteEntry) c.mu.Unlock() - return e.out, metadataValidators{etag: e.etag}, nil + return el.Value.(*rewriteEntry).rewrittenMetadata, nil } if f, ok := c.inFlight[key]; ok { c.mu.Unlock() select { case <-ctx.Done(): - return nil, metadataValidators{}, ctx.Err() + return rewrittenMetadata{}, ctx.Err() case <-f.done: - return f.out, metadataValidators{etag: f.etag}, f.err + return f.m, f.err } } f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} @@ -189,31 +200,46 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewri c.mu.Lock() delete(c.inFlight, key) if f.err == nil { - c.add(&rewriteEntry{key: key, out: f.out, etag: f.etag}) + c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m}) } c.mu.Unlock() close(f.done) }() - var fv metadataValidators - f.out, fv, f.err = rewriteWithETag(in, rewrite) - f.etag = fv.etag - return f.out, fv, f.err + f.m, f.err = rewriteWithETag(in, rewrite) + if f.err == nil { + f.m.gzipped = gzipIfSmaller(f.m.body) + } + return f.m, f.err } // rewriteWithETag runs rewrite and computes the ETag of what it returns. -func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, metadataValidators, error) { +func rewriteWithETag(in []byte, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { out, err := rewrite(in) if err != nil { - return nil, metadataValidators{}, err + return rewrittenMetadata{}, err } - return out, metadataValidators{etag: metadataETag(out)}, nil + return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: metadataETag(out)}}, nil +} + +// gzipIfSmaller returns body gzip-compressed, or nil when compressing does +// not make it smaller. +func gzipIfSmaller(body []byte) []byte { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + if _, err := zw.Write(body); err != nil { + return nil + } + if err := zw.Close(); err != nil || buf.Len() >= len(body) { + return nil + } + return buf.Bytes() } // add stores e and evicts least recently used entries until the cache is -// back under its limit. Output larger than the whole cache is not stored. +// back under its limit. An entry larger than the whole cache is not stored. // Called with mu held. func (c *rewriteCache) add(e *rewriteEntry) { - n := int64(len(e.out)) + n := e.size() if n > c.maxBytes { return } @@ -224,7 +250,7 @@ func (c *rewriteCache) add(e *rewriteEntry) { e := oldest.Value.(*rewriteEntry) c.order.Remove(oldest) delete(c.entries, e.key) - c.size -= int64(len(e.out)) + c.size -= e.size() } } @@ -252,24 +278,23 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if err != nil { return rewrittenMetadata{}, err } - return rewrittenMetadata{out, metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil + return rewrittenMetadata{body: out, metadataValidators: metadataValidators{etag: keptVersionsETag(digest, proxyURL, name, kept)}}, nil } rewriteOnly := func(b []byte) ([]byte, error) { out, _, err := rewrite(b) return out, err } - var out []byte - var v metadataValidators + var m rewrittenMetadata var err error if p.rewrites == nil || p.cooldownEnabled() { - out, v, err = rewriteWithETag(in, rewriteOnly) + m, err = rewriteWithETag(in, rewriteOnly) } else { - out, v, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) + m, err = p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewriteOnly) } if err == nil && !p.cooldownEnabled() { - v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } - return rewrittenMetadata{out, v}, err + return m, err } // uncachedRewrite gives a rewrite made outside the cache its validators: an @@ -280,7 +305,7 @@ func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) re if !p.cooldownEnabled() { v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) } - return rewrittenMetadata{out, v} + return rewrittenMetadata{body: out, metadataValidators: v} } func (p *Proxy) cooldownEnabled() bool { @@ -307,29 +332,78 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri if !entry.FetchedAt.Valid || time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return rewrittenMetadata{}, false } - out, v, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) + m, ok := p.rewrites.get(rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String)) if !ok { return rewrittenMetadata{}, false } - v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) metrics.RecordCacheHit(ecosystem) - return rewrittenMetadata{out, v}, true + return m, true } // serveRewrittenMetadata writes rewritten metadata with its validators, or -// 304 Not Modified when they show the client's copy is current. +// 304 Not Modified when they show the client's copy is current. A client +// accepting gzip gets the compressed copy when there is one. Being another +// representation, it has an ETag of its own. func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { - w.Header().Set(headerETag, m.etag) + body, etag := m.body, m.etag + compressed := m.gzipped != nil && acceptsGzip(r) + if compressed { + body, etag = m.gzipped, gzipETag(m.etag) + } + w.Header().Set("Vary", headerAcceptEncoding) + w.Header().Set(headerETag, etag) if !m.lastModified.IsZero() { w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) } - if notModified(r, m.etag, m.lastModified) { + if notModified(r, etag, m.lastModified) { w.WriteHeader(http.StatusNotModified) return } w.Header().Set(headerContentType, contentTypeJSON) + if compressed { + w.Header().Set(headerContentEncoding, "gzip") + } w.WriteHeader(http.StatusOK) - _, _ = w.Write(m.body) + _, _ = w.Write(body) +} + +// gzipETag returns the ETag of the gzip representation of the document +// whose ETag is etag. +func gzipETag(etag string) string { + return strings.TrimSuffix(etag, `"`) + `-gzip"` +} + +// acceptsGzip reports whether the request's Accept-Encoding allows a gzip +// response: gzip or x-gzip, or else "*", with a quality above zero. +func acceptsGzip(r *http.Request) bool { + wildcard := false + for _, value := range r.Header.Values(headerAcceptEncoding) { + for coding := range strings.SplitSeq(value, ",") { + name, params, _ := strings.Cut(coding, ";") + switch strings.ToLower(strings.TrimSpace(name)) { + case "gzip", "x-gzip": + return codingQualityPositive(params) + case "*": + wildcard = codingQualityPositive(params) + } + } + } + return wildcard +} + +// codingQualityPositive reports whether an Accept-Encoding entry's +// parameters give it a quality above zero. No q parameter means 1. +func codingQualityPositive(params string) bool { + for param := range strings.SplitSeq(params, ";") { + key, value, _ := strings.Cut(param, "=") + if strings.TrimSpace(key) != "q" { + continue + } + q, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + return err == nil && q > 0 + } + return true } // notModified reports whether a conditional request's validators match. diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 473f80a9..85843123 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -42,9 +42,9 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) - if err != nil || string(out) != "DOC" { - t.Fatalf("rewrite = %q, %v", out, err) + m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil || string(m.body) != "DOC" { + t.Fatalf("rewrite = %q, %v", m.body, err) } } if got := r.calls.Load(); got != 1 { @@ -58,9 +58,9 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - out, _, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) - if err != nil || string(out) != strings.ToUpper(doc) { - t.Fatalf("rewrite(%s) = %q, %v", doc, out, err) + m, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + if err != nil || string(m.body) != strings.ToUpper(doc) { + t.Fatalf("rewrite(%s) = %q, %v", doc, m.body, err) } } if got := r.calls.Load(); got != 2 { @@ -80,11 +80,11 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) if err != nil { t.Error(err) } - outs <- string(out) + outs <- string(m.body) } go run() <-r.entered @@ -113,21 +113,21 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if out, _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { - t.Errorf("cached rewrite = %q, %v", out, err) + if m, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(m.body) != "DOC" { + t.Errorf("cached rewrite = %q, %v", m.body, err) } if got := r.calls.Load(); got != 1 { t.Errorf("rewrites = %d, want 1", got) @@ -140,18 +140,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) } - _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent - _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb before := r.calls.Load() - _, _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) - _, _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -162,7 +162,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -175,7 +175,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { t.Fatal("expected the rewrite error") } }