Serve cached npm and Composer metadata gzip-compressed

config.example.yaml

@@ -19,8 +19,8 @@ base_url: "http://localhost:8080"
 # artifact_max_size: ""
 
 # Memory for rewritten npm and Composer metadata, so each upstream document is
-# rewritten once rather than on every request. Set to "0" to rewrite on every
-# request. Default: "256MB".
+# rewritten and gzip-compressed once rather than on every request. Set to "0"
+# to rewrite on every request. Default: "256MB".
 # metadata_rewrite_cache_size: "256MB"
 
 # Public URL where the web UI is reached. Defaults to base_url when unset.

docs/configuration.md

@@ -966,6 +966,8 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`.
 
 The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time.
 
+Each cached document is also kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. The compressed copy counts against the cache size and typically adds about an eighth to each entry. Documents rewritten outside the cache are served uncompressed.
+
 ```yaml
 metadata_rewrite_cache_size: "256MB"   # default; "0" rewrites on every request
 ```

internal/handler/metadata_gzip_test.go

@@ -0,0 +1,227 @@
+package handler
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/git-pkgs/cooldown"
+)
+
+// compressibleDocuments sets upstream documents with enough versions that
+// gzip makes them smaller.
+func compressibleDocuments(u *validatorUpstream) {
+	var npm, composer []string
+	for i := range 50 {
+		v := fmt.Sprintf("1.0.%d", i)
+		npm = append(npm, `"`+v+`":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-`+v+`.tgz"}}`)
+		composer = append(composer, `{"version":"`+v+`","dist":{"type":"zip","url":"https://example.com/pkg-`+v+`.zip"}}`)
+	}
+	u.set(
+		`{"name":"left-pad","versions":{`+strings.Join(npm, ",")+`}}`,
+		`{"packages":{"vendor/pkg":[`+strings.Join(composer, ",")+`]}}`,
+	)
+}
+
+func gunzip(t *testing.T, rec *httptest.ResponseRecorder) string {
+	t.Helper()
+	zr, err := gzip.NewReader(bytes.NewReader(rec.Body.Bytes()))
+	if err != nil {
+		t.Fatalf("body is not gzip: %v", err)
+	}
+	body, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatalf("reading gzip body: %v", err)
+	}
+	return string(body)
+}
+
+var acceptGzip = http.Header{"Accept-Encoding": {"gzip, deflate, br"}}
+
+// TestRewrittenMetadataGzip checks that a client accepting gzip gets the
+// cached compressed copy of rewritten metadata, on the first fetch and on
+// the rewrite fast path, with an ETag of its own so it never matches the
+// uncompressed representation.
+func TestRewrittenMetadataGzip(t *testing.T) {
+	for _, c := range validatorCases {
+		for _, ttl := range []time.Duration{time.Hour, 0} {
+			t.Run(c.ecosystem+"/ttl="+ttl.String(), func(t *testing.T) {
+				u := newValidatorUpstream(t)
+				compressibleDocuments(u)
+				handler := c.handler(validatorProxy(t, u, ttl), u)
+
+				identity := serve(handler, c.path, nil)
+				for range 2 {
+					expectGzipOf(t, serve(handler, c.path, acceptGzip), identity)
+				}
+			})
+		}
+	}
+}
+
+// expectGzipOf checks that rec is the gzip representation of the
+// uncompressed response identity.
+func expectGzipOf(t *testing.T, rec, identity *httptest.ResponseRecorder) {
+	t.Helper()
+	if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "gzip" {
+		t.Fatalf("status %d, Content-Encoding %q; want 200 with gzip", rec.Code, rec.Header().Get("Content-Encoding"))
+	}
+	if rec.Body.Len() >= identity.Body.Len() {
+		t.Errorf("gzip body is %d bytes, uncompressed %d", rec.Body.Len(), identity.Body.Len())
+	}
+	if got := gunzip(t, rec); got != identity.Body.String() {
+		t.Errorf("decompressed body differs from the uncompressed response")
+	}
+	if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
+		t.Errorf("Vary = %q, want Accept-Encoding", got)
+	}
+	if etag := rec.Header().Get("ETag"); etag == "" || etag == identity.Header().Get("ETag") {
+		t.Errorf("gzip ETag %q must differ from the uncompressed ETag %q", etag, identity.Header().Get("ETag"))
+	}
+	if rec.Header().Get("Last-Modified") != identity.Header().Get("Last-Modified") {
+		t.Errorf("Last-Modified %q, want %q", rec.Header().Get("Last-Modified"), identity.Header().Get("Last-Modified"))
+	}
+}
+
+// TestRewrittenMetadataGzipRevalidation checks that each representation
+// revalidates with its own ETag, also when a reverse proxy weakened it.
+func TestRewrittenMetadataGzipRevalidation(t *testing.T) {
+	for _, c := range validatorCases {
+		t.Run(c.ecosystem, func(t *testing.T) {
+			u := newValidatorUpstream(t)
+			compressibleDocuments(u)
+			handler := c.handler(validatorProxy(t, u, time.Hour), u)
+			gzipETag := serve(handler, c.path, acceptGzip).Header().Get("ETag")
+			identityETag := serve(handler, c.path, nil).Header().Get("ETag")
+
+			for _, ifNoneMatch := range []string{gzipETag, "W/" + gzipETag} {
+				rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {ifNoneMatch}})
+				expectNotModified(t, rec, gzipETag)
+				if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
+					t.Errorf("304 Vary = %q, want Accept-Encoding", got)
+				}
+			}
+			if rec := serve(handler, c.path, http.Header{"Accept-Encoding": {"gzip"}, "If-None-Match": {identityETag}}); rec.Code != http.StatusOK {
+				t.Errorf("gzip request revalidating the uncompressed ETag: status %d, want 200", rec.Code)
+			}
+			if rec := serve(handler, c.path, http.Header{"If-None-Match": {gzipETag}}); rec.Code != http.StatusOK {
+				t.Errorf("uncompressed request revalidating the gzip ETag: status %d, want 200", rec.Code)
+			}
+		})
+	}
+}
+
+// TestRewrittenMetadataWithoutGzip checks that clients that do not accept
+// gzip get the uncompressed document, which still varies on Accept-Encoding.
+func TestRewrittenMetadataWithoutGzip(t *testing.T) {
+	for _, c := range validatorCases {
+		t.Run(c.ecosystem, func(t *testing.T) {
+			u := newValidatorUpstream(t)
+			compressibleDocuments(u)
+			handler := c.handler(validatorProxy(t, u, time.Hour), u)
+
+			for _, acceptEncoding := range []string{"", "identity", "gzip;q=0", "br"} {
+				header := http.Header{}
+				if acceptEncoding != "" {
+					header.Set("Accept-Encoding", acceptEncoding)
+				}
+				rec := serve(handler, c.path, header)
+				if rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" {
+					t.Errorf("Accept-Encoding %q: status %d, Content-Encoding %q; want 200 uncompressed", acceptEncoding, rec.Code, rec.Header().Get("Content-Encoding"))
+				}
+				if got := rec.Header().Get("Vary"); got != "Accept-Encoding" {
+					t.Errorf("Accept-Encoding %q: Vary = %q, want Accept-Encoding", acceptEncoding, got)
+				}
+			}
+		})
+	}
+}
+
+// TestRewrittenMetadataGzipOnlyFromRewriteCache checks that rewrites made
+// outside the cache, with it disabled or with cooldown on, stay uncompressed:
+// compressing them on every request is better left to a reverse proxy.
+func TestRewrittenMetadataGzipOnlyFromRewriteCache(t *testing.T) {
+	setups := map[string]func(*Proxy){
+		"cache disabled": func(p *Proxy) { p.SetMetadataRewriteCacheSize(0) },
+		"cooldown":       func(p *Proxy) { p.Cooldown = &cooldown.Config{Default: "3d"} },
+	}
+	for _, c := range validatorCases {
+		for name, setup := range setups {
+			t.Run(c.ecosystem+"/"+name, func(t *testing.T) {
+				u := newValidatorUpstream(t)
+				compressibleDocuments(u)
+				proxy := validatorProxy(t, u, time.Hour)
+				setup(proxy)
+				handler := c.handler(proxy, u)
+
+				for range 2 {
+					if rec := serve(handler, c.path, acceptGzip); rec.Code != http.StatusOK || rec.Header().Get("Content-Encoding") != "" {
+						t.Errorf("status %d, Content-Encoding %q; want 200 uncompressed", rec.Code, rec.Header().Get("Content-Encoding"))
+					}
+				}
+			})
+		}
+	}
+}
+
+// TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller checks that the cache keeps a
+// compressed copy of a rewrite, counted against its size, unless compressing
+// does not make the rewrite smaller.
+func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) {
+	c := newRewriteCache(1 << 20)
+	identity := func(b []byte) ([]byte, error) { return b, nil }
+	large := []byte(strings.Repeat(`{"version":"1.0.0"},`, 100))
+
+	for _, doc := range [][]byte{large, []byte("abc")} {
+		key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc)
+		if _, err := c.rewrite(t.Context(), key, doc, identity); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	m, ok := c.get(rewriteCacheKey("npm", "http://proxy", string(large[:3]), large))
+	if !ok || m.gzipped == nil {
+		t.Fatalf("large rewrite: cached %v, gzip copy %d bytes; want a gzip copy", ok, len(m.gzipped))
+	}
+	if c.size != int64(len(large)+len(m.gzipped)+3) {
+		t.Errorf("cache size = %d, want both rewrites and the gzip copy (%d)", c.size, len(large)+len(m.gzipped)+3)
+	}
+	if small, ok := c.get(rewriteCacheKey("npm", "http://proxy", "abc", []byte("abc"))); !ok || small.gzipped != nil {
+		t.Errorf("small rewrite: cached %v, gzip copy %d bytes; want none", ok, len(small.gzipped))
+	}
+}
+
+func TestAcceptsGzip(t *testing.T) {
+	cases := map[string]bool{
+		"":                        false,
+		"identity":                false,
+		"br, deflate":             false,
+		"gzip":                    true,
+		"GZIP":                    true,
+		"x-gzip":                  true,
+		"deflate, gzip;q=1.0":     true,
+		"gzip; q=0.5":             true,
+		"gzip;q=0":                false,
+		"gzip;q=0.000":            false,
+		"*":                       true,
+		"*;q=0":                   false,
+		"br;q=1, *;q=0.5":         true,
+		"gzip;q=0, *":             false,
+		"gzip;q=invalid, deflate": false,
+	}
+	for header, want := range cases {
+		r := httptest.NewRequest(http.MethodGet, "/", nil)
+		if header != "" {
+			r.Header.Set("Accept-Encoding", header)
+		}
+		if got := acceptsGzip(r); got != want {
+			t.Errorf("acceptsGzip(%q) = %v, want %v", header, got, want)
+		}
+	}
+}

internal/handler/metadata_validators_test.go

@@ -481,17 +481,17 @@ func TestRewriteCacheETag(t *testing.T) {
 	exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil }
 	key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc"))
 
-	out, etag, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim)
-	if err != nil || string(out) != "doc!" {
-		t.Fatalf("rewrite = %q, %v", out, err)
+	m, err := c.rewrite(t.Context(), key, []byte("doc"), exclaim)
+	if err != nil || string(m.body) != "doc!" {
+		t.Fatalf("rewrite = %q, %v", m.body, err)
 	}
-	if want := metadataETag([]byte("doc!")); etag.etag != want {
-		t.Errorf("rewrite ETag = %q, want %q", etag.etag, want)
+	if want := metadataETag([]byte("doc!")); m.etag != want {
+		t.Errorf("rewrite ETag = %q, want %q", m.etag, want)
 	}
-	if _, cached, ok := c.get(key); !ok || cached != etag {
-		t.Errorf("get validators = %+v, %v; want %+v", cached, ok, etag)
+	if cached, ok := c.get(key); !ok || cached.metadataValidators != m.metadataValidators {
+		t.Errorf("get validators = %+v, %v; want %+v", cached.metadataValidators, ok, m.metadataValidators)
 	}
-	if metadataETag([]byte("other")) == etag.etag {
+	if metadataETag([]byte("other")) == m.etag {
 		t.Error("different output has the same ETag")
 	}
 }

internal/handler/npm_rewrite_test.go

@@ -247,7 +247,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) {
 	body := []byte(`{}`)
 	proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body})
 	key := rewriteCacheKey("npm", "http://proxy", "demo", body)
-	if _, _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil {
+	if _, err := proxy.rewrites.rewrite(t.Context(), key, body, func(b []byte) ([]byte, error) { return b, nil }); err != nil {
 		t.Fatal(err)
 	}
 	if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok {