Harden against resource exhaustion: partial blobs, artifact size, concurrent diffs

cmd/proxy/main.go

@@ -521,7 +521,7 @@ func runMirror() {
 	// Build proxy (reuses same pipeline as serve)
 	fetcher := fetch.NewFetcher()
 	resolver := fetch.NewResolver()
-	proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
+	proxy := handler.NewProxy(db, store, handler.LimitArtifactSize(fetcher, cfg.ParseArtifactMaxSize()), resolver, logger)
 	proxy.Denylist, err = denylist.New(cfg.Denylist.Packages)
 	if err != nil {
 		_ = db.Close()

config.example.yaml

@@ -14,6 +14,10 @@ base_url: "http://localhost:8080"
 # Set to "0" to disable the timeout. Default: "30s".
 # http_timeout: "30s"
 
+# Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused
+# and not cached. Default: unlimited.
+# artifact_max_size: ""
+
 # Memory for rewritten npm and Composer metadata, so each upstream document is
 # rewritten once rather than on every request. Set to "0" to rewrite on every
 # request. Default: "256MB".

docs/configuration.md

@@ -892,6 +892,16 @@ metadata_max_size: "100MB"   # default
 
 Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
 
+### Artifact size limit
+
+Artifacts have no size limit by default. `artifact_max_size` refuses any artifact larger than the limit, so a client cannot fill storage or tie up the proxy by requesting huge downloads. An artifact whose upstream `Content-Length` is over the limit is refused before its body is read; one without a declared size fails once the download passes the limit. Either way nothing is cached and the client gets `502 Bad Gateway`, except when artifacts are streamed: one without a declared size has already started sending when it passes the limit, so the client sees the download cut off. The limit applies to `serve`, `mirror` and streamed artifacts.
+
+```yaml
+artifact_max_size: "2GB"   # default: unlimited ("" or "0")
+```
+
+Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`.
+
 ### Rewritten metadata cache
 
 The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time.

docs/swagger/docs.go

@@ -497,6 +497,12 @@ const docTemplate = `{
                         "schema": {
                             "$ref": "#/definitions/server.ErrorResponse"
                         }
+                    },
+                    "503": {
+                        "description": "Service Unavailable",
+                        "schema": {
+                            "$ref": "#/definitions/server.ErrorResponse"
+                        }
                     }
                 }
             }

docs/swagger/swagger.json

@@ -490,6 +490,12 @@
                         "schema": {
                             "$ref": "#/definitions/server.ErrorResponse"
                         }
+                    },
+                    "503": {
+                        "description": "Service Unavailable",
+                        "schema": {
+                            "$ref": "#/definitions/server.ErrorResponse"
+                        }
                     }
                 }
             }

internal/config/config.go

@@ -167,6 +167,11 @@ type Config struct {
 	// size return ErrMetadataTooLarge. Default: "100MB".
 	MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
 
+	// ArtifactMaxSize is the largest upstream artifact the proxy will fetch
+	// (e.g. "2GB"). Larger artifacts are refused and not cached. Default: ""
+	// (unlimited). "0" is also unlimited.
+	ArtifactMaxSize string `json:"artifact_max_size" yaml:"artifact_max_size"`
+
 	// MetadataRewriteCacheSize is how much rewritten npm and Composer
 	// metadata to keep in memory, so a document is rewritten once rather than
 	// on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to
@@ -1175,6 +1180,7 @@ func (c *Config) LoadFromEnv() {
 	setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
 	setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
 	setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
+	setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE")
 	setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE")
 	setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
 	setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY")
@@ -1272,7 +1278,7 @@ func (c *Config) Validate() error {
 		}
 	}
 
-	if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
+	if err := c.validateDownloadSizeLimits(); err != nil {
 		return err
 	}
 
@@ -1444,6 +1450,27 @@ func validateMetadataMaxSize(s string) error {
 	return nil
 }
 
+func (c *Config) validateDownloadSizeLimits() error {
+	if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
+		return err
+	}
+	return validateArtifactMaxSize(c.ArtifactMaxSize)
+}
+
+func validateArtifactMaxSize(s string) error {
+	if s == "" {
+		return nil
+	}
+	size, err := ParseSize(s)
+	if err != nil {
+		return fmt.Errorf("invalid artifact_max_size: %w", err)
+	}
+	if size < 0 {
+		return fmt.Errorf("invalid artifact_max_size %q: must not be negative", s)
+	}
+	return nil
+}
+
 func validateMetadataRewriteCacheSize(s string) error {
 	if s == "" || s == "0" {
 		return nil
@@ -1487,6 +1514,19 @@ func (c *Config) ParseMetadataMaxSize() int64 {
 	return size
 }
 
+// ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for
+// no limit.
+func (c *Config) ParseArtifactMaxSize() int64 {
+	if c.ArtifactMaxSize == "" {
+		return 0
+	}
+	size, err := ParseSize(c.ArtifactMaxSize)
+	if err != nil || size < 0 {
+		return 0
+	}
+	return size
+}
+
 // ParseHTTPTimeout returns the upstream HTTP client timeout.
 // Returns 30s if unset, 0 (no timeout) if explicitly set to "0".
 func (c *Config) ParseHTTPTimeout() time.Duration {

internal/config/config_test.go

@@ -1669,3 +1669,52 @@ func TestLoadCacheArtifactsFromEnv(t *testing.T) {
 		t.Error("Storage.CacheArtifacts should be false")
 	}
 }
+
+func TestParseArtifactMaxSize(t *testing.T) {
+	tests := []struct {
+		name string
+		size string
+		want int64
+	}{
+		{"unset is unlimited", "", 0},
+		{"zero is unlimited", "0", 0},
+		{"explicit value", "2GB", 2 << 30},
+		{"bytes", "1024", 1024},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Default()
+			cfg.ArtifactMaxSize = tt.size
+			if got := cfg.ParseArtifactMaxSize(); got != tt.want {
+				t.Errorf("ParseArtifactMaxSize() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateArtifactMaxSize(t *testing.T) {
+	for _, invalid := range []string{"not-a-size", "-1"} {
+		cfg := Default()
+		cfg.ArtifactMaxSize = invalid
+		if err := cfg.Validate(); err == nil {
+			t.Errorf("expected validation error for artifact_max_size %q", invalid)
+		}
+	}
+	for _, valid := range []string{"", "0", "2GB"} {
+		cfg := Default()
+		cfg.ArtifactMaxSize = valid
+		if err := cfg.Validate(); err != nil {
+			t.Errorf("unexpected error for artifact_max_size %q: %v", valid, err)
+		}
+	}
+}
+
+func TestLoadArtifactMaxSizeFromEnv(t *testing.T) {
+	cfg := Default()
+	t.Setenv("PROXY_ARTIFACT_MAX_SIZE", "2GB")
+	cfg.LoadFromEnv()
+
+	if cfg.ArtifactMaxSize != "2GB" {
+		t.Errorf("ArtifactMaxSize = %q, want %q", cfg.ArtifactMaxSize, "2GB")
+	}
+}

internal/handler/artifact_size.go

@@ -0,0 +1,76 @@
+package handler
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/git-pkgs/registries/fetch"
+)
+
+// ErrArtifactTooLarge is returned for an upstream artifact larger than the
+// configured artifact size limit.
+var ErrArtifactTooLarge = errors.New("artifact exceeds size limit")
+
+// LimitArtifactSize wraps fetcher so no artifact over maxSize bytes is
+// fetched. An artifact whose declared size is over the limit is refused
+// before its body is read; one of unknown size fails once reading passes the
+// limit, so the store or stream consuming it fails and nothing is cached.
+// A maxSize of zero or less returns fetcher unchanged.
+func LimitArtifactSize(fetcher fetch.FetcherInterface, maxSize int64) fetch.FetcherInterface { //nolint:ireturn // returns fetcher itself when there is no limit
+	if maxSize <= 0 {
+		return fetcher
+	}
+	return &sizeLimitedFetcher{FetcherInterface: fetcher, maxSize: maxSize}
+}
+
+type sizeLimitedFetcher struct {
+	fetch.FetcherInterface
+	maxSize int64
+}
+
+func (f *sizeLimitedFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) {
+	return f.limit(f.FetcherInterface.Fetch(ctx, url))
+}
+
+func (f *sizeLimitedFetcher) FetchWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.Artifact, error) {
+	return f.limit(f.FetcherInterface.FetchWithHeaders(ctx, url, headers))
+}
+
+func (f *sizeLimitedFetcher) limit(artifact *fetch.Artifact, err error) (*fetch.Artifact, error) {
+	if err != nil {
+		return nil, err
+	}
+	if artifact.Size > f.maxSize {
+		_ = artifact.Body.Close()
+		return nil, fmt.Errorf("%w: upstream declared %d bytes, limit is %d", ErrArtifactTooLarge, artifact.Size, f.maxSize)
+	}
+	limited := *artifact
+	limited.Body = &sizeLimitedBody{ReadCloser: artifact.Body, remaining: f.maxSize}
+	return &limited, nil
+}
+
+// sizeLimitedBody fails with ErrArtifactTooLarge once more than the limit has
+// been read. It reads one byte past the limit to tell an artifact of exactly
+// the limit from a larger one.
+type sizeLimitedBody struct {
+	io.ReadCloser
+	remaining int64
+}
+
+func (b *sizeLimitedBody) Read(p []byte) (int, error) {
+	if b.remaining < 0 {
+		return 0, ErrArtifactTooLarge
+	}
+	if int64(len(p)) > b.remaining+1 {
+		p = p[:b.remaining+1]
+	}
+	n, err := b.ReadCloser.Read(p)
+	b.remaining -= int64(n)
+	if b.remaining < 0 {
+		return n + int(b.remaining), ErrArtifactTooLarge
+	}
+	return n, err
+}

internal/handler/artifact_size_test.go

@@ -0,0 +1,115 @@
+package handler
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/git-pkgs/registries/fetch"
+)
+
+type closeRecorder struct {
+	io.Reader
+	closed bool
+}
+
+func (c *closeRecorder) Close() error {
+	c.closed = true
+	return nil
+}
+
+func TestLimitArtifactSizeUnlimitedReturnsFetcher(t *testing.T) {
+	fetcher := &mockFetcher{}
+	if got := LimitArtifactSize(fetcher, 0); got != fetcher {
+		t.Errorf("LimitArtifactSize(f, 0) = %T, want the fetcher itself", got)
+	}
+}
+
+func TestLimitArtifactSizeRefusesDeclaredOversize(t *testing.T) {
+	body := &closeRecorder{Reader: strings.NewReader("0123456789")}
+	limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{Body: body, Size: 10}}, 4)
+
+	_, err := limited.Fetch(context.Background(), "https://registry.example/a.tgz")
+	if !errors.Is(err, ErrArtifactTooLarge) {
+		t.Fatalf("Fetch error = %v, want ErrArtifactTooLarge", err)
+	}
+	if !body.closed {
+		t.Error("refused body was not closed")
+	}
+}
+
+func TestLimitArtifactSizeFailsUndeclaredOversizeWhileReading(t *testing.T) {
+	limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{
+		Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1,
+	}}, 4)
+
+	artifact, err := limited.Fetch(context.Background(), "https://registry.example/a.tgz")
+	if err != nil {
+		t.Fatalf("Fetch failed: %v", err)
+	}
+	if _, err := io.ReadAll(artifact.Body); !errors.Is(err, ErrArtifactTooLarge) {
+		t.Errorf("reading body error = %v, want ErrArtifactTooLarge", err)
+	}
+}
+
+func TestLimitArtifactSizeAllowsArtifactAtLimit(t *testing.T) {
+	limited := LimitArtifactSize(&mockFetcher{artifact: &fetch.Artifact{
+		Body: io.NopCloser(strings.NewReader("0123")), Size: -1,
+	}}, 4)
+
+	artifact, err := limited.FetchWithHeaders(context.Background(), "https://registry.example/a.tgz", nil)
+	if err != nil {
+		t.Fatalf("Fetch failed: %v", err)
+	}
+	data, err := io.ReadAll(artifact.Body)
+	if err != nil || string(data) != "0123" {
+		t.Errorf("body = %q, %v; want %q, nil", data, err, "0123")
+	}
+}
+
+func TestOversizedArtifactIsNotCached(t *testing.T) {
+	proxy, db, store, fetcher := setupTestProxy(t)
+	fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1}
+	proxy.Fetcher = LimitArtifactSize(fetcher, 4)
+
+	_, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://registry.example/big-1.0.0.tgz")
+	if !errors.Is(err, ErrArtifactTooLarge) {
+		t.Fatalf("GetOrFetchArtifactFromURL error = %v, want ErrArtifactTooLarge", err)
+	}
+	if len(store.files) != 0 {
+		t.Errorf("storage holds %d objects after an oversized fetch, want 0", len(store.files))
+	}
+	record, err := db.GetArtifact("pkg:npm/big@1.0.0", "big-1.0.0.tgz")
+	if err != nil {
+		t.Fatalf("GetArtifact failed: %v", err)
+	}
+	if record != nil {
+		t.Error("oversized artifact was recorded in the cache database")
+	}
+
+	rec := httptest.NewRecorder()
+	proxy.serveArtifactError(rec, err, "failed to fetch package")
+	if rec.Code != http.StatusBadGateway {
+		t.Errorf("status = %d, want %d", rec.Code, http.StatusBadGateway)
+	}
+}
+
+func TestOversizedStreamedArtifactFailsTheStream(t *testing.T) {
+	proxy, _, _, fetcher := setupTestProxy(t)
+	proxy.StreamArtifacts = true
+	fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("0123456789")), Size: -1}
+	proxy.Fetcher = LimitArtifactSize(fetcher, 4)
+
+	result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "big", "1.0.0", "big-1.0.0.tgz", "https://registry.example/big-1.0.0.tgz")
+	if err != nil {
+		t.Fatalf("GetOrFetchArtifactFromURL failed: %v", err)
+	}
+	defer func() { _ = result.Reader.Close() }()
+	if _, err := io.ReadAll(result.Reader); !errors.Is(err, ErrArtifactTooLarge) {
+		t.Errorf("streamed body error = %v, want ErrArtifactTooLarge", err)
+	}
+}