From 87505c01c199aa370c5d30ca12380dede6dff25d Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 12:35:22 +0200 Subject: [PATCH 01/17] Mark pushed artifacts as local and keep them out of eviction Artifacts get a source column: upstream for cached copies of a registry, local for archives pushed to the proxy. Existing rows become upstream through the new 012_add_artifact_source migration. Local artifacts aren't a cache, so the LRU size limit doesn't count or evict them and storage retention never selects them. They are only removed by deleting them. A local artifact has no upstream URL and stores it as empty. Refs #133 --- internal/database/local_artifacts_test.go | 154 ++++++++++++++++++++++ internal/database/queries.go | 37 +++--- internal/database/retention.go | 1 + internal/database/schema.go | 29 +++- internal/database/types.go | 16 +++ 5 files changed, 217 insertions(+), 20 deletions(-) create mode 100644 internal/database/local_artifacts_test.go diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go new file mode 100644 index 00000000..cba4b640 --- /dev/null +++ b/internal/database/local_artifacts_test.go @@ -0,0 +1,154 @@ +package database + +import ( + "database/sql" + "path/filepath" + "testing" + "time" +) + +func seedSourcedArtifact(t *testing.T, db *DB, name, source string, size int64, fetched time.Time) { + t.Helper() + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@1.0.0" + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + art := &Artifact{ + VersionPURL: versionPURL, + Filename: name + "-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/" + name + "/1.0.0/" + name + "-1.0.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + Source: source, + } + if source != SourceLocal { + art.UpstreamURL = "https://registry.example/" + name + "-1.0.0.tgz" + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +func TestArtifactSourceDefaultsToUpstream(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "cached", "", 100, time.Now()) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, time.Now()) + + cached, err := db.GetArtifact("pkg:npm/cached@1.0.0", "cached-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact(cached): %v", err) + } + if cached.Source != SourceUpstream { + t.Errorf("cached artifact source = %q, want %q", cached.Source, SourceUpstream) + } + + pushed, err := db.GetArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact(pushed): %v", err) + } + if pushed.Source != SourceLocal { + t.Errorf("pushed artifact source = %q, want %q", pushed.Source, SourceLocal) + } + if pushed.UpstreamURL != "" { + t.Errorf("pushed artifact upstream_url = %q, want empty", pushed.UpstreamURL) + } + }) +} + +func TestLocalArtifactsAreNotEvictable(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "cached", SourceUpstream, 1000, time.Now()) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 5000, time.Now()) + + total, err := db.GetTotalCacheSize() + if err != nil { + t.Fatalf("GetTotalCacheSize: %v", err) + } + if total != 1000 { + t.Errorf("total cache size = %d, want 1000 (local artifacts excluded)", total) + } + + lru, err := db.GetLeastRecentlyUsedArtifacts(10) + if err != nil { + t.Fatalf("GetLeastRecentlyUsedArtifacts: %v", err) + } + if len(lru) != 1 || lru[0].Filename != "cached-1.0.0.tgz" { + t.Errorf("LRU artifacts = %+v, want only cached-1.0.0.tgz", lru) + } + }) +} + +func TestLocalArtifactsAreNotRetentionCandidates(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + old := time.Now().Add(-48 * time.Hour) + cutoff := time.Now().Add(-24 * time.Hour) + seedSourcedArtifact(t, db, "cached", SourceUpstream, 100, old) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, old) + + maxID, err := db.MaxArtifactID() + if err != nil { + t.Fatalf("MaxArtifactID: %v", err) + } + candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + got := candidateNames(candidates) + if _, ok := got["pushed-1.0.0.tgz"]; ok { + t.Error("local artifact returned as a retention candidate") + } + if _, ok := got["cached-1.0.0.tgz"]; !ok { + t.Error("upstream artifact missing from retention candidates") + } + + cleared, err := db.ClearExpiredArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz", "npm/pushed/1.0.0/pushed-1.0.0.tgz", cutoff) + if err != nil || cleared { + t.Errorf("ClearExpiredArtifact(local) = %v, %v, want false, nil", cleared, err) + } + }) +} + +func TestMigrationAddsArtifactSource(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN source`); err != nil { + t.Fatalf("drop source column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '012_add_artifact_source'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if _, err := sqlDB.Exec(`INSERT INTO artifacts (version_purl, filename, upstream_url, created_at, updated_at) + VALUES ('pkg:npm/old@1.0.0', 'old-1.0.0.tgz', 'https://registry.example/old-1.0.0.tgz', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("seed artifact: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + + art, err := db.GetArtifact("pkg:npm/old@1.0.0", "old-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if art.Source != SourceUpstream { + t.Errorf("existing artifact source = %q, want %q", art.Source, SourceUpstream) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 2652bc17..53943f15 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -70,7 +70,8 @@ func (db *DB) UpsertPackage(pkg *Package) error { repository_url = EXCLUDED.repository_url, registry_url = EXCLUDED.registry_url, enriched_at = EXCLUDED.enriched_at, - updated_at = EXCLUDED.updated_at + updated_at = EXCLUDED.updated_at, + source = EXCLUDED.source ` } else { query = ` @@ -86,7 +87,8 @@ func (db *DB) UpsertPackage(pkg *Package) error { repository_url = excluded.repository_url, registry_url = excluded.registry_url, enriched_at = excluded.enriched_at, - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + source = excluded.source ` } @@ -216,7 +218,7 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE version_purl = ? AND filename = ? `) err := db.Get(&a, query, versionPURL, filename) @@ -284,7 +286,7 @@ func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE storage_path = ? `) err := db.Get(&a, query, storagePath) @@ -302,7 +304,7 @@ func (db *DB) GetArtifactsByVersionPURL(versionPURL string) ([]Artifact, error) query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE version_purl = ? ORDER BY filename `) @@ -367,8 +369,8 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + created_at, updated_at, source) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = EXCLUDED.upstream_url, storage_path = EXCLUDED.storage_path, @@ -376,15 +378,16 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er size = EXCLUDED.size, content_type = EXCLUDED.content_type, fetched_at = EXCLUDED.fetched_at, - updated_at = EXCLUDED.updated_at - WHERE artifacts.storage_path IS NOT DISTINCT FROM $13 + updated_at = EXCLUDED.updated_at, + source = EXCLUDED.source + WHERE artifacts.storage_path IS NOT DISTINCT FROM $14 ` } else { query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + created_at, updated_at, source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = excluded.upstream_url, storage_path = excluded.storage_path, @@ -392,14 +395,16 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er size = excluded.size, content_type = excluded.content_type, fetched_at = excluded.fetched_at, - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + source = excluded.source WHERE artifacts.storage_path IS ? ` } res, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, previous, + a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, + a.sourceOrUpstream(), previous, ) if err != nil { return false, fmt.Errorf("upserting artifact: %w", err) @@ -445,9 +450,9 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts - WHERE storage_path IS NOT NULL + WHERE storage_path IS NOT NULL AND source <> 'local' ORDER BY last_accessed_at ASC NULLS FIRST LIMIT ? `) @@ -460,7 +465,7 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { func (db *DB) GetTotalCacheSize() (int64, error) { var total sql.NullInt64 - err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL`) + err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local'`) if err != nil { return 0, err } diff --git a/internal/database/retention.go b/internal/database/retention.go index cd6e11fe..90fffd2a 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -36,6 +36,7 @@ func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { // accessed since. It takes the cutoff twice. const expiredCondition = ` storage_path IS NOT NULL + AND source <> 'local' AND fetched_at < ? AND (last_accessed_at IS NULL OR last_accessed_at < ?)` diff --git a/internal/database/schema.go b/internal/database/schema.go index b8d3f93f..76beaf12 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -72,7 +72,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME + updated_at DATETIME, + source TEXT NOT NULL DEFAULT 'upstream' ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -190,7 +191,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP + updated_at TIMESTAMP, + source TEXT NOT NULL DEFAULT 'upstream' ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -267,7 +269,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME + updated_at DATETIME, + source TEXT NOT NULL DEFAULT 'upstream' ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -288,7 +291,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP + updated_at TIMESTAMP, + source TEXT NOT NULL DEFAULT 'upstream' ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -401,6 +405,7 @@ var migrations = []migration{ {"009_add_pending_deletes", migrateAddPendingDeletes}, {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, {"011_add_metadata_dates", migrateAddMetadataDates}, + {"012_add_artifact_source", migrateAddArtifactSource}, } // isTableNotFound returns true if the error indicates a missing table. @@ -778,3 +783,19 @@ func (db *DB) EnsureMetadataCacheTable() error { } return nil } + +// migrateAddArtifactSource records whether an artifact is a cached copy of an +// upstream registry or was pushed to the proxy. Existing rows are all upstream. +func migrateAddArtifactSource(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "source") + if err != nil { + return fmt.Errorf("checking column source: %w", err) + } + if hasCol { + return nil + } + if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN source TEXT NOT NULL DEFAULT 'upstream'`); err != nil { + return fmt.Errorf("adding column source to artifacts: %w", err) + } + return nil +} diff --git a/internal/database/types.go b/internal/database/types.go index 9c4fdd6a..501d49e7 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -125,6 +125,14 @@ func decodePURLComponent(s string) string { return decoded } +// Artifact sources. Upstream artifacts are a cache of a registry and may be +// evicted; local artifacts were pushed to the proxy and are only removed by a +// delete. +const ( + SourceUpstream = "upstream" + SourceLocal = "local" +) + // Artifact represents a cached artifact in the database. // This table is proxy-specific and not part of git-pkgs. type Artifact struct { @@ -141,6 +149,14 @@ type Artifact struct { LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Source string `db:"source" json:"source"` +} + +func (a *Artifact) sourceOrUpstream() string { + if a.Source == "" { + return SourceUpstream + } + return a.Source } // IsCached returns true if the artifact has been fetched and stored locally. From ea696a7abc43a3fe3b5a21a738f212b618696cb5 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 12:35:22 +0200 Subject: [PATCH 02/17] Add a hosted config block and a bearer token check for writes hosted.enabled turns on pushing packages to the proxy and is off by default, like mirror_api. A pushed version is served in place of the upstream version with the same number, so anyone who can write could swap a package for every client. Enabling it therefore requires hosted.token, which supports ${VAR_NAME} expansion and can be set with PROXY_HOSTED_TOKEN. hosted.max_upload_size caps one archive and defaults to 100MB. requireBearerToken compares SHA-256 digests of the presented and configured tokens in constant time and answers 401 with a WWW-Authenticate challenge otherwise. Refs #133 --- internal/config/config.go | 12 +++++ internal/config/hosted.go | 55 ++++++++++++++++++++ internal/config/hosted_test.go | 79 +++++++++++++++++++++++++++++ internal/server/errors.go | 11 ++-- internal/server/hosted_auth.go | 37 ++++++++++++++ internal/server/hosted_auth_test.go | 57 +++++++++++++++++++++ 6 files changed, 246 insertions(+), 5 deletions(-) create mode 100644 internal/config/hosted.go create mode 100644 internal/config/hosted_test.go create mode 100644 internal/server/hosted_auth.go create mode 100644 internal/server/hosted_auth_test.go diff --git a/internal/config/config.go b/internal/config/config.go index 87cb713f..fd483899 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -190,6 +190,9 @@ type Config struct { // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` + // Hosted configures pushing packages to the proxy (/api/packages). + Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -1209,6 +1212,9 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") + setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") + setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") @@ -1335,6 +1341,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") case c.MirrorAPI: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") + case c.Hosted.Enabled: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } return nil } @@ -1364,6 +1372,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Hosted.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/hosted.go b/internal/config/hosted.go new file mode 100644 index 00000000..16aeced0 --- /dev/null +++ b/internal/config/hosted.go @@ -0,0 +1,55 @@ +package config + +import "fmt" + +const defaultHostedMaxUploadSize = "100MB" + +// HostedConfig configures packages pushed to the proxy through the +// /api/packages endpoints. Pushed versions are served in place of upstream +// versions with the same number, so writes always require a token. +type HostedConfig struct { + // Enabled registers the /api/packages endpoints. Off by default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to push and delete packages. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxUploadSize caps a single pushed archive (e.g., "100MB"). Must be > 0. + // Default: "100MB". + MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"` + + maxUploadBytes int64 +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (h *HostedConfig) TokenExpanded() string { + return expandEnv(h.Token) +} + +// MaxUploadBytes returns the parsed MaxUploadSize. Validate must run first. +func (h *HostedConfig) MaxUploadBytes() int64 { + return h.maxUploadBytes +} + +// Validate checks the hosted package settings, applying the default upload +// size if unset. +func (h *HostedConfig) Validate() error { + if h.MaxUploadSize == "" { + h.MaxUploadSize = defaultHostedMaxUploadSize + } + size, err := ParseSize(h.MaxUploadSize) + if err != nil { + return fmt.Errorf("invalid hosted.max_upload_size: %w", err) + } + if size <= 0 { + return fmt.Errorf("invalid hosted.max_upload_size %q: must be > 0", h.MaxUploadSize) + } + h.maxUploadBytes = size + + if h.Enabled && h.TokenExpanded() == "" { + return fmt.Errorf("hosted.token is required when hosted.enabled is true") + } + return nil +} diff --git a/internal/config/hosted_test.go b/internal/config/hosted_test.go new file mode 100644 index 00000000..787935dd --- /dev/null +++ b/internal/config/hosted_test.go @@ -0,0 +1,79 @@ +package config + +import "testing" + +func TestHostedConfigValidate(t *testing.T) { + t.Setenv("HOSTED_TEST_TOKEN", "from-env") + + tests := []struct { + name string + hosted HostedConfig + wantErr bool + }{ + {name: "disabled without token", hosted: HostedConfig{}}, + {name: "enabled without token", hosted: HostedConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", hosted: HostedConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"}}, + {name: "token env reference unset", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_UNSET}"}, wantErr: true}, + {name: "invalid upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "lots"}, wantErr: true}, + {name: "zero upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "0"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.hosted.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestHostedConfigDefaults(t *testing.T) { + h := HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"} + t.Setenv("HOSTED_TEST_TOKEN", "from-env") + if err := h.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := h.TokenExpanded(); got != "from-env" { + t.Errorf("TokenExpanded() = %q, want from-env", got) + } + if got := h.MaxUploadBytes(); got != 100<<20 { + t.Errorf("MaxUploadBytes() = %d, want %d", got, 100<<20) + } + + h.MaxUploadSize = "2GB" + if err := h.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := h.MaxUploadBytes(); got != 2<<30 { + t.Errorf("MaxUploadBytes() = %d, want %d", got, int64(2)<<30) + } +} + +func TestValidateRejectsHostedWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Hosted = HostedConfig{Enabled: true, Token: "s3cret"} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted hosted.enabled with storage.cache_artifacts: false") + } +} + +func TestLoadFromEnvHosted(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_HOSTED_ENABLED", "true") + t.Setenv("PROXY_HOSTED_TOKEN", "env-token") + t.Setenv("PROXY_HOSTED_MAX_UPLOAD_SIZE", "50MB") + + cfg.LoadFromEnv() + + if !cfg.Hosted.Enabled { + t.Error("Hosted.Enabled not loaded from PROXY_HOSTED_ENABLED") + } + if cfg.Hosted.Token != "env-token" { + t.Errorf("Hosted.Token = %q, want env-token", cfg.Hosted.Token) + } + if cfg.Hosted.MaxUploadSize != "50MB" { + t.Errorf("Hosted.MaxUploadSize = %q, want 50MB", cfg.Hosted.MaxUploadSize) + } +} diff --git a/internal/server/errors.go b/internal/server/errors.go index d30bff92..7cbe20ed 100644 --- a/internal/server/errors.go +++ b/internal/server/errors.go @@ -8,11 +8,12 @@ import ( // Error codes returned in API error responses. These are stable identifiers // that clients can match on; the message text is for humans and may change. const ( - ErrCodeBadRequest = "BAD_REQUEST" - ErrCodeNotFound = "NOT_FOUND" - ErrCodeUpstream = "UPSTREAM_ERROR" - ErrCodeInternal = "INTERNAL_ERROR" - ErrCodeBusy = "BUSY" + ErrCodeBadRequest = "BAD_REQUEST" + ErrCodeNotFound = "NOT_FOUND" + ErrCodeUnauthorized = "UNAUTHORIZED" + ErrCodeUpstream = "UPSTREAM_ERROR" + ErrCodeInternal = "INTERNAL_ERROR" + ErrCodeBusy = "BUSY" ) // ErrorResponse is the JSON body returned for API errors. diff --git a/internal/server/hosted_auth.go b/internal/server/hosted_auth.go new file mode 100644 index 00000000..b47a2971 --- /dev/null +++ b/internal/server/hosted_auth.go @@ -0,0 +1,37 @@ +package server + +import ( + "crypto/sha256" + "crypto/subtle" + "net/http" + "strings" +) + +const bearerScheme = "bearer " + +// requireBearerToken rejects requests that do not carry +// "Authorization: Bearer ". Both tokens are hashed before comparing so +// the check takes the same time whatever the length of the presented token. +func requireBearerToken(token string) func(http.Handler) http.Handler { + want := sha256.Sum256([]byte(token)) + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + presented, ok := bearerToken(r) + got := sha256.Sum256([]byte(presented)) + if !ok || subtle.ConstantTimeCompare(got[:], want[:]) != 1 { + w.Header().Set("WWW-Authenticate", `Bearer realm="proxy"`) + writeError(w, http.StatusUnauthorized, ErrCodeUnauthorized, "a valid bearer token is required") + return + } + next.ServeHTTP(w, r) + }) + } +} + +func bearerToken(r *http.Request) (string, bool) { + header := r.Header.Get("Authorization") + if len(header) <= len(bearerScheme) || !strings.EqualFold(header[:len(bearerScheme)], bearerScheme) { + return "", false + } + return header[len(bearerScheme):], true +} diff --git a/internal/server/hosted_auth_test.go b/internal/server/hosted_auth_test.go new file mode 100644 index 00000000..33c0edfb --- /dev/null +++ b/internal/server/hosted_auth_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRequireBearerToken(t *testing.T) { + tests := []struct { + name string + authorization string + wantStatus int + }{ + {name: "missing header", authorization: "", wantStatus: http.StatusUnauthorized}, + {name: "wrong token", authorization: "Bearer guess", wantStatus: http.StatusUnauthorized}, + {name: "token prefix only", authorization: "Bearer s3cre", wantStatus: http.StatusUnauthorized}, + {name: "basic scheme", authorization: "Basic czNjcmV0", wantStatus: http.StatusUnauthorized}, + {name: "empty bearer", authorization: "Bearer ", wantStatus: http.StatusUnauthorized}, + {name: "valid token", authorization: "Bearer s3cret", wantStatus: http.StatusNoContent}, + {name: "scheme is case-insensitive", authorization: "bearer s3cret", wantStatus: http.StatusNoContent}, + } + + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + handler := requireBearerToken("s3cret")(next) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/api/packages/npm/request/2.88.3", nil) + if tt.authorization != "" { + req.Header.Set("Authorization", tt.authorization) + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", rec.Code, tt.wantStatus) + } + if tt.wantStatus != http.StatusUnauthorized { + return + } + if got := rec.Header().Get("WWW-Authenticate"); got != `Bearer realm="proxy"` { + t.Errorf("WWW-Authenticate = %q, want Bearer realm=\"proxy\"", got) + } + var body ErrorResponse + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decoding error body: %v", err) + } + if body.Code != ErrCodeUnauthorized { + t.Errorf("error code = %q, want %q", body.Code, ErrCodeUnauthorized) + } + }) + } +} From 5e66688873f8905632e17869d560195bca01ac3d Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:11:49 +0200 Subject: [PATCH 03/17] Push, list and delete npm and Composer archives through /api/hosted With hosted.enabled, PUT /api/hosted/{ecosystem}/{name}/{version} stores the request body as the archive for that version, marked local. It replaces any cached upstream copy and is served from the cache without contacting the upstream. GET /api/hosted/{ecosystem}/{name} lists what was pushed and DELETE removes a version again, after which clients get the upstream archive if there is one. Pushing and deleting need the bearer token; listing is open like the rest of the API. Download handlers look archives up by the filename clients request, so a push is stored under the name its ecosystem derives from name and version: {name}-{version}.tgz for npm, without the scope, and {package}-{version}.zip for Composer. Other ecosystems are refused for now. The Composer download handler resolves the upstream metadata before it looks at the cache and answers 404 for versions the upstream doesn't list. It now serves a pushed archive first. Only artifacts marked local take that path, so cached upstream archives still go through the metadata and commit checks. The version's integrity is set to the pushed bytes, so the cache's integrity check verifies them rather than an upstream digest. Names and versions with path separators, dot segments or control characters are refused before anything is stored. Refs #133 --- cmd/proxy/main.go | 5 + docs/swagger/docs.go | 239 +++++++++++++++++ docs/swagger/swagger.json | 239 +++++++++++++++++ internal/database/local_artifacts.go | 63 +++++ internal/database/local_artifacts_test.go | 53 ++++ internal/handler/composer.go | 10 + internal/handler/hosted.go | 263 +++++++++++++++++++ internal/handler/hosted_test.go | 181 +++++++++++++ internal/server/hosted_api.go | 192 ++++++++++++++ internal/server/hosted_api_test.go | 302 ++++++++++++++++++++++ internal/server/server.go | 6 + 11 files changed, 1553 insertions(+) create mode 100644 internal/database/local_artifacts.go create mode 100644 internal/handler/hosted.go create mode 100644 internal/handler/hosted_test.go create mode 100644 internal/server/hosted_api.go create mode 100644 internal/server/hosted_api_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 47efdc77..9f8d243a 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -5,6 +5,11 @@ // @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing. // @BasePath / // +// @securityDefinitions.apikey BearerAuth +// @in header +// @name Authorization +// @description "Bearer " followed by hosted.token +// // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. // diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 798b50ac..5e54d3c6 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -115,6 +115,174 @@ const docTemplate = `{ } } }, + "/api/hosted/{ecosystem}/{name}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "List pushed package archives", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.HostedListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/hosted/{ecosystem}/{name}/{version}": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "consumes": [ + "application/octet-stream" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "Push a package archive", + "parameters": [ + { + "type": "string", + "description": "Ecosystem (npm or composer)", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name, including an npm scope or the Composer vendor", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/server.HostedPushResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "tags": [ + "hosted" + ], + "summary": "Delete a pushed version", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/outdated": { "post": { "consumes": [ @@ -509,6 +677,26 @@ const docTemplate = `{ } }, "definitions": { + "handler.HostedArtifact": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -640,6 +828,49 @@ const docTemplate = `{ } } }, + "server.HostedListResponse": { + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.HostedArtifact" + } + }, + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "server.HostedPushResponse": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.OutdatedPackage": { "type": "object", "properties": { @@ -872,6 +1103,14 @@ const docTemplate = `{ } } } + }, + "securityDefinitions": { + "BearerAuth": { + "description": "\"Bearer \" followed by hosted.token", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } } }` diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index fd175bf9..65a271b2 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -108,6 +108,174 @@ } } }, + "/api/hosted/{ecosystem}/{name}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "List pushed package archives", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.HostedListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/hosted/{ecosystem}/{name}/{version}": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "consumes": [ + "application/octet-stream" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "Push a package archive", + "parameters": [ + { + "type": "string", + "description": "Ecosystem (npm or composer)", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name, including an npm scope or the Composer vendor", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/server.HostedPushResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "tags": [ + "hosted" + ], + "summary": "Delete a pushed version", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/outdated": { "post": { "consumes": [ @@ -502,6 +670,26 @@ } }, "definitions": { + "handler.HostedArtifact": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -633,6 +821,49 @@ } } }, + "server.HostedListResponse": { + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.HostedArtifact" + } + }, + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "server.HostedPushResponse": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.OutdatedPackage": { "type": "object", "properties": { @@ -865,5 +1096,13 @@ } } } + }, + "securityDefinitions": { + "BearerAuth": { + "description": "\"Bearer \" followed by hosted.token", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } } } \ No newline at end of file diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go new file mode 100644 index 00000000..3eee1fc2 --- /dev/null +++ b/internal/database/local_artifacts.go @@ -0,0 +1,63 @@ +package database + +import ( + "fmt" + "time" +) + +// GetLocalArtifacts returns the artifacts pushed to the proxy for a package, +// ordered by version and filename. +func (db *DB) GetLocalArtifacts(packagePURL string) ([]Artifact, error) { + var artifacts []Artifact + query := db.Rebind(` + SELECT a.id, a.version_purl, a.filename, a.upstream_url, a.storage_path, a.content_hash, + a.size, a.content_type, a.fetched_at, a.hit_count, a.last_accessed_at, + a.created_at, a.updated_at, a.source + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + WHERE v.package_purl = ? AND a.source = 'local' + ORDER BY a.version_purl, a.filename + `) + if err := db.Select(&artifacts, query, packagePURL); err != nil { + return nil, err + } + return artifacts, nil +} + +// DeleteLocalArtifacts removes the pushed artifacts of a version and queues +// their stored objects for deletion in the same transaction, so a failed +// queue keeps the records instead of orphaning the objects. Artifacts cached +// from an upstream are left alone. It returns how many records it removed. +func (db *DB) DeleteLocalArtifacts(versionPURL string) (int, error) { + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + var paths []string + selectQuery := db.Rebind(` + SELECT storage_path FROM artifacts + WHERE version_purl = ? AND source = 'local' AND storage_path IS NOT NULL + `) + if err := tx.Select(&paths, selectQuery, versionPURL); err != nil { + return 0, fmt.Errorf("reading local artifacts: %w", err) + } + + res, err := tx.Exec(db.Rebind(`DELETE FROM artifacts WHERE version_purl = ? AND source = 'local'`), versionPURL) + if err != nil { + return 0, fmt.Errorf("deleting local artifacts: %w", err) + } + deleted, err := res.RowsAffected() + if err != nil { + return 0, err + } + + queuedAt := time.Now().UTC() + for _, path := range paths { + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), path, queuedAt); err != nil { + return 0, fmt.Errorf("queueing %s for deletion: %w", path, err) + } + } + return int(deleted), tx.Commit() +} diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index cba4b640..3947b524 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -152,3 +152,56 @@ func TestMigrationAddsArtifactSource(t *testing.T) { t.Errorf("existing artifact source = %q, want %q", art.Source, SourceUpstream) } } + +func TestGetLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "lodash", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) + + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil { + t.Fatalf("GetLocalArtifacts: %v", err) + } + if len(local) != 1 || local[0].Filename != "request-1.0.0.tgz" { + t.Fatalf("local artifacts of request = %+v, want only request-1.0.0.tgz", local) + } + + upstreamOnly, err := db.GetLocalArtifacts("pkg:npm/express") + if err != nil { + t.Fatalf("GetLocalArtifacts(express): %v", err) + } + if len(upstreamOnly) != 0 { + t.Errorf("local artifacts of express = %+v, want none", upstreamOnly) + } + }) +} + +func TestDeleteLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) + const pushedPath = "npm/request/1.0.0/request-1.0.0.tgz" + + deleted, err := db.DeleteLocalArtifacts("pkg:npm/request@1.0.0") + if err != nil || deleted != 1 { + t.Fatalf("DeleteLocalArtifacts(request) = %d, %v, want 1, nil", deleted, err) + } + art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") + if err != nil || art != nil { + t.Errorf("GetArtifact after delete = %+v, %v, want nil, nil", art, err) + } + var queued int + if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), pushedPath); err != nil || queued != 1 { + t.Errorf("queued deletes for %s = %d, %v, want 1", pushedPath, queued, err) + } + + deleted, err = db.DeleteLocalArtifacts("pkg:npm/express@1.0.0") + if err != nil || deleted != 0 { + t.Errorf("DeleteLocalArtifacts(upstream) = %d, %v, want 0, nil", deleted, err) + } + if art, err := db.GetArtifact("pkg:npm/express@1.0.0", "express-1.0.0.tgz"); err != nil || art == nil { + t.Errorf("upstream artifact removed by DeleteLocalArtifacts: %+v, %v", art, err) + } + }) +} diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 5b9ab965..6a324297 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -563,6 +563,16 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.proxy.Logger.Info("composer download request", "package", packageName, "version", version, "filename", filename) + local, err := h.proxy.GetLocalArtifact(r.Context(), "composer", packageName, version, filename) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read pushed package") + return + } + if local != nil { + ServeArtifactRequest(w, r, local) + return + } + // We need to fetch the metadata to get the actual download URL since // Packagist URLs include a hash. Packagist serves dev versions (e.g. // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go new file mode 100644 index 00000000..bad457a3 --- /dev/null +++ b/internal/handler/hosted.go @@ -0,0 +1,263 @@ +package handler + +import ( + "context" + "database/sql" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/proxy/internal/storage" +) + +var ( + ErrHostedUnsupported = errors.New("pushing packages is not supported for this ecosystem") + ErrHostedInvalidName = errors.New("invalid package name or version") + ErrHostedVersionNotFound = errors.New("no pushed artifacts for this version") +) + +// HostedTarget is where a pushed archive is stored: the package name and +// version it is cached under and the filename clients request it by. Download +// handlers look artifacts up by exactly these values, so a push under any +// other name would never be served. +type HostedTarget struct { + Name string + Version string + Filename string +} + +// HostedArtifact describes one pushed archive. +type HostedArtifact struct { + Version string `json:"version"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + PushedAt time.Time `json:"pushed_at"` +} + +// hostedNaming describes how an ecosystem names pushed archives: which +// package names it has, split on "/", and the filename its clients request. +type hostedNaming struct { + validName func(segments []string) bool + filename func(name, version string) string +} + +var hostedNamings = map[string]hostedNaming{ + "npm": { + validName: func(segments []string) bool { + return len(segments) == 1 || (len(segments) == 2 && strings.HasPrefix(segments[0], "@")) + }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".tgz" }, + }, + "composer": { + validName: func(segments []string) bool { return len(segments) == vendorPackageParts }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".zip" }, + }, +} + +// HostedSupported reports whether packages can be pushed for ecosystem. +func HostedSupported(ecosystem string) bool { + _, ok := hostedNamings[ecosystem] + return ok +} + +func lastSegment(name string) string { + return name[strings.LastIndex(name, "/")+1:] +} + +// HostedTargetFor validates a push of name@version to ecosystem and returns +// where it is stored. +func HostedTargetFor(ecosystem, name, version string) (HostedTarget, error) { + naming, ok := hostedNamings[ecosystem] + if !ok { + return HostedTarget{}, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) + } + segments := strings.Split(name, "/") + if !naming.validName(segments) { + return HostedTarget{}, ErrHostedInvalidName + } + for _, segment := range append(segments, version) { + if !validHostedSegment(segment) { + return HostedTarget{}, ErrHostedInvalidName + } + } + return HostedTarget{Name: name, Version: version, Filename: naming.filename(name, version)}, nil +} + +func validHostedSegment(s string) bool { + if s == "" || s == "." || s == ".." || strings.ContainsAny(s, `/\`) { + return false + } + for _, c := range s { + if c < ' ' || c == 0x7f { + return false + } + } + return true +} + +// PushLocalArtifact stores body as the artifact for target and records it as +// local, replacing any cached upstream copy. Download handlers serve it from +// the cache without contacting the upstream, and eviction never removes it. +func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType string) (HostedArtifact, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) + if err != nil { + return HostedArtifact{}, err + } + if p.versionDenied(ecosystem, target.Name, target.Version) { + return HostedArtifact{}, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + + storagePath := storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename) + storeStart := time.Now() + size, hash, err := p.Storage.Store(ctx, storagePath, body) + metrics.RecordStorageOperation("write", time.Since(storeStart)) + if err != nil { + metrics.RecordStorageError("write", metrics.FailureDetail{ + PURL: versionPURL, Filename: target.Filename, Path: storagePath, Err: err}) + return HostedArtifact{}, fmt.Errorf("storing artifact: %w", err) + } + + if p.Scanners != nil && p.Scanners.Enabled() { + if err := p.runScan(ctx, ecosystem, target.Name, target.Version, target.Filename, versionPURL, storagePath, size, contentType); err != nil { + p.discardStored(ctx, storagePath) + return HostedArtifact{}, err + } + } + + pushedAt := time.Now() + if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, storagePath, hash, size, contentType, pushedAt); err != nil { + p.discardStored(ctx, storagePath) + return HostedArtifact{}, err + } + + return HostedArtifact{ + Version: target.Version, + Filename: target.Filename, + SHA256: hash, + Size: size, + PushedAt: pushedAt.UTC(), + }, nil +} + +func (p *Proxy) discardStored(ctx context.Context, storagePath string) { + if err := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); err != nil { + p.Logger.Warn("failed to delete pushed artifact from storage", "path", storagePath, "error", err) + } +} + +// recordLocalArtifact writes the package, version and artifact records. The +// version's integrity is set to the pushed bytes, so the cache's integrity +// check verifies them rather than an upstream archive's digest. An existing +// package record is kept as it is, since it may describe the upstream package. +func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL, storagePath, hash string, size int64, contentType string, pushedAt time.Time) error { + existing, err := p.DB.GetPackageByPURL(pkgPURL) + if err != nil { + return fmt.Errorf("reading package: %w", err) + } + if existing == nil { + if err := p.DB.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}); err != nil { + return fmt.Errorf("upserting package: %w", err) + } + } + + sri, err := hexDigestSRI(hash) + if err != nil { + return err + } + if err := p.DB.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + Integrity: sql.NullString{String: sri, Valid: true}, + }); err != nil { + return fmt.Errorf("upserting version: %w", err) + } + + if err := p.DB.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: target.Filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, + FetchedAt: sql.NullTime{Time: pushedAt, Valid: true}, + Source: database.SourceLocal, + }); err != nil { + return fmt.Errorf("upserting artifact: %w", err) + } + return nil +} + +func hexDigestSRI(hexDigest string) (string, error) { + raw, err := hex.DecodeString(hexDigest) + if err != nil { + return "", fmt.Errorf("decoding stored digest: %w", err) + } + return "sha256-" + base64.StdEncoding.EncodeToString(raw), nil +} + +// LocalArtifacts lists the archives pushed for a package. +func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) { + pkgPURL := packageurl.MakeString(ecosystem, name, "") + if pkgPURL == "" { + return nil, fmt.Errorf("%w: %s %q", errUnsupportedPackageIdentity, ecosystem, name) + } + rows, err := p.DB.GetLocalArtifacts(pkgPURL) + if err != nil { + return nil, fmt.Errorf("listing local artifacts: %w", err) + } + listed := make([]HostedArtifact, 0, len(rows)) + for _, row := range rows { + listed = append(listed, HostedArtifact{ + Version: database.VersionFromPURL(row.VersionPURL), + Filename: row.Filename, + SHA256: row.ContentHash.String, + Size: row.Size.Int64, + PushedAt: row.FetchedAt.Time.UTC(), + }) + } + return listed, nil +} + +// DeleteLocalVersion removes the archives pushed for name@version. If the +// upstream has the version, clients get the upstream archive again. +func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return err + } + deleted, err := p.DB.DeleteLocalArtifacts(versionPURL) + if err != nil { + return fmt.Errorf("deleting local artifacts: %w", err) + } + if deleted == 0 { + return ErrHostedVersionNotFound + } + return nil +} + +// GetLocalArtifact serves a pushed artifact from the cache, or returns nil +// when the artifact was not pushed. Handlers that must consult the upstream +// before serving a cached copy call it first, so pushed versions the upstream +// does not list are still served. +func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return nil, err + } + record, err := p.DB.GetArtifact(versionPURL, filename) + if err != nil { + return nil, fmt.Errorf("checking artifact source: %w", err) + } + if record == nil || record.Source != database.SourceLocal { + return nil, nil + } + return p.checkCache(ctx, pkgPURL, versionPURL, filename) +} diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go new file mode 100644 index 00000000..b327b89c --- /dev/null +++ b/internal/handler/hosted_test.go @@ -0,0 +1,181 @@ +package handler + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +func TestHostedTargetFor(t *testing.T) { + tests := []struct { + name string + ecosystem, pkg, version string + want HostedTarget + wantErr error + }{ + {name: "npm", ecosystem: "npm", pkg: "request", version: "2.88.3", + want: HostedTarget{Name: "request", Version: "2.88.3", Filename: "request-2.88.3.tgz"}}, + {name: "npm scoped", ecosystem: "npm", pkg: "@types/node", version: "20.1.0", + want: HostedTarget{Name: "@types/node", Version: "20.1.0", Filename: "node-20.1.0.tgz"}}, + {name: "composer", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", + want: HostedTarget{Name: "monolog/monolog", Version: "3.5.0", Filename: "monolog-3.5.0.zip"}}, + {name: "composer without vendor", ecosystem: "composer", pkg: "monolog", version: "3.5.0", + wantErr: ErrHostedInvalidName}, + {name: "composer with extra segment", ecosystem: "composer", pkg: "a/b/c", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "unsupported ecosystem", ecosystem: "cargo", pkg: "serde", version: "1.0.200", + wantErr: ErrHostedUnsupported}, + {name: "unknown ecosystem", ecosystem: "maven", pkg: "junit", version: "4.13", + wantErr: ErrHostedUnsupported}, + {name: "npm scope without @", ecosystem: "npm", pkg: "types/node", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "dot-dot name segment", ecosystem: "npm", pkg: "@scope/..", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "dot-dot version", ecosystem: "npm", pkg: "request", version: "..", + wantErr: ErrHostedInvalidName}, + {name: "control character", ecosystem: "npm", pkg: "request", version: "1.0.0\n", + wantErr: ErrHostedInvalidName}, + {name: "empty version", ecosystem: "npm", pkg: "request", version: "", + wantErr: ErrHostedInvalidName}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("HostedTargetFor() error = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("HostedTargetFor() error = %v", err) + } + if got != tt.want { + t.Errorf("HostedTargetFor() = %+v, want %+v", got, tt.want) + } + }) + } +} + +const pushedTarball = "patched request tarball" + +func pushRequest(t *testing.T, proxy *Proxy) HostedArtifact { + t.Helper() + target, err := HostedTargetFor("npm", "request", "2.88.3") + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + pushed, err := proxy.PushLocalArtifact(context.Background(), "npm", target, strings.NewReader(pushedTarball), "application/gzip") + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return pushed +} + +func TestPushLocalArtifactIsServedWithoutUpstream(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + + pushed := pushRequest(t, proxy) + if pushed.Version != "2.88.3" || pushed.Filename != "request-2.88.3.tgz" || pushed.Size != int64(len(pushedTarball)) || len(pushed.SHA256) != 64 { + t.Errorf("pushed artifact = %+v", pushed) + } + + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz", + "https://registry.npmjs.org/request/-/request-2.88.3.tgz") + if err != nil { + t.Fatalf("GetOrFetchArtifactFromURL: %v", err) + } + if got := readResult(result); got != pushedTarball { + t.Errorf("served body = %q, want the pushed tarball", got) + } + if fetcher.fetchCalled { + t.Error("serving a pushed artifact contacted the upstream") + } + + art, err := db.GetArtifact("pkg:npm/request@2.88.3", "request-2.88.3.tgz") + if err != nil || art == nil { + t.Fatalf("GetArtifact: %+v, %v", art, err) + } + if art.Source != database.SourceLocal || art.UpstreamURL != "" { + t.Errorf("artifact source/upstream_url = %q/%q, want local/empty", art.Source, art.UpstreamURL) + } +} + +func TestPushLocalArtifactReplacesCachedUpstreamCopy(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + + const upstreamURL = "https://registry.npmjs.org/request/-/request-2.88.3.tgz" + first, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz", upstreamURL) + if err != nil { + t.Fatalf("caching the upstream copy: %v", err) + } + _ = readResult(first) + + pushRequest(t, proxy) + + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz", upstreamURL) + if err != nil { + t.Fatalf("GetOrFetchArtifactFromURL: %v", err) + } + if got := readResult(result); got != pushedTarball { + t.Errorf("served body = %q, want the pushed tarball", got) + } +} + +func TestLocalArtifactsAndDeleteLocalVersion(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + pushed := pushRequest(t, proxy) + + listed, err := proxy.LocalArtifacts("npm", "request") + if err != nil { + t.Fatalf("LocalArtifacts: %v", err) + } + if len(listed) != 1 || listed[0].Version != pushed.Version || listed[0].SHA256 != pushed.SHA256 { + t.Fatalf("LocalArtifacts = %+v, want the pushed artifact %+v", listed, pushed) + } + + if err := proxy.DeleteLocalVersion("npm", "request", "2.88.3"); err != nil { + t.Fatalf("DeleteLocalVersion: %v", err) + } + if cached, err := proxy.GetCachedArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz"); err != nil || cached != nil { + t.Errorf("GetCachedArtifact after delete = %+v, %v, want nil, nil", cached, err) + } + if listed, err := proxy.LocalArtifacts("npm", "request"); err != nil || len(listed) != 0 { + t.Errorf("LocalArtifacts after delete = %+v, %v, want none", listed, err) + } + + if err := proxy.DeleteLocalVersion("npm", "request", "2.88.3"); !errors.Is(err, ErrHostedVersionNotFound) { + t.Errorf("second DeleteLocalVersion error = %v, want %v", err, ErrHostedVersionNotFound) + } +} + +func TestGetLocalArtifactIgnoresUpstreamCopies(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + cached, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz", + "https://registry.npmjs.org/request/-/request-2.88.3.tgz") + if err != nil { + t.Fatalf("caching the upstream copy: %v", err) + } + _ = readResult(cached) + + local, err := proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz") + if err != nil || local != nil { + t.Fatalf("GetLocalArtifact for an upstream copy = %+v, %v, want nil, nil", local, err) + } + + pushRequest(t, proxy) + local, err = proxy.GetLocalArtifact(context.Background(), "npm", "request", "2.88.3", "request-2.88.3.tgz") + if err != nil || local == nil { + t.Fatalf("GetLocalArtifact for a pushed artifact = %+v, %v", local, err) + } + if got := readResult(local); got != pushedTarball { + t.Errorf("served body = %q, want the pushed tarball", got) + } +} diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go new file mode 100644 index 00000000..962bc177 --- /dev/null +++ b/internal/server/hosted_api.go @@ -0,0 +1,192 @@ +package server + +import ( + "bufio" + "errors" + "io" + "log/slog" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +// HostedAPIHandler serves /api/hosted, where operators push archives that the +// proxy serves in place of, or in addition to, what the upstream has. +type HostedAPIHandler struct { + proxy *handler.Proxy + maxUploadBytes int64 + logger *slog.Logger +} + +// NewHostedAPIHandler creates the /api/hosted handler. +func NewHostedAPIHandler(proxy *handler.Proxy, maxUploadBytes int64, logger *slog.Logger) *HostedAPIHandler { + return &HostedAPIHandler{proxy: proxy, maxUploadBytes: maxUploadBytes, logger: logger} +} + +// HostedPushResponse describes a pushed archive. +type HostedPushResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + handler.HostedArtifact +} + +// HostedListResponse lists the archives pushed for a package. +type HostedListResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Artifacts []handler.HostedArtifact `json:"artifacts"` +} + +// registerHostedRoutes mounts /api/hosted. Listing is open like the rest of +// the API; pushing and deleting require the bearer token. +func registerHostedRoutes(r chi.Router, h *HostedAPIHandler, token string) { + r.Get("/api/hosted/{ecosystem}/*", h.HandleList) + r.Group(func(w chi.Router) { + w.Use(requireBearerToken(token)) + w.Put("/api/hosted/{ecosystem}/*", h.HandlePush) + w.Delete("/api/hosted/{ecosystem}/*", h.HandleDelete) + }) +} + +// HandlePush handles PUT /api/hosted/{ecosystem}/{name}/{version} +// @Summary Push a package archive +// @Description Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. +// @Tags hosted +// @Accept octet-stream +// @Produce json +// @Security BearerAuth +// @Param ecosystem path string true "Ecosystem (npm or composer)" +// @Param name path string true "Package name, including an npm scope or the Composer vendor" +// @Param version path string true "Version" +// @Success 201 {object} HostedPushResponse +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name}/{version} [put] +func (h *HostedAPIHandler) HandlePush(w http.ResponseWriter, r *http.Request) { + ecosystem, name, version, ok := hostedVersionPath(w, r) + if !ok { + return + } + target, err := handler.HostedTargetFor(ecosystem, name, version) + if err != nil { + badRequest(w, err.Error()) + return + } + + body := bufio.NewReader(http.MaxBytesReader(w, r.Body, h.maxUploadBytes)) + if _, err := body.Peek(1); errors.Is(err, io.EOF) { + badRequest(w, "request body is empty") + return + } + + pushed, err := h.proxy.PushLocalArtifact(r.Context(), ecosystem, target, body, r.Header.Get("Content-Type")) + if err != nil { + h.pushError(w, err) + return + } + h.logger.Info("pushed package archive", + "ecosystem", ecosystem, "name", target.Name, "version", target.Version, + "filename", target.Filename, "sha256", pushed.SHA256) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + writeJSON(w, HostedPushResponse{Ecosystem: ecosystem, Name: target.Name, HostedArtifact: pushed}) +} + +func (h *HostedAPIHandler) pushError(w http.ResponseWriter, err error) { + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "archive exceeds hosted.max_upload_size") + case errors.Is(err, handler.ErrVersionDenied), errors.Is(err, handler.ErrArtifactBlocked): + writeError(w, http.StatusForbidden, ErrCodeBadRequest, err.Error()) + default: + h.logger.Error("failed to push package archive", "error", err) + internalError(w, "failed to store archive") + } +} + +// HandleList handles GET /api/hosted/{ecosystem}/{name} +// @Summary List pushed package archives +// @Tags hosted +// @Produce json +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name" +// @Success 200 {object} HostedListResponse +// @Failure 400 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name} [get] +func (h *HostedAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { + ecosystem := chi.URLParam(r, "ecosystem") + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return + } + name := strings.Join(segments, "/") + if !handler.HostedSupported(ecosystem) { + badRequest(w, handler.ErrHostedUnsupported.Error()) + return + } + + listed, err := h.proxy.LocalArtifacts(ecosystem, name) + if err != nil { + h.logger.Error("failed to list pushed archives", "error", err) + internalError(w, "failed to list pushed archives") + return + } + writeJSON(w, HostedListResponse{Ecosystem: ecosystem, Name: name, Artifacts: listed}) +} + +// HandleDelete handles DELETE /api/hosted/{ecosystem}/{name}/{version} +// @Summary Delete a pushed version +// @Description Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. +// @Tags hosted +// @Security BearerAuth +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name" +// @Param version path string true "Version" +// @Success 204 +// @Failure 401 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name}/{version} [delete] +func (h *HostedAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { + ecosystem, name, version, ok := hostedVersionPath(w, r) + if !ok { + return + } + err := h.proxy.DeleteLocalVersion(ecosystem, name, version) + switch { + case errors.Is(err, handler.ErrHostedVersionNotFound): + notFound(w, err.Error()) + case err != nil: + h.logger.Error("failed to delete pushed version", "error", err) + internalError(w, "failed to delete pushed version") + default: + h.logger.Info("deleted pushed version", "ecosystem", ecosystem, "name", name, "version", version) + w.WriteHeader(http.StatusNoContent) + } +} + +// minVersionPathSegments is one name segment plus the version. +const minVersionPathSegments = 2 + +// hostedVersionPath splits {name}/{version} from the wildcard. The version is +// always the last segment; the name may span several (npm scopes, Composer +// vendors). +func hostedVersionPath(w http.ResponseWriter, r *http.Request) (ecosystem, name, version string, ok bool) { + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return "", "", "", false + } + if len(segments) < minVersionPathSegments { + badRequest(w, "path must be /api/hosted/{ecosystem}/{name}/{version}") + return "", "", "", false + } + last := len(segments) - 1 + return chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last], true +} diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go new file mode 100644 index 00000000..9051572e --- /dev/null +++ b/internal/server/hosted_api_test.go @@ -0,0 +1,302 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +const ( + testHostedToken = "push-token" + testHostedMaxUpload = 64 + pushedBody = "patched tarball" +) + +type hostedTestServer struct { + router http.Handler + fetches *int + upstreamRequests *int +} + +// countingFetcher fails every fetch, so a test fails loudly if the proxy goes +// upstream for something it should serve from the pushed copy. +type countingFetcher struct{ calls *int } + +func (f countingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f countingFetcher) FetchWithHeaders(context.Context, string, http.Header) (*fetch.Artifact, error) { + *f.calls++ + return nil, fetch.ErrNotFound +} + +func (f countingFetcher) Head(context.Context, string) (int64, string, error) { + *f.calls++ + return 0, "", fetch.ErrNotFound +} + +func newHostedTestServer(t *testing.T) hostedTestServer { + t.Helper() + dir := t.TempDir() + db, err := database.Create(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("database.Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "artifacts")) + if err != nil { + t.Fatalf("storage.OpenBucket: %v", err) + } + + calls := 0 + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(db, store, countingFetcher{calls: &calls}, fetch.NewResolver(), logger) + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.NotFound(w, nil) + })) + t.Cleanup(upstream.Close) + + r := chi.NewRouter() + registerHostedRoutes(r, NewHostedAPIHandler(proxy, testHostedMaxUpload, logger), testHostedToken) + r.Mount("/npm", http.StripPrefix("/npm", handler.NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes())) + composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + r.Mount("/composer", http.StripPrefix("/composer", composer.Routes())) + return hostedTestServer{router: r, fetches: &calls, upstreamRequests: &upstreamRequests} +} + +func (s hostedTestServer) do(t *testing.T, method, target, token, body string) *httptest.ResponseRecorder { + t.Helper() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reader) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.router.ServeHTTP(rec, req) + return rec +} + +func TestHostedPushIsServedByTheEcosystemHandler(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, pushedBody) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + var pushed HostedPushResponse + if err := json.NewDecoder(rec.Body).Decode(&pushed); err != nil { + t.Fatalf("decoding PUT response: %v", err) + } + if pushed.Ecosystem != "npm" || pushed.Name != "request" || pushed.Version != "2.88.3" || + pushed.Filename != "request-2.88.3.tgz" || pushed.Size != int64(len(pushedBody)) { + t.Errorf("PUT response = %+v", pushed) + } + + rec = s.do(t, http.MethodGet, "/npm/request/-/request-2.88.3.tgz", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } + if *s.fetches != 0 { + t.Errorf("download contacted the upstream %d times", *s.fetches) + } +} + +func TestHostedPushIsServedByTheComposerHandler(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, pushedBody) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec = s.do(t, http.MethodGet, "/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + t.Fatalf("download status = %d, body %q, want 200 and the pushed archive", rec.Code, rec.Body) + } + if *s.upstreamRequests != 0 || *s.fetches != 0 { + t.Errorf("download contacted the upstream (%d metadata requests, %d fetches)", *s.upstreamRequests, *s.fetches) + } +} + +func TestComposerDownloadOfUnpushedFileStillNeedsUpstreamMetadata(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, pushedBody); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/composer/files/monolog/monolog/3.5.0/other.zip", "", "") + if rec.Code != http.StatusNotFound { + t.Errorf("download of another filename status = %d, want 404 from the upstream lookup", rec.Code) + } + if *s.upstreamRequests == 0 { + t.Error("a filename that was not pushed was served without the upstream metadata lookup") + } +} + +func TestHostedPushScopedNpmPackage(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/npm/@types/node/20.1.0", testHostedToken, pushedBody) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + rec = s.do(t, http.MethodGet, "/npm/@types/node/-/node-20.1.0.tgz", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } +} + +func TestHostedListAndDelete(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, pushedBody); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/api/hosted/npm/request", "", "") + if rec.Code != http.StatusOK { + t.Fatalf("GET status = %d, body %s", rec.Code, rec.Body) + } + var listed HostedListResponse + if err := json.NewDecoder(rec.Body).Decode(&listed); err != nil { + t.Fatalf("decoding GET response: %v", err) + } + if listed.Ecosystem != "npm" || listed.Name != "request" || len(listed.Artifacts) != 1 || listed.Artifacts[0].Version != "2.88.3" { + t.Errorf("GET response = %+v", listed) + } + + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", "", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("DELETE without token status = %d, want 401", rec.Code) + } + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", testHostedToken, ""); rec.Code != http.StatusNoContent { + t.Errorf("DELETE status = %d, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", testHostedToken, ""); rec.Code != http.StatusNotFound { + t.Errorf("second DELETE status = %d, want 404", rec.Code) + } +} + +func TestHostedPushRejections(t *testing.T) { + tests := []struct { + name string + target string + token string + body string + wantStatus int + }{ + {name: "no token", target: "/api/hosted/npm/request/2.88.3", body: pushedBody, wantStatus: http.StatusUnauthorized}, + {name: "wrong token", target: "/api/hosted/npm/request/2.88.3", token: "guess", body: pushedBody, wantStatus: http.StatusUnauthorized}, + {name: "unsupported ecosystem", target: "/api/hosted/cargo/serde/1.0.200", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, + {name: "composer name without vendor", target: "/api/hosted/composer/monolog/3.5.0", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, + {name: "missing version", target: "/api/hosted/npm/request", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, + {name: "empty body", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, wantStatus: http.StatusBadRequest}, + {name: "too large", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: strings.Repeat("x", testHostedMaxUpload+1), wantStatus: http.StatusRequestEntityTooLarge}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newHostedTestServer(t) + rec := s.do(t, http.MethodPut, tt.target, tt.token, tt.body) + if rec.Code != tt.wantStatus { + t.Fatalf("PUT status = %d, want %d, body %s", rec.Code, tt.wantStatus, rec.Body) + } + listed := s.do(t, http.MethodGet, "/api/hosted/npm/request", "", "") + if strings.Contains(listed.Body.String(), "2.88.3") { + t.Error("rejected push was stored") + } + }) + } +} + +func TestHostedListUnsupportedEcosystem(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodGet, "/api/hosted/maven/junit", "", ""); rec.Code != http.StatusBadRequest { + t.Errorf("GET status = %d, want 400", rec.Code) + } +} + +func startHostedServer(t *testing.T, hosted config.HostedConfig) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Hosted = hosted + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + <-startErr + }) + return cfg.BaseURL +} + +func putArchive(t *testing.T, url, token string) int { + t.Helper() + req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(pushedBody)) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("PUT %s: %v", url, err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +func TestHostedRoutesFollowConfig(t *testing.T) { + disabled := startHostedServer(t, config.HostedConfig{}) + if status := putArchive(t, disabled+"/api/hosted/npm/request/2.88.3", testHostedToken); status == http.StatusCreated { + t.Error("push accepted with hosted.enabled false") + } + + enabled := startHostedServer(t, config.HostedConfig{Enabled: true, Token: testHostedToken}) + if status := putArchive(t, enabled+"/api/hosted/npm/request/2.88.3", ""); status != http.StatusUnauthorized { + t.Errorf("push without token status = %d, want 401", status) + } + if status := putArchive(t, enabled+"/api/hosted/npm/request/2.88.3", testHostedToken); status != http.StatusCreated { + t.Errorf("push with token status = %d, want 201", status) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 055d19d4..000d3313 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -365,6 +365,12 @@ func (s *Server) serve(listener net.Listener) error { go jobStore.StartCleanup(bgCtx) } + // Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED) + if s.cfg.Hosted.Enabled { + hostedAPI := NewHostedAPIHandler(proxy, s.cfg.Hosted.MaxUploadBytes(), s.logger) + registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded()) + } + s.http = &http.Server{ Addr: s.cfg.Listen, Handler: r, From 80fdd16719c5ee6a130b8aa3cf2076647495a1fa Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:11:49 +0200 Subject: [PATCH 04/17] Add push, list and delete commands for hosted packages proxy push, proxy list and proxy delete call the /api/hosted endpoints of a running proxy. The proxy URL comes from -url or PROXY_URL and the token from PROXY_HOSTED_TOKEN, so it never ends up in shell history or process listings. Document the hosted config block, the endpoints, the supported ecosystems and how pushed archives interact with eviction. Refs #133 --- README.md | 22 ++++ cmd/proxy/hosted.go | 218 +++++++++++++++++++++++++++++++++++++++ cmd/proxy/hosted_test.go | 142 +++++++++++++++++++++++++ cmd/proxy/main.go | 22 ++++ config.example.yaml | 8 ++ docs/configuration.md | 57 ++++++++++ 6 files changed, 469 insertions(+) create mode 100644 cmd/proxy/hosted.go create mode 100644 cmd/proxy/hosted_test.go diff --git a/README.md b/README.md index 3aecba4f..74f33735 100644 --- a/README.md +++ b/README.md @@ -1026,6 +1026,18 @@ curl http://localhost:8080/api/mirror/mirror-1 curl -X DELETE http://localhost:8080/api/mirror/mirror-1 ``` +### push, list, delete + +Push package archives to a running proxy with `hosted.enabled`, list them and delete them. See [Hosted Packages](docs/configuration.md#hosted-packages). + +```bash +export PROXY_URL=https://proxy.example.com PROXY_HOSTED_TOKEN=... +proxy push npm request 2.88.3 ./request-2.88.3.tgz +proxy push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip +proxy list npm request +proxy delete npm request 2.88.3 +``` + ### stats Show cache statistics without running the server. @@ -1119,6 +1131,16 @@ Recently cached: | `GET /api/mirror/{id}` | Get job status and progress | | `DELETE /api/mirror/{id}` | Cancel a running job | +### Hosted API + +Registered when `hosted.enabled` is set. Pushing and deleting require `Authorization: Bearer `. + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/hosted/{ecosystem}/{name}/{version}` | Push the request body as the archive for a version (npm and composer) | +| `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | +| `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | + ### Enrichment API The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection. diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go new file mode 100644 index 00000000..2db238fe --- /dev/null +++ b/cmd/proxy/hosted.go @@ -0,0 +1,218 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "text/tabwriter" + "time" +) + +const ( + defaultProxyURL = "http://localhost:8080" + hostedTokenEnv = "PROXY_HOSTED_TOKEN" + hostedClientTimeout = 10 * time.Minute + + pushArgCount = 4 // ecosystem, name, version, file + listArgCount = 2 // ecosystem, name + deleteArgCount = 3 // ecosystem, name, version + tablePadding = 2 +) + +// hostedArtifact mirrors the JSON the /api/hosted endpoints return. +type hostedArtifact struct { + Version string `json:"version"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + PushedAt time.Time `json:"pushed_at"` +} + +type hostedPushResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + hostedArtifact +} + +type hostedListResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Artifacts []hostedArtifact `json:"artifacts"` +} + +type hostedCommand struct { + flags *flag.FlagSet + proxyURL *string + usage string +} + +func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(io.Discard) + proxyURL := getenv("PROXY_URL") + if proxyURL == "" { + proxyURL = defaultProxyURL + } + cmd := hostedCommand{flags: fs, usage: usage} + cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage) + fs.SetOutput(os.Stderr) + fs.PrintDefaults() + } + return cmd +} + +// parse parses flags and checks for exactly want positional arguments. +func (c hostedCommand) parse(args []string, want int) ([]string, error) { + if err := c.flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + c.flags.Usage() + } + return nil, fmt.Errorf("%w\nusage: %s", err, c.usage) + } + if c.flags.NArg() != want { + return nil, fmt.Errorf("usage: %s", c.usage) + } + return c.flags.Args(), nil +} + +func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string { + segments := []string{url.PathEscape(ecosystem)} + for segment := range strings.SplitSeq(name, "/") { + segments = append(segments, url.PathEscape(segment)) + } + for _, v := range version { + segments = append(segments, url.PathEscape(v)) + } + return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/") +} + +func hostedToken(getenv func(string) string) (string, error) { + token := getenv(hostedTokenEnv) + if token == "" { + return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv) + } + return token, nil +} + +func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("push", "proxy push [-url URL] ", + "Push a package archive to the proxy", getenv) + positional, err := cmd.parse(args, pushArgCount) + if err != nil { + return err + } + token, err := hostedToken(getenv) + if err != nil { + return err + } + ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3] + + file, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + req, err := http.NewRequest(http.MethodPut, cmd.endpoint(ecosystem, name, version), file) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/octet-stream") + + var pushed hostedPushResponse + if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil { + return err + } + _, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n", + pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256) + return err +} + +func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("list", "proxy list [-url URL] ", "List pushed package archives", getenv) + positional, err := cmd.parse(args, listArgCount) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil) + if err != nil { + return err + } + + var listed hostedListResponse + if err := doHosted(client, req, http.StatusOK, &listed); err != nil { + return err + } + tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0) + _, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED") + for _, a := range listed.Artifacts { + _, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339)) + } + return tw.Flush() +} + +func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("delete", "proxy delete [-url URL] ", "Delete a pushed version", getenv) + positional, err := cmd.parse(args, deleteArgCount) + if err != nil { + return err + } + token, err := hostedToken(getenv) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2]), nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + + if err := doHosted(client, req, http.StatusNoContent, nil); err != nil { + return err + } + _, err = fmt.Fprintf(stdout, "deleted %s %s@%s\n", positional[0], positional[1], positional[2]) + return err +} + +// doHosted sends req and decodes a successful response into out. Any other +// status is returned as an error carrying the server's message. +func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error { + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != wantStatus { + var apiErr struct { + Message string `json:"message"` + } + if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" { + return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status) + } + return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} + +// runHostedCommand runs one of the push, list and delete commands against the +// proxy's /api/hosted endpoints and exits non-zero on failure. +func runHostedCommand(run func([]string, func(string) string, io.Writer, *http.Client) error) { + client := &http.Client{Timeout: hostedClientTimeout} + if err := run(os.Args[1:], os.Getenv, os.Stdout, client); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go new file mode 100644 index 00000000..b15f915f --- /dev/null +++ b/cmd/proxy/hosted_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +type recordedRequest struct { + method, path, query, authorization, body string +} + +func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) { + t.Helper() + recorded := &recordedRequest{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + *recorded = recordedRequest{ + method: r.Method, + path: r.URL.EscapedPath(), + query: r.URL.RawQuery, + authorization: r.Header.Get("Authorization"), + body: string(body), + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, response) + })) + t.Cleanup(srv.Close) + return srv, recorded +} + +func hostedEnv(vars map[string]string) func(string) string { + return func(key string) string { return vars[key] } +} + +func writeArchive(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("writing archive: %v", err) + } + return path +} + +func TestRunPush(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusCreated, + `{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`) + archive := writeArchive(t, "node-20.1.0.tgz", "tarball") + var stdout bytes.Buffer + + err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client()) + if err != nil { + t.Fatalf("runPush: %v", err) + } + if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "" { + t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query) + } + if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") { + t.Errorf("output = %q, want filename and sha256", out) + } +} + +func TestRunPushErrors(t *testing.T) { + srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`) + archive := writeArchive(t, "serde-1.0.200.crate", "crate") + + err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") { + t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err) + } + + err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") { + t.Errorf("rejected push error = %v, want the server's message", err) + } + + err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "usage") { + t.Errorf("push with missing arguments error = %v, want usage", err) + } +} + +func TestRunPushUsesProxyURLFromEnv(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`) + archive := writeArchive(t, "request-2.88.3.tgz", "tarball") + + err := runPush([]string{"npm", "request", "2.88.3", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client()) + if err != nil { + t.Fatalf("runPush: %v", err) + } + if recorded.path != "/api/hosted/npm/request/2.88.3" { + t.Errorf("path = %q", recorded.path) + } +} + +func TestRunList(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusOK, + `{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`) + var stdout bytes.Buffer + + if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil { + t.Fatalf("runList: %v", err) + } + if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" { + t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) + } + if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") { + t.Errorf("output = %q", out) + } +} + +func TestRunDelete(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusNoContent, "") + + err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err != nil { + t.Fatalf("runDelete: %v", err) + } + if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" { + t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) + } + + missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`) + err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client()) + if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") { + t.Errorf("delete of a missing version error = %v", err) + } +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 9f8d243a..786819ae 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -22,6 +22,9 @@ // serve Start the proxy server (default if no command given) // stats Show cache statistics // mirror Pre-populate cache from PURLs, SBOMs, or registries +// push Push a package archive to a running proxy (needs hosted.enabled) +// list List package archives pushed to a running proxy +// delete Delete a pushed version from a running proxy // // Serve Flags: // @@ -87,6 +90,10 @@ // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size // PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval // PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s") +// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted +// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting +// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") +// PROXY_URL - Proxy URL for push, list and delete // // Example: // @@ -159,6 +166,18 @@ func main() { os.Args = append(os.Args[:1], os.Args[2:]...) runMirror() return + case "push": + os.Args = append(os.Args[:1], os.Args[2:]...) + runHostedCommand(runPush) + return + case "list": + os.Args = append(os.Args[:1], os.Args[2:]...) + runHostedCommand(runList) + return + case "delete": + os.Args = append(os.Args[:1], os.Args[2:]...) + runHostedCommand(runDelete) + return case "-version", "--version": fmt.Printf("proxy %s (%s)\n", Version, Commit) os.Exit(0) @@ -181,6 +200,9 @@ Commands: serve Start the proxy server (default) stats Show cache statistics mirror Pre-populate cache from PURLs, SBOMs, or registries + push Push a package archive to a running proxy + list List package archives pushed to a running proxy + delete Delete a pushed version from a running proxy Run 'proxy -help' for more information on a command. diff --git a/config.example.yaml b/config.example.yaml index 52ebc26d..1b8e691e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -364,6 +364,14 @@ gradle: # How often eviction runs when max_age or max_size is set sweep_interval: "10m" +# Pushing package archives to /api/hosted (disabled by default). +# A pushed version replaces the upstream one for every client, so a token is +# required to push and delete. +# hosted: +# enabled: true +# token: "${PROXY_HOSTED_TOKEN}" +# max_upload_size: "100MB" + # Health endpoint configuration. health: # Minimum time between storage backend probes. diff --git a/docs/configuration.md b/docs/configuration.md index ae117f63..2b93975e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -982,6 +982,63 @@ Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`. Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). Independently of this setting, the shared transport gives up on an upstream that has not sent response headers within 60 seconds. +## Hosted Packages + +Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. + +The `/api/hosted` endpoints are disabled by default. Because a pushed version replaces the upstream one for every client, enabling them requires a token: + +```yaml +hosted: + enabled: true + token: "${PROXY_HOSTED_TOKEN}" + max_upload_size: "100MB" +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) | +| `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` | +| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive (default `100MB`) | + +`hosted.enabled` cannot be combined with `storage.cache_artifacts: false`. + +### Pushing a patched package + +1. Build the archive with the original package name inside it, e.g. `request-2.88.3.tgz`. +2. Push it: + + ```bash + export PROXY_HOSTED_TOKEN=... + proxy push -url https://proxy.example.com npm request 2.88.3 ./request-2.88.3.tgz + ``` + + or with curl: + + ```bash + curl -X PUT --data-binary @request-2.88.3.tgz \ + -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ + https://proxy.example.com/api/hosted/npm/request/2.88.3 + ``` + +3. Check what is pushed: `proxy list npm request` or `GET /api/hosted/npm/request`. +4. Remove it again: `proxy delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. + +### Supported ecosystems + +Download handlers look archives up by the filename clients request, so a push is stored under that name: + +| Ecosystem | Name | Filename | +|-----------|------|----------| +| `npm` | `{name}` or `@{scope}/{name}` | `{name}-{version}.tgz`, without the scope | +| `composer` | `{vendor}/{package}` | `{package}-{version}.zip` | + +Other ecosystems are not supported yet. + +### Storage and eviction + +Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: From e7449d15408c35fa4ee9609ee14326c11c9830ec Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:19:58 +0200 Subject: [PATCH 05/17] Read the package manifest of pushed archives Resolvers take a version's dependencies from the registry metadata, not from the archive, so listing a pushed version needs its manifest. A push now reads package.json from the top directory of an npm tarball, or composer.json from the root or the single top directory of a Composer zip, and keeps it in the new artifacts.manifest column (migration 013_add_artifact_manifest). The manifest has to name the pushed package and version. A Composer manifest may leave the version out, and its name is compared case-insensitively. An archive that can't be read or whose manifest names something else is refused with 400 and removed from storage. The version's integrity is now the archive's sha512, which npm clients expect in dist.integrity. Refs #134 --- internal/database/local_artifacts.go | 13 +- internal/database/local_artifacts_test.go | 68 ++++++++++ internal/database/queries.go | 18 +-- internal/database/schema.go | 29 ++++- internal/database/types.go | 1 + internal/handler/handler.go | 1 + internal/handler/hosted.go | 133 +++++++++++++++----- internal/handler/hosted_manifest.go | 144 ++++++++++++++++++++++ internal/handler/hosted_test.go | 136 +++++++++++++++++++- internal/server/hosted_api.go | 2 + internal/server/hosted_api_test.go | 35 +++--- internal/server/hosted_archives_test.go | 48 ++++++++ 12 files changed, 561 insertions(+), 67 deletions(-) create mode 100644 internal/handler/hosted_manifest.go create mode 100644 internal/server/hosted_archives_test.go diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go index 3eee1fc2..83197768 100644 --- a/internal/database/local_artifacts.go +++ b/internal/database/local_artifacts.go @@ -1,18 +1,25 @@ package database import ( + "database/sql" "fmt" "time" ) +// LocalArtifact is a pushed artifact with the integrity of its version. +type LocalArtifact struct { + Artifact + Integrity sql.NullString `db:"integrity"` +} + // GetLocalArtifacts returns the artifacts pushed to the proxy for a package, // ordered by version and filename. -func (db *DB) GetLocalArtifacts(packagePURL string) ([]Artifact, error) { - var artifacts []Artifact +func (db *DB) GetLocalArtifacts(packagePURL string) ([]LocalArtifact, error) { + var artifacts []LocalArtifact query := db.Rebind(` SELECT a.id, a.version_purl, a.filename, a.upstream_url, a.storage_path, a.content_hash, a.size, a.content_type, a.fetched_at, a.hit_count, a.last_accessed_at, - a.created_at, a.updated_at, a.source + a.created_at, a.updated_at, a.source, a.manifest, v.integrity FROM artifacts a JOIN versions v ON v.purl = a.version_purl WHERE v.package_purl = ? AND a.source = 'local' diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 3947b524..798d2e27 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -205,3 +205,71 @@ func TestDeleteLocalArtifacts(t *testing.T) { } }) } + +func TestLocalArtifactManifestAndIntegrity(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: "pkg:npm/request@2.88.3", + PackagePURL: "pkg:npm/request", + Integrity: sql.NullString{String: "sha512-abc", Valid: true}, + }); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + const manifest = `{"name":"request","version":"2.88.3"}` + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/request@2.88.3", + Filename: "request-2.88.3.tgz", + StoragePath: sql.NullString{String: "npm/request/2.88.3/x/request-2.88.3.tgz", Valid: true}, + ContentHash: sql.NullString{String: "00", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + Source: SourceLocal, + Manifest: sql.NullString{String: manifest, Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil || len(local) != 1 { + t.Fatalf("GetLocalArtifacts = %+v, %v", local, err) + } + if local[0].Manifest.String != manifest || local[0].Integrity.String != "sha512-abc" { + t.Errorf("manifest/integrity = %q/%q", local[0].Manifest.String, local[0].Integrity.String) + } + }) +} + +func TestMigrationAddsArtifactManifest(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN manifest`); err != nil { + t.Fatalf("drop manifest column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '013_add_artifact_manifest'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + has, err := db.HasColumn("artifacts", "manifest") + if err != nil || !has { + t.Errorf("artifacts.manifest after migration = %v, %v", has, err) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 53943f15..2d322a1a 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -369,8 +369,8 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + created_at, updated_at, source, manifest) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = EXCLUDED.upstream_url, storage_path = EXCLUDED.storage_path, @@ -379,15 +379,16 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er content_type = EXCLUDED.content_type, fetched_at = EXCLUDED.fetched_at, updated_at = EXCLUDED.updated_at, - source = EXCLUDED.source - WHERE artifacts.storage_path IS NOT DISTINCT FROM $14 + source = EXCLUDED.source, + manifest = EXCLUDED.manifest + WHERE artifacts.storage_path IS NOT DISTINCT FROM $15 ` } else { query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at, source) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + created_at, updated_at, source, manifest) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = excluded.upstream_url, storage_path = excluded.storage_path, @@ -396,7 +397,8 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er content_type = excluded.content_type, fetched_at = excluded.fetched_at, updated_at = excluded.updated_at, - source = excluded.source + source = excluded.source, + manifest = excluded.manifest WHERE artifacts.storage_path IS ? ` } @@ -404,7 +406,7 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er res, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, - a.sourceOrUpstream(), previous, + a.sourceOrUpstream(), a.Manifest, previous, ) if err != nil { return false, fmt.Errorf("upserting artifact: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index 76beaf12..ce124907 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -73,7 +73,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( last_accessed_at DATETIME, created_at DATETIME, updated_at DATETIME, - source TEXT NOT NULL DEFAULT 'upstream' + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -192,7 +193,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( last_accessed_at TIMESTAMP, created_at TIMESTAMP, updated_at TIMESTAMP, - source TEXT NOT NULL DEFAULT 'upstream' + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -270,7 +272,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( last_accessed_at DATETIME, created_at DATETIME, updated_at DATETIME, - source TEXT NOT NULL DEFAULT 'upstream' + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -292,7 +295,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( last_accessed_at TIMESTAMP, created_at TIMESTAMP, updated_at TIMESTAMP, - source TEXT NOT NULL DEFAULT 'upstream' + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -406,6 +410,7 @@ var migrations = []migration{ {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, {"011_add_metadata_dates", migrateAddMetadataDates}, {"012_add_artifact_source", migrateAddArtifactSource}, + {"013_add_artifact_manifest", migrateAddArtifactManifest}, } // isTableNotFound returns true if the error indicates a missing table. @@ -799,3 +804,19 @@ func migrateAddArtifactSource(db *DB) error { } return nil } + +// migrateAddArtifactManifest stores the manifest read from a pushed archive, +// which the metadata of its ecosystem lists the version with. +func migrateAddArtifactManifest(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "manifest") + if err != nil { + return fmt.Errorf("checking column manifest: %w", err) + } + if hasCol { + return nil + } + if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN manifest TEXT`); err != nil { + return fmt.Errorf("adding column manifest to artifacts: %w", err) + } + return nil +} diff --git a/internal/database/types.go b/internal/database/types.go index 501d49e7..e931f5b5 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -150,6 +150,7 @@ type Artifact struct { CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` Source string `db:"source" json:"source"` + Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` } func (a *Artifact) sourceOrUpstream() string { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c8dee1c6..c00a9309 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -162,6 +162,7 @@ type Proxy struct { MetadataTTL time.Duration MetadataMaxSize int64 GradleReadOnly bool + HostedEnabled bool GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index bad457a3..d4e4d9f8 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -2,9 +2,10 @@ package handler import ( "context" + "crypto/sha512" "database/sql" "encoding/base64" - "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -106,6 +107,8 @@ func validHostedSegment(s string) bool { // PushLocalArtifact stores body as the artifact for target and records it as // local, replacing any cached upstream copy. Download handlers serve it from // the cache without contacting the upstream, and eviction never removes it. +// The archive's manifest must name target; it is kept so the ecosystem's +// metadata can list the version with its dependencies. func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType string) (HostedArtifact, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) if err != nil { @@ -115,38 +118,70 @@ func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target return HostedArtifact{}, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } - storagePath := storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename) + push := storedPush{ + storagePath: storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename), + contentType: contentType, + } + sha512Hash := sha512.New() storeStart := time.Now() - size, hash, err := p.Storage.Store(ctx, storagePath, body) + push.size, push.sha256, err = p.Storage.Store(ctx, push.storagePath, io.TeeReader(body, sha512Hash)) metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { metrics.RecordStorageError("write", metrics.FailureDetail{ - PURL: versionPURL, Filename: target.Filename, Path: storagePath, Err: err}) + PURL: versionPURL, Filename: target.Filename, Path: push.storagePath, Err: err}) return HostedArtifact{}, fmt.Errorf("storing artifact: %w", err) } + push.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha512Hash.Sum(nil)) - if p.Scanners != nil && p.Scanners.Enabled() { - if err := p.runScan(ctx, ecosystem, target.Name, target.Version, target.Filename, versionPURL, storagePath, size, contentType); err != nil { - p.discardStored(ctx, storagePath) - return HostedArtifact{}, err - } + if err := p.checkPush(ctx, ecosystem, target, versionPURL, &push); err != nil { + p.discardStored(ctx, push.storagePath) + return HostedArtifact{}, err } - pushedAt := time.Now() - if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, storagePath, hash, size, contentType, pushedAt); err != nil { - p.discardStored(ctx, storagePath) + push.pushedAt = time.Now() + if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, push); err != nil { + p.discardStored(ctx, push.storagePath) return HostedArtifact{}, err } return HostedArtifact{ Version: target.Version, Filename: target.Filename, - SHA256: hash, - Size: size, - PushedAt: pushedAt.UTC(), + SHA256: push.sha256, + Size: push.size, + PushedAt: push.pushedAt.UTC(), }, nil } +// storedPush is a pushed archive once it is in storage. +type storedPush struct { + storagePath string + sha256 string + integrity string + size int64 + contentType string + manifest json.RawMessage + pushedAt time.Time +} + +// checkPush reads the manifest of a stored push and runs the scanners on it. +func (p *Proxy) checkPush(ctx context.Context, ecosystem string, target HostedTarget, versionPURL string, push *storedPush) error { + stored, err := p.Storage.Open(ctx, push.storagePath) + if err != nil { + return fmt.Errorf("reading pushed archive: %w", err) + } + push.manifest, err = readHostedManifest(ecosystem, stored, target) + _ = stored.Close() + if err != nil { + return err + } + + if p.Scanners != nil && p.Scanners.Enabled() { + return p.runScan(ctx, ecosystem, target.Name, target.Version, target.Filename, versionPURL, push.storagePath, push.size, push.contentType) + } + return nil +} + func (p *Proxy) discardStored(ctx context.Context, storagePath string) { if err := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); err != nil { p.Logger.Warn("failed to delete pushed artifact from storage", "path", storagePath, "error", err) @@ -157,7 +192,7 @@ func (p *Proxy) discardStored(ctx context.Context, storagePath string) { // version's integrity is set to the pushed bytes, so the cache's integrity // check verifies them rather than an upstream archive's digest. An existing // package record is kept as it is, since it may describe the upstream package. -func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL, storagePath, hash string, size int64, contentType string, pushedAt time.Time) error { +func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL string, push storedPush) error { existing, err := p.DB.GetPackageByPURL(pkgPURL) if err != nil { return fmt.Errorf("reading package: %w", err) @@ -168,14 +203,10 @@ func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPU } } - sri, err := hexDigestSRI(hash) - if err != nil { - return err - } if err := p.DB.UpsertVersion(&database.Version{ PURL: versionPURL, PackagePURL: pkgPURL, - Integrity: sql.NullString{String: sri, Valid: true}, + Integrity: sql.NullString{String: push.integrity, Valid: true}, }); err != nil { return fmt.Errorf("upserting version: %w", err) } @@ -183,26 +214,19 @@ func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPU if err := p.DB.UpsertArtifact(&database.Artifact{ VersionPURL: versionPURL, Filename: target.Filename, - StoragePath: sql.NullString{String: storagePath, Valid: true}, - ContentHash: sql.NullString{String: hash, Valid: true}, - Size: sql.NullInt64{Int64: size, Valid: true}, - ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, - FetchedAt: sql.NullTime{Time: pushedAt, Valid: true}, + StoragePath: sql.NullString{String: push.storagePath, Valid: true}, + ContentHash: sql.NullString{String: push.sha256, Valid: true}, + Size: sql.NullInt64{Int64: push.size, Valid: true}, + ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, + FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, Source: database.SourceLocal, + Manifest: sql.NullString{String: string(push.manifest), Valid: true}, }); err != nil { return fmt.Errorf("upserting artifact: %w", err) } return nil } -func hexDigestSRI(hexDigest string) (string, error) { - raw, err := hex.DecodeString(hexDigest) - if err != nil { - return "", fmt.Errorf("decoding stored digest: %w", err) - } - return "sha256-" + base64.StdEncoding.EncodeToString(raw), nil -} - // LocalArtifacts lists the archives pushed for a package. func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) { pkgPURL := packageurl.MakeString(ecosystem, name, "") @@ -226,6 +250,47 @@ func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) return listed, nil } +// localVersion is a pushed version as ecosystem metadata lists it. +type localVersion struct { + Version string + Filename string + SHA256 string + Integrity string + Manifest json.RawMessage + PushedAt time.Time +} + +// localVersions returns the versions pushed for a package, or nil when hosted +// packages are disabled, so metadata requests cost no extra lookup then. +func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { + if !p.HostedEnabled { + return nil, nil + } + pkgPURL := packageurl.MakeString(ecosystem, name, "") + if pkgPURL == "" { + return nil, nil + } + rows, err := p.DB.GetLocalArtifacts(pkgPURL) + if err != nil { + return nil, fmt.Errorf("listing local versions: %w", err) + } + versions := make([]localVersion, 0, len(rows)) + for _, row := range rows { + if !row.Manifest.Valid { + continue + } + versions = append(versions, localVersion{ + Version: database.VersionFromPURL(row.VersionPURL), + Filename: row.Filename, + SHA256: row.ContentHash.String, + Integrity: row.Integrity.String, + Manifest: json.RawMessage(row.Manifest.String), + PushedAt: row.FetchedAt.Time.UTC(), + }) + } + return versions, nil +} + // DeleteLocalVersion removes the archives pushed for name@version. If the // upstream has the version, clients get the upstream archive again. func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { diff --git a/internal/handler/hosted_manifest.go b/internal/handler/hosted_manifest.go new file mode 100644 index 00000000..db6d0b6c --- /dev/null +++ b/internal/handler/hosted_manifest.go @@ -0,0 +1,144 @@ +package handler + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "strings" +) + +var ( + ErrHostedInvalidArchive = errors.New("archive has no readable package manifest") + ErrHostedManifestMismatch = errors.New("package manifest does not match the pushed name and version") +) + +// maxManifestSize bounds how much of a manifest file is read. Real manifests +// are a few kilobytes; the bound keeps a hostile archive from claiming memory. +const maxManifestSize = 1 << 20 + +// readHostedManifest reads the package manifest out of a pushed archive and +// checks that it describes target. The manifest is what the ecosystem's +// metadata lists the version with, so its dependencies reach the client. +func readHostedManifest(ecosystem string, archive io.Reader, target HostedTarget) (json.RawMessage, error) { + switch ecosystem { + case "npm": + return readNPMManifest(archive, target) + case "composer": + return readComposerManifest(archive, target) + default: + return nil, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) + } +} + +// readNPMManifest reads package.json from the top directory of an npm +// tarball. npm packs it under package/, but published tarballs use other +// directory names too. +func readNPMManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { + gz, err := gzip.NewReader(archive) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + defer func() { _ = gz.Close() }() + + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if err != nil { + return nil, fmt.Errorf("%w: no package.json in tarball", ErrHostedInvalidArchive) + } + if header.Typeflag != tar.TypeReg || !isTopLevelManifest(header.Name, "package.json") { + continue + } + manifest, fields, err := decodeManifest(tr) + if err != nil { + return nil, err + } + if fields.Name != target.Name || fields.Version != target.Version { + return nil, fmt.Errorf("%w: package.json is %s@%s", ErrHostedManifestMismatch, fields.Name, fields.Version) + } + return manifest, nil + } +} + +// readComposerManifest reads composer.json from the root of a Composer dist +// zip, or from its single top directory as GitHub zipballs have it. Package +// names are case-insensitive, and composer.json only names a version when +// the package pins one. +func readComposerManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { + file, err := os.CreateTemp("", "hosted-composer-*.zip") + if err != nil { + return nil, err + } + defer func() { + _ = file.Close() + _ = os.Remove(file.Name()) + }() + size, err := io.Copy(file, archive) + if err != nil { + return nil, err + } + zr, err := zip.NewReader(file, size) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + + for _, entry := range zr.File { + if entry.FileInfo().IsDir() || (entry.Name != "composer.json" && !isTopLevelManifest(entry.Name, "composer.json")) { + continue + } + rc, err := entry.Open() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + manifest, fields, err := decodeManifest(rc) + _ = rc.Close() + if err != nil { + return nil, err + } + if !strings.EqualFold(fields.Name, target.Name) || (fields.Version != "" && fields.Version != target.Version) { + return nil, fmt.Errorf("%w: composer.json is %s %s", ErrHostedManifestMismatch, fields.Name, fields.Version) + } + return manifest, nil + } + return nil, fmt.Errorf("%w: no composer.json in zip", ErrHostedInvalidArchive) +} + +// isTopLevelManifest reports whether name is filename inside one top +// directory, such as package/package.json. +func isTopLevelManifest(name, filename string) bool { + dir, file := path.Split(path.Clean(name)) + return file == filename && dir != "" && !strings.Contains(strings.TrimSuffix(dir, "/"), "/") +} + +type manifestFields struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// decodeManifest reads a JSON object manifest and returns it compacted along +// with its name and version. +func decodeManifest(r io.Reader) (json.RawMessage, manifestFields, error) { + raw, err := io.ReadAll(io.LimitReader(r, maxManifestSize+1)) + if err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + if len(raw) > maxManifestSize { + return nil, manifestFields{}, fmt.Errorf("%w: manifest exceeds %d bytes", ErrHostedInvalidArchive, maxManifestSize) + } + var fields manifestFields + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: manifest is not a JSON object: %w", ErrHostedInvalidArchive, err) + } + var compact bytes.Buffer + if err := json.Compact(&compact, raw); err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + return compact.Bytes(), fields, nil +} diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index b327b89c..89cdd812 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -1,6 +1,10 @@ package handler import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" "context" "errors" "io" @@ -62,7 +66,51 @@ func TestHostedTargetFor(t *testing.T) { } } -const pushedTarball = "patched request tarball" +// npmTarball builds an npm package tarball holding manifest as +// package/package.json. Building an archive in memory cannot fail, so errors +// panic. +func npmTarball(manifest string) []byte { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, f := range [][2]string{{"package/package.json", manifest}, {"package/index.js", "module.exports = 1\n"}} { + must(tw.WriteHeader(&tar.Header{Name: f[0], Mode: 0o644, Size: int64(len(f[1]))})) + _, err := io.WriteString(tw, f[1]) + must(err) + } + must(tw.Close()) + must(gz.Close()) + return buf.Bytes() +} + +// composerZip builds a Composer dist archive holding manifest as composer.json +// under dir, or at the root when dir is empty. +func composerZip(dir, manifest string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + prefix := "" + if dir != "" { + prefix = dir + "/" + } + for _, f := range [][2]string{{prefix + "composer.json", manifest}, {prefix + "src/Logger.php", "=8.1"}}`)}, + {name: "composer manifest in a top directory", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", + archive: composerZip("Seldaek-monolog-abc123", `{"name":"Monolog/Monolog","version":"3.5.0"}`)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + target, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + _, err = proxy.PushLocalArtifact(context.Background(), tt.ecosystem, target, bytes.NewReader(tt.archive), "") + if tt.wantErr == nil { + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return + } + if !errors.Is(err, tt.wantErr) { + t.Fatalf("PushLocalArtifact error = %v, want %v", err, tt.wantErr) + } + if listed, _ := proxy.LocalArtifacts(tt.ecosystem, tt.pkg); len(listed) != 0 { + t.Errorf("rejected archive was recorded: %+v", listed) + } + if len(store.files) != 0 { + t.Errorf("rejected archive left %d objects in storage", len(store.files)) + } + }) + } +} diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go index 962bc177..e9033d64 100644 --- a/internal/server/hosted_api.go +++ b/internal/server/hosted_api.go @@ -102,6 +102,8 @@ func (h *HostedAPIHandler) pushError(w http.ResponseWriter, err error) { switch { case errors.As(err, &tooLarge): writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "archive exceeds hosted.max_upload_size") + case errors.Is(err, handler.ErrHostedInvalidArchive), errors.Is(err, handler.ErrHostedManifestMismatch): + badRequest(w, err.Error()) case errors.Is(err, handler.ErrVersionDenied), errors.Is(err, handler.ErrArtifactBlocked): writeError(w, http.StatusForbidden, ErrCodeBadRequest, err.Error()) default: diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 9051572e..0409c64a 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -23,8 +23,7 @@ import ( const ( testHostedToken = "push-token" - testHostedMaxUpload = 64 - pushedBody = "patched tarball" + testHostedMaxUpload = 4096 ) type hostedTestServer struct { @@ -101,7 +100,7 @@ func (s hostedTestServer) do(t *testing.T, method, target, token, body string) * func TestHostedPushIsServedByTheEcosystemHandler(t *testing.T) { s := newHostedTestServer(t) - rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, pushedBody) + rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, requestTarball) if rec.Code != http.StatusCreated { t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) } @@ -110,12 +109,12 @@ func TestHostedPushIsServedByTheEcosystemHandler(t *testing.T) { t.Fatalf("decoding PUT response: %v", err) } if pushed.Ecosystem != "npm" || pushed.Name != "request" || pushed.Version != "2.88.3" || - pushed.Filename != "request-2.88.3.tgz" || pushed.Size != int64(len(pushedBody)) { + pushed.Filename != "request-2.88.3.tgz" || pushed.Size != int64(len(requestTarball)) { t.Errorf("PUT response = %+v", pushed) } rec = s.do(t, http.MethodGet, "/npm/request/-/request-2.88.3.tgz", "", "") - if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + if rec.Code != http.StatusOK || rec.Body.String() != requestTarball { t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) } if *s.fetches != 0 { @@ -126,13 +125,13 @@ func TestHostedPushIsServedByTheEcosystemHandler(t *testing.T) { func TestHostedPushIsServedByTheComposerHandler(t *testing.T) { s := newHostedTestServer(t) - rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, pushedBody) + rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, monologZip) if rec.Code != http.StatusCreated { t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) } rec = s.do(t, http.MethodGet, "/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", "", "") - if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + if rec.Code != http.StatusOK || rec.Body.String() != monologZip { t.Fatalf("download status = %d, body %q, want 200 and the pushed archive", rec.Code, rec.Body) } if *s.upstreamRequests != 0 || *s.fetches != 0 { @@ -142,7 +141,7 @@ func TestHostedPushIsServedByTheComposerHandler(t *testing.T) { func TestComposerDownloadOfUnpushedFileStillNeedsUpstreamMetadata(t *testing.T) { s := newHostedTestServer(t) - if rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, pushedBody); rec.Code != http.StatusCreated { + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, monologZip); rec.Code != http.StatusCreated { t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) } @@ -158,19 +157,19 @@ func TestComposerDownloadOfUnpushedFileStillNeedsUpstreamMetadata(t *testing.T) func TestHostedPushScopedNpmPackage(t *testing.T) { s := newHostedTestServer(t) - rec := s.do(t, http.MethodPut, "/api/hosted/npm/@types/node/20.1.0", testHostedToken, pushedBody) + rec := s.do(t, http.MethodPut, "/api/hosted/npm/@types/node/20.1.0", testHostedToken, nodeTarball) if rec.Code != http.StatusCreated { t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) } rec = s.do(t, http.MethodGet, "/npm/@types/node/-/node-20.1.0.tgz", "", "") - if rec.Code != http.StatusOK || rec.Body.String() != pushedBody { + if rec.Code != http.StatusOK || rec.Body.String() != nodeTarball { t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) } } func TestHostedListAndDelete(t *testing.T) { s := newHostedTestServer(t) - if rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, pushedBody); rec.Code != http.StatusCreated { + if rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, requestTarball); rec.Code != http.StatusCreated { t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) } @@ -205,11 +204,13 @@ func TestHostedPushRejections(t *testing.T) { body string wantStatus int }{ - {name: "no token", target: "/api/hosted/npm/request/2.88.3", body: pushedBody, wantStatus: http.StatusUnauthorized}, - {name: "wrong token", target: "/api/hosted/npm/request/2.88.3", token: "guess", body: pushedBody, wantStatus: http.StatusUnauthorized}, - {name: "unsupported ecosystem", target: "/api/hosted/cargo/serde/1.0.200", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, - {name: "composer name without vendor", target: "/api/hosted/composer/monolog/3.5.0", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, - {name: "missing version", target: "/api/hosted/npm/request", token: testHostedToken, body: pushedBody, wantStatus: http.StatusBadRequest}, + {name: "no token", target: "/api/hosted/npm/request/2.88.3", body: requestTarball, wantStatus: http.StatusUnauthorized}, + {name: "wrong token", target: "/api/hosted/npm/request/2.88.3", token: "guess", body: requestTarball, wantStatus: http.StatusUnauthorized}, + {name: "unsupported ecosystem", target: "/api/hosted/cargo/serde/1.0.200", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "composer name without vendor", target: "/api/hosted/composer/monolog/3.5.0", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "missing version", target: "/api/hosted/npm/request", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "not an archive", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: "plain text", wantStatus: http.StatusBadRequest}, + {name: "manifest of another version", target: "/api/hosted/npm/request/2.88.4", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, {name: "empty body", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, wantStatus: http.StatusBadRequest}, {name: "too large", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: strings.Repeat("x", testHostedMaxUpload+1), wantStatus: http.StatusRequestEntityTooLarge}, } @@ -271,7 +272,7 @@ func startHostedServer(t *testing.T, hosted config.HostedConfig) string { func putArchive(t *testing.T, url, token string) int { t.Helper() - req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(pushedBody)) + req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(requestTarball)) if err != nil { t.Fatalf("building request: %v", err) } diff --git a/internal/server/hosted_archives_test.go b/internal/server/hosted_archives_test.go new file mode 100644 index 00000000..85150aa2 --- /dev/null +++ b/internal/server/hosted_archives_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "io" +) + +// npmTarball builds an npm package tarball holding manifest as +// package/package.json. Building an archive in memory cannot fail, so errors +// panic. +func npmTarball(manifest string) string { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + must(tw.WriteHeader(&tar.Header{Name: "package/package.json", Mode: 0o644, Size: int64(len(manifest))})) + _, err := io.WriteString(tw, manifest) + must(err) + must(tw.Close()) + must(gz.Close()) + return buf.String() +} + +// composerZip builds a Composer dist archive holding manifest as composer.json. +func composerZip(manifest string) string { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("composer.json") + must(err) + _, err = io.WriteString(w, manifest) + must(err) + must(zw.Close()) + return buf.String() +} + +func must(err error) { + if err != nil { + panic(err) + } +} + +var ( + requestTarball = npmTarball(`{"name":"request","version":"2.88.3","dependencies":{"qs":"~6.5.2"}}`) + nodeTarball = npmTarball(`{"name":"@types/node","version":"20.1.0"}`) + monologZip = composerZip(`{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"}}`) +) From c5669495732275b2d79d291734bb9cb377d50f36 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:23:29 +0200 Subject: [PATCH 06/17] List pushed npm versions in package metadata With hosted packages enabled, the packument of a package with pushed versions lists each of them, built from its package.json with dist pointing at the proxy and carrying the archive's sha512, and the time it was pushed. A pushed version takes the place of an upstream entry with the same version. Cooldown and the denylist still filter only upstream versions, and dist-tags stay as upstream has them. When the upstream doesn't know the package, the packument holds only the pushed versions, with latest set to the highest of them. Requests for a single pushed version are answered from its entry. Rewrites that include pushed versions bypass the rewrite cache, which is keyed by the upstream document alone, so a push or delete shows on the next request and changes the ETag. Packages without pushed versions take the cached path as before, and with hosted packages disabled there is no extra lookup. Refs #134 --- internal/handler/npm.go | 83 +++++++++-- internal/handler/npm_hosted.go | 122 +++++++++++++++ internal/handler/npm_hosted_test.go | 221 ++++++++++++++++++++++++++++ internal/handler/rewrite_cache.go | 11 ++ internal/server/hosted_api_test.go | 29 +++- internal/server/server.go | 1 + 6 files changed, 457 insertions(+), 10 deletions(-) create mode 100644 internal/handler/npm_hosted.go create mode 100644 internal/handler/npm_hosted_test.go diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 92629d24..7cd5380d 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -215,12 +215,29 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } + specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) + local, err := h.npmLocalEntries(packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed npm versions", "package", packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") + return + } + if e, ok := findNPMLocalEntry(local, specifier); versionLookup && ok { + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(e.entry) + return + } + if len(local) > 0 && !versionLookup { + h.serveMetadataWithLocal(w, r, packageName, upstreamURL, accept, metadataCacheKey, local) + return + } + if stored, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { serveRewrittenMetadata(w, r, stored) return } - specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) if versionLookup && h.npmVersionMetadataUsesFilteredPackument(packageName) { h.handleFilteredNPMVersionMetadata(w, r, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey) return @@ -396,6 +413,14 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e // rewriteMetadataKeeping is rewriteMetadata that also reports the versions it // kept. func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { + return h.rewriteMetadataWithLocal(packageName, body, nil) +} + +// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the +// pushed versions in local, in place of any upstream entry with the same +// version. Cooldown and the denylist only filter upstream versions, and +// dist-tags stay as upstream has them. +func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) { doc, err := indexNPMPackument(body) if err != nil { return nil, nil, err @@ -439,18 +464,26 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] h.applyCooldownFiltering(metadata, versions, packageName) h.applyDenylistFiltering(metadata, versions, packageName) + for _, e := range local { + delete(versions, e.version) + } // Untouched members are copied as they are; the three the filters // handled are written from their filtered state, and only when it changed. out, err := doc.write(func(out *bytes.Buffer, i int) error { switch { case i == doc.versionsAt: - return h.writeNPMVersions(out, packageName, versionsRaw, versions) - case i == doc.timeAt && timeMap != nil && len(timeMap) != timeEntries: - return writeFilteredJSONObject(out, doc.value(i), func(key string) bool { + return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) + case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): + pushedAt := make([][2]string, 0, len(local)) + for _, e := range local { + pushedAt = append(pushedAt, [2]string{e.version, e.time}) + } + return writeFilteredJSONObjectWith(out, doc.value(i), func(key string) bool { _, ok := timeMap[key] - return ok - }) + _, pushed := findNPMLocalEntry(local, key) + return ok && !pushed + }, pushedAt) case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): encoded, err := json.Marshal(distTags) out.Write(encoded) @@ -460,7 +493,11 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] return nil } }) - return out, slices.Collect(maps.Keys(versions)), err + kept := slices.Collect(maps.Keys(versions)) + for _, e := range local { + kept = append(kept, e.version) + } + return out, kept, err } // npmPackument is a packument's top-level members as offsets into its bytes, @@ -547,8 +584,9 @@ func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([ } // writeNPMVersions writes the versions object, keeping the versions still in -// keep and pointing each kept version's tarball at this proxy. -func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any) error { +// keep and pointing each kept version's tarball at this proxy, followed by the +// pushed versions in local. +func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error { out.WriteByte('{') first := true err := forEachJSONMember(versionsRaw, func(m jsonMember) error { @@ -568,6 +606,16 @@ func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, ver h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) return nil }) + for _, e := range local { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(e.version) + out.Write(key) + out.WriteByte(':') + out.Write(e.entry) + } out.WriteByte('}') return err } @@ -610,6 +658,12 @@ func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version str // writeFilteredJSONObject writes the object obj holds, keeping the members // whose decoded key keep accepts. func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bool) error { + return writeFilteredJSONObjectWith(out, obj, keep, nil) +} + +// writeFilteredJSONObjectWith is writeFilteredJSONObject followed by extra +// string members. +func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { out.WriteByte('{') first := true err := forEachJSONMember(obj, func(m jsonMember) error { @@ -627,6 +681,17 @@ func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bo out.Write(obj[m.keyStart:m.valEnd]) return nil }) + for _, member := range extra { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(member[0]) + value, _ := json.Marshal(member[1]) + out.Write(key) + out.WriteByte(':') + out.Write(value) + } out.WriteByte('}') return err } diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go new file mode 100644 index 00000000..6e0e0b62 --- /dev/null +++ b/internal/handler/npm_hosted.go @@ -0,0 +1,122 @@ +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/Masterminds/semver/v3" +) + +// npmLocalEntry is a pushed version as a packument lists it: the version +// entry built from its package.json, and the time it was pushed. +type npmLocalEntry struct { + version string + entry json.RawMessage + time string +} + +// npmLocalEntries returns the pushed versions of a package as packument +// entries, or nil when there are none or hosted packages are disabled. +func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error) { + local, err := h.proxy.localVersions("npm", packageName) + if err != nil || len(local) == 0 { + return nil, err + } + entries := make([]npmLocalEntry, 0, len(local)) + for _, lv := range local { + var manifest map[string]any + if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { + return nil, fmt.Errorf("decoding manifest of %s@%s: %w", packageName, lv.Version, err) + } + manifest["name"] = packageName + manifest["version"] = lv.Version + manifest["_id"] = packageName + "@" + lv.Version + manifest["dist"] = map[string]string{ + "tarball": h.proxyTarballURL(packageName, lv.Version, lv.Filename), + "integrity": lv.Integrity, + } + entry, err := json.Marshal(manifest) + if err != nil { + return nil, err + } + entries = append(entries, npmLocalEntry{version: lv.Version, entry: entry, time: lv.PushedAt.Format(time.RFC3339)}) + } + return entries, nil +} + +func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) { + for _, e := range local { + if e.version == version { + return e, true + } + } + return npmLocalEntry{}, false +} + +// serveMetadataWithLocal serves a packument with the pushed versions of the +// package added. It bypasses the rewrite cache, whose entries are keyed by the +// upstream document alone, so a push or delete shows up on the next request. +// When the upstream does not know the package, the packument holds only the +// pushed versions. +func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string, local []npmLocalEntry) { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + var out []byte + var err error + switch { + case errors.Is(upstream.err, ErrUpstreamNotFound): + out, err = localNPMPackument(packageName, local) + case upstream.err != nil: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", upstream.err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return + default: + out, _, err = h.rewriteMetadataWithLocal(packageName, upstream.body, local) + } + if err != nil { + if errors.Is(err, ErrVersionDenied) { + JSONError(w, http.StatusForbidden, err.Error()) + return + } + h.proxy.Logger.Error("failed to add pushed versions to npm metadata", "package", packageName, "error", err) + JSONError(w, http.StatusBadGateway, "failed to rewrite package metadata") + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) +} + +// localNPMPackument builds a packument from pushed versions alone. latest is +// the highest of them, or the last pushed when a version is not semver. +func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error) { + versions := make(map[string]json.RawMessage, len(local)) + times := make(map[string]string, len(local)) + latest, latestTime := "", "" + var latestSemver *semver.Version + semverOnly := true + for _, e := range local { + versions[e.version] = e.entry + times[e.version] = e.time + if e.time > latestTime { + latest, latestTime = e.version, e.time + } + v, err := semver.NewVersion(e.version) + if err != nil { + semverOnly = false + continue + } + if latestSemver == nil || v.GreaterThan(latestSemver) { + latestSemver = v + } + } + if semverOnly && latestSemver != nil { + latest = latestSemver.Original() + } + return json.Marshal(map[string]any{ + "name": packageName, + "dist-tags": map[string]string{"latest": latest}, + "versions": versions, + "time": times, + }) +} diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go new file mode 100644 index 00000000..df4e32d0 --- /dev/null +++ b/internal/handler/npm_hosted_test.go @@ -0,0 +1,221 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +const requestPackument = `{ + "name": "request", + "dist-tags": {"latest": "2.88.2"}, + "versions": { + "2.88.2": {"name": "request", "version": "2.88.2", "dependencies": {"qs": "~6.5.1"}, + "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-upstream2"}}, + "2.88.3": {"name": "request", "version": "2.88.3", "dependencies": {"old": "1.0.0"}, + "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.3.tgz", "integrity": "sha512-upstream3"}} + }, + "time": {"modified": "2020-02-11T00:00:00Z", "2.88.2": "2020-02-11T00:00:00Z", "2.88.3": "2020-02-12T00:00:00Z"} +}` + +type npmHostedFixture struct { + proxy *Proxy + handler http.Handler +} + +// newNPMHostedFixture serves packuments from upstream, keyed by request path, +// and answers 404 for anything else. +func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixture { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := upstream[r.URL.EscapedPath()] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HTTPClient = srv.Client() + return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes()} +} + +func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { + t.Helper() + target, err := HostedTargetFor("npm", name, version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "npm", target, bytes.NewReader(npmTarball(manifest)), "") + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return pushed +} + +func (f npmHostedFixture) get(t *testing.T, path string) *httptest.ResponseRecorder { + t.Helper() + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +type npmTestPackument struct { + Name string `json:"name"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]npmTestVersion `json:"versions"` + Time map[string]string `json:"time"` + Extra map[string]json.RawMessage `json:"-"` +} + +type npmTestVersion struct { + Name string `json:"name"` + Version string `json:"version"` + ID string `json:"_id"` + Dependencies map[string]string `json:"dependencies"` + Dist struct { + Tarball string `json:"tarball"` + Integrity string `json:"integrity"` + } `json:"dist"` +} + +func decodePackument(t *testing.T, rec *httptest.ResponseRecorder) npmTestPackument { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body) + } + var doc npmTestPackument + if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding packument: %v\n%s", err, rec.Body) + } + return doc +} + +func TestNPMMetadataListsPushedVersion(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + pushed := f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) + + doc := decodePackument(t, f.get(t, "/request")) + + local, ok := doc.Versions["2.88.4"] + if !ok { + t.Fatalf("pushed version missing from versions: %v", doc.Versions) + } + if local.Name != "request" || local.Version != "2.88.4" || local.ID != "request@2.88.4" || local.Dependencies["qs"] != "~6.5.2" { + t.Errorf("pushed version entry = %+v", local) + } + if local.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { + t.Errorf("tarball = %q", local.Dist.Tarball) + } + if !strings.HasPrefix(local.Dist.Integrity, "sha512-") { + t.Errorf("integrity = %q, want the pushed archive's sha512", local.Dist.Integrity) + } + if doc.Time["2.88.4"] != pushed.PushedAt.Format(time.RFC3339) { + t.Errorf("time[2.88.4] = %q, want %q", doc.Time["2.88.4"], pushed.PushedAt.Format(time.RFC3339)) + } + if doc.Versions["2.88.2"].Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.2.tgz" { + t.Errorf("upstream version not rewritten: %+v", doc.Versions["2.88.2"]) + } + if doc.DistTags["latest"] != "2.88.2" { + t.Errorf("dist-tags.latest = %q, want upstream's 2.88.2", doc.DistTags["latest"]) + } +} + +func TestNPMMetadataPushedVersionReplacesUpstream(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3","dependencies":{"qs":"~6.5.2"}}`) + + rec := f.get(t, "/request") + if body := rec.Body.String(); strings.Contains(body, "sha512-upstream3") || strings.Count(body, `"2.88.3":{`) != 1 { + t.Fatalf("upstream entry for the pushed version still listed:\n%s", body) + } + local := decodePackument(t, rec).Versions["2.88.3"] + if local.Dependencies["qs"] != "~6.5.2" || local.Dependencies["old"] != "" { + t.Errorf("dependencies = %v, want the pushed package.json's", local.Dependencies) + } +} + +func TestNPMMetadataForPackageOnlyPushed(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.push(t, "@acme/private", "1.10.0", `{"name":"@acme/private","version":"1.10.0"}`) + f.push(t, "@acme/private", "1.9.0", `{"name":"@acme/private","version":"1.9.0"}`) + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if doc.Name != "@acme/private" || len(doc.Versions) != 2 { + t.Errorf("packument = %+v", doc) + } + if doc.DistTags["latest"] != "1.10.0" { + t.Errorf("dist-tags.latest = %q, want the highest pushed version 1.10.0", doc.DistTags["latest"]) + } + if doc.Versions["1.9.0"].Dist.Tarball != "http://proxy.local/npm/@acme%2Fprivate/-/private-1.9.0.tgz" { + t.Errorf("tarball = %q", doc.Versions["1.9.0"].Dist.Tarball) + } + + if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { + t.Errorf("unknown package status = %d, want 404", rec.Code) + } +} + +func TestNPMVersionMetadataForPushedVersion(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) + + rec := f.get(t, "/request/2.88.4") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body) + } + var version npmTestVersion + if err := json.Unmarshal(rec.Body.Bytes(), &version); err != nil { + t.Fatalf("decoding version: %v", err) + } + if version.Version != "2.88.4" || version.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { + t.Errorf("version document = %+v", version) + } +} + +func TestNPMMetadataFollowsPushAndDelete(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.proxy.CacheMetadata = true + f.proxy.MetadataTTL = time.Hour + f.proxy.SetMetadataRewriteCacheSize(1 << 20) + + before := f.get(t, "/request") + if before.Code != http.StatusOK { + t.Fatalf("status = %d", before.Code) + } + + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) + pushed := f.get(t, "/request") + if _, ok := decodePackument(t, pushed).Versions["2.88.4"]; !ok { + t.Fatal("pushed version not listed while the upstream document is cached") + } + if pushed.Header().Get("ETag") == before.Header().Get("ETag") { + t.Error("ETag unchanged after a push") + } + + if err := f.proxy.DeleteLocalVersion("npm", "request", "2.88.4"); err != nil { + t.Fatalf("DeleteLocalVersion: %v", err) + } + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { + t.Error("deleted version still listed") + } +} + +func TestNPMMetadataIgnoresPushesWhenHostedDisabled(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) + f.proxy.HostedEnabled = false + + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { + t.Error("pushed version listed with hosted packages disabled") + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 7348bbe5..42273c3b 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -272,6 +272,17 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str return rewrittenMetadata{out, v}, err } +// uncachedRewrite gives a rewrite made outside the cache its validators: an +// ETag of the output itself and, as cachedRewrite does, a date that replicas +// agree on. +func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) rewrittenMetadata { + v := metadataValidators{etag: metadataETag(out)} + if !p.cooldownEnabled() { + v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + } + return rewrittenMetadata{out, v} +} + func (p *Proxy) cooldownEnabled() bool { return p.Cooldown != nil && p.Cooldown.Enabled() } diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index 0409c64a..d4016fe3 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -251,6 +251,10 @@ func startHostedServer(t *testing.T, hosted config.HostedConfig) string { cfg.Database.Path = filepath.Join(tempDir, "proxy.db") cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") cfg.Hosted = hosted + upstream := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(upstream.Close) + cfg.Upstream.NPM = upstream.URL + cfg.Upstream.AllowLoopback = true if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) } @@ -272,7 +276,12 @@ func startHostedServer(t *testing.T, hosted config.HostedConfig) string { func putArchive(t *testing.T, url, token string) int { t.Helper() - req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(requestTarball)) + return putBody(t, url, token, requestTarball) +} + +func putBody(t *testing.T, url, token, body string) int { + t.Helper() + req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(body)) if err != nil { t.Fatalf("building request: %v", err) } @@ -301,3 +310,21 @@ func TestHostedRoutesFollowConfig(t *testing.T) { t.Errorf("push with token status = %d, want 201", status) } } + +func TestHostedNpmPackageIsListedByTheRunningServer(t *testing.T) { + base := startHostedServer(t, config.HostedConfig{Enabled: true, Token: testHostedToken}) + private := npmTarball(`{"name":"@acme/private","version":"1.0.0"}`) + if status := putBody(t, base+"/api/hosted/npm/@acme/private/1.0.0", testHostedToken, private); status != http.StatusCreated { + t.Fatalf("push status = %d, want 201", status) + } + + resp, err := http.Get(base + "/npm/@acme%2fprivate") + if err != nil { + t.Fatalf("GET packument: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"1.0.0"`) { + t.Errorf("packument status = %d, body %s", resp.StatusCode, body) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 000d3313..ac773c56 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -268,6 +268,7 @@ func (s *Server) serve(listener net.Listener) error { proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly + proxy.HostedEnabled = s.cfg.Hosted.Enabled proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.DirectServe = s.cfg.Storage.DirectServe From 112fc590b792850e7990e27dcb27ad8a6ee7226f Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:27:13 +0200 Subject: [PATCH 07/17] List pushed Composer versions in package metadata With hosted packages enabled, a package's p2 metadata lists its pushed versions, built from composer.json without the root-only keys, with a zip dist at the proxy and the time they were pushed. dist.reference is the archive's sha256 so a re-push isn't served from Composer's cache, and shasum is left empty. Dev versions go to the ~dev file and tagged versions to the other. A pushed version takes the place of an upstream entry with the same version, with or without a leading v. Pushed entries follow the upstream ones and go through the same minified writer, so fields an earlier upstream entry set and the pushed one lacks are written as __unset rather than inherited. When the upstream doesn't know the package, the document holds only the pushed versions. Like npm, documents with pushed versions bypass the rewrite cache. Refs #134 --- internal/handler/composer.go | 63 ++++++++- internal/handler/composer_hosted.go | 108 +++++++++++++++ internal/handler/composer_hosted_test.go | 167 +++++++++++++++++++++++ internal/server/hosted_api_test.go | 32 +++++ 4 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 internal/handler/composer_hosted.go create mode 100644 internal/handler/composer_hosted_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6a324297..b8c0a752 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -148,6 +148,18 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R repoURL, cacheKey := h.sourceFor(packageName) upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) + basePackage := strings.TrimSuffix(packageName, composerDevFile) + local, err := h.composerLocalEntries(basePackage, basePackage != packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed composer versions", "package", basePackage, "error", err) + http.Error(w, "failed to read pushed versions", http.StatusInternalServerError) + return + } + if len(local) > 0 { + h.serveMetadataWithLocal(w, r, basePackage, upstreamURL, cacheKey, local) + return + } + if stored, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { serveRewrittenMetadata(w, r, stored) return @@ -195,6 +207,15 @@ func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { // rewriteMetadataKeeping is rewriteMetadata that also reports the versions // it kept, each as its package name and version. func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, error) { + return h.rewriteMetadataWithLocal(body, "", nil) +} + +// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the +// pushed versions in local under localPackage, in place of any upstream entry +// with the same version. Cooldown only filters upstream versions. A document +// that does not list localPackage at all is replaced by one holding only the +// pushed versions. +func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage string, local []composerLocalEntry) ([]byte, []string, error) { if !json.Valid(body) { return nil, nil, errors.New("composer metadata is not valid JSON") } @@ -206,6 +227,7 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, var out bytes.Buffer var kept []string + listed := false out.Grow(len(body) + len(body)/8) err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { packages := m.value(body) @@ -221,12 +243,20 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, if err != nil { return false, err } - return true, h.writeVersions(out, packageName, versions, minified, &kept) + var pushed []composerLocalEntry + if packageName == localPackage { + pushed, listed = local, true + } + return true, h.writeVersions(out, packageName, versions, minified, &kept, pushed) }) }) if err != nil { return nil, nil, err } + if len(local) > 0 && !listed { + document, err := localComposerDocument(localPackage, local) + return document, nil, err + } return out.Bytes(), kept, nil } @@ -239,7 +269,10 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, // entry itself unless filtered versions were dropped before it; then it also // carries the changes those versions made. Every version gets its own dist, // since the proxied URL names the version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string) error { +// +// The pushed versions in local follow the upstream ones and take the place of +// upstream entries with the same version. +func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []composerLocalEntry) error { packagePURL := canonicalPackagePURL("composer", packageName) upstream, written := newComposerFields(), newComposerFields() devReset, first := false, true @@ -265,7 +298,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v } version := upstream.string("version") - if h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { + if pushedComposerVersion(local, version) || h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { return nil } @@ -284,8 +317,30 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v *kept = append(*kept, packageName+"\x00"+version) return nil }) + if err != nil { + return err + } + for _, e := range local { + pushed := newComposerFields() + if err := pushed.apply(e.entry, false); err != nil { + return err + } + if !minified { + written.reset() + } + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + composerDevReset + `",`) + devReset = false + } + h.writeVersion(out, packageName, e.version, pushed, written) + *kept = append(*kept, packageName+"\x00"+e.version) + } out.WriteByte(']') - return err + return nil } // writeVersion writes the fields of upstream that differ from written, with diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go new file mode 100644 index 00000000..5a94f828 --- /dev/null +++ b/internal/handler/composer_hosted.go @@ -0,0 +1,108 @@ +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +// composerRootOnlyKeys are composer.json keys that only apply to the root +// project. Repositories leave them out of the version entries they serve. +var composerRootOnlyKeys = []string{"repositories", "config", "minimum-stability", "prefer-stable"} + +// composerLocalEntry is a pushed version as Composer metadata lists it. +type composerLocalEntry struct { + version string + entry json.RawMessage +} + +// composerLocalEntries returns the pushed versions of a package for one of +// its metadata files: the dev versions for the ~dev file, the tagged ones for +// the other. +func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([]composerLocalEntry, error) { + local, err := h.proxy.localVersions("composer", packageName) + if err != nil || len(local) == 0 { + return nil, err + } + var entries []composerLocalEntry + for _, lv := range local { + if isDevVersion(lv.Version) != dev { + continue + } + var manifest map[string]any + if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { + return nil, fmt.Errorf("decoding manifest of %s %s: %w", packageName, lv.Version, err) + } + for _, key := range composerRootOnlyKeys { + delete(manifest, key) + } + manifest["name"] = packageName + manifest["version"] = lv.Version + manifest["time"] = lv.PushedAt.Format(time.RFC3339) + manifest["dist"] = map[string]string{ + "type": "zip", + "url": h.localDistURL(packageName, lv.Version, lv.Filename), + "reference": lv.SHA256, + "shasum": "", + } + entry, err := json.Marshal(manifest) + if err != nil { + return nil, err + } + entries = append(entries, composerLocalEntry{version: lv.Version, entry: entry}) + } + return entries, nil +} + +func (h *ComposerHandler) localDistURL(packageName, version, filename string) string { + return fmt.Sprintf("%s/composer/files/%s/%s/%s", h.proxyURL, packageName, url.PathEscape(version), filename) +} + +// pushedComposerVersion reports whether version is one of the pushed ones. +// Tags are listed with and without a leading v, and both name one version. +func pushedComposerVersion(local []composerLocalEntry, version string) bool { + for _, e := range local { + if strings.TrimPrefix(e.version, "v") == strings.TrimPrefix(version, "v") { + return true + } + } + return false +} + +// serveMetadataWithLocal serves a package's metadata with its pushed versions +// added, bypassing the rewrite cache like the npm handler does. When the +// upstream does not know the package, the document holds only the pushed +// versions. +func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, cacheKey string, local []composerLocalEntry) { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON) + var out []byte + var err error + switch { + case errors.Is(upstream.err, ErrUpstreamNotFound): + out, err = localComposerDocument(packageName, local) + case upstream.err != nil: + h.proxy.Logger.Error("upstream request failed", "error", upstream.err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + default: + out, _, err = h.rewriteMetadataWithLocal(upstream.body, packageName, local) + } + if err != nil { + h.proxy.Logger.Error("failed to add pushed versions to composer metadata", "package", packageName, "error", err) + http.Error(w, "failed to rewrite package metadata", http.StatusBadGateway) + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) +} + +func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { + entries := make([]json.RawMessage, 0, len(local)) + for _, e := range local { + entries = append(entries, e.entry) + } + return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) +} diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go new file mode 100644 index 00000000..09e44c2c --- /dev/null +++ b/internal/handler/composer_hosted_test.go @@ -0,0 +1,167 @@ +package handler + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const monologMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ + {"name":"monolog/monolog","version":"3.5.0","version_normalized":"3.5.0.0", + "require":{"php":">=8.1","psr/log":"^2.0"},"support":{"issues":"https://github.com/Seldaek/monolog/issues"}, + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/aaa","reference":"aaa","shasum":""}, + "time":"2023-10-27T15:32:31+00:00"}, + {"version":"3.4.0","version_normalized":"3.4.0.0", + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/bbb","reference":"bbb","shasum":""}} +]}}` + +const monologDevMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ + {"name":"monolog/monolog","version":"dev-main","version_normalized":"dev-main", + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/ccc","reference":"ccc","shasum":""}} +]}}` + +type composerHostedFixture struct { + proxy *Proxy + handler http.Handler +} + +func newComposerHostedFixture(t *testing.T, upstream map[string]string) composerHostedFixture { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := upstream[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HTTPClient = srv.Client() + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.local", srv.URL, srv.URL) + return composerHostedFixture{proxy: proxy, handler: h.Routes()} +} + +func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { + t.Helper() + target, err := HostedTargetFor("composer", name, version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "composer", target, bytes.NewReader(composerZip("", manifest)), "") + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return pushed +} + +func (f composerHostedFixture) versions(t *testing.T, path, packageName string) map[string]map[string]any { + t.Helper() + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, body %s", path, rec.Code, rec.Body) + } + byVersion := map[string]map[string]any{} + for _, v := range composerVersions(t, rec.Body.Bytes(), packageName) { + version, _ := v["version"].(string) + if _, dup := byVersion[version]; dup { + t.Fatalf("version %s listed twice:\n%s", version, rec.Body) + } + byVersion[version] = v + } + return byVersion +} + +func TestComposerMetadataListsPushedVersion(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + pushed := f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"},"repositories":[{"type":"vcs","url":"x"}]}`) + + versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + local, ok := versions["3.6.0"] + if !ok { + t.Fatalf("pushed version missing: %v", versions) + } + if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { + t.Errorf("require = %v, want the pushed composer.json's", local["require"]) + } + for _, inherited := range []string{"version_normalized", "support", "repositories"} { + if _, ok := local[inherited]; ok { + t.Errorf("pushed version carries %s = %v", inherited, local[inherited]) + } + } + dist, _ := local["dist"].(map[string]any) + if dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.6.0/monolog-3.6.0.zip" || + dist["type"] != "zip" || dist["reference"] != pushed.SHA256 || dist["shasum"] != "" { + t.Errorf("dist = %v", dist) + } + if local["time"] != pushed.PushedAt.Format(time.RFC3339) { + t.Errorf("time = %v, want %s", local["time"], pushed.PushedAt.Format(time.RFC3339)) + } + if _, ok := versions["3.4.0"]; !ok { + t.Error("upstream version 3.4.0 missing") + } +} + +func TestComposerMetadataPushedVersionReplacesUpstream(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"}}`) + + versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + local := versions["3.5.0"] + if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { + t.Errorf("require = %v, want the pushed composer.json's", local["require"]) + } + if dist, _ := local["dist"].(map[string]any); dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip" { + t.Errorf("dist = %v", dist) + } + if v34 := versions["3.4.0"]; v34["version_normalized"] != "3.4.0.0" { + t.Errorf("upstream 3.4.0 expanded to %v", v34) + } +} + +func TestComposerMetadataSplitsPushedDevVersions(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{ + "/p2/monolog/monolog.json": monologMetadata, + "/p2/monolog/monolog~dev.json": monologDevMetadata, + }) + f.push(t, "monolog/monolog", "dev-patched", `{"name":"monolog/monolog"}`) + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + + tagged := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + dev := f.versions(t, "/p2/monolog/monolog~dev.json", "monolog/monolog") + if _, ok := tagged["dev-patched"]; ok { + t.Error("pushed dev version listed with the tagged versions") + } + if _, ok := dev["dev-patched"]; !ok { + t.Errorf("pushed dev version missing from the dev file: %v", dev) + } + if _, ok := dev["3.6.0"]; ok { + t.Error("pushed tagged version listed with the dev versions") + } + if _, ok := dev["dev-main"]; !ok { + t.Error("upstream dev-main missing") + } +} + +func TestComposerMetadataForPackageOnlyPushed(t *testing.T) { + f := newComposerHostedFixture(t, nil) + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private","require":{"php":">=8.1"}}`) + + versions := f.versions(t, "/p2/acme/private.json", "acme/private") + if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { + t.Errorf("versions = %v, want only 1.0.0", versions) + } + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/p2/acme/private~dev.json", nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("dev file of a package with no pushed dev versions status = %d, want 404", rec.Code) + } +} diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go index d4016fe3..101ff318 100644 --- a/internal/server/hosted_api_test.go +++ b/internal/server/hosted_api_test.go @@ -66,6 +66,7 @@ func newHostedTestServer(t *testing.T) hostedTestServer { calls := 0 logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy := handler.NewProxy(db, store, countingFetcher{calls: &calls}, fetch.NewResolver(), logger) + proxy.HostedEnabled = true upstreamRequests := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -328,3 +329,34 @@ func TestHostedNpmPackageIsListedByTheRunningServer(t *testing.T) { t.Errorf("packument status = %d, body %s", resp.StatusCode, body) } } + +func TestHostedComposerVersionIsInstallableFromMetadata(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/acme/private/1.0.0", testHostedToken, + composerZip(`{"name":"acme/private","require":{"php":">=8.1"}}`)); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/composer/p2/acme/private.json", "", "") + if rec.Code != http.StatusOK { + t.Fatalf("metadata status = %d, body %s", rec.Code, rec.Body) + } + var doc struct { + Packages map[string][]struct { + Dist struct { + URL string `json:"url"` + } `json:"dist"` + } `json:"packages"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding metadata: %v", err) + } + versions := doc.Packages["acme/private"] + if len(versions) != 1 { + t.Fatalf("metadata = %s", rec.Body) + } + distPath := strings.TrimPrefix(versions[0].Dist.URL, "http://proxy.test") + if rec := s.do(t, http.MethodGet, distPath, "", ""); rec.Code != http.StatusOK { + t.Errorf("download of %s status = %d", distPath, rec.Code) + } +} From c0b2a200c2230e31cab0712156d86001aab29830 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:27:27 +0200 Subject: [PATCH 08/17] Document how pushed versions appear in package metadata Refs #134 --- docs/configuration.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2b93975e..c7dc83c4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1005,7 +1005,7 @@ hosted: ### Pushing a patched package -1. Build the archive with the original package name inside it, e.g. `request-2.88.3.tgz`. +1. Build the archive with the original package name and the version you push in its manifest (`package.json` or `composer.json`), e.g. `request-2.88.3.tgz`. 2. Push it: ```bash @@ -1021,8 +1021,9 @@ hosted: https://proxy.example.com/api/hosted/npm/request/2.88.3 ``` -3. Check what is pushed: `proxy list npm request` or `GET /api/hosted/npm/request`. -4. Remove it again: `proxy delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. +3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too. +4. Check what is pushed: `proxy list npm request` or `GET /api/hosted/npm/request`. +5. Remove it again: `proxy delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. ### Supported ecosystems @@ -1035,6 +1036,20 @@ Download handlers look archives up by the filename clients request, so a push is Other ecosystems are not supported yet. +The archive must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400. + +### Package metadata + +Pushed versions are listed in the package metadata clients resolve against, with the dependencies from their manifest: + +- A pushed version takes the place of an upstream version with the same number. +- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them. +- A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. +- Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. +- npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. + +Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. + ### Storage and eviction Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. From 96241b910b0feb8602f06de78a301b5a7763dc95 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:34:52 +0200 Subject: [PATCH 09/17] Serve pushed npm tarballs before the cooldown check The packument lists pushed versions without cooldown, but the tarball download checked cooldown against the upstream's publish times before looking at the cache. A pushed override of a version the upstream published recently was listed and then withheld with 404. Pushed tarballs are now served right after the denylist check. Refs #134 --- internal/handler/npm.go | 12 ++++++++++ internal/handler/npm_hosted_test.go | 36 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 7cd5380d..311d1ba4 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -826,6 +826,18 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) return } + // Pushed versions are listed without cooldown, so they are served without + // it too, even when the upstream published the same version recently. + local, err := h.proxy.GetLocalArtifact(r.Context(), "npm", packageName, version, filename) + if err != nil { + h.proxy.Logger.Error("failed to read pushed npm tarball", "error", err) + JSONError(w, http.StatusInternalServerError, "failed to read pushed package") + return + } + if local != nil { + ServeArtifactRequest(w, r, local) + return + } registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { upstreamURL := registryURL + "/" + url.PathEscape(packageName) diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index df4e32d0..4a4da72a 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" "time" + + "github.com/git-pkgs/cooldown" ) const requestPackument = `{ @@ -219,3 +221,37 @@ func TestNPMMetadataIgnoresPushesWhenHostedDisabled(t *testing.T) { t.Error("pushed version listed with hosted packages disabled") } } + +func TestNPMDownloadOfPushedPackageWithCooldown(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + + rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz") + if rec.Code != http.StatusOK { + t.Fatalf("download status = %d, body %s, want the pushed tarball", rec.Code, rec.Body) + } + if _, ok := decodePackument(t, f.get(t, "/@acme/private")).Versions["1.0.0"]; !ok { + t.Error("pushed version missing from the packument with cooldown enabled") + } +} + +func TestNPMDownloadOfPushedOverrideOfVersionInCooldown(t *testing.T) { + recent := time.Now().Add(-time.Hour).Format(time.RFC3339) + packument := `{"name":"request","dist-tags":{"latest":"2.88.2"}, + "versions":{ + "2.88.2":{"name":"request","version":"2.88.2","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.2.tgz"}}, + "2.88.3":{"name":"request","version":"2.88.3","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.3.tgz"}}}, + "time":{"2.88.2":"2020-02-11T00:00:00Z","2.88.3":"` + recent + `"}}` + f := newNPMHostedFixture(t, map[string]string{"/request": packument}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.3"]; !ok { + t.Fatal("pushed version missing from the packument") + } + rec := f.get(t, "/request/-/request-2.88.3.tgz") + if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(`{"name":"request","version":"2.88.3"}`)) { + t.Errorf("download status = %d, want 200 and the pushed tarball: the packument lists it, so the download must not be withheld", rec.Code) + } +} From 0916eabe37069548c4e85a29c14ae4df8d44dee3 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:37:58 +0200 Subject: [PATCH 10/17] Fix leftover /api/packages references to the hosted endpoints Refs #133 --- internal/config/config.go | 2 +- internal/config/hosted.go | 4 ++-- internal/server/hosted_auth_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index fd483899..44bb630a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -190,7 +190,7 @@ type Config struct { // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` - // Hosted configures pushing packages to the proxy (/api/packages). + // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` // Gradle configures Gradle HttpBuildCache behavior. diff --git a/internal/config/hosted.go b/internal/config/hosted.go index 16aeced0..e9258589 100644 --- a/internal/config/hosted.go +++ b/internal/config/hosted.go @@ -5,10 +5,10 @@ import "fmt" const defaultHostedMaxUploadSize = "100MB" // HostedConfig configures packages pushed to the proxy through the -// /api/packages endpoints. Pushed versions are served in place of upstream +// /api/hosted endpoints. Pushed versions are served in place of upstream // versions with the same number, so writes always require a token. type HostedConfig struct { - // Enabled registers the /api/packages endpoints. Off by default. + // Enabled registers the /api/hosted endpoints. Off by default. Enabled bool `json:"enabled" yaml:"enabled"` // Token is the bearer token required to push and delete packages. diff --git a/internal/server/hosted_auth_test.go b/internal/server/hosted_auth_test.go index 33c0edfb..6a642b07 100644 --- a/internal/server/hosted_auth_test.go +++ b/internal/server/hosted_auth_test.go @@ -29,7 +29,7 @@ func TestRequireBearerToken(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodPut, "/api/packages/npm/request/2.88.3", nil) + req := httptest.NewRequest(http.MethodPut, "/api/hosted/npm/request/2.88.3", nil) if tt.authorization != "" { req.Header.Set("Authorization", tt.authorization) } From 69313c26d993c7efc05dda3c79687c1ddbd0b153 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 13:42:34 +0200 Subject: [PATCH 11/17] Serve hosted-only npm scopes and Composer vendors from pushed packages alone A package the upstream doesn't know is served from its pushed versions, but the upstream is still asked about it. Once someone publishes the same name there, their versions are listed next to the pushed ones and a version range can resolve to them. hosted.npm_scopes and hosted.composer_vendors (PROXY_HOSTED_NPM_SCOPES, PROXY_HOSTED_COMPOSER_VENDORS) name namespaces whose packages are served from pushed versions alone. Their metadata is built from what was pushed, npm latest included, and anything not pushed answers 404. The upstream is never asked about them. Packages in other namespaces are proxied as before. The npm metadata handler's decisions about pushed versions move into serveLocalMetadata. Refs #133 Refs #134 --- cmd/proxy/main.go | 2 + config.example.yaml | 4 + docs/configuration.md | 15 ++++ internal/config/config.go | 2 + internal/config/hosted.go | 24 +++++- internal/config/hosted_test.go | 20 ++++- internal/handler/composer.go | 8 ++ internal/handler/composer_hosted.go | 16 ++++ internal/handler/composer_hosted_test.go | 40 +++++++++- internal/handler/handler.go | 5 ++ internal/handler/hosted.go | 25 +++++++ internal/handler/npm.go | 19 ++--- internal/handler/npm_hosted.go | 95 +++++++++++++++++++++--- internal/handler/npm_hosted_test.go | 44 ++++++++++- internal/server/server.go | 2 + 15 files changed, 288 insertions(+), 33 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 786819ae..136f3ab7 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -93,6 +93,8 @@ // PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted // PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") +// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only +// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only // PROXY_URL - Proxy URL for push, list and delete // // Example: diff --git a/config.example.yaml b/config.example.yaml index 1b8e691e..c41f30f3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -371,6 +371,10 @@ gradle: # enabled: true # token: "${PROXY_HOSTED_TOKEN}" # max_upload_size: "100MB" +# # Namespaces served from pushed packages only; the upstream is never asked +# # about them, so nobody can mix in versions by publishing the same name. +# npm_scopes: ["@acme"] +# composer_vendors: ["acme"] # Health endpoint configuration. health: diff --git a/docs/configuration.md b/docs/configuration.md index c7dc83c4..d1cca7f6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -993,6 +993,8 @@ hosted: enabled: true token: "${PROXY_HOSTED_TOKEN}" max_upload_size: "100MB" + npm_scopes: ["@acme"] + composer_vendors: ["acme"] ``` | Config | Environment | Description | @@ -1000,6 +1002,8 @@ hosted: | `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) | | `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` | | `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive (default `100MB`) | +| `hosted.npm_scopes` | `PROXY_HOSTED_NPM_SCOPES` | npm scopes served from pushed packages only (comma-separated in the environment) | +| `hosted.composer_vendors` | `PROXY_HOSTED_COMPOSER_VENDORS` | Composer vendors served from pushed packages only | `hosted.enabled` cannot be combined with `storage.cache_artifacts: false`. @@ -1050,6 +1054,17 @@ Pushed versions are listed in the package metadata clients resolve against, with Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. +### Hosted-only namespaces + +A package the upstream doesn't know is served from its pushed versions, but the proxy still asks the upstream about it on every request. If someone later publishes the same name there, their versions are listed next to yours, and a range such as `^1.0.0` can resolve to one of them (dependency confusion). + +For private packages: + +1. Prefer a namespace you own on the public registry: an npm scope registered on npmjs.com, or a Composer vendor with packages on Packagist. Nobody else can publish under it. +2. List the namespace in `hosted.npm_scopes` or `hosted.composer_vendors`. Its packages are served from pushed versions alone: the upstream is never asked about them, and anything not pushed answers 404. + +Other packages, including ones in other scopes and vendors, are proxied as usual. + ### Storage and eviction Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. diff --git a/internal/config/config.go b/internal/config/config.go index 44bb630a..a9d7c212 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1215,6 +1215,8 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") + setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") + setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") diff --git a/internal/config/hosted.go b/internal/config/hosted.go index e9258589..9b67e288 100644 --- a/internal/config/hosted.go +++ b/internal/config/hosted.go @@ -1,6 +1,9 @@ package config -import "fmt" +import ( + "fmt" + "strings" +) const defaultHostedMaxUploadSize = "100MB" @@ -20,6 +23,15 @@ type HostedConfig struct { // Default: "100MB". MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"` + // NPMScopes are npm scopes (e.g. "@acme") served only from pushed + // packages. The upstream registry is never asked about them, so a package + // published there under the same name can't be mixed in. + NPMScopes []string `json:"npm_scopes" yaml:"npm_scopes"` + + // ComposerVendors are Composer vendors (e.g. "acme") served only from + // pushed packages, like NPMScopes. + ComposerVendors []string `json:"composer_vendors" yaml:"composer_vendors"` + maxUploadBytes int64 } @@ -51,5 +63,15 @@ func (h *HostedConfig) Validate() error { if h.Enabled && h.TokenExpanded() == "" { return fmt.Errorf("hosted.token is required when hosted.enabled is true") } + for _, scope := range h.NPMScopes { + if len(scope) < 2 || scope[0] != '@' || strings.Contains(scope, "/") { + return fmt.Errorf("invalid hosted.npm_scopes entry %q: must be a scope such as \"@acme\"", scope) + } + } + for _, vendor := range h.ComposerVendors { + if vendor == "" || strings.Contains(vendor, "/") { + return fmt.Errorf("invalid hosted.composer_vendors entry %q: must be a vendor such as \"acme\"", vendor) + } + } return nil } diff --git a/internal/config/hosted_test.go b/internal/config/hosted_test.go index 787935dd..e7ad4515 100644 --- a/internal/config/hosted_test.go +++ b/internal/config/hosted_test.go @@ -1,6 +1,9 @@ package config -import "testing" +import ( + "slices" + "testing" +) func TestHostedConfigValidate(t *testing.T) { t.Setenv("HOSTED_TEST_TOKEN", "from-env") @@ -17,6 +20,13 @@ func TestHostedConfigValidate(t *testing.T) { {name: "token env reference unset", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_UNSET}"}, wantErr: true}, {name: "invalid upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "lots"}, wantErr: true}, {name: "zero upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "0"}, wantErr: true}, + {name: "npm scopes", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme", "@acme-internal"}}}, + {name: "npm scope without @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"acme"}}, wantErr: true}, + {name: "npm scope with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme/private"}}, wantErr: true}, + {name: "bare @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@"}}, wantErr: true}, + {name: "composer vendors", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme", "acme-internal"}}}, + {name: "composer vendor with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme/private"}}, wantErr: true}, + {name: "empty composer vendor", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{""}}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -64,6 +74,8 @@ func TestLoadFromEnvHosted(t *testing.T) { t.Setenv("PROXY_HOSTED_ENABLED", "true") t.Setenv("PROXY_HOSTED_TOKEN", "env-token") t.Setenv("PROXY_HOSTED_MAX_UPLOAD_SIZE", "50MB") + t.Setenv("PROXY_HOSTED_NPM_SCOPES", "@acme, @acme-internal") + t.Setenv("PROXY_HOSTED_COMPOSER_VENDORS", "acme") cfg.LoadFromEnv() @@ -76,4 +88,10 @@ func TestLoadFromEnvHosted(t *testing.T) { if cfg.Hosted.MaxUploadSize != "50MB" { t.Errorf("Hosted.MaxUploadSize = %q, want 50MB", cfg.Hosted.MaxUploadSize) } + if !slices.Equal(cfg.Hosted.NPMScopes, []string{"@acme", "@acme-internal"}) { + t.Errorf("Hosted.NPMScopes = %q", cfg.Hosted.NPMScopes) + } + if !slices.Equal(cfg.Hosted.ComposerVendors, []string{"acme"}) { + t.Errorf("Hosted.ComposerVendors = %q", cfg.Hosted.ComposerVendors) + } } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index b8c0a752..9dac530c 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -155,6 +155,10 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R http.Error(w, "failed to read pushed versions", http.StatusInternalServerError) return } + if h.proxy.hostedOnly("composer", basePackage) { + h.serveHostedOnlyMetadata(w, r, basePackage, cacheKey, local) + return + } if len(local) > 0 { h.serveMetadataWithLocal(w, r, basePackage, upstreamURL, cacheKey, local) return @@ -627,6 +631,10 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) ServeArtifactRequest(w, r, local) return } + if h.proxy.hostedOnly("composer", packageName) { + http.Error(w, "version not found", http.StatusNotFound) + return + } // We need to fetch the metadata to get the actual download URL since // Packagist URLs include a hash. Packagist serves dev versions (e.g. diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 5a94f828..a42a27a1 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -99,6 +99,22 @@ func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http. serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) } +// serveHostedOnlyMetadata answers a metadata request for a package of a +// hosted-only vendor from its pushed versions alone. +func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, cacheKey string, local []composerLocalEntry) { + if len(local) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return + } + out, err := localComposerDocument(packageName, local) + if err != nil { + h.proxy.Logger.Error("failed to build composer metadata from pushed versions", "package", packageName, "error", err) + http.Error(w, "failed to build package metadata", http.StatusInternalServerError) + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) +} + func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { entries := make([]json.RawMessage, 0, len(local)) for _, e := range local { diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 09e44c2c..aa7b0a7b 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -24,13 +24,16 @@ const monologDevMetadata = `{"minified":"composer/2.0","packages":{"monolog/mono ]}}` type composerHostedFixture struct { - proxy *Proxy - handler http.Handler + proxy *Proxy + handler http.Handler + upstreamRequests *int } func newComposerHostedFixture(t *testing.T, upstream map[string]string) composerHostedFixture { t.Helper() + upstreamRequests := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ body, ok := upstream[r.URL.Path] if !ok { http.NotFound(w, r) @@ -45,7 +48,7 @@ func newComposerHostedFixture(t *testing.T, upstream map[string]string) composer proxy.HostedEnabled = true proxy.HTTPClient = srv.Client() h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.local", srv.URL, srv.URL) - return composerHostedFixture{proxy: proxy, handler: h.Routes()} + return composerHostedFixture{proxy: proxy, handler: h.Routes(), upstreamRequests: &upstreamRequests} } func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { @@ -165,3 +168,34 @@ func TestComposerMetadataForPackageOnlyPushed(t *testing.T) { t.Errorf("dev file of a package with no pushed dev versions status = %d, want 404", rec.Code) } } + +func TestComposerHostedOnlyVendorNeverAsksUpstream(t *testing.T) { + squatter := `{"packages":{"acme/private":[{"name":"acme/private","version":"1.0.1", + "dist":{"type":"zip","url":"https://api.github.com/repos/evil/private/zipball/ddd","reference":"ddd"}}]}}` + f := newComposerHostedFixture(t, map[string]string{ + "/p2/acme/private.json": squatter, + "/p2/monolog/monolog.json": monologMetadata, + }) + f.proxy.HostedComposerVendors = []string{"acme"} + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) + + versions := f.versions(t, "/p2/acme/private.json", "acme/private") + if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { + t.Errorf("versions = %v, want only the pushed 1.0.0", versions) + } + for _, path := range []string{"/p2/acme/private~dev.json", "/p2/acme/other.json", "/files/acme/private/1.0.1/ddd.zip"} { + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("GET %s status = %d, want 404", path, rec.Code) + } + } + if *f.upstreamRequests != 0 { + t.Errorf("hosted-only vendor sent %d requests upstream", *f.upstreamRequests) + } + + f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + if *f.upstreamRequests == 0 { + t.Error("package outside the vendor was not fetched upstream") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c00a9309..e46134f3 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -175,6 +175,11 @@ type Proxy struct { DirectServeBaseURL string HTTPClient *http.Client + // HostedNPMScopes and HostedComposerVendors name namespaces served from + // pushed packages alone, without asking the upstream. + HostedNPMScopes []string + HostedComposerVendors []string + // StreamArtifacts streams artifacts from upstream without storing them. // Each request fetches its own copy: there is no cache to check and // nothing for concurrent misses to share. diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index d4e4d9f8..2923cdfd 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "time" @@ -250,6 +251,30 @@ func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) return listed, nil } +// hostedOnly reports whether a package is in an npm scope or Composer vendor +// that is served from pushed packages alone. The upstream is never asked +// about such a package, so nobody can mix in versions by publishing the same +// name there. +func (p *Proxy) hostedOnly(ecosystem, name string) bool { + if !p.HostedEnabled { + return false + } + namespace, _, namespaced := strings.Cut(name, "/") + if !namespaced { + return false + } + switch ecosystem { + case "npm": + return slices.Contains(p.HostedNPMScopes, namespace) + case "composer": + return slices.ContainsFunc(p.HostedComposerVendors, func(vendor string) bool { + return strings.EqualFold(vendor, namespace) + }) + default: + return false + } +} + // localVersion is a pushed version as ecosystem metadata lists it. type localVersion struct { Version string diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 311d1ba4..3284b88f 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -216,20 +216,7 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques } specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) - local, err := h.npmLocalEntries(packageName) - if err != nil { - h.proxy.Logger.Error("failed to read pushed npm versions", "package", packageName, "error", err) - JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") - return - } - if e, ok := findNPMLocalEntry(local, specifier); versionLookup && ok { - w.Header().Set(headerContentType, contentTypeJSON) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.entry) - return - } - if len(local) > 0 && !versionLookup { - h.serveMetadataWithLocal(w, r, packageName, upstreamURL, accept, metadataCacheKey, local) + if h.serveLocalMetadata(w, r, npmMetadataRequest{packageName, specifier, versionLookup, upstreamURL, accept, metadataCacheKey}) { return } @@ -838,6 +825,10 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, local) return } + if h.proxy.hostedOnly("npm", packageName) { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { upstreamURL := registryURL + "/" + url.PathEscape(packageName) diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index 6e0e0b62..e8998c90 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -56,6 +56,42 @@ func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bo return npmLocalEntry{}, false } +// npmMetadataRequest is a packument or version metadata request. +type npmMetadataRequest struct { + packageName string + specifier string + versionLookup bool + upstreamURL string + accept string + metadataCacheKey string +} + +// serveLocalMetadata answers the metadata requests that pushed versions +// decide, and reports whether it did: a request for a pushed version, any +// request for a package in a hosted-only scope, and a packument of a package +// with pushed versions. Everything else takes the usual upstream path. +func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, req npmMetadataRequest) bool { + local, err := h.npmLocalEntries(req.packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed npm versions", "package", req.packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") + return true + } + switch e, pushed := findNPMLocalEntry(local, req.specifier); { + case req.versionLookup && pushed: + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(e.entry) + case h.proxy.hostedOnly("npm", req.packageName): + h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local) + case len(local) > 0 && !req.versionLookup: + h.serveMetadataWithLocal(w, r, req.packageName, req.upstreamURL, req.accept, req.metadataCacheKey, local) + default: + return false + } + return true +} + // serveMetadataWithLocal serves a packument with the pushed versions of the // package added. It bypasses the rewrite cache, whose entries are keyed by the // upstream document alone, so a push or delete shows up on the next request. @@ -87,17 +123,59 @@ func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Reque serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) } -// localNPMPackument builds a packument from pushed versions alone. latest is -// the highest of them, or the last pushed when a version is not semver. +// serveHostedOnlyMetadata answers a metadata request for a package in a +// hosted-only scope from its pushed versions alone. +func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier string, versionLookup bool, metadataCacheKey string, local []npmLocalEntry) { + if len(local) == 0 { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return + } + if versionLookup { + if specifier == "latest" { + specifier = latestNPMVersion(local) + } + e, ok := findNPMLocalEntry(local, specifier) + if !ok { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(e.entry) + return + } + out, err := localNPMPackument(packageName, local) + if err != nil { + h.proxy.Logger.Error("failed to build npm metadata from pushed versions", "package", packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to build package metadata") + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) +} + +// localNPMPackument builds a packument from pushed versions alone. func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error) { versions := make(map[string]json.RawMessage, len(local)) times := make(map[string]string, len(local)) + for _, e := range local { + versions[e.version] = e.entry + times[e.version] = e.time + } + return json.Marshal(map[string]any{ + "name": packageName, + "dist-tags": map[string]string{"latest": latestNPMVersion(local)}, + "versions": versions, + "time": times, + }) +} + +// latestNPMVersion returns the highest pushed version, or the last pushed +// when a version is not semver. +func latestNPMVersion(local []npmLocalEntry) string { latest, latestTime := "", "" var latestSemver *semver.Version semverOnly := true for _, e := range local { - versions[e.version] = e.entry - times[e.version] = e.time if e.time > latestTime { latest, latestTime = e.version, e.time } @@ -111,12 +189,7 @@ func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error } } if semverOnly && latestSemver != nil { - latest = latestSemver.Original() + return latestSemver.Original() } - return json.Marshal(map[string]any{ - "name": packageName, - "dist-tags": map[string]string{"latest": latest}, - "versions": versions, - "time": times, - }) + return latest } diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index 4a4da72a..1bf0d462 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -26,15 +26,18 @@ const requestPackument = `{ }` type npmHostedFixture struct { - proxy *Proxy - handler http.Handler + proxy *Proxy + handler http.Handler + upstreamRequests *int } // newNPMHostedFixture serves packuments from upstream, keyed by request path, // and answers 404 for anything else. func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixture { t.Helper() + upstreamRequests := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ body, ok := upstream[r.URL.EscapedPath()] if !ok { http.NotFound(w, r) @@ -48,7 +51,7 @@ func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixt proxy, _, _, _ := setupTestProxy(t) proxy.HostedEnabled = true proxy.HTTPClient = srv.Client() - return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes()} + return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes(), upstreamRequests: &upstreamRequests} } func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { @@ -255,3 +258,38 @@ func TestNPMDownloadOfPushedOverrideOfVersionInCooldown(t *testing.T) { t.Errorf("download status = %d, want 200 and the pushed tarball: the packument lists it, so the download must not be withheld", rec.Code) } } + +func TestNPMHostedOnlyScopeNeverAsksUpstream(t *testing.T) { + squatter := `{"name":"@acme/private","dist-tags":{"latest":"1.0.1"},"versions":{"1.0.1":{"name":"@acme/private","version":"1.0.1", + "dist":{"tarball":"https://registry.npmjs.org/@acme/private/-/private-1.0.1.tgz"}}}}` + f := newNPMHostedFixture(t, map[string]string{"/@acme%2fprivate": squatter, "/@acme%2Fprivate": squatter, "/request": requestPackument}) + f.proxy.HostedNPMScopes = []string{"@acme"} + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if len(doc.Versions) != 1 || doc.DistTags["latest"] != "1.0.0" { + t.Errorf("packument = %+v, want only the pushed 1.0.0", doc) + } + if rec := f.get(t, "/@acme/private/latest"); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"version":"1.0.0"`) { + t.Errorf("latest status = %d, body %s", rec.Code, rec.Body) + } + if rec := f.get(t, "/@acme/private/1.0.1"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed version status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/private/-/private-1.0.1.tgz"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed tarball status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed package status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz"); rec.Code != http.StatusOK { + t.Errorf("pushed tarball status = %d, want 200", rec.Code) + } + if *f.upstreamRequests != 0 { + t.Errorf("hosted-only scope sent %d requests upstream", *f.upstreamRequests) + } + + if rec := f.get(t, "/request"); rec.Code != http.StatusOK || *f.upstreamRequests == 0 { + t.Errorf("package outside the scope status = %d after %d upstream requests, want it fetched upstream", rec.Code, *f.upstreamRequests) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index ac773c56..89a60905 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -269,6 +269,8 @@ func (s *Server) serve(listener net.Listener) error { proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.HostedEnabled = s.cfg.Hosted.Enabled + proxy.HostedNPMScopes = s.cfg.Hosted.NPMScopes + proxy.HostedComposerVendors = s.cfg.Hosted.ComposerVendors proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.DirectServe = s.cfg.Storage.DirectServe From 3dae0fbae0456a9d49d55a15c1dfe1412227606c Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 15:13:02 +0200 Subject: [PATCH 12/17] Stop UpsertPackage from clearing packages.source The artifact source change added source = excluded.source to the packages upsert as well. source isn't in that INSERT's column list, so every package upsert set the column to NULL. The packages upsert is back to what it was, with a test that an existing source survives it. Refs #133 --- internal/database/local_artifacts_test.go | 22 ++++++++++++++++++++++ internal/database/queries.go | 6 ++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go index 798d2e27..81d5ec8b 100644 --- a/internal/database/local_artifacts_test.go +++ b/internal/database/local_artifacts_test.go @@ -273,3 +273,25 @@ func TestMigrationAddsArtifactManifest(t *testing.T) { t.Errorf("artifacts.manifest after migration = %v, %v", has, err) } } + +func TestUpsertPackageKeepsSource(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if _, err := db.Exec(db.Rebind(`UPDATE packages SET source = ? WHERE purl = ?`), "deps.dev", pkg.PURL); err != nil { + t.Fatalf("setting source: %v", err) + } + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("second UpsertPackage: %v", err) + } + got, err := db.GetPackageByPURL(pkg.PURL) + if err != nil || got == nil { + t.Fatalf("GetPackageByPURL: %+v, %v", got, err) + } + if got.Source.String != "deps.dev" { + t.Errorf("source after upsert = %q, want deps.dev", got.Source.String) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 2d322a1a..cd7c51ce 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -70,8 +70,7 @@ func (db *DB) UpsertPackage(pkg *Package) error { repository_url = EXCLUDED.repository_url, registry_url = EXCLUDED.registry_url, enriched_at = EXCLUDED.enriched_at, - updated_at = EXCLUDED.updated_at, - source = EXCLUDED.source + updated_at = EXCLUDED.updated_at ` } else { query = ` @@ -87,8 +86,7 @@ func (db *DB) UpsertPackage(pkg *Package) error { repository_url = excluded.repository_url, registry_url = excluded.registry_url, enriched_at = excluded.enriched_at, - updated_at = excluded.updated_at, - source = excluded.source + updated_at = excluded.updated_at ` } From e0ab227f145640bdb49d02179035b54969edb749 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 15:16:55 +0200 Subject: [PATCH 13/17] Never replace a pushed artifact with the upstream copy checkCache answered nil both when a stored object couldn't be opened and after an integrity failure had cleared the record. Download handlers then fetched the upstream archive for the same version and filename, and recording it turned the pushed row into an upstream one without its manifest and queued the pushed object for deletion. One storage error while downloading a patched version served the unpatched upstream archive and dropped the push. checkCache now looks at the record on a miss. For a pushed artifact it returns ErrHostedArtifactUnavailable when the object is recorded but couldn't be served, and ErrHostedArtifactMissing when no object is recorded any more. Every path that would fetch upstream goes through it, so none of them can replace the pushed artifact. Downloads answer 503 and 404 for the two. Refs #133 --- internal/handler/composer_hosted_test.go | 12 ++++++ internal/handler/handler.go | 38 ++++++++++++++++ internal/handler/hosted.go | 6 +++ internal/handler/hosted_test.go | 55 ++++++++++++++++++++++++ internal/handler/npm.go | 3 +- internal/handler/npm_hosted_test.go | 10 +++++ 6 files changed, 122 insertions(+), 2 deletions(-) diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index aa7b0a7b..cb35b721 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -199,3 +199,15 @@ func TestComposerHostedOnlyVendorNeverAsksUpstream(t *testing.T) { t.Error("package outside the vendor was not fetched upstream") } } + +func TestComposerDownloadOfUnreadablePushedArchive(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog"}`) + removeStored(t, f.proxy, "pkg:composer/monolog/monolog@3.5.0", "monolog-3.5.0.zip") + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream archive", rec.Code, rec.Body) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e46134f3..4265fd34 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -347,7 +347,36 @@ func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) e // checkCache looks up an artifact in the cache. Returns nil if not cached. // With StreamArtifacts set it always reports a miss, so entries stored before the // mode was enabled are never served. +// +// A miss on a pushed artifact is an error rather than a miss: the caller would +// otherwise fetch the upstream copy and replace the pushed one with it. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { + result, err := p.lookupCache(ctx, pkgPURL, versionPURL, filename) + if err != nil || result != nil || p.StreamArtifacts { + return result, err + } + return nil, p.localArtifactMiss(versionPURL, filename) +} + +// localArtifactMiss returns ErrHostedArtifactUnavailable when the artifact +// was pushed and is stored but couldn't be served, ErrHostedArtifactMissing +// when it was pushed and its stored object is gone, and nil otherwise. +func (p *Proxy) localArtifactMiss(versionPURL, filename string) error { + record, err := p.DB.GetArtifact(versionPURL, filename) + if err != nil { + return fmt.Errorf("checking artifact source: %w", err) + } + if record == nil || record.Source != database.SourceLocal { + return nil + } + if record.StoragePath.Valid { + return fmt.Errorf("%w: %s %s", ErrHostedArtifactUnavailable, versionPURL, filename) + } + return fmt.Errorf("%w: %s %s", ErrHostedArtifactMissing, versionPURL, filename) +} + +// lookupCache looks up an artifact in the cache. Returns nil if not cached. +func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } @@ -1144,6 +1173,15 @@ func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg s JSONError(w, http.StatusForbidden, err.Error()) return } + if errors.Is(err, ErrHostedArtifactMissing) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if errors.Is(err, ErrHostedArtifactUnavailable) { + p.Logger.Error("failed to read pushed artifact", "error", err) + http.Error(w, "pushed package is temporarily unavailable", http.StatusServiceUnavailable) + return + } p.Logger.Error("failed to get artifact", "error", err) http.Error(w, clientMsg, http.StatusBadGateway) } diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 2923cdfd..98c797aa 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -23,6 +23,12 @@ var ( ErrHostedUnsupported = errors.New("pushing packages is not supported for this ecosystem") ErrHostedInvalidName = errors.New("invalid package name or version") ErrHostedVersionNotFound = errors.New("no pushed artifacts for this version") + + // ErrHostedArtifactUnavailable and ErrHostedArtifactMissing report a + // pushed artifact the cache can't serve. They stop the caller from + // fetching the upstream copy, which would replace the pushed one. + ErrHostedArtifactUnavailable = errors.New("pushed artifact is stored but can't be read") + ErrHostedArtifactMissing = errors.New("pushed artifact is no longer stored") ) // HostedTarget is where a pushed archive is stored: the package name and diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index 89cdd812..2ee8c5d9 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -313,3 +313,58 @@ func TestPushLocalArtifactChecksManifest(t *testing.T) { }) } } + +// removeStored deletes the stored object of an artifact behind the cache's back. +func removeStored(t *testing.T, proxy *Proxy, versionPURL, filename string) { + t.Helper() + record, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || record == nil { + t.Fatalf("GetArtifact: %+v, %v", record, err) + } + if err := proxy.Storage.Delete(context.Background(), record.StoragePath.String); err != nil { + t.Fatalf("deleting stored object: %v", err) + } +} + +func TestPushedArtifactIsNeverReplacedFromUpstream(t *testing.T) { + const versionPURL, filename = "pkg:npm/request@2.88.3", "request-2.88.3.tgz" + tests := []struct { + name string + damage func(t *testing.T, proxy *Proxy) + wantErr error + }{ + {name: "stored object missing", wantErr: ErrHostedArtifactUnavailable, + damage: func(t *testing.T, proxy *Proxy) { removeStored(t, proxy, versionPURL, filename) }}, + {name: "record cleared after an integrity failure", wantErr: ErrHostedArtifactMissing, + damage: func(t *testing.T, proxy *Proxy) { + record, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || record == nil { + t.Fatalf("GetArtifact: %+v, %v", record, err) + } + if err := proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { + t.Fatalf("DiscardArtifact: %v", err) + } + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + pushRequest(t, proxy) + tt.damage(t, proxy) + + _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", filename, + "https://registry.npmjs.org/request/-/request-2.88.3.tgz") + if !errors.Is(err, tt.wantErr) { + t.Errorf("GetOrFetchArtifactFromURL error = %v, want %v", err, tt.wantErr) + } + if fetcher.fetchCalled { + t.Error("a miss on a pushed artifact fetched the upstream copy") + } + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil || len(local) != 1 || !local[0].Manifest.Valid { + t.Errorf("pushed records after the miss = %+v, %v, want the pushed one with its manifest", local, err) + } + }) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 3284b88f..308f6aff 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -817,8 +817,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // it too, even when the upstream published the same version recently. local, err := h.proxy.GetLocalArtifact(r.Context(), "npm", packageName, version, filename) if err != nil { - h.proxy.Logger.Error("failed to read pushed npm tarball", "error", err) - JSONError(w, http.StatusInternalServerError, "failed to read pushed package") + h.proxy.serveArtifactError(w, err, "failed to read pushed package") return } if local != nil { diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index 1bf0d462..17a0a6ca 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -293,3 +293,13 @@ func TestNPMHostedOnlyScopeNeverAsksUpstream(t *testing.T) { t.Errorf("package outside the scope status = %d after %d upstream requests, want it fetched upstream", rec.Code, *f.upstreamRequests) } } + +func TestNPMDownloadOfUnreadablePushedTarball(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") + + if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) + } +} From 2a957b92e90c09ddda3086bfd3aa4c3ead916c63 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 15:17:48 +0200 Subject: [PATCH 14/17] Prefer the root composer.json of a pushed archive The manifest lookup took the first composer.json at the root or inside a top directory, in zip order. Dist archives keep composer.json files for tests and examples, so tests/composer.json stored before the root file refused the push with a name mismatch. The root composer.json now always wins. Without one, the manifest inside a top directory is used only when there is exactly one; more than one refuses the archive as ambiguous. Refs #134 --- internal/handler/hosted_manifest.go | 53 +++++++++++++++++++++-------- internal/handler/hosted_test.go | 47 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 14 deletions(-) diff --git a/internal/handler/hosted_manifest.go b/internal/handler/hosted_manifest.go index db6d0b6c..3bd9b38a 100644 --- a/internal/handler/hosted_manifest.go +++ b/internal/handler/hosted_manifest.go @@ -89,25 +89,50 @@ func readComposerManifest(archive io.Reader, target HostedTarget) (json.RawMessa return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) } - for _, entry := range zr.File { - if entry.FileInfo().IsDir() || (entry.Name != "composer.json" && !isTopLevelManifest(entry.Name, "composer.json")) { + entry, err := composerManifestEntry(zr.File) + if err != nil { + return nil, err + } + rc, err := entry.Open() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + manifest, fields, err := decodeManifest(rc) + _ = rc.Close() + if err != nil { + return nil, err + } + if !strings.EqualFold(fields.Name, target.Name) || (fields.Version != "" && fields.Version != target.Version) { + return nil, fmt.Errorf("%w: composer.json is %s %s", ErrHostedManifestMismatch, fields.Name, fields.Version) + } + return manifest, nil +} + +// composerManifestEntry picks the package's composer.json: the one at the +// root, or else the only one directly inside a top directory. Archives keep +// other composer.json files for tests and examples, in any order, so the +// first one found isn't necessarily the package's. +func composerManifestEntry(files []*zip.File) (*zip.File, error) { + var topLevel []*zip.File + for _, entry := range files { + if entry.FileInfo().IsDir() { continue } - rc, err := entry.Open() - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + if entry.Name == "composer.json" { + return entry, nil } - manifest, fields, err := decodeManifest(rc) - _ = rc.Close() - if err != nil { - return nil, err + if isTopLevelManifest(entry.Name, "composer.json") { + topLevel = append(topLevel, entry) } - if !strings.EqualFold(fields.Name, target.Name) || (fields.Version != "" && fields.Version != target.Version) { - return nil, fmt.Errorf("%w: composer.json is %s %s", ErrHostedManifestMismatch, fields.Name, fields.Version) - } - return manifest, nil } - return nil, fmt.Errorf("%w: no composer.json in zip", ErrHostedInvalidArchive) + switch len(topLevel) { + case 0: + return nil, fmt.Errorf("%w: no composer.json in zip", ErrHostedInvalidArchive) + case 1: + return topLevel[0], nil + default: + return nil, fmt.Errorf("%w: no composer.json at the root and %d in top directories", ErrHostedInvalidArchive, len(topLevel)) + } } // isTopLevelManifest reports whether name is filename inside one top diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go index 2ee8c5d9..a4e7145e 100644 --- a/internal/handler/hosted_test.go +++ b/internal/handler/hosted_test.go @@ -368,3 +368,50 @@ func TestPushedArtifactIsNeverReplacedFromUpstream(t *testing.T) { }) } } + +// zipWith builds a zip holding files in the given order. +func zipWith(files ...[2]string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, f := range files { + w, err := zw.Create(f[0]) + must(err) + _, err = io.WriteString(w, f[1]) + must(err) + } + must(zw.Close()) + return buf.Bytes() +} + +func TestComposerManifestLookup(t *testing.T) { + const monolog = `{"name":"monolog/monolog"}` + const fixture = `{"name":"acme/test-fixture"}` + tests := []struct { + name string + archive []byte + wantErr error + }{ + {name: "root manifest after a nested one", + archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"composer.json", monolog})}, + {name: "top directory manifest next to a deeper one", + archive: zipWith([2]string{"monolog-abc/tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog})}, + {name: "two top directories with manifests", + archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog}), + wantErr: ErrHostedInvalidArchive}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + target, err := HostedTargetFor("composer", "monolog/monolog", "3.5.0") + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + _, err = readHostedManifest("composer", bytes.NewReader(tt.archive), target) + if tt.wantErr == nil && err != nil { + t.Errorf("readHostedManifest error = %v, want the root or single top directory manifest", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Errorf("readHostedManifest error = %v, want %v", err, tt.wantErr) + } + }) + } +} From eca224bb56ea81857b128ed3a38aa9e0a979ee48 Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 15:18:49 +0200 Subject: [PATCH 15/17] Leave denied pushed versions out of package metadata Pushing and downloading refuse denied versions, but npm and Composer metadata listed every pushed version. A version denied after it was pushed stayed listed, so resolvers picked it and then got 403 on the download. localVersions now leaves denied versions out, which covers both ecosystems and the hosted-only namespaces; npm latest falls back to the highest allowed version. /api/hosted still lists them, so operators can see and delete them. Refs #134 --- internal/handler/composer_hosted_test.go | 10 ++++++++++ internal/handler/hosted.go | 7 +++++-- internal/handler/npm_hosted_test.go | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index cb35b721..43932a30 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -211,3 +211,13 @@ func TestComposerDownloadOfUnreadablePushedArchive(t *testing.T) { t.Errorf("download status = %d, body %s, want 503 rather than the upstream archive", rec.Code, rec.Body) } } + +func TestComposerMetadataLeavesOutDeniedPushedVersions(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + setTestDenylist(t, f.proxy, "pkg:composer/monolog/monolog@3.6.0") + + if _, ok := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog")["3.6.0"]; ok { + t.Error("denied pushed version listed") + } +} diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index 98c797aa..dfbbda4c 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -293,6 +293,8 @@ type localVersion struct { // localVersions returns the versions pushed for a package, or nil when hosted // packages are disabled, so metadata requests cost no extra lookup then. +// Denied versions are left out, so metadata never lists a version whose +// download is refused. func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { if !p.HostedEnabled { return nil, nil @@ -307,11 +309,12 @@ func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { } versions := make([]localVersion, 0, len(rows)) for _, row := range rows { - if !row.Manifest.Valid { + version := database.VersionFromPURL(row.VersionPURL) + if !row.Manifest.Valid || p.versionDenied(ecosystem, name, version) { continue } versions = append(versions, localVersion{ - Version: database.VersionFromPURL(row.VersionPURL), + Version: version, Filename: row.Filename, SHA256: row.ContentHash.String, Integrity: row.Integrity.String, diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index 17a0a6ca..b93fabdc 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -303,3 +303,18 @@ func TestNPMDownloadOfUnreadablePushedTarball(t *testing.T) { t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) } } + +func TestNPMMetadataLeavesOutDeniedPushedVersions(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + f.push(t, "@acme/private", "1.1.0", `{"name":"@acme/private","version":"1.1.0"}`) + setTestDenylist(t, f.proxy, "pkg:npm/@acme/private@1.1.0") + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if _, ok := doc.Versions["1.1.0"]; ok || len(doc.Versions) != 1 { + t.Errorf("versions = %v, want the denied 1.1.0 left out", doc.Versions) + } + if doc.DistTags["latest"] != "1.0.0" { + t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) + } +} From 8265c7d44b9322d9bba597dfc6f77f4b983129ea Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Tue, 13 Oct 2026 15:20:57 +0200 Subject: [PATCH 16/17] Nest the hosted commands under proxy hosted proxy list and proxy delete sat next to serve, stats and mirror and read as cache operations. The commands are now proxy hosted push, proxy hosted list and proxy hosted delete, before a release makes the names hard to change. proxy hosted without a known subcommand prints the three. Refs #133 --- README.md | 10 +++++----- cmd/proxy/hosted.go | 39 ++++++++++++++++++++++++++++++++------- cmd/proxy/hosted_test.go | 25 +++++++++++++++++++++++++ cmd/proxy/main.go | 22 +++++----------------- docs/configuration.md | 6 +++--- 5 files changed, 70 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 74f33735..66d1972c 100644 --- a/README.md +++ b/README.md @@ -1026,16 +1026,16 @@ curl http://localhost:8080/api/mirror/mirror-1 curl -X DELETE http://localhost:8080/api/mirror/mirror-1 ``` -### push, list, delete +### hosted Push package archives to a running proxy with `hosted.enabled`, list them and delete them. See [Hosted Packages](docs/configuration.md#hosted-packages). ```bash export PROXY_URL=https://proxy.example.com PROXY_HOSTED_TOKEN=... -proxy push npm request 2.88.3 ./request-2.88.3.tgz -proxy push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip -proxy list npm request -proxy delete npm request 2.88.3 +proxy hosted push npm request 2.88.3 ./request-2.88.3.tgz +proxy hosted push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip +proxy hosted list npm request +proxy hosted delete npm request 2.88.3 ``` ### stats diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go index 2db238fe..bbc947df 100644 --- a/cmd/proxy/hosted.go +++ b/cmd/proxy/hosted.go @@ -103,7 +103,7 @@ func hostedToken(getenv func(string) string) (string, error) { } func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("push", "proxy push [-url URL] ", + cmd := newHostedCommand("push", "proxy hosted push [-url URL] ", "Push a package archive to the proxy", getenv) positional, err := cmd.parse(args, pushArgCount) if err != nil { @@ -138,7 +138,7 @@ func runPush(args []string, getenv func(string) string, stdout io.Writer, client } func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("list", "proxy list [-url URL] ", "List pushed package archives", getenv) + cmd := newHostedCommand("list", "proxy hosted list [-url URL] ", "List pushed package archives", getenv) positional, err := cmd.parse(args, listArgCount) if err != nil { return err @@ -161,7 +161,7 @@ func runList(args []string, getenv func(string) string, stdout io.Writer, client } func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { - cmd := newHostedCommand("delete", "proxy delete [-url URL] ", "Delete a pushed version", getenv) + cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] ", "Delete a pushed version", getenv) positional, err := cmd.parse(args, deleteArgCount) if err != nil { return err @@ -207,11 +207,36 @@ func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) e return json.NewDecoder(resp.Body).Decode(out) } -// runHostedCommand runs one of the push, list and delete commands against the -// proxy's /api/hosted endpoints and exits non-zero on failure. -func runHostedCommand(run func([]string, func(string) string, io.Writer, *http.Client) error) { +const hostedUsage = "usage: proxy hosted [flags] ...\n\n" + + " push Push a package archive to the proxy\n" + + " list List pushed package archives\n" + + " delete Delete a pushed version" + +type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error + +var hostedSubcommands = map[string]hostedSubcommand{ + "push": runPush, + "list": runList, + "delete": runDelete, +} + +// runHosted runs the hosted subcommand named by args[0]. +func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + if len(args) == 0 { + return errors.New(hostedUsage) + } + run, ok := hostedSubcommands[args[0]] + if !ok { + return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage) + } + return run(args[1:], getenv, stdout, client) +} + +// runHostedCommand runs proxy hosted against the proxy's /api/hosted +// endpoints and exits non-zero on failure. +func runHostedCommand() { client := &http.Client{Timeout: hostedClientTimeout} - if err := run(os.Args[1:], os.Getenv, os.Stdout, client); err != nil { + if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go index b15f915f..fef71608 100644 --- a/cmd/proxy/hosted_test.go +++ b/cmd/proxy/hosted_test.go @@ -140,3 +140,28 @@ func TestRunDelete(t *testing.T) { t.Errorf("delete of a missing version error = %v", err) } } + +func TestRunHostedDispatchesSubcommands(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`) + + if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil { + t.Fatalf("runHosted list: %v", err) + } + if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + + for _, args := range [][]string{nil, {"bogus"}} { + err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted ") { + t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err) + } + } +} + +func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) { + err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient) + if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") { + t.Errorf("push usage error = %v, want it to name proxy hosted push", err) + } +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 136f3ab7..4fdbf373 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -22,9 +22,7 @@ // serve Start the proxy server (default if no command given) // stats Show cache statistics // mirror Pre-populate cache from PURLs, SBOMs, or registries -// push Push a package archive to a running proxy (needs hosted.enabled) -// list List package archives pushed to a running proxy -// delete Delete a pushed version from a running proxy +// hosted Push, list or delete packages on a running proxy (needs hosted.enabled) // // Serve Flags: // @@ -95,7 +93,7 @@ // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") // PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only // PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only -// PROXY_URL - Proxy URL for push, list and delete +// PROXY_URL - Proxy URL for proxy hosted // // Example: // @@ -168,17 +166,9 @@ func main() { os.Args = append(os.Args[:1], os.Args[2:]...) runMirror() return - case "push": + case "hosted": os.Args = append(os.Args[:1], os.Args[2:]...) - runHostedCommand(runPush) - return - case "list": - os.Args = append(os.Args[:1], os.Args[2:]...) - runHostedCommand(runList) - return - case "delete": - os.Args = append(os.Args[:1], os.Args[2:]...) - runHostedCommand(runDelete) + runHostedCommand() return case "-version", "--version": fmt.Printf("proxy %s (%s)\n", Version, Commit) @@ -202,9 +192,7 @@ Commands: serve Start the proxy server (default) stats Show cache statistics mirror Pre-populate cache from PURLs, SBOMs, or registries - push Push a package archive to a running proxy - list List package archives pushed to a running proxy - delete Delete a pushed version from a running proxy + hosted Push, list or delete packages on a running proxy Run 'proxy -help' for more information on a command. diff --git a/docs/configuration.md b/docs/configuration.md index d1cca7f6..9aa307be 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1014,7 +1014,7 @@ hosted: ```bash export PROXY_HOSTED_TOKEN=... - proxy push -url https://proxy.example.com npm request 2.88.3 ./request-2.88.3.tgz + proxy hosted push -url https://proxy.example.com npm request 2.88.3 ./request-2.88.3.tgz ``` or with curl: @@ -1026,8 +1026,8 @@ hosted: ``` 3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too. -4. Check what is pushed: `proxy list npm request` or `GET /api/hosted/npm/request`. -5. Remove it again: `proxy delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. +4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. +5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. ### Supported ecosystems From c0ac418160eb9581ec967dd13fadc2b59acc7d7a Mon Sep 17 00:00:00 2001 From: simon chrzanowski Date: Wed, 14 Oct 2026 13:21:57 +0200 Subject: [PATCH 17/17] Serve pushed artifacts when streaming upstream artifacts - GetLocalArtifact went through checkCache, which reports a miss when StreamArtifacts is set, so npm and Composer downloads fetched the upstream archive instead of the pushed one - Split readStored out of lookupCache and read pushed artifacts through it regardless of StreamArtifacts - Keep the denylist check and the 503/404 answers for unreadable or missing pushed artifacts - Test npm and Composer downloads of pushed archives with StreamArtifacts set Refs #133 --- internal/handler/composer_hosted_test.go | 26 ++++++++++++++++-- internal/handler/handler.go | 6 ++++ internal/handler/hosted.go | 14 ++++++++-- internal/handler/npm_hosted_test.go | 35 ++++++++++++++++++++++-- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 43932a30..048c291c 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -3,10 +3,14 @@ package handler import ( "bytes" "context" + "io" "net/http" "net/http/httptest" + "strings" "testing" "time" + + "github.com/git-pkgs/registries/fetch" ) const monologMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ @@ -27,6 +31,7 @@ type composerHostedFixture struct { proxy *Proxy handler http.Handler upstreamRequests *int + fetcher *mockFetcher } func newComposerHostedFixture(t *testing.T, upstream map[string]string) composerHostedFixture { @@ -44,11 +49,11 @@ func newComposerHostedFixture(t *testing.T, upstream map[string]string) composer })) t.Cleanup(srv.Close) - proxy, _, _, _ := setupTestProxy(t) + proxy, _, _, fetcher := setupTestProxy(t) proxy.HostedEnabled = true proxy.HTTPClient = srv.Client() h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.local", srv.URL, srv.URL) - return composerHostedFixture{proxy: proxy, handler: h.Routes(), upstreamRequests: &upstreamRequests} + return composerHostedFixture{proxy: proxy, handler: h.Routes(), upstreamRequests: &upstreamRequests, fetcher: fetcher} } func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { @@ -221,3 +226,20 @@ func TestComposerMetadataLeavesOutDeniedPushedVersions(t *testing.T) { t.Error("denied pushed version listed") } } + +func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { + const manifest = `{"name":"monolog/monolog"}` + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream archive")), ContentType: "application/zip"} + f.push(t, "monolog/monolog", "3.5.0", manifest) + f.proxy.StreamArtifacts = true + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != string(composerZip("", manifest)) { + t.Errorf("download status = %d, want 200 and the pushed archive", rec.Code) + } + if f.fetcher.fetchCalled { + t.Error("a pushed archive was fetched upstream while streaming artifacts") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4265fd34..45c7006b 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -383,6 +383,12 @@ func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename if p.StreamArtifacts { return nil, nil } + return p.readStored(ctx, pkgPURL, versionPURL, filename) +} + +// readStored opens the stored object of an artifact, whether or not +// StreamArtifacts is set. Returns nil if no usable object is stored. +func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go index dfbbda4c..41ccb930 100644 --- a/internal/handler/hosted.go +++ b/internal/handler/hosted.go @@ -342,10 +342,11 @@ func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { return nil } -// GetLocalArtifact serves a pushed artifact from the cache, or returns nil +// GetLocalArtifact serves a pushed artifact from storage, or returns nil // when the artifact was not pushed. Handlers that must consult the upstream // before serving a cached copy call it first, so pushed versions the upstream -// does not list are still served. +// does not list are still served. Pushed artifacts are not a cache, so they +// are served even when StreamArtifacts is set. func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { @@ -358,5 +359,12 @@ func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, if record == nil || record.Source != database.SourceLocal { return nil, nil } - return p.checkCache(ctx, pkgPURL, versionPURL, filename) + if p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + result, err := p.readStored(ctx, pkgPURL, versionPURL, filename) + if err != nil || result != nil { + return result, err + } + return nil, p.localArtifactMiss(versionPURL, filename) } diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go index b93fabdc..7102046c 100644 --- a/internal/handler/npm_hosted_test.go +++ b/internal/handler/npm_hosted_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "io" "net/http" "net/http/httptest" "strings" @@ -11,6 +12,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) const requestPackument = `{ @@ -29,6 +31,7 @@ type npmHostedFixture struct { proxy *Proxy handler http.Handler upstreamRequests *int + fetcher *mockFetcher } // newNPMHostedFixture serves packuments from upstream, keyed by request path, @@ -48,10 +51,11 @@ func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixt })) t.Cleanup(srv.Close) - proxy, _, _, _ := setupTestProxy(t) + proxy, _, _, fetcher := setupTestProxy(t) proxy.HostedEnabled = true proxy.HTTPClient = srv.Client() - return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes(), upstreamRequests: &upstreamRequests} + return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes(), + upstreamRequests: &upstreamRequests, fetcher: fetcher} } func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { @@ -318,3 +322,30 @@ func TestNPMMetadataLeavesOutDeniedPushedVersions(t *testing.T) { t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) } } + +func TestNPMDownloadOfPushedTarballWhileStreamingArtifacts(t *testing.T) { + const manifest = `{"name":"request","version":"2.88.3"}` + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + f.push(t, "request", "2.88.3", manifest) + f.proxy.StreamArtifacts = true + + rec := f.get(t, "/request/-/request-2.88.3.tgz") + if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(manifest)) { + t.Errorf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } + if f.fetcher.fetchCalled { + t.Error("a pushed tarball was fetched upstream while streaming artifacts") + } +} + +func TestNPMDownloadOfUnreadablePushedTarballWhileStreamingArtifacts(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") + f.proxy.StreamArtifacts = true + + if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) + } +}