Add package routes for private Composer and npm registries

README.md

@@ -161,6 +161,8 @@ therefore come from upstream's database, not from the proxy's own vulnerability
 data, and versions withheld by [cooldown](#version-cooldown) are not excluded
 from the report.
 
+To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes).
+
 ### Cargo
 
 Create or edit `~/.cargo/config.toml`:
@@ -344,6 +346,8 @@ Or set globally:
 composer config -g repositories.proxy composer http://localhost:8080/composer
 ```
 
+To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes).
+
 ### Conan (C/C++)
 
 Add the proxy as a remote:

config.example.yaml

@@ -140,6 +140,11 @@ upstream:
   # gate on publish age, e.g. Yarn's npmMinimalAgeGate. Default: false.
   # npm_full_metadata: true
 
+  # npm packages served only from a private registry, by name pattern.
+  # Matching packages are never looked up on the npm registry above.
+  # npm_routes:
+  #   "@example/*": "https://npm.example.com/registry"
+
   # Cargo sparse index URL
   cargo: "https://index.crates.io"
 
@@ -185,6 +190,11 @@ upstream:
   # Packagist repository URL
   composer_repository: "https://repo.packagist.org"
 
+  # Composer packages served only from a private repository, by name pattern.
+  # Matching packages are never looked up on composer_repository.
+  # composer_routes:
+  #   "example/*": "https://composer.example.com/packages"
+
   # Conan registry URL
   conan: "https://center.conan.io"
 

docs/configuration.md

@@ -229,6 +229,36 @@ upstream:
 
 `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint.
 
+### Private package routes
+
+Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream:
+
+```yaml
+upstream:
+  composer_routes:
+    "example/*": "https://packages.example.com/composer"
+  npm_routes:
+    "@example/*": "https://packages.example.com/npm"
+  allow_private_hosts:
+    - "packages.example.com"
+  auth:
+    "https://packages.example.com":
+      type: bearer
+      token: "${PRIVATE_REGISTRY_TOKEN}"
+```
+
+- Patterns use Go's [`path.Match`](https://pkg.go.dev/path#Match) syntax and are matched case-insensitively. `*` does not cross `/`, so `example/*` matches `example/library` but not `example-fork/library`. The longest matching pattern wins.
+- A Composer route URL is the repository base that serves `/p2/{vendor}/{package}.json`. A route also covers the package's dev versions in `/p2/{vendor}/{package}~dev.json`; patterns are matched against the package name without `~dev`.
+- An npm route URL is the registry base that serves `/{package}`. Tarball URLs in its packuments must be on the same host and below the same path.
+- A routed package is not looked up anywhere else. If the private registry does not have it, the client gets a 404. If the registry fails or rejects the credentials, the client gets a 502. A package with the same name on the public registry is never served in its place (dependency confusion).
+- Metadata and artifacts of routed packages are cached under keys that include the route's URL. Entries cached from the public registry before a route was added are never served for a routed package.
+- With cooldown enabled, the publish time of a routed npm version always comes from the route's metadata. Times recorded for the public package of the same name, or for a route's previous URL, are not used, and routed downloads do not record times of their own.
+- Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint.
+- Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages.
+- Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only.
+- npm audit and signing-key requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included.
+- Routes can only be set in the configuration file, not with environment variables.
+
 Helm HTTP repositories and additional OCI registries are configured as named maps:
 
 ```yaml

internal/config/config.go

@@ -66,6 +66,7 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -512,6 +513,13 @@ type UpstreamConfig struct {
 	// Default: false (the abbreviated format is preferred).
 	NPMFullMetadata bool `json:"npm_full_metadata" yaml:"npm_full_metadata"`
 
+	// NPMRoutes maps package name patterns to npm registry URLs. A package
+	// whose name matches a pattern is fetched only from that registry, never
+	// from upstream.npm, so a public package cannot stand in for it. Patterns
+	// use path.Match syntax, e.g. "@example/*"; the longest matching pattern
+	// wins.
+	NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"`
+
 	// Cargo is the upstream cargo index URL.
 	// Default: https://index.crates.io
 	Cargo string `json:"cargo" yaml:"cargo"`
@@ -573,6 +581,14 @@ type UpstreamConfig struct {
 	// Default: https://repo.packagist.org
 	ComposerRepository string `json:"composer_repository" yaml:"composer_repository"`
 
+	// ComposerRoutes maps package name patterns to Composer repository URLs
+	// (the base that serves /p2/{vendor}/{package}.json). A package whose
+	// name matches a pattern is fetched only from that repository, never
+	// from upstream.composer_repository, so a public package cannot stand in
+	// for it. Patterns use path.Match syntax, e.g. "example/*"; the longest
+	// matching pattern wins.
+	ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"`
+
 	// Conan is the upstream Conan registry URL.
 	// Default: https://center.conan.io
 	Conan string `json:"conan" yaml:"conan"`
@@ -692,6 +708,12 @@ func (u *UpstreamConfig) Validate() error {
 	if err := validateNamedUpstreams("upstream.helm", u.Helm); err != nil {
 		return err
 	}
+	if err := validatePackageRoutes("upstream.composer_routes", u.ComposerRoutes); err != nil {
+		return err
+	}
+	if err := validatePackageRoutes("upstream.npm_routes", u.NPMRoutes); err != nil {
+		return err
+	}
 	if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil {
 		return err
 	}
@@ -784,6 +806,18 @@ func validateNamedUpstreams(field string, upstreams map[string]string) error {
 	return nil
 }
 
+func validatePackageRoutes(field string, routes map[string]string) error {
+	for pattern, upstreamURL := range routes {
+		if _, err := path.Match(pattern, ""); pattern == "" || err != nil {
+			return fmt.Errorf("invalid %s pattern %q", field, pattern)
+		}
+		if err := validateAbsoluteURL(field+"."+pattern, upstreamURL); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func parseAuthURL(value string) (*url.URL, error) {
 	parsed, err := url.Parse(value)
 	if err != nil || !parsed.IsAbs() || parsed.Hostname() == "" || parsed.Opaque != "" {

internal/config/config_test.go

@@ -1302,6 +1302,61 @@ func TestValidateUpstreamAuthURLs(t *testing.T) {
 	})
 }
 
+func TestValidatePackageRoutes(t *testing.T) {
+	tests := []struct {
+		name    string
+		modify  func(*Config)
+		wantErr bool
+	}{
+		{
+			name: "valid Composer and npm routes",
+			modify: func(cfg *Config) {
+				cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com/packages"}
+				cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "https://npm.example.com/registry"}
+			},
+		},
+		{
+			name: "Composer route URL is not absolute",
+			modify: func(cfg *Config) {
+				cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "composer.example.com"}
+			},
+			wantErr: true,
+		},
+		{
+			name: "npm route URL is not absolute",
+			modify: func(cfg *Config) {
+				cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": "npm.example.com"}
+			},
+			wantErr: true,
+		},
+		{
+			name: "Composer route pattern is malformed",
+			modify: func(cfg *Config) {
+				cfg.Upstream.ComposerRoutes = map[string]string{"acme/[": "https://composer.example.com"}
+			},
+			wantErr: true,
+		},
+		{
+			name: "npm route pattern is empty",
+			modify: func(cfg *Config) {
+				cfg.Upstream.NPMRoutes = map[string]string{"": "https://npm.example.com"}
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Default()
+			tt.modify(cfg)
+			err := cfg.Validate()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func TestValidateNamedUpstreams(t *testing.T) {
 	tests := []struct {
 		name    string

internal/handler/composer.go

@@ -19,6 +19,7 @@ const (
 	composerMinified   = "composer/2.0"
 	composerUnset      = "__unset"
 	composerDevReset   = "~dev"
+	composerDevFile    = "~dev" // suffix of the metadata file holding dev versions
 	vendorPackageParts = 2
 )
 
@@ -30,6 +31,7 @@ type ComposerHandler struct {
 	upstreamURL string
 	repoURL     string
 	proxyURL    string
+	routes      packageRoutes
 }
 
 // NewComposerHandler creates a new Composer protocol handler.
@@ -51,6 +53,13 @@ func NewComposerHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, repoUR
 	return h
 }
 
+// WithPackageRoutes sends packages matching a pattern to a dedicated Composer
+// repository instead of the default one. See packageRoutes.
+func (h *ComposerHandler) WithPackageRoutes(routes map[string]string) *ComposerHandler {
+	h.routes = newPackageRoutes(routes)
+	return h
+}
+
 // Routes returns the HTTP handler for Composer requests.
 func (h *ComposerHandler) Routes() http.Handler {
 	mux := http.NewServeMux()
@@ -86,6 +95,18 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ
 	_ = json.NewEncoder(w).Encode(index)
 }
 
+// sourceFor returns the repository serving a package and the key its metadata
+// is cached under. packageName may carry the ~dev suffix of the file holding
+// the package's dev versions: routes match the package without it, so its dev
+// versions come from the same repository, while the cache key keeps it so the
+// two files stay apart.
+func (h *ComposerHandler) sourceFor(packageName string) (repoURL, cacheKey string) {
+	if route, routed := h.routes.match(strings.TrimSuffix(packageName, composerDevFile)); routed {
+		return route.url, route.cacheKey(packageName)
+	}
+	return h.repoURL, packageName
+}
+
 // handlePackageMetadata proxies and rewrites package metadata.
 func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) {
 	// Parse path: /p2/{vendor}/{package}.json
@@ -102,15 +123,16 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R
 
 	h.proxy.Logger.Info("composer metadata request", "package", packageName)
 
-	upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
+	repoURL, cacheKey := h.sourceFor(packageName)
+	upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg)
 
-	if rewritten, ok := h.proxy.storedRewrite("composer", packageName, h.proxyURL, packageName); ok {
+	if rewritten, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok {
 		w.Header().Set(headerContentType, "application/json")
 		_, _ = w.Write(rewritten)
 		return
 	}
 
-	body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", packageName, upstreamURL)
+	body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "composer", cacheKey, upstreamURL)
 	if err != nil {
 		if errors.Is(err, ErrUpstreamNotFound) {
 			http.Error(w, "not found", http.StatusNotFound)
@@ -121,7 +143,7 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, packageName, body, h.rewriteMetadata)
+	rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, cacheKey, body, h.rewriteMetadata)
 	if err != nil {
 		if r.Context().Err() != nil {
 			return // the client left while waiting on a shared rewrite
@@ -192,6 +214,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v
 	packagePURL := canonicalPackagePURL("composer", packageName)
 	upstream, written := newComposerFields(), newComposerFields()
 	devReset, first := false, true
+	_, routed := h.routes.match(packageName)
 
 	out.WriteByte('[')
 	err := forEachJSONElement(versions, func(entry []byte) error {
@@ -225,6 +248,9 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v
 			out.WriteString(`"` + composerDevReset + `",`)
 			devReset = false
 		}
+		if routed {
+			disableDefaultNotification(upstream)
+		}
 		h.writeVersion(out, packageName, version, upstream, written)
 		return nil
 	})
@@ -296,6 +322,17 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version
 	return false
 }
 
+// disableDefaultNotification stops Composer from reporting installs of a routed
+// package to the default repository's notify-batch URL, which would disclose
+// the package name to it. Composer only falls back to notify-batch when a
+// version has no notification-url of its own, and skips an empty one. Setting
+// it on the expanded fields keeps it set for the versions that inherit them.
+func disableDefaultNotification(fields *composerFields) {
+	if fields.get("notification-url") == nil {
+		fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`))
+	}
+}
+
 // rewriteDist returns a version's dist object with its url pointing at this
 // proxy. A dist without a string url is returned unchanged.
 func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte {
@@ -475,7 +512,8 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request)
 		"package", packageName, "version", version,
 		"download_url", downloadURL)
 
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, filename, downloadURL)
+	cacheFilename := h.routes.cacheFilename(packageName, filename)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "composer", packageName, version, cacheFilename, downloadURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch package")
 		return
@@ -497,8 +535,9 @@ func isDevVersion(version string) bool {
 // file, tagged releases from the regular file; the other file is included as a
 // fallback so an unexpected classification still resolves.
 func (h *ComposerHandler) metadataURLsForVersion(vendor, pkg, version string) []string {
-	stable := fmt.Sprintf("%s/p2/%s/%s.json", h.repoURL, vendor, pkg)
-	dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", h.repoURL, vendor, pkg)
+	repoURL, _ := h.sourceFor(vendor + "/" + pkg)
+	stable := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg)
+	dev := fmt.Sprintf("%s/p2/%s/%s~dev.json", repoURL, vendor, pkg)
 
 	if isDevVersion(version) {
 		return []string{dev, stable}