diff --git a/README.md b/README.md index 3aecba4f..66d1972c 100644 --- a/README.md +++ b/README.md @@ -1026,6 +1026,18 @@ curl http://localhost:8080/api/mirror/mirror-1 curl -X DELETE http://localhost:8080/api/mirror/mirror-1 ``` +### hosted + +Push package archives to a running proxy with `hosted.enabled`, list them and delete them. See [Hosted Packages](docs/configuration.md#hosted-packages). + +```bash +export PROXY_URL=https://proxy.example.com PROXY_HOSTED_TOKEN=... +proxy hosted push npm request 2.88.3 ./request-2.88.3.tgz +proxy hosted push composer monolog/monolog 3.5.0 ./monolog-3.5.0.zip +proxy hosted list npm request +proxy hosted delete npm request 2.88.3 +``` + ### stats Show cache statistics without running the server. @@ -1119,6 +1131,16 @@ Recently cached: | `GET /api/mirror/{id}` | Get job status and progress | | `DELETE /api/mirror/{id}` | Cancel a running job | +### Hosted API + +Registered when `hosted.enabled` is set. Pushing and deleting require `Authorization: Bearer `. + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/hosted/{ecosystem}/{name}/{version}` | Push the request body as the archive for a version (npm and composer) | +| `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | +| `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | + ### Enrichment API The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection. diff --git a/cmd/proxy/hosted.go b/cmd/proxy/hosted.go new file mode 100644 index 00000000..bbc947df --- /dev/null +++ b/cmd/proxy/hosted.go @@ -0,0 +1,243 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "text/tabwriter" + "time" +) + +const ( + defaultProxyURL = "http://localhost:8080" + hostedTokenEnv = "PROXY_HOSTED_TOKEN" + hostedClientTimeout = 10 * time.Minute + + pushArgCount = 4 // ecosystem, name, version, file + listArgCount = 2 // ecosystem, name + deleteArgCount = 3 // ecosystem, name, version + tablePadding = 2 +) + +// hostedArtifact mirrors the JSON the /api/hosted endpoints return. +type hostedArtifact struct { + Version string `json:"version"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + PushedAt time.Time `json:"pushed_at"` +} + +type hostedPushResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + hostedArtifact +} + +type hostedListResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Artifacts []hostedArtifact `json:"artifacts"` +} + +type hostedCommand struct { + flags *flag.FlagSet + proxyURL *string + usage string +} + +func newHostedCommand(name, usage, summary string, getenv func(string) string) hostedCommand { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(io.Discard) + proxyURL := getenv("PROXY_URL") + if proxyURL == "" { + proxyURL = defaultProxyURL + } + cmd := hostedCommand{flags: fs, usage: usage} + cmd.proxyURL = fs.String("url", proxyURL, "Proxy URL (env PROXY_URL)") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "git-pkgs proxy - %s\n\nUsage: %s\n\nFlags:\n", summary, usage) + fs.SetOutput(os.Stderr) + fs.PrintDefaults() + } + return cmd +} + +// parse parses flags and checks for exactly want positional arguments. +func (c hostedCommand) parse(args []string, want int) ([]string, error) { + if err := c.flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + c.flags.Usage() + } + return nil, fmt.Errorf("%w\nusage: %s", err, c.usage) + } + if c.flags.NArg() != want { + return nil, fmt.Errorf("usage: %s", c.usage) + } + return c.flags.Args(), nil +} + +func (c hostedCommand) endpoint(ecosystem, name string, version ...string) string { + segments := []string{url.PathEscape(ecosystem)} + for segment := range strings.SplitSeq(name, "/") { + segments = append(segments, url.PathEscape(segment)) + } + for _, v := range version { + segments = append(segments, url.PathEscape(v)) + } + return strings.TrimSuffix(*c.proxyURL, "/") + "/api/hosted/" + strings.Join(segments, "/") +} + +func hostedToken(getenv func(string) string) (string, error) { + token := getenv(hostedTokenEnv) + if token == "" { + return "", fmt.Errorf("%s must be set to the proxy's hosted.token", hostedTokenEnv) + } + return token, nil +} + +func runPush(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("push", "proxy hosted push [-url URL] ", + "Push a package archive to the proxy", getenv) + positional, err := cmd.parse(args, pushArgCount) + if err != nil { + return err + } + token, err := hostedToken(getenv) + if err != nil { + return err + } + ecosystem, name, version, path := positional[0], positional[1], positional[2], positional[3] + + file, err := os.Open(path) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + req, err := http.NewRequest(http.MethodPut, cmd.endpoint(ecosystem, name, version), file) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/octet-stream") + + var pushed hostedPushResponse + if err := doHosted(client, req, http.StatusCreated, &pushed); err != nil { + return err + } + _, err = fmt.Fprintf(stdout, "pushed %s %s@%s as %s (%d bytes, sha256 %s)\n", + pushed.Ecosystem, pushed.Name, pushed.Version, pushed.Filename, pushed.Size, pushed.SHA256) + return err +} + +func runList(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("list", "proxy hosted list [-url URL] ", "List pushed package archives", getenv) + positional, err := cmd.parse(args, listArgCount) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodGet, cmd.endpoint(positional[0], positional[1]), nil) + if err != nil { + return err + } + + var listed hostedListResponse + if err := doHosted(client, req, http.StatusOK, &listed); err != nil { + return err + } + tw := tabwriter.NewWriter(stdout, 0, 0, tablePadding, ' ', 0) + _, _ = fmt.Fprintln(tw, "VERSION\tFILENAME\tSIZE\tSHA256\tPUSHED") + for _, a := range listed.Artifacts { + _, _ = fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", a.Version, a.Filename, a.Size, a.SHA256, a.PushedAt.Format(time.RFC3339)) + } + return tw.Flush() +} + +func runDelete(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + cmd := newHostedCommand("delete", "proxy hosted delete [-url URL] ", "Delete a pushed version", getenv) + positional, err := cmd.parse(args, deleteArgCount) + if err != nil { + return err + } + token, err := hostedToken(getenv) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodDelete, cmd.endpoint(positional[0], positional[1], positional[2]), nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + + if err := doHosted(client, req, http.StatusNoContent, nil); err != nil { + return err + } + _, err = fmt.Fprintf(stdout, "deleted %s %s@%s\n", positional[0], positional[1], positional[2]) + return err +} + +// doHosted sends req and decodes a successful response into out. Any other +// status is returned as an error carrying the server's message. +func doHosted(client *http.Client, req *http.Request, wantStatus int, out any) error { + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != wantStatus { + var apiErr struct { + Message string `json:"message"` + } + if json.NewDecoder(resp.Body).Decode(&apiErr) == nil && apiErr.Message != "" { + return fmt.Errorf("%s %s: %s (%s)", req.Method, req.URL.Path, apiErr.Message, resp.Status) + } + return fmt.Errorf("%s %s: %s", req.Method, req.URL.Path, resp.Status) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} + +const hostedUsage = "usage: proxy hosted [flags] ...\n\n" + + " push Push a package archive to the proxy\n" + + " list List pushed package archives\n" + + " delete Delete a pushed version" + +type hostedSubcommand func([]string, func(string) string, io.Writer, *http.Client) error + +var hostedSubcommands = map[string]hostedSubcommand{ + "push": runPush, + "list": runList, + "delete": runDelete, +} + +// runHosted runs the hosted subcommand named by args[0]. +func runHosted(args []string, getenv func(string) string, stdout io.Writer, client *http.Client) error { + if len(args) == 0 { + return errors.New(hostedUsage) + } + run, ok := hostedSubcommands[args[0]] + if !ok { + return fmt.Errorf("unknown hosted command %q\n%s", args[0], hostedUsage) + } + return run(args[1:], getenv, stdout, client) +} + +// runHostedCommand runs proxy hosted against the proxy's /api/hosted +// endpoints and exits non-zero on failure. +func runHostedCommand() { + client := &http.Client{Timeout: hostedClientTimeout} + if err := runHosted(os.Args[1:], os.Getenv, os.Stdout, client); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/proxy/hosted_test.go b/cmd/proxy/hosted_test.go new file mode 100644 index 00000000..fef71608 --- /dev/null +++ b/cmd/proxy/hosted_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +type recordedRequest struct { + method, path, query, authorization, body string +} + +func hostedTestServer(t *testing.T, status int, response string) (*httptest.Server, *recordedRequest) { + t.Helper() + recorded := &recordedRequest{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + *recorded = recordedRequest{ + method: r.Method, + path: r.URL.EscapedPath(), + query: r.URL.RawQuery, + authorization: r.Header.Get("Authorization"), + body: string(body), + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, response) + })) + t.Cleanup(srv.Close) + return srv, recorded +} + +func hostedEnv(vars map[string]string) func(string) string { + return func(key string) string { return vars[key] } +} + +func writeArchive(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("writing archive: %v", err) + } + return path +} + +func TestRunPush(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusCreated, + `{"ecosystem":"npm","name":"@types/node","version":"20.1.0","filename":"node-20.1.0.tgz","sha256":"abc123","size":7}`) + archive := writeArchive(t, "node-20.1.0.tgz", "tarball") + var stdout bytes.Buffer + + err := runPush([]string{"-url", srv.URL, "npm", "@types/node", "20.1.0", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), &stdout, srv.Client()) + if err != nil { + t.Fatalf("runPush: %v", err) + } + if recorded.method != http.MethodPut || recorded.path != "/api/hosted/npm/@types/node/20.1.0" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + if recorded.authorization != "Bearer tok" || recorded.body != "tarball" || recorded.query != "" { + t.Errorf("authorization/body/query = %q/%q/%q", recorded.authorization, recorded.body, recorded.query) + } + if out := stdout.String(); !strings.Contains(out, "node-20.1.0.tgz") || !strings.Contains(out, "abc123") { + t.Errorf("output = %q, want filename and sha256", out) + } +} + +func TestRunPushErrors(t *testing.T) { + srv, _ := hostedTestServer(t, http.StatusBadRequest, `{"code":"BAD_REQUEST","message":"pushing packages is not supported for this ecosystem: cargo"}`) + archive := writeArchive(t, "serde-1.0.200.crate", "crate") + + err := runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, hostedEnv(nil), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "PROXY_HOSTED_TOKEN") { + t.Errorf("push without token error = %v, want a PROXY_HOSTED_TOKEN hint", err) + } + + err = runPush([]string{"-url", srv.URL, "cargo", "serde", "1.0.200", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "not supported for this ecosystem") { + t.Errorf("rejected push error = %v, want the server's message", err) + } + + err = runPush([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "usage") { + t.Errorf("push with missing arguments error = %v, want usage", err) + } +} + +func TestRunPushUsesProxyURLFromEnv(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusCreated, `{}`) + archive := writeArchive(t, "request-2.88.3.tgz", "tarball") + + err := runPush([]string{"npm", "request", "2.88.3", archive}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok", "PROXY_URL": srv.URL}), io.Discard, srv.Client()) + if err != nil { + t.Fatalf("runPush: %v", err) + } + if recorded.path != "/api/hosted/npm/request/2.88.3" { + t.Errorf("path = %q", recorded.path) + } +} + +func TestRunList(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusOK, + `{"ecosystem":"npm","name":"request","artifacts":[{"version":"2.88.3","filename":"request-2.88.3.tgz","sha256":"abc123","size":7,"pushed_at":"2026-10-13T10:00:00Z"}]}`) + var stdout bytes.Buffer + + if err := runList([]string{"-url", srv.URL, "npm", "request"}, hostedEnv(nil), &stdout, srv.Client()); err != nil { + t.Fatalf("runList: %v", err) + } + if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" || recorded.authorization != "" { + t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) + } + if out := stdout.String(); !strings.Contains(out, "2.88.3") || !strings.Contains(out, "request-2.88.3.tgz") { + t.Errorf("output = %q", out) + } +} + +func TestRunDelete(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusNoContent, "") + + err := runDelete([]string{"-url", srv.URL, "npm", "request", "2.88.3"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, srv.Client()) + if err != nil { + t.Fatalf("runDelete: %v", err) + } + if recorded.method != http.MethodDelete || recorded.path != "/api/hosted/npm/request/2.88.3" || recorded.authorization != "Bearer tok" { + t.Errorf("request = %s %s (authorization %q)", recorded.method, recorded.path, recorded.authorization) + } + + missing, _ := hostedTestServer(t, http.StatusNotFound, `{"code":"NOT_FOUND","message":"no pushed artifacts for this version"}`) + err = runDelete([]string{"-url", missing.URL, "npm", "request", "2.88.3"}, + hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, missing.Client()) + if err == nil || !strings.Contains(err.Error(), "no pushed artifacts") { + t.Errorf("delete of a missing version error = %v", err) + } +} + +func TestRunHostedDispatchesSubcommands(t *testing.T) { + srv, recorded := hostedTestServer(t, http.StatusOK, `{"ecosystem":"npm","name":"request","artifacts":[]}`) + + if err := runHosted([]string{"list", "-url", srv.URL, "npm", "request"}, hostedEnv(nil), io.Discard, srv.Client()); err != nil { + t.Fatalf("runHosted list: %v", err) + } + if recorded.method != http.MethodGet || recorded.path != "/api/hosted/npm/request" { + t.Errorf("request = %s %s", recorded.method, recorded.path) + } + + for _, args := range [][]string{nil, {"bogus"}} { + err := runHosted(args, hostedEnv(nil), io.Discard, srv.Client()) + if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted ") { + t.Errorf("runHosted(%q) error = %v, want the hosted usage", args, err) + } + } +} + +func TestHostedSubcommandUsageNamesTheHostedCommand(t *testing.T) { + err := runPush([]string{"npm"}, hostedEnv(map[string]string{"PROXY_HOSTED_TOKEN": "tok"}), io.Discard, http.DefaultClient) + if err == nil || !strings.Contains(err.Error(), "usage: proxy hosted push") { + t.Errorf("push usage error = %v, want it to name proxy hosted push", err) + } +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 47efdc77..4fdbf373 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -5,6 +5,11 @@ // @description HTTP API for package enrichment, vulnerability lookup, cache stats, and source browsing. // @BasePath / // +// @securityDefinitions.apikey BearerAuth +// @in header +// @name Authorization +// @description "Bearer " followed by hosted.token +// // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. // @@ -17,6 +22,7 @@ // serve Start the proxy server (default if no command given) // stats Show cache statistics // mirror Pre-populate cache from PURLs, SBOMs, or registries +// hosted Push, list or delete packages on a running proxy (needs hosted.enabled) // // Serve Flags: // @@ -82,6 +88,12 @@ // PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size // PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval // PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s") +// PROXY_HOSTED_ENABLED - Enable pushing packages to /api/hosted +// PROXY_HOSTED_TOKEN - Bearer token for pushing and deleting +// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") +// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only +// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only +// PROXY_URL - Proxy URL for proxy hosted // // Example: // @@ -154,6 +166,10 @@ func main() { os.Args = append(os.Args[:1], os.Args[2:]...) runMirror() return + case "hosted": + os.Args = append(os.Args[:1], os.Args[2:]...) + runHostedCommand() + return case "-version", "--version": fmt.Printf("proxy %s (%s)\n", Version, Commit) os.Exit(0) @@ -176,6 +192,7 @@ Commands: serve Start the proxy server (default) stats Show cache statistics mirror Pre-populate cache from PURLs, SBOMs, or registries + hosted Push, list or delete packages on a running proxy Run 'proxy -help' for more information on a command. diff --git a/config.example.yaml b/config.example.yaml index 52ebc26d..c41f30f3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -364,6 +364,18 @@ gradle: # How often eviction runs when max_age or max_size is set sweep_interval: "10m" +# Pushing package archives to /api/hosted (disabled by default). +# A pushed version replaces the upstream one for every client, so a token is +# required to push and delete. +# hosted: +# enabled: true +# token: "${PROXY_HOSTED_TOKEN}" +# max_upload_size: "100MB" +# # Namespaces served from pushed packages only; the upstream is never asked +# # about them, so nobody can mix in versions by publishing the same name. +# npm_scopes: ["@acme"] +# composer_vendors: ["acme"] + # Health endpoint configuration. health: # Minimum time between storage backend probes. diff --git a/docs/configuration.md b/docs/configuration.md index ae117f63..9aa307be 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -982,6 +982,93 @@ Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`. Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). Independently of this setting, the shared transport gives up on an upstream that has not sent response headers within 60 seconds. +## Hosted Packages + +Operators can push a package archive into the proxy under a name and version. The proxy serves it in place of the upstream archive with the same version, or in addition to the upstream's versions. Use it to ship a patched build of a dependency without running a separate registry. + +The `/api/hosted` endpoints are disabled by default. Because a pushed version replaces the upstream one for every client, enabling them requires a token: + +```yaml +hosted: + enabled: true + token: "${PROXY_HOSTED_TOKEN}" + max_upload_size: "100MB" + npm_scopes: ["@acme"] + composer_vendors: ["acme"] +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `hosted.enabled` | `PROXY_HOSTED_ENABLED` | Register the `/api/hosted` endpoints (default `false`) | +| `hosted.token` | `PROXY_HOSTED_TOKEN` | Bearer token required to push and delete; required when enabled, supports `${VAR}` | +| `hosted.max_upload_size` | `PROXY_HOSTED_MAX_UPLOAD_SIZE` | Maximum size of one pushed archive (default `100MB`) | +| `hosted.npm_scopes` | `PROXY_HOSTED_NPM_SCOPES` | npm scopes served from pushed packages only (comma-separated in the environment) | +| `hosted.composer_vendors` | `PROXY_HOSTED_COMPOSER_VENDORS` | Composer vendors served from pushed packages only | + +`hosted.enabled` cannot be combined with `storage.cache_artifacts: false`. + +### Pushing a patched package + +1. Build the archive with the original package name and the version you push in its manifest (`package.json` or `composer.json`), e.g. `request-2.88.3.tgz`. +2. Push it: + + ```bash + export PROXY_HOSTED_TOKEN=... + proxy hosted push -url https://proxy.example.com npm request 2.88.3 ./request-2.88.3.tgz + ``` + + or with curl: + + ```bash + curl -X PUT --data-binary @request-2.88.3.tgz \ + -H "Authorization: Bearer $PROXY_HOSTED_TOKEN" \ + https://proxy.example.com/api/hosted/npm/request/2.88.3 + ``` + +3. Install it as usual, e.g. `npm install request@2.88.3`. The version is listed in the package metadata, so ranges like `^2.88.0` resolve to it too. +4. Check what is pushed: `proxy hosted list npm request` or `GET /api/hosted/npm/request`. +5. Remove it again: `proxy hosted delete npm request 2.88.3` or `DELETE /api/hosted/npm/request/2.88.3`. If the upstream has the version, clients get the upstream archive again. + +### Supported ecosystems + +Download handlers look archives up by the filename clients request, so a push is stored under that name: + +| Ecosystem | Name | Filename | +|-----------|------|----------| +| `npm` | `{name}` or `@{scope}/{name}` | `{name}-{version}.tgz`, without the scope | +| `composer` | `{vendor}/{package}` | `{package}-{version}.zip` | + +Other ecosystems are not supported yet. + +The archive must contain the package manifest: `package.json` in the top directory of an npm tarball, or `composer.json` at the root or in the single top directory of a Composer zip. Its name, and its version when it has one, must match the push. Otherwise the push is refused with 400. + +### Package metadata + +Pushed versions are listed in the package metadata clients resolve against, with the dependencies from their manifest: + +- A pushed version takes the place of an upstream version with the same number. +- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them. +- A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. +- Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. +- npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. + +Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. + +### Hosted-only namespaces + +A package the upstream doesn't know is served from its pushed versions, but the proxy still asks the upstream about it on every request. If someone later publishes the same name there, their versions are listed next to yours, and a range such as `^1.0.0` can resolve to one of them (dependency confusion). + +For private packages: + +1. Prefer a namespace you own on the public registry: an npm scope registered on npmjs.com, or a Composer vendor with packages on Packagist. Nobody else can publish under it. +2. List the namespace in `hosted.npm_scopes` or `hosted.composer_vendors`. Its packages are served from pushed versions alone: the upstream is never asked about them, and anything not pushed answers 404. + +Other packages, including ones in other scopes and vendors, are proxied as usual. + +### Storage and eviction + +Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 798b50ac..5e54d3c6 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -115,6 +115,174 @@ const docTemplate = `{ } } }, + "/api/hosted/{ecosystem}/{name}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "List pushed package archives", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.HostedListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/hosted/{ecosystem}/{name}/{version}": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "consumes": [ + "application/octet-stream" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "Push a package archive", + "parameters": [ + { + "type": "string", + "description": "Ecosystem (npm or composer)", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name, including an npm scope or the Composer vendor", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/server.HostedPushResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "tags": [ + "hosted" + ], + "summary": "Delete a pushed version", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/outdated": { "post": { "consumes": [ @@ -509,6 +677,26 @@ const docTemplate = `{ } }, "definitions": { + "handler.HostedArtifact": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -640,6 +828,49 @@ const docTemplate = `{ } } }, + "server.HostedListResponse": { + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.HostedArtifact" + } + }, + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "server.HostedPushResponse": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.OutdatedPackage": { "type": "object", "properties": { @@ -872,6 +1103,14 @@ const docTemplate = `{ } } } + }, + "securityDefinitions": { + "BearerAuth": { + "description": "\"Bearer \" followed by hosted.token", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } } }` diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index fd175bf9..65a271b2 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -108,6 +108,174 @@ } } }, + "/api/hosted/{ecosystem}/{name}": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "List pushed package archives", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.HostedListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/hosted/{ecosystem}/{name}/{version}": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted.", + "consumes": [ + "application/octet-stream" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosted" + ], + "summary": "Push a package archive", + "parameters": [ + { + "type": "string", + "description": "Ecosystem (npm or composer)", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name, including an npm scope or the Composer vendor", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/server.HostedPushResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again.", + "tags": [ + "hosted" + ], + "summary": "Delete a pushed version", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/outdated": { "post": { "consumes": [ @@ -502,6 +670,26 @@ } }, "definitions": { + "handler.HostedArtifact": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.BrowseFileInfo": { "type": "object", "properties": { @@ -633,6 +821,49 @@ } } }, + "server.HostedListResponse": { + "type": "object", + "properties": { + "artifacts": { + "type": "array", + "items": { + "$ref": "#/definitions/handler.HostedArtifact" + } + }, + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "server.HostedPushResponse": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pushed_at": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, "server.OutdatedPackage": { "type": "object", "properties": { @@ -865,5 +1096,13 @@ } } } + }, + "securityDefinitions": { + "BearerAuth": { + "description": "\"Bearer \" followed by hosted.token", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } } } \ No newline at end of file diff --git a/internal/config/config.go b/internal/config/config.go index 87cb713f..a9d7c212 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -190,6 +190,9 @@ type Config struct { // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` + // Hosted configures pushing packages to the proxy (/api/hosted). + Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -1209,6 +1212,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") + setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") + setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") + setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") + setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") @@ -1335,6 +1343,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") case c.MirrorAPI: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") + case c.Hosted.Enabled: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } return nil } @@ -1364,6 +1374,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Hosted.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/hosted.go b/internal/config/hosted.go new file mode 100644 index 00000000..9b67e288 --- /dev/null +++ b/internal/config/hosted.go @@ -0,0 +1,77 @@ +package config + +import ( + "fmt" + "strings" +) + +const defaultHostedMaxUploadSize = "100MB" + +// HostedConfig configures packages pushed to the proxy through the +// /api/hosted endpoints. Pushed versions are served in place of upstream +// versions with the same number, so writes always require a token. +type HostedConfig struct { + // Enabled registers the /api/hosted endpoints. Off by default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to push and delete packages. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxUploadSize caps a single pushed archive (e.g., "100MB"). Must be > 0. + // Default: "100MB". + MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"` + + // NPMScopes are npm scopes (e.g. "@acme") served only from pushed + // packages. The upstream registry is never asked about them, so a package + // published there under the same name can't be mixed in. + NPMScopes []string `json:"npm_scopes" yaml:"npm_scopes"` + + // ComposerVendors are Composer vendors (e.g. "acme") served only from + // pushed packages, like NPMScopes. + ComposerVendors []string `json:"composer_vendors" yaml:"composer_vendors"` + + maxUploadBytes int64 +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (h *HostedConfig) TokenExpanded() string { + return expandEnv(h.Token) +} + +// MaxUploadBytes returns the parsed MaxUploadSize. Validate must run first. +func (h *HostedConfig) MaxUploadBytes() int64 { + return h.maxUploadBytes +} + +// Validate checks the hosted package settings, applying the default upload +// size if unset. +func (h *HostedConfig) Validate() error { + if h.MaxUploadSize == "" { + h.MaxUploadSize = defaultHostedMaxUploadSize + } + size, err := ParseSize(h.MaxUploadSize) + if err != nil { + return fmt.Errorf("invalid hosted.max_upload_size: %w", err) + } + if size <= 0 { + return fmt.Errorf("invalid hosted.max_upload_size %q: must be > 0", h.MaxUploadSize) + } + h.maxUploadBytes = size + + if h.Enabled && h.TokenExpanded() == "" { + return fmt.Errorf("hosted.token is required when hosted.enabled is true") + } + for _, scope := range h.NPMScopes { + if len(scope) < 2 || scope[0] != '@' || strings.Contains(scope, "/") { + return fmt.Errorf("invalid hosted.npm_scopes entry %q: must be a scope such as \"@acme\"", scope) + } + } + for _, vendor := range h.ComposerVendors { + if vendor == "" || strings.Contains(vendor, "/") { + return fmt.Errorf("invalid hosted.composer_vendors entry %q: must be a vendor such as \"acme\"", vendor) + } + } + return nil +} diff --git a/internal/config/hosted_test.go b/internal/config/hosted_test.go new file mode 100644 index 00000000..e7ad4515 --- /dev/null +++ b/internal/config/hosted_test.go @@ -0,0 +1,97 @@ +package config + +import ( + "slices" + "testing" +) + +func TestHostedConfigValidate(t *testing.T) { + t.Setenv("HOSTED_TEST_TOKEN", "from-env") + + tests := []struct { + name string + hosted HostedConfig + wantErr bool + }{ + {name: "disabled without token", hosted: HostedConfig{}}, + {name: "enabled without token", hosted: HostedConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", hosted: HostedConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"}}, + {name: "token env reference unset", hosted: HostedConfig{Enabled: true, Token: "${HOSTED_TEST_UNSET}"}, wantErr: true}, + {name: "invalid upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "lots"}, wantErr: true}, + {name: "zero upload size", hosted: HostedConfig{Enabled: true, Token: "s3cret", MaxUploadSize: "0"}, wantErr: true}, + {name: "npm scopes", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme", "@acme-internal"}}}, + {name: "npm scope without @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"acme"}}, wantErr: true}, + {name: "npm scope with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@acme/private"}}, wantErr: true}, + {name: "bare @", hosted: HostedConfig{Enabled: true, Token: "s3cret", NPMScopes: []string{"@"}}, wantErr: true}, + {name: "composer vendors", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme", "acme-internal"}}}, + {name: "composer vendor with package", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{"acme/private"}}, wantErr: true}, + {name: "empty composer vendor", hosted: HostedConfig{Enabled: true, Token: "s3cret", ComposerVendors: []string{""}}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.hosted.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestHostedConfigDefaults(t *testing.T) { + h := HostedConfig{Enabled: true, Token: "${HOSTED_TEST_TOKEN}"} + t.Setenv("HOSTED_TEST_TOKEN", "from-env") + if err := h.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := h.TokenExpanded(); got != "from-env" { + t.Errorf("TokenExpanded() = %q, want from-env", got) + } + if got := h.MaxUploadBytes(); got != 100<<20 { + t.Errorf("MaxUploadBytes() = %d, want %d", got, 100<<20) + } + + h.MaxUploadSize = "2GB" + if err := h.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := h.MaxUploadBytes(); got != 2<<30 { + t.Errorf("MaxUploadBytes() = %d, want %d", got, int64(2)<<30) + } +} + +func TestValidateRejectsHostedWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Hosted = HostedConfig{Enabled: true, Token: "s3cret"} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted hosted.enabled with storage.cache_artifacts: false") + } +} + +func TestLoadFromEnvHosted(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_HOSTED_ENABLED", "true") + t.Setenv("PROXY_HOSTED_TOKEN", "env-token") + t.Setenv("PROXY_HOSTED_MAX_UPLOAD_SIZE", "50MB") + t.Setenv("PROXY_HOSTED_NPM_SCOPES", "@acme, @acme-internal") + t.Setenv("PROXY_HOSTED_COMPOSER_VENDORS", "acme") + + cfg.LoadFromEnv() + + if !cfg.Hosted.Enabled { + t.Error("Hosted.Enabled not loaded from PROXY_HOSTED_ENABLED") + } + if cfg.Hosted.Token != "env-token" { + t.Errorf("Hosted.Token = %q, want env-token", cfg.Hosted.Token) + } + if cfg.Hosted.MaxUploadSize != "50MB" { + t.Errorf("Hosted.MaxUploadSize = %q, want 50MB", cfg.Hosted.MaxUploadSize) + } + if !slices.Equal(cfg.Hosted.NPMScopes, []string{"@acme", "@acme-internal"}) { + t.Errorf("Hosted.NPMScopes = %q", cfg.Hosted.NPMScopes) + } + if !slices.Equal(cfg.Hosted.ComposerVendors, []string{"acme"}) { + t.Errorf("Hosted.ComposerVendors = %q", cfg.Hosted.ComposerVendors) + } +} diff --git a/internal/database/local_artifacts.go b/internal/database/local_artifacts.go new file mode 100644 index 00000000..83197768 --- /dev/null +++ b/internal/database/local_artifacts.go @@ -0,0 +1,70 @@ +package database + +import ( + "database/sql" + "fmt" + "time" +) + +// LocalArtifact is a pushed artifact with the integrity of its version. +type LocalArtifact struct { + Artifact + Integrity sql.NullString `db:"integrity"` +} + +// GetLocalArtifacts returns the artifacts pushed to the proxy for a package, +// ordered by version and filename. +func (db *DB) GetLocalArtifacts(packagePURL string) ([]LocalArtifact, error) { + var artifacts []LocalArtifact + query := db.Rebind(` + SELECT a.id, a.version_purl, a.filename, a.upstream_url, a.storage_path, a.content_hash, + a.size, a.content_type, a.fetched_at, a.hit_count, a.last_accessed_at, + a.created_at, a.updated_at, a.source, a.manifest, v.integrity + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + WHERE v.package_purl = ? AND a.source = 'local' + ORDER BY a.version_purl, a.filename + `) + if err := db.Select(&artifacts, query, packagePURL); err != nil { + return nil, err + } + return artifacts, nil +} + +// DeleteLocalArtifacts removes the pushed artifacts of a version and queues +// their stored objects for deletion in the same transaction, so a failed +// queue keeps the records instead of orphaning the objects. Artifacts cached +// from an upstream are left alone. It returns how many records it removed. +func (db *DB) DeleteLocalArtifacts(versionPURL string) (int, error) { + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + var paths []string + selectQuery := db.Rebind(` + SELECT storage_path FROM artifacts + WHERE version_purl = ? AND source = 'local' AND storage_path IS NOT NULL + `) + if err := tx.Select(&paths, selectQuery, versionPURL); err != nil { + return 0, fmt.Errorf("reading local artifacts: %w", err) + } + + res, err := tx.Exec(db.Rebind(`DELETE FROM artifacts WHERE version_purl = ? AND source = 'local'`), versionPURL) + if err != nil { + return 0, fmt.Errorf("deleting local artifacts: %w", err) + } + deleted, err := res.RowsAffected() + if err != nil { + return 0, err + } + + queuedAt := time.Now().UTC() + for _, path := range paths { + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), path, queuedAt); err != nil { + return 0, fmt.Errorf("queueing %s for deletion: %w", path, err) + } + } + return int(deleted), tx.Commit() +} diff --git a/internal/database/local_artifacts_test.go b/internal/database/local_artifacts_test.go new file mode 100644 index 00000000..81d5ec8b --- /dev/null +++ b/internal/database/local_artifacts_test.go @@ -0,0 +1,297 @@ +package database + +import ( + "database/sql" + "path/filepath" + "testing" + "time" +) + +func seedSourcedArtifact(t *testing.T, db *DB, name, source string, size int64, fetched time.Time) { + t.Helper() + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@1.0.0" + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + art := &Artifact{ + VersionPURL: versionPURL, + Filename: name + "-1.0.0.tgz", + StoragePath: sql.NullString{String: "npm/" + name + "/1.0.0/" + name + "-1.0.0.tgz", Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: fetched, Valid: true}, + Source: source, + } + if source != SourceLocal { + art.UpstreamURL = "https://registry.example/" + name + "-1.0.0.tgz" + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +func TestArtifactSourceDefaultsToUpstream(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "cached", "", 100, time.Now()) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, time.Now()) + + cached, err := db.GetArtifact("pkg:npm/cached@1.0.0", "cached-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact(cached): %v", err) + } + if cached.Source != SourceUpstream { + t.Errorf("cached artifact source = %q, want %q", cached.Source, SourceUpstream) + } + + pushed, err := db.GetArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact(pushed): %v", err) + } + if pushed.Source != SourceLocal { + t.Errorf("pushed artifact source = %q, want %q", pushed.Source, SourceLocal) + } + if pushed.UpstreamURL != "" { + t.Errorf("pushed artifact upstream_url = %q, want empty", pushed.UpstreamURL) + } + }) +} + +func TestLocalArtifactsAreNotEvictable(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "cached", SourceUpstream, 1000, time.Now()) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 5000, time.Now()) + + total, err := db.GetTotalCacheSize() + if err != nil { + t.Fatalf("GetTotalCacheSize: %v", err) + } + if total != 1000 { + t.Errorf("total cache size = %d, want 1000 (local artifacts excluded)", total) + } + + lru, err := db.GetLeastRecentlyUsedArtifacts(10) + if err != nil { + t.Fatalf("GetLeastRecentlyUsedArtifacts: %v", err) + } + if len(lru) != 1 || lru[0].Filename != "cached-1.0.0.tgz" { + t.Errorf("LRU artifacts = %+v, want only cached-1.0.0.tgz", lru) + } + }) +} + +func TestLocalArtifactsAreNotRetentionCandidates(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + old := time.Now().Add(-48 * time.Hour) + cutoff := time.Now().Add(-24 * time.Hour) + seedSourcedArtifact(t, db, "cached", SourceUpstream, 100, old) + seedSourcedArtifact(t, db, "pushed", SourceLocal, 100, old) + + maxID, err := db.MaxArtifactID() + if err != nil { + t.Fatalf("MaxArtifactID: %v", err) + } + candidates, err := db.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatalf("GetRetentionCandidates: %v", err) + } + got := candidateNames(candidates) + if _, ok := got["pushed-1.0.0.tgz"]; ok { + t.Error("local artifact returned as a retention candidate") + } + if _, ok := got["cached-1.0.0.tgz"]; !ok { + t.Error("upstream artifact missing from retention candidates") + } + + cleared, err := db.ClearExpiredArtifact("pkg:npm/pushed@1.0.0", "pushed-1.0.0.tgz", "npm/pushed/1.0.0/pushed-1.0.0.tgz", cutoff) + if err != nil || cleared { + t.Errorf("ClearExpiredArtifact(local) = %v, %v, want false, nil", cleared, err) + } + }) +} + +func TestMigrationAddsArtifactSource(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN source`); err != nil { + t.Fatalf("drop source column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '012_add_artifact_source'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if _, err := sqlDB.Exec(`INSERT INTO artifacts (version_purl, filename, upstream_url, created_at, updated_at) + VALUES ('pkg:npm/old@1.0.0', 'old-1.0.0.tgz', 'https://registry.example/old-1.0.0.tgz', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("seed artifact: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + + art, err := db.GetArtifact("pkg:npm/old@1.0.0", "old-1.0.0.tgz") + if err != nil { + t.Fatalf("GetArtifact: %v", err) + } + if art.Source != SourceUpstream { + t.Errorf("existing artifact source = %q, want %q", art.Source, SourceUpstream) + } +} + +func TestGetLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "lodash", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) + + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil { + t.Fatalf("GetLocalArtifacts: %v", err) + } + if len(local) != 1 || local[0].Filename != "request-1.0.0.tgz" { + t.Fatalf("local artifacts of request = %+v, want only request-1.0.0.tgz", local) + } + + upstreamOnly, err := db.GetLocalArtifacts("pkg:npm/express") + if err != nil { + t.Fatalf("GetLocalArtifacts(express): %v", err) + } + if len(upstreamOnly) != 0 { + t.Errorf("local artifacts of express = %+v, want none", upstreamOnly) + } + }) +} + +func TestDeleteLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedSourcedArtifact(t, db, "request", SourceLocal, 100, time.Now()) + seedSourcedArtifact(t, db, "express", SourceUpstream, 100, time.Now()) + const pushedPath = "npm/request/1.0.0/request-1.0.0.tgz" + + deleted, err := db.DeleteLocalArtifacts("pkg:npm/request@1.0.0") + if err != nil || deleted != 1 { + t.Fatalf("DeleteLocalArtifacts(request) = %d, %v, want 1, nil", deleted, err) + } + art, err := db.GetArtifact("pkg:npm/request@1.0.0", "request-1.0.0.tgz") + if err != nil || art != nil { + t.Errorf("GetArtifact after delete = %+v, %v, want nil, nil", art, err) + } + var queued int + if err := db.Get(&queued, db.Rebind(`SELECT COUNT(*) FROM pending_deletes WHERE path = ?`), pushedPath); err != nil || queued != 1 { + t.Errorf("queued deletes for %s = %d, %v, want 1", pushedPath, queued, err) + } + + deleted, err = db.DeleteLocalArtifacts("pkg:npm/express@1.0.0") + if err != nil || deleted != 0 { + t.Errorf("DeleteLocalArtifacts(upstream) = %d, %v, want 0, nil", deleted, err) + } + if art, err := db.GetArtifact("pkg:npm/express@1.0.0", "express-1.0.0.tgz"); err != nil || art == nil { + t.Errorf("upstream artifact removed by DeleteLocalArtifacts: %+v, %v", art, err) + } + }) +} + +func TestLocalArtifactManifestAndIntegrity(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{ + PURL: "pkg:npm/request@2.88.3", + PackagePURL: "pkg:npm/request", + Integrity: sql.NullString{String: "sha512-abc", Valid: true}, + }); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + const manifest = `{"name":"request","version":"2.88.3"}` + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/request@2.88.3", + Filename: "request-2.88.3.tgz", + StoragePath: sql.NullString{String: "npm/request/2.88.3/x/request-2.88.3.tgz", Valid: true}, + ContentHash: sql.NullString{String: "00", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + Source: SourceLocal, + Manifest: sql.NullString{String: manifest, Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil || len(local) != 1 { + t.Fatalf("GetLocalArtifacts = %+v, %v", local, err) + } + if local[0].Manifest.String != manifest || local[0].Integrity.String != "sha512-abc" { + t.Errorf("manifest/integrity = %q/%q", local[0].Manifest.String, local[0].Integrity.String) + } + }) +} + +func TestMigrationAddsArtifactManifest(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE artifacts DROP COLUMN manifest`); err != nil { + t.Fatalf("drop manifest column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '013_add_artifact_manifest'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + has, err := db.HasColumn("artifacts", "manifest") + if err != nil || !has { + t.Errorf("artifacts.manifest after migration = %v, %v", has, err) + } +} + +func TestUpsertPackageKeepsSource(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + pkg := &Package{PURL: "pkg:npm/request", Ecosystem: "npm", Name: "request"} + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if _, err := db.Exec(db.Rebind(`UPDATE packages SET source = ? WHERE purl = ?`), "deps.dev", pkg.PURL); err != nil { + t.Fatalf("setting source: %v", err) + } + if err := db.UpsertPackage(pkg); err != nil { + t.Fatalf("second UpsertPackage: %v", err) + } + got, err := db.GetPackageByPURL(pkg.PURL) + if err != nil || got == nil { + t.Fatalf("GetPackageByPURL: %+v, %v", got, err) + } + if got.Source.String != "deps.dev" { + t.Errorf("source after upsert = %q, want deps.dev", got.Source.String) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 2652bc17..cd7c51ce 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -216,7 +216,7 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE version_purl = ? AND filename = ? `) err := db.Get(&a, query, versionPURL, filename) @@ -284,7 +284,7 @@ func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE storage_path = ? `) err := db.Get(&a, query, storagePath) @@ -302,7 +302,7 @@ func (db *DB) GetArtifactsByVersionPURL(versionPURL string) ([]Artifact, error) query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts WHERE version_purl = ? ORDER BY filename `) @@ -367,8 +367,8 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + created_at, updated_at, source, manifest) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = EXCLUDED.upstream_url, storage_path = EXCLUDED.storage_path, @@ -376,15 +376,17 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er size = EXCLUDED.size, content_type = EXCLUDED.content_type, fetched_at = EXCLUDED.fetched_at, - updated_at = EXCLUDED.updated_at - WHERE artifacts.storage_path IS NOT DISTINCT FROM $13 + updated_at = EXCLUDED.updated_at, + source = EXCLUDED.source, + manifest = EXCLUDED.manifest + WHERE artifacts.storage_path IS NOT DISTINCT FROM $15 ` } else { query = ` INSERT INTO artifacts (version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + created_at, updated_at, source, manifest) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(version_purl, filename) DO UPDATE SET upstream_url = excluded.upstream_url, storage_path = excluded.storage_path, @@ -392,14 +394,17 @@ func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, er size = excluded.size, content_type = excluded.content_type, fetched_at = excluded.fetched_at, - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + source = excluded.source, + manifest = excluded.manifest WHERE artifacts.storage_path IS ? ` } res, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, previous, + a.Size, a.ContentType, inUTC(a.FetchedAt), a.HitCount, inUTC(a.LastAccessedAt), now, now, + a.sourceOrUpstream(), a.Manifest, previous, ) if err != nil { return false, fmt.Errorf("upserting artifact: %w", err) @@ -445,9 +450,9 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { query := db.Rebind(` SELECT id, version_purl, filename, upstream_url, storage_path, content_hash, size, content_type, fetched_at, hit_count, last_accessed_at, - created_at, updated_at + created_at, updated_at, source FROM artifacts - WHERE storage_path IS NOT NULL + WHERE storage_path IS NOT NULL AND source <> 'local' ORDER BY last_accessed_at ASC NULLS FIRST LIMIT ? `) @@ -460,7 +465,7 @@ func (db *DB) GetLeastRecentlyUsedArtifacts(limit int) ([]Artifact, error) { func (db *DB) GetTotalCacheSize() (int64, error) { var total sql.NullInt64 - err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL`) + err := db.Get(&total, `SELECT SUM(size) FROM artifacts WHERE storage_path IS NOT NULL AND source <> 'local'`) if err != nil { return 0, err } diff --git a/internal/database/retention.go b/internal/database/retention.go index cd6e11fe..90fffd2a 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -36,6 +36,7 @@ func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { // accessed since. It takes the cutoff twice. const expiredCondition = ` storage_path IS NOT NULL + AND source <> 'local' AND fetched_at < ? AND (last_accessed_at IS NULL OR last_accessed_at < ?)` diff --git a/internal/database/schema.go b/internal/database/schema.go index b8d3f93f..ce124907 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -72,7 +72,9 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME + updated_at DATETIME, + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -190,7 +192,9 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP + updated_at TIMESTAMP, + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -267,7 +271,9 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count INTEGER DEFAULT 0, last_accessed_at DATETIME, created_at DATETIME, - updated_at DATETIME + updated_at DATETIME, + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -288,7 +294,9 @@ CREATE TABLE IF NOT EXISTS artifacts ( hit_count BIGINT DEFAULT 0, last_accessed_at TIMESTAMP, created_at TIMESTAMP, - updated_at TIMESTAMP + updated_at TIMESTAMP, + source TEXT NOT NULL DEFAULT 'upstream', + manifest TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -401,6 +409,8 @@ var migrations = []migration{ {"009_add_pending_deletes", migrateAddPendingDeletes}, {"010_add_pending_deletes_queued_at_index", migrateAddPendingDeletesQueuedAtIndex}, {"011_add_metadata_dates", migrateAddMetadataDates}, + {"012_add_artifact_source", migrateAddArtifactSource}, + {"013_add_artifact_manifest", migrateAddArtifactManifest}, } // isTableNotFound returns true if the error indicates a missing table. @@ -778,3 +788,35 @@ func (db *DB) EnsureMetadataCacheTable() error { } return nil } + +// migrateAddArtifactSource records whether an artifact is a cached copy of an +// upstream registry or was pushed to the proxy. Existing rows are all upstream. +func migrateAddArtifactSource(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "source") + if err != nil { + return fmt.Errorf("checking column source: %w", err) + } + if hasCol { + return nil + } + if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN source TEXT NOT NULL DEFAULT 'upstream'`); err != nil { + return fmt.Errorf("adding column source to artifacts: %w", err) + } + return nil +} + +// migrateAddArtifactManifest stores the manifest read from a pushed archive, +// which the metadata of its ecosystem lists the version with. +func migrateAddArtifactManifest(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "manifest") + if err != nil { + return fmt.Errorf("checking column manifest: %w", err) + } + if hasCol { + return nil + } + if _, err := db.Exec(`ALTER TABLE artifacts ADD COLUMN manifest TEXT`); err != nil { + return fmt.Errorf("adding column manifest to artifacts: %w", err) + } + return nil +} diff --git a/internal/database/types.go b/internal/database/types.go index 9c4fdd6a..e931f5b5 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -125,6 +125,14 @@ func decodePURLComponent(s string) string { return decoded } +// Artifact sources. Upstream artifacts are a cache of a registry and may be +// evicted; local artifacts were pushed to the proxy and are only removed by a +// delete. +const ( + SourceUpstream = "upstream" + SourceLocal = "local" +) + // Artifact represents a cached artifact in the database. // This table is proxy-specific and not part of git-pkgs. type Artifact struct { @@ -141,6 +149,15 @@ type Artifact struct { LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Source string `db:"source" json:"source"` + Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` +} + +func (a *Artifact) sourceOrUpstream() string { + if a.Source == "" { + return SourceUpstream + } + return a.Source } // IsCached returns true if the artifact has been fetched and stored locally. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 5b9ab965..9dac530c 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -148,6 +148,22 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R repoURL, cacheKey := h.sourceFor(packageName) upstreamURL := fmt.Sprintf("%s/p2/%s/%s.json", repoURL, vendor, pkg) + basePackage := strings.TrimSuffix(packageName, composerDevFile) + local, err := h.composerLocalEntries(basePackage, basePackage != packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed composer versions", "package", basePackage, "error", err) + http.Error(w, "failed to read pushed versions", http.StatusInternalServerError) + return + } + if h.proxy.hostedOnly("composer", basePackage) { + h.serveHostedOnlyMetadata(w, r, basePackage, cacheKey, local) + return + } + if len(local) > 0 { + h.serveMetadataWithLocal(w, r, basePackage, upstreamURL, cacheKey, local) + return + } + if stored, ok := h.proxy.storedRewrite("composer", cacheKey, h.proxyURL, cacheKey); ok { serveRewrittenMetadata(w, r, stored) return @@ -195,6 +211,15 @@ func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) { // rewriteMetadataKeeping is rewriteMetadata that also reports the versions // it kept, each as its package name and version. func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, error) { + return h.rewriteMetadataWithLocal(body, "", nil) +} + +// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the +// pushed versions in local under localPackage, in place of any upstream entry +// with the same version. Cooldown only filters upstream versions. A document +// that does not list localPackage at all is replaced by one holding only the +// pushed versions. +func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage string, local []composerLocalEntry) ([]byte, []string, error) { if !json.Valid(body) { return nil, nil, errors.New("composer metadata is not valid JSON") } @@ -206,6 +231,7 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, var out bytes.Buffer var kept []string + listed := false out.Grow(len(body) + len(body)/8) err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { packages := m.value(body) @@ -221,12 +247,20 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, if err != nil { return false, err } - return true, h.writeVersions(out, packageName, versions, minified, &kept) + var pushed []composerLocalEntry + if packageName == localPackage { + pushed, listed = local, true + } + return true, h.writeVersions(out, packageName, versions, minified, &kept, pushed) }) }) if err != nil { return nil, nil, err } + if len(local) > 0 && !listed { + document, err := localComposerDocument(localPackage, local) + return document, nil, err + } return out.Bytes(), kept, nil } @@ -239,7 +273,10 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, // entry itself unless filtered versions were dropped before it; then it also // carries the changes those versions made. Every version gets its own dist, // since the proxied URL names the version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string) error { +// +// The pushed versions in local follow the upstream ones and take the place of +// upstream entries with the same version. +func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []composerLocalEntry) error { packagePURL := canonicalPackagePURL("composer", packageName) upstream, written := newComposerFields(), newComposerFields() devReset, first := false, true @@ -265,7 +302,7 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v } version := upstream.string("version") - if h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { + if pushedComposerVersion(local, version) || h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { return nil } @@ -284,8 +321,30 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v *kept = append(*kept, packageName+"\x00"+version) return nil }) + if err != nil { + return err + } + for _, e := range local { + pushed := newComposerFields() + if err := pushed.apply(e.entry, false); err != nil { + return err + } + if !minified { + written.reset() + } + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + composerDevReset + `",`) + devReset = false + } + h.writeVersion(out, packageName, e.version, pushed, written) + *kept = append(*kept, packageName+"\x00"+e.version) + } out.WriteByte(']') - return err + return nil } // writeVersion writes the fields of upstream that differ from written, with @@ -563,6 +622,20 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) h.proxy.Logger.Info("composer download request", "package", packageName, "version", version, "filename", filename) + local, err := h.proxy.GetLocalArtifact(r.Context(), "composer", packageName, version, filename) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read pushed package") + return + } + if local != nil { + ServeArtifactRequest(w, r, local) + return + } + if h.proxy.hostedOnly("composer", packageName) { + http.Error(w, "version not found", http.StatusNotFound) + return + } + // We need to fetch the metadata to get the actual download URL since // Packagist URLs include a hash. Packagist serves dev versions (e.g. // "3.x-dev", "dev-master") from a separate "~dev" metadata file, while diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go new file mode 100644 index 00000000..a42a27a1 --- /dev/null +++ b/internal/handler/composer_hosted.go @@ -0,0 +1,124 @@ +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +// composerRootOnlyKeys are composer.json keys that only apply to the root +// project. Repositories leave them out of the version entries they serve. +var composerRootOnlyKeys = []string{"repositories", "config", "minimum-stability", "prefer-stable"} + +// composerLocalEntry is a pushed version as Composer metadata lists it. +type composerLocalEntry struct { + version string + entry json.RawMessage +} + +// composerLocalEntries returns the pushed versions of a package for one of +// its metadata files: the dev versions for the ~dev file, the tagged ones for +// the other. +func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([]composerLocalEntry, error) { + local, err := h.proxy.localVersions("composer", packageName) + if err != nil || len(local) == 0 { + return nil, err + } + var entries []composerLocalEntry + for _, lv := range local { + if isDevVersion(lv.Version) != dev { + continue + } + var manifest map[string]any + if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { + return nil, fmt.Errorf("decoding manifest of %s %s: %w", packageName, lv.Version, err) + } + for _, key := range composerRootOnlyKeys { + delete(manifest, key) + } + manifest["name"] = packageName + manifest["version"] = lv.Version + manifest["time"] = lv.PushedAt.Format(time.RFC3339) + manifest["dist"] = map[string]string{ + "type": "zip", + "url": h.localDistURL(packageName, lv.Version, lv.Filename), + "reference": lv.SHA256, + "shasum": "", + } + entry, err := json.Marshal(manifest) + if err != nil { + return nil, err + } + entries = append(entries, composerLocalEntry{version: lv.Version, entry: entry}) + } + return entries, nil +} + +func (h *ComposerHandler) localDistURL(packageName, version, filename string) string { + return fmt.Sprintf("%s/composer/files/%s/%s/%s", h.proxyURL, packageName, url.PathEscape(version), filename) +} + +// pushedComposerVersion reports whether version is one of the pushed ones. +// Tags are listed with and without a leading v, and both name one version. +func pushedComposerVersion(local []composerLocalEntry, version string) bool { + for _, e := range local { + if strings.TrimPrefix(e.version, "v") == strings.TrimPrefix(version, "v") { + return true + } + } + return false +} + +// serveMetadataWithLocal serves a package's metadata with its pushed versions +// added, bypassing the rewrite cache like the npm handler does. When the +// upstream does not know the package, the document holds only the pushed +// versions. +func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, cacheKey string, local []composerLocalEntry) { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "composer", cacheKey, upstreamURL, contentTypeJSON) + var out []byte + var err error + switch { + case errors.Is(upstream.err, ErrUpstreamNotFound): + out, err = localComposerDocument(packageName, local) + case upstream.err != nil: + h.proxy.Logger.Error("upstream request failed", "error", upstream.err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + default: + out, _, err = h.rewriteMetadataWithLocal(upstream.body, packageName, local) + } + if err != nil { + h.proxy.Logger.Error("failed to add pushed versions to composer metadata", "package", packageName, "error", err) + http.Error(w, "failed to rewrite package metadata", http.StatusBadGateway) + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) +} + +// serveHostedOnlyMetadata answers a metadata request for a package of a +// hosted-only vendor from its pushed versions alone. +func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, cacheKey string, local []composerLocalEntry) { + if len(local) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return + } + out, err := localComposerDocument(packageName, local) + if err != nil { + h.proxy.Logger.Error("failed to build composer metadata from pushed versions", "package", packageName, "error", err) + http.Error(w, "failed to build package metadata", http.StatusInternalServerError) + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) +} + +func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { + entries := make([]json.RawMessage, 0, len(local)) + for _, e := range local { + entries = append(entries, e.entry) + } + return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) +} diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go new file mode 100644 index 00000000..048c291c --- /dev/null +++ b/internal/handler/composer_hosted_test.go @@ -0,0 +1,245 @@ +package handler + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +const monologMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ + {"name":"monolog/monolog","version":"3.5.0","version_normalized":"3.5.0.0", + "require":{"php":">=8.1","psr/log":"^2.0"},"support":{"issues":"https://github.com/Seldaek/monolog/issues"}, + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/aaa","reference":"aaa","shasum":""}, + "time":"2023-10-27T15:32:31+00:00"}, + {"version":"3.4.0","version_normalized":"3.4.0.0", + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/bbb","reference":"bbb","shasum":""}} +]}}` + +const monologDevMetadata = `{"minified":"composer/2.0","packages":{"monolog/monolog":[ + {"name":"monolog/monolog","version":"dev-main","version_normalized":"dev-main", + "dist":{"type":"zip","url":"https://api.github.com/repos/Seldaek/monolog/zipball/ccc","reference":"ccc","shasum":""}} +]}}` + +type composerHostedFixture struct { + proxy *Proxy + handler http.Handler + upstreamRequests *int + fetcher *mockFetcher +} + +func newComposerHostedFixture(t *testing.T, upstream map[string]string) composerHostedFixture { + t.Helper() + upstreamRequests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + body, ok := upstream[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HTTPClient = srv.Client() + h := NewComposerHandlerWithUpstreams(proxy, "http://proxy.local", srv.URL, srv.URL) + return composerHostedFixture{proxy: proxy, handler: h.Routes(), upstreamRequests: &upstreamRequests, fetcher: fetcher} +} + +func (f composerHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { + t.Helper() + target, err := HostedTargetFor("composer", name, version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "composer", target, bytes.NewReader(composerZip("", manifest)), "") + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return pushed +} + +func (f composerHostedFixture) versions(t *testing.T, path, packageName string) map[string]map[string]any { + t.Helper() + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s status = %d, body %s", path, rec.Code, rec.Body) + } + byVersion := map[string]map[string]any{} + for _, v := range composerVersions(t, rec.Body.Bytes(), packageName) { + version, _ := v["version"].(string) + if _, dup := byVersion[version]; dup { + t.Fatalf("version %s listed twice:\n%s", version, rec.Body) + } + byVersion[version] = v + } + return byVersion +} + +func TestComposerMetadataListsPushedVersion(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + pushed := f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"},"repositories":[{"type":"vcs","url":"x"}]}`) + + versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + local, ok := versions["3.6.0"] + if !ok { + t.Fatalf("pushed version missing: %v", versions) + } + if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { + t.Errorf("require = %v, want the pushed composer.json's", local["require"]) + } + for _, inherited := range []string{"version_normalized", "support", "repositories"} { + if _, ok := local[inherited]; ok { + t.Errorf("pushed version carries %s = %v", inherited, local[inherited]) + } + } + dist, _ := local["dist"].(map[string]any) + if dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.6.0/monolog-3.6.0.zip" || + dist["type"] != "zip" || dist["reference"] != pushed.SHA256 || dist["shasum"] != "" { + t.Errorf("dist = %v", dist) + } + if local["time"] != pushed.PushedAt.Format(time.RFC3339) { + t.Errorf("time = %v, want %s", local["time"], pushed.PushedAt.Format(time.RFC3339)) + } + if _, ok := versions["3.4.0"]; !ok { + t.Error("upstream version 3.4.0 missing") + } +} + +func TestComposerMetadataPushedVersionReplacesUpstream(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"}}`) + + versions := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + local := versions["3.5.0"] + if require, _ := local["require"].(map[string]any); require["psr/log"] != "^3.0" { + t.Errorf("require = %v, want the pushed composer.json's", local["require"]) + } + if dist, _ := local["dist"].(map[string]any); dist["url"] != "http://proxy.local/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip" { + t.Errorf("dist = %v", dist) + } + if v34 := versions["3.4.0"]; v34["version_normalized"] != "3.4.0.0" { + t.Errorf("upstream 3.4.0 expanded to %v", v34) + } +} + +func TestComposerMetadataSplitsPushedDevVersions(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{ + "/p2/monolog/monolog.json": monologMetadata, + "/p2/monolog/monolog~dev.json": monologDevMetadata, + }) + f.push(t, "monolog/monolog", "dev-patched", `{"name":"monolog/monolog"}`) + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + + tagged := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + dev := f.versions(t, "/p2/monolog/monolog~dev.json", "monolog/monolog") + if _, ok := tagged["dev-patched"]; ok { + t.Error("pushed dev version listed with the tagged versions") + } + if _, ok := dev["dev-patched"]; !ok { + t.Errorf("pushed dev version missing from the dev file: %v", dev) + } + if _, ok := dev["3.6.0"]; ok { + t.Error("pushed tagged version listed with the dev versions") + } + if _, ok := dev["dev-main"]; !ok { + t.Error("upstream dev-main missing") + } +} + +func TestComposerMetadataForPackageOnlyPushed(t *testing.T) { + f := newComposerHostedFixture(t, nil) + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private","require":{"php":">=8.1"}}`) + + versions := f.versions(t, "/p2/acme/private.json", "acme/private") + if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { + t.Errorf("versions = %v, want only 1.0.0", versions) + } + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/p2/acme/private~dev.json", nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("dev file of a package with no pushed dev versions status = %d, want 404", rec.Code) + } +} + +func TestComposerHostedOnlyVendorNeverAsksUpstream(t *testing.T) { + squatter := `{"packages":{"acme/private":[{"name":"acme/private","version":"1.0.1", + "dist":{"type":"zip","url":"https://api.github.com/repos/evil/private/zipball/ddd","reference":"ddd"}}]}}` + f := newComposerHostedFixture(t, map[string]string{ + "/p2/acme/private.json": squatter, + "/p2/monolog/monolog.json": monologMetadata, + }) + f.proxy.HostedComposerVendors = []string{"acme"} + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) + + versions := f.versions(t, "/p2/acme/private.json", "acme/private") + if _, ok := versions["1.0.0"]; !ok || len(versions) != 1 { + t.Errorf("versions = %v, want only the pushed 1.0.0", versions) + } + for _, path := range []string{"/p2/acme/private~dev.json", "/p2/acme/other.json", "/files/acme/private/1.0.1/ddd.zip"} { + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("GET %s status = %d, want 404", path, rec.Code) + } + } + if *f.upstreamRequests != 0 { + t.Errorf("hosted-only vendor sent %d requests upstream", *f.upstreamRequests) + } + + f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog") + if *f.upstreamRequests == 0 { + t.Error("package outside the vendor was not fetched upstream") + } +} + +func TestComposerDownloadOfUnreadablePushedArchive(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.5.0", `{"name":"monolog/monolog"}`) + removeStored(t, f.proxy, "pkg:composer/monolog/monolog@3.5.0", "monolog-3.5.0.zip") + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream archive", rec.Code, rec.Body) + } +} + +func TestComposerMetadataLeavesOutDeniedPushedVersions(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + setTestDenylist(t, f.proxy, "pkg:composer/monolog/monolog@3.6.0") + + if _, ok := f.versions(t, "/p2/monolog/monolog.json", "monolog/monolog")["3.6.0"]; ok { + t.Error("denied pushed version listed") + } +} + +func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { + const manifest = `{"name":"monolog/monolog"}` + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream archive")), ContentType: "application/zip"} + f.push(t, "monolog/monolog", "3.5.0", manifest) + f.proxy.StreamArtifacts = true + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != string(composerZip("", manifest)) { + t.Errorf("download status = %d, want 200 and the pushed archive", rec.Code) + } + if f.fetcher.fetchCalled { + t.Error("a pushed archive was fetched upstream while streaming artifacts") + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c8dee1c6..45c7006b 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -162,6 +162,7 @@ type Proxy struct { MetadataTTL time.Duration MetadataMaxSize int64 GradleReadOnly bool + HostedEnabled bool GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. @@ -174,6 +175,11 @@ type Proxy struct { DirectServeBaseURL string HTTPClient *http.Client + // HostedNPMScopes and HostedComposerVendors name namespaces served from + // pushed packages alone, without asking the upstream. + HostedNPMScopes []string + HostedComposerVendors []string + // StreamArtifacts streams artifacts from upstream without storing them. // Each request fetches its own copy: there is no cache to check and // nothing for concurrent misses to share. @@ -341,13 +347,48 @@ func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) e // checkCache looks up an artifact in the cache. Returns nil if not cached. // With StreamArtifacts set it always reports a miss, so entries stored before the // mode was enabled are never served. +// +// A miss on a pushed artifact is an error rather than a miss: the caller would +// otherwise fetch the upstream copy and replace the pushed one with it. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { + result, err := p.lookupCache(ctx, pkgPURL, versionPURL, filename) + if err != nil || result != nil || p.StreamArtifacts { + return result, err + } + return nil, p.localArtifactMiss(versionPURL, filename) +} + +// localArtifactMiss returns ErrHostedArtifactUnavailable when the artifact +// was pushed and is stored but couldn't be served, ErrHostedArtifactMissing +// when it was pushed and its stored object is gone, and nil otherwise. +func (p *Proxy) localArtifactMiss(versionPURL, filename string) error { + record, err := p.DB.GetArtifact(versionPURL, filename) + if err != nil { + return fmt.Errorf("checking artifact source: %w", err) + } + if record == nil || record.Source != database.SourceLocal { + return nil + } + if record.StoragePath.Valid { + return fmt.Errorf("%w: %s %s", ErrHostedArtifactUnavailable, versionPURL, filename) + } + return fmt.Errorf("%w: %s %s", ErrHostedArtifactMissing, versionPURL, filename) +} + +// lookupCache looks up an artifact in the cache. Returns nil if not cached. +func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { if p.Denylist.Denied(versionPURL) { return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) } if p.StreamArtifacts { return nil, nil } + return p.readStored(ctx, pkgPURL, versionPURL, filename) +} + +// readStored opens the stored object of an artifact, whether or not +// StreamArtifacts is set. Returns nil if no usable object is stored. +func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) @@ -1138,6 +1179,15 @@ func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg s JSONError(w, http.StatusForbidden, err.Error()) return } + if errors.Is(err, ErrHostedArtifactMissing) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if errors.Is(err, ErrHostedArtifactUnavailable) { + p.Logger.Error("failed to read pushed artifact", "error", err) + http.Error(w, "pushed package is temporarily unavailable", http.StatusServiceUnavailable) + return + } p.Logger.Error("failed to get artifact", "error", err) http.Error(w, clientMsg, http.StatusBadGateway) } diff --git a/internal/handler/hosted.go b/internal/handler/hosted.go new file mode 100644 index 00000000..41ccb930 --- /dev/null +++ b/internal/handler/hosted.go @@ -0,0 +1,370 @@ +package handler + +import ( + "context" + "crypto/sha512" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/proxy/internal/storage" +) + +var ( + ErrHostedUnsupported = errors.New("pushing packages is not supported for this ecosystem") + ErrHostedInvalidName = errors.New("invalid package name or version") + ErrHostedVersionNotFound = errors.New("no pushed artifacts for this version") + + // ErrHostedArtifactUnavailable and ErrHostedArtifactMissing report a + // pushed artifact the cache can't serve. They stop the caller from + // fetching the upstream copy, which would replace the pushed one. + ErrHostedArtifactUnavailable = errors.New("pushed artifact is stored but can't be read") + ErrHostedArtifactMissing = errors.New("pushed artifact is no longer stored") +) + +// HostedTarget is where a pushed archive is stored: the package name and +// version it is cached under and the filename clients request it by. Download +// handlers look artifacts up by exactly these values, so a push under any +// other name would never be served. +type HostedTarget struct { + Name string + Version string + Filename string +} + +// HostedArtifact describes one pushed archive. +type HostedArtifact struct { + Version string `json:"version"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + PushedAt time.Time `json:"pushed_at"` +} + +// hostedNaming describes how an ecosystem names pushed archives: which +// package names it has, split on "/", and the filename its clients request. +type hostedNaming struct { + validName func(segments []string) bool + filename func(name, version string) string +} + +var hostedNamings = map[string]hostedNaming{ + "npm": { + validName: func(segments []string) bool { + return len(segments) == 1 || (len(segments) == 2 && strings.HasPrefix(segments[0], "@")) + }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".tgz" }, + }, + "composer": { + validName: func(segments []string) bool { return len(segments) == vendorPackageParts }, + filename: func(name, version string) string { return lastSegment(name) + "-" + version + ".zip" }, + }, +} + +// HostedSupported reports whether packages can be pushed for ecosystem. +func HostedSupported(ecosystem string) bool { + _, ok := hostedNamings[ecosystem] + return ok +} + +func lastSegment(name string) string { + return name[strings.LastIndex(name, "/")+1:] +} + +// HostedTargetFor validates a push of name@version to ecosystem and returns +// where it is stored. +func HostedTargetFor(ecosystem, name, version string) (HostedTarget, error) { + naming, ok := hostedNamings[ecosystem] + if !ok { + return HostedTarget{}, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) + } + segments := strings.Split(name, "/") + if !naming.validName(segments) { + return HostedTarget{}, ErrHostedInvalidName + } + for _, segment := range append(segments, version) { + if !validHostedSegment(segment) { + return HostedTarget{}, ErrHostedInvalidName + } + } + return HostedTarget{Name: name, Version: version, Filename: naming.filename(name, version)}, nil +} + +func validHostedSegment(s string) bool { + if s == "" || s == "." || s == ".." || strings.ContainsAny(s, `/\`) { + return false + } + for _, c := range s { + if c < ' ' || c == 0x7f { + return false + } + } + return true +} + +// PushLocalArtifact stores body as the artifact for target and records it as +// local, replacing any cached upstream copy. Download handlers serve it from +// the cache without contacting the upstream, and eviction never removes it. +// The archive's manifest must name target; it is kept so the ecosystem's +// metadata can list the version with its dependencies. +func (p *Proxy) PushLocalArtifact(ctx context.Context, ecosystem string, target HostedTarget, body io.Reader, contentType string) (HostedArtifact, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, target.Name, target.Version) + if err != nil { + return HostedArtifact{}, err + } + if p.versionDenied(ecosystem, target.Name, target.Version) { + return HostedArtifact{}, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + + push := storedPush{ + storagePath: storage.FetchPath(ecosystem, target.Name, target.Version, storage.NewFetchID(), target.Filename), + contentType: contentType, + } + sha512Hash := sha512.New() + storeStart := time.Now() + push.size, push.sha256, err = p.Storage.Store(ctx, push.storagePath, io.TeeReader(body, sha512Hash)) + metrics.RecordStorageOperation("write", time.Since(storeStart)) + if err != nil { + metrics.RecordStorageError("write", metrics.FailureDetail{ + PURL: versionPURL, Filename: target.Filename, Path: push.storagePath, Err: err}) + return HostedArtifact{}, fmt.Errorf("storing artifact: %w", err) + } + push.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha512Hash.Sum(nil)) + + if err := p.checkPush(ctx, ecosystem, target, versionPURL, &push); err != nil { + p.discardStored(ctx, push.storagePath) + return HostedArtifact{}, err + } + + push.pushedAt = time.Now() + if err := p.recordLocalArtifact(ecosystem, target, pkgPURL, versionPURL, push); err != nil { + p.discardStored(ctx, push.storagePath) + return HostedArtifact{}, err + } + + return HostedArtifact{ + Version: target.Version, + Filename: target.Filename, + SHA256: push.sha256, + Size: push.size, + PushedAt: push.pushedAt.UTC(), + }, nil +} + +// storedPush is a pushed archive once it is in storage. +type storedPush struct { + storagePath string + sha256 string + integrity string + size int64 + contentType string + manifest json.RawMessage + pushedAt time.Time +} + +// checkPush reads the manifest of a stored push and runs the scanners on it. +func (p *Proxy) checkPush(ctx context.Context, ecosystem string, target HostedTarget, versionPURL string, push *storedPush) error { + stored, err := p.Storage.Open(ctx, push.storagePath) + if err != nil { + return fmt.Errorf("reading pushed archive: %w", err) + } + push.manifest, err = readHostedManifest(ecosystem, stored, target) + _ = stored.Close() + if err != nil { + return err + } + + if p.Scanners != nil && p.Scanners.Enabled() { + return p.runScan(ctx, ecosystem, target.Name, target.Version, target.Filename, versionPURL, push.storagePath, push.size, push.contentType) + } + return nil +} + +func (p *Proxy) discardStored(ctx context.Context, storagePath string) { + if err := p.Storage.Delete(context.WithoutCancel(ctx), storagePath); err != nil { + p.Logger.Warn("failed to delete pushed artifact from storage", "path", storagePath, "error", err) + } +} + +// recordLocalArtifact writes the package, version and artifact records. The +// version's integrity is set to the pushed bytes, so the cache's integrity +// check verifies them rather than an upstream archive's digest. An existing +// package record is kept as it is, since it may describe the upstream package. +func (p *Proxy) recordLocalArtifact(ecosystem string, target HostedTarget, pkgPURL, versionPURL string, push storedPush) error { + existing, err := p.DB.GetPackageByPURL(pkgPURL) + if err != nil { + return fmt.Errorf("reading package: %w", err) + } + if existing == nil { + if err := p.DB.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: target.Name}); err != nil { + return fmt.Errorf("upserting package: %w", err) + } + } + + if err := p.DB.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + Integrity: sql.NullString{String: push.integrity, Valid: true}, + }); err != nil { + return fmt.Errorf("upserting version: %w", err) + } + + if err := p.DB.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: target.Filename, + StoragePath: sql.NullString{String: push.storagePath, Valid: true}, + ContentHash: sql.NullString{String: push.sha256, Valid: true}, + Size: sql.NullInt64{Int64: push.size, Valid: true}, + ContentType: sql.NullString{String: push.contentType, Valid: push.contentType != ""}, + FetchedAt: sql.NullTime{Time: push.pushedAt, Valid: true}, + Source: database.SourceLocal, + Manifest: sql.NullString{String: string(push.manifest), Valid: true}, + }); err != nil { + return fmt.Errorf("upserting artifact: %w", err) + } + return nil +} + +// LocalArtifacts lists the archives pushed for a package. +func (p *Proxy) LocalArtifacts(ecosystem, name string) ([]HostedArtifact, error) { + pkgPURL := packageurl.MakeString(ecosystem, name, "") + if pkgPURL == "" { + return nil, fmt.Errorf("%w: %s %q", errUnsupportedPackageIdentity, ecosystem, name) + } + rows, err := p.DB.GetLocalArtifacts(pkgPURL) + if err != nil { + return nil, fmt.Errorf("listing local artifacts: %w", err) + } + listed := make([]HostedArtifact, 0, len(rows)) + for _, row := range rows { + listed = append(listed, HostedArtifact{ + Version: database.VersionFromPURL(row.VersionPURL), + Filename: row.Filename, + SHA256: row.ContentHash.String, + Size: row.Size.Int64, + PushedAt: row.FetchedAt.Time.UTC(), + }) + } + return listed, nil +} + +// hostedOnly reports whether a package is in an npm scope or Composer vendor +// that is served from pushed packages alone. The upstream is never asked +// about such a package, so nobody can mix in versions by publishing the same +// name there. +func (p *Proxy) hostedOnly(ecosystem, name string) bool { + if !p.HostedEnabled { + return false + } + namespace, _, namespaced := strings.Cut(name, "/") + if !namespaced { + return false + } + switch ecosystem { + case "npm": + return slices.Contains(p.HostedNPMScopes, namespace) + case "composer": + return slices.ContainsFunc(p.HostedComposerVendors, func(vendor string) bool { + return strings.EqualFold(vendor, namespace) + }) + default: + return false + } +} + +// localVersion is a pushed version as ecosystem metadata lists it. +type localVersion struct { + Version string + Filename string + SHA256 string + Integrity string + Manifest json.RawMessage + PushedAt time.Time +} + +// localVersions returns the versions pushed for a package, or nil when hosted +// packages are disabled, so metadata requests cost no extra lookup then. +// Denied versions are left out, so metadata never lists a version whose +// download is refused. +func (p *Proxy) localVersions(ecosystem, name string) ([]localVersion, error) { + if !p.HostedEnabled { + return nil, nil + } + pkgPURL := packageurl.MakeString(ecosystem, name, "") + if pkgPURL == "" { + return nil, nil + } + rows, err := p.DB.GetLocalArtifacts(pkgPURL) + if err != nil { + return nil, fmt.Errorf("listing local versions: %w", err) + } + versions := make([]localVersion, 0, len(rows)) + for _, row := range rows { + version := database.VersionFromPURL(row.VersionPURL) + if !row.Manifest.Valid || p.versionDenied(ecosystem, name, version) { + continue + } + versions = append(versions, localVersion{ + Version: version, + Filename: row.Filename, + SHA256: row.ContentHash.String, + Integrity: row.Integrity.String, + Manifest: json.RawMessage(row.Manifest.String), + PushedAt: row.FetchedAt.Time.UTC(), + }) + } + return versions, nil +} + +// DeleteLocalVersion removes the archives pushed for name@version. If the +// upstream has the version, clients get the upstream archive again. +func (p *Proxy) DeleteLocalVersion(ecosystem, name, version string) error { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return err + } + deleted, err := p.DB.DeleteLocalArtifacts(versionPURL) + if err != nil { + return fmt.Errorf("deleting local artifacts: %w", err) + } + if deleted == 0 { + return ErrHostedVersionNotFound + } + return nil +} + +// GetLocalArtifact serves a pushed artifact from storage, or returns nil +// when the artifact was not pushed. Handlers that must consult the upstream +// before serving a cached copy call it first, so pushed versions the upstream +// does not list are still served. Pushed artifacts are not a cache, so they +// are served even when StreamArtifacts is set. +func (p *Proxy) GetLocalArtifact(ctx context.Context, ecosystem, name, version, filename string) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return nil, err + } + record, err := p.DB.GetArtifact(versionPURL, filename) + if err != nil { + return nil, fmt.Errorf("checking artifact source: %w", err) + } + if record == nil || record.Source != database.SourceLocal { + return nil, nil + } + if p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + result, err := p.readStored(ctx, pkgPURL, versionPURL, filename) + if err != nil || result != nil { + return result, err + } + return nil, p.localArtifactMiss(versionPURL, filename) +} diff --git a/internal/handler/hosted_manifest.go b/internal/handler/hosted_manifest.go new file mode 100644 index 00000000..3bd9b38a --- /dev/null +++ b/internal/handler/hosted_manifest.go @@ -0,0 +1,169 @@ +package handler + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "strings" +) + +var ( + ErrHostedInvalidArchive = errors.New("archive has no readable package manifest") + ErrHostedManifestMismatch = errors.New("package manifest does not match the pushed name and version") +) + +// maxManifestSize bounds how much of a manifest file is read. Real manifests +// are a few kilobytes; the bound keeps a hostile archive from claiming memory. +const maxManifestSize = 1 << 20 + +// readHostedManifest reads the package manifest out of a pushed archive and +// checks that it describes target. The manifest is what the ecosystem's +// metadata lists the version with, so its dependencies reach the client. +func readHostedManifest(ecosystem string, archive io.Reader, target HostedTarget) (json.RawMessage, error) { + switch ecosystem { + case "npm": + return readNPMManifest(archive, target) + case "composer": + return readComposerManifest(archive, target) + default: + return nil, fmt.Errorf("%w: %s", ErrHostedUnsupported, ecosystem) + } +} + +// readNPMManifest reads package.json from the top directory of an npm +// tarball. npm packs it under package/, but published tarballs use other +// directory names too. +func readNPMManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { + gz, err := gzip.NewReader(archive) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + defer func() { _ = gz.Close() }() + + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if err != nil { + return nil, fmt.Errorf("%w: no package.json in tarball", ErrHostedInvalidArchive) + } + if header.Typeflag != tar.TypeReg || !isTopLevelManifest(header.Name, "package.json") { + continue + } + manifest, fields, err := decodeManifest(tr) + if err != nil { + return nil, err + } + if fields.Name != target.Name || fields.Version != target.Version { + return nil, fmt.Errorf("%w: package.json is %s@%s", ErrHostedManifestMismatch, fields.Name, fields.Version) + } + return manifest, nil + } +} + +// readComposerManifest reads composer.json from the root of a Composer dist +// zip, or from its single top directory as GitHub zipballs have it. Package +// names are case-insensitive, and composer.json only names a version when +// the package pins one. +func readComposerManifest(archive io.Reader, target HostedTarget) (json.RawMessage, error) { + file, err := os.CreateTemp("", "hosted-composer-*.zip") + if err != nil { + return nil, err + } + defer func() { + _ = file.Close() + _ = os.Remove(file.Name()) + }() + size, err := io.Copy(file, archive) + if err != nil { + return nil, err + } + zr, err := zip.NewReader(file, size) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + + entry, err := composerManifestEntry(zr.File) + if err != nil { + return nil, err + } + rc, err := entry.Open() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + manifest, fields, err := decodeManifest(rc) + _ = rc.Close() + if err != nil { + return nil, err + } + if !strings.EqualFold(fields.Name, target.Name) || (fields.Version != "" && fields.Version != target.Version) { + return nil, fmt.Errorf("%w: composer.json is %s %s", ErrHostedManifestMismatch, fields.Name, fields.Version) + } + return manifest, nil +} + +// composerManifestEntry picks the package's composer.json: the one at the +// root, or else the only one directly inside a top directory. Archives keep +// other composer.json files for tests and examples, in any order, so the +// first one found isn't necessarily the package's. +func composerManifestEntry(files []*zip.File) (*zip.File, error) { + var topLevel []*zip.File + for _, entry := range files { + if entry.FileInfo().IsDir() { + continue + } + if entry.Name == "composer.json" { + return entry, nil + } + if isTopLevelManifest(entry.Name, "composer.json") { + topLevel = append(topLevel, entry) + } + } + switch len(topLevel) { + case 0: + return nil, fmt.Errorf("%w: no composer.json in zip", ErrHostedInvalidArchive) + case 1: + return topLevel[0], nil + default: + return nil, fmt.Errorf("%w: no composer.json at the root and %d in top directories", ErrHostedInvalidArchive, len(topLevel)) + } +} + +// isTopLevelManifest reports whether name is filename inside one top +// directory, such as package/package.json. +func isTopLevelManifest(name, filename string) bool { + dir, file := path.Split(path.Clean(name)) + return file == filename && dir != "" && !strings.Contains(strings.TrimSuffix(dir, "/"), "/") +} + +type manifestFields struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// decodeManifest reads a JSON object manifest and returns it compacted along +// with its name and version. +func decodeManifest(r io.Reader) (json.RawMessage, manifestFields, error) { + raw, err := io.ReadAll(io.LimitReader(r, maxManifestSize+1)) + if err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + if len(raw) > maxManifestSize { + return nil, manifestFields{}, fmt.Errorf("%w: manifest exceeds %d bytes", ErrHostedInvalidArchive, maxManifestSize) + } + var fields manifestFields + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: manifest is not a JSON object: %w", ErrHostedInvalidArchive, err) + } + var compact bytes.Buffer + if err := json.Compact(&compact, raw); err != nil { + return nil, manifestFields{}, fmt.Errorf("%w: %w", ErrHostedInvalidArchive, err) + } + return compact.Bytes(), fields, nil +} diff --git a/internal/handler/hosted_test.go b/internal/handler/hosted_test.go new file mode 100644 index 00000000..a4e7145e --- /dev/null +++ b/internal/handler/hosted_test.go @@ -0,0 +1,417 @@ +package handler + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +func TestHostedTargetFor(t *testing.T) { + tests := []struct { + name string + ecosystem, pkg, version string + want HostedTarget + wantErr error + }{ + {name: "npm", ecosystem: "npm", pkg: "request", version: "2.88.3", + want: HostedTarget{Name: "request", Version: "2.88.3", Filename: "request-2.88.3.tgz"}}, + {name: "npm scoped", ecosystem: "npm", pkg: "@types/node", version: "20.1.0", + want: HostedTarget{Name: "@types/node", Version: "20.1.0", Filename: "node-20.1.0.tgz"}}, + {name: "composer", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", + want: HostedTarget{Name: "monolog/monolog", Version: "3.5.0", Filename: "monolog-3.5.0.zip"}}, + {name: "composer without vendor", ecosystem: "composer", pkg: "monolog", version: "3.5.0", + wantErr: ErrHostedInvalidName}, + {name: "composer with extra segment", ecosystem: "composer", pkg: "a/b/c", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "unsupported ecosystem", ecosystem: "cargo", pkg: "serde", version: "1.0.200", + wantErr: ErrHostedUnsupported}, + {name: "unknown ecosystem", ecosystem: "maven", pkg: "junit", version: "4.13", + wantErr: ErrHostedUnsupported}, + {name: "npm scope without @", ecosystem: "npm", pkg: "types/node", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "dot-dot name segment", ecosystem: "npm", pkg: "@scope/..", version: "1.0.0", + wantErr: ErrHostedInvalidName}, + {name: "dot-dot version", ecosystem: "npm", pkg: "request", version: "..", + wantErr: ErrHostedInvalidName}, + {name: "control character", ecosystem: "npm", pkg: "request", version: "1.0.0\n", + wantErr: ErrHostedInvalidName}, + {name: "empty version", ecosystem: "npm", pkg: "request", version: "", + wantErr: ErrHostedInvalidName}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("HostedTargetFor() error = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("HostedTargetFor() error = %v", err) + } + if got != tt.want { + t.Errorf("HostedTargetFor() = %+v, want %+v", got, tt.want) + } + }) + } +} + +// npmTarball builds an npm package tarball holding manifest as +// package/package.json. Building an archive in memory cannot fail, so errors +// panic. +func npmTarball(manifest string) []byte { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, f := range [][2]string{{"package/package.json", manifest}, {"package/index.js", "module.exports = 1\n"}} { + must(tw.WriteHeader(&tar.Header{Name: f[0], Mode: 0o644, Size: int64(len(f[1]))})) + _, err := io.WriteString(tw, f[1]) + must(err) + } + must(tw.Close()) + must(gz.Close()) + return buf.Bytes() +} + +// composerZip builds a Composer dist archive holding manifest as composer.json +// under dir, or at the root when dir is empty. +func composerZip(dir, manifest string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + prefix := "" + if dir != "" { + prefix = dir + "/" + } + for _, f := range [][2]string{{prefix + "composer.json", manifest}, {prefix + "src/Logger.php", "=8.1"}}`)}, + {name: "composer manifest in a top directory", ecosystem: "composer", pkg: "monolog/monolog", version: "3.5.0", + archive: composerZip("Seldaek-monolog-abc123", `{"name":"Monolog/Monolog","version":"3.5.0"}`)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + target, err := HostedTargetFor(tt.ecosystem, tt.pkg, tt.version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + _, err = proxy.PushLocalArtifact(context.Background(), tt.ecosystem, target, bytes.NewReader(tt.archive), "") + if tt.wantErr == nil { + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return + } + if !errors.Is(err, tt.wantErr) { + t.Fatalf("PushLocalArtifact error = %v, want %v", err, tt.wantErr) + } + if listed, _ := proxy.LocalArtifacts(tt.ecosystem, tt.pkg); len(listed) != 0 { + t.Errorf("rejected archive was recorded: %+v", listed) + } + if len(store.files) != 0 { + t.Errorf("rejected archive left %d objects in storage", len(store.files)) + } + }) + } +} + +// removeStored deletes the stored object of an artifact behind the cache's back. +func removeStored(t *testing.T, proxy *Proxy, versionPURL, filename string) { + t.Helper() + record, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || record == nil { + t.Fatalf("GetArtifact: %+v, %v", record, err) + } + if err := proxy.Storage.Delete(context.Background(), record.StoragePath.String); err != nil { + t.Fatalf("deleting stored object: %v", err) + } +} + +func TestPushedArtifactIsNeverReplacedFromUpstream(t *testing.T) { + const versionPURL, filename = "pkg:npm/request@2.88.3", "request-2.88.3.tgz" + tests := []struct { + name string + damage func(t *testing.T, proxy *Proxy) + wantErr error + }{ + {name: "stored object missing", wantErr: ErrHostedArtifactUnavailable, + damage: func(t *testing.T, proxy *Proxy) { removeStored(t, proxy, versionPURL, filename) }}, + {name: "record cleared after an integrity failure", wantErr: ErrHostedArtifactMissing, + damage: func(t *testing.T, proxy *Proxy) { + record, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || record == nil { + t.Fatalf("GetArtifact: %+v, %v", record, err) + } + if err := proxy.DB.DiscardArtifact(versionPURL, filename, record.StoragePath.String); err != nil { + t.Fatalf("DiscardArtifact: %v", err) + } + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + pushRequest(t, proxy) + tt.damage(t, proxy) + + _, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "npm", "request", "2.88.3", filename, + "https://registry.npmjs.org/request/-/request-2.88.3.tgz") + if !errors.Is(err, tt.wantErr) { + t.Errorf("GetOrFetchArtifactFromURL error = %v, want %v", err, tt.wantErr) + } + if fetcher.fetchCalled { + t.Error("a miss on a pushed artifact fetched the upstream copy") + } + local, err := db.GetLocalArtifacts("pkg:npm/request") + if err != nil || len(local) != 1 || !local[0].Manifest.Valid { + t.Errorf("pushed records after the miss = %+v, %v, want the pushed one with its manifest", local, err) + } + }) + } +} + +// zipWith builds a zip holding files in the given order. +func zipWith(files ...[2]string) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, f := range files { + w, err := zw.Create(f[0]) + must(err) + _, err = io.WriteString(w, f[1]) + must(err) + } + must(zw.Close()) + return buf.Bytes() +} + +func TestComposerManifestLookup(t *testing.T) { + const monolog = `{"name":"monolog/monolog"}` + const fixture = `{"name":"acme/test-fixture"}` + tests := []struct { + name string + archive []byte + wantErr error + }{ + {name: "root manifest after a nested one", + archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"composer.json", monolog})}, + {name: "top directory manifest next to a deeper one", + archive: zipWith([2]string{"monolog-abc/tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog})}, + {name: "two top directories with manifests", + archive: zipWith([2]string{"tests/composer.json", fixture}, [2]string{"monolog-abc/composer.json", monolog}), + wantErr: ErrHostedInvalidArchive}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + target, err := HostedTargetFor("composer", "monolog/monolog", "3.5.0") + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + _, err = readHostedManifest("composer", bytes.NewReader(tt.archive), target) + if tt.wantErr == nil && err != nil { + t.Errorf("readHostedManifest error = %v, want the root or single top directory manifest", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Errorf("readHostedManifest error = %v, want %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 92629d24..308f6aff 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -215,12 +215,16 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques accept = contentTypeJSON } + specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) + if h.serveLocalMetadata(w, r, npmMetadataRequest{packageName, specifier, versionLookup, upstreamURL, accept, metadataCacheKey}) { + return + } + if stored, ok := h.proxy.storedRewrite("npm", metadataCacheKey, h.proxyURL, metadataCacheKey); ok { serveRewrittenMetadata(w, r, stored) return } - specifier, versionLookup := npmVersionSpecifier(packagePath, packageName) if versionLookup && h.npmVersionMetadataUsesFilteredPackument(packageName) { h.handleFilteredNPMVersionMetadata(w, r, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey) return @@ -396,6 +400,14 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e // rewriteMetadataKeeping is rewriteMetadata that also reports the versions it // kept. func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { + return h.rewriteMetadataWithLocal(packageName, body, nil) +} + +// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the +// pushed versions in local, in place of any upstream entry with the same +// version. Cooldown and the denylist only filter upstream versions, and +// dist-tags stay as upstream has them. +func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) { doc, err := indexNPMPackument(body) if err != nil { return nil, nil, err @@ -439,18 +451,26 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] h.applyCooldownFiltering(metadata, versions, packageName) h.applyDenylistFiltering(metadata, versions, packageName) + for _, e := range local { + delete(versions, e.version) + } // Untouched members are copied as they are; the three the filters // handled are written from their filtered state, and only when it changed. out, err := doc.write(func(out *bytes.Buffer, i int) error { switch { case i == doc.versionsAt: - return h.writeNPMVersions(out, packageName, versionsRaw, versions) - case i == doc.timeAt && timeMap != nil && len(timeMap) != timeEntries: - return writeFilteredJSONObject(out, doc.value(i), func(key string) bool { + return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) + case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): + pushedAt := make([][2]string, 0, len(local)) + for _, e := range local { + pushedAt = append(pushedAt, [2]string{e.version, e.time}) + } + return writeFilteredJSONObjectWith(out, doc.value(i), func(key string) bool { _, ok := timeMap[key] - return ok - }) + _, pushed := findNPMLocalEntry(local, key) + return ok && !pushed + }, pushedAt) case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): encoded, err := json.Marshal(distTags) out.Write(encoded) @@ -460,7 +480,11 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] return nil } }) - return out, slices.Collect(maps.Keys(versions)), err + kept := slices.Collect(maps.Keys(versions)) + for _, e := range local { + kept = append(kept, e.version) + } + return out, kept, err } // npmPackument is a packument's top-level members as offsets into its bytes, @@ -547,8 +571,9 @@ func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([ } // writeNPMVersions writes the versions object, keeping the versions still in -// keep and pointing each kept version's tarball at this proxy. -func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any) error { +// keep and pointing each kept version's tarball at this proxy, followed by the +// pushed versions in local. +func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error { out.WriteByte('{') first := true err := forEachJSONMember(versionsRaw, func(m jsonMember) error { @@ -568,6 +593,16 @@ func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, ver h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) return nil }) + for _, e := range local { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(e.version) + out.Write(key) + out.WriteByte(':') + out.Write(e.entry) + } out.WriteByte('}') return err } @@ -610,6 +645,12 @@ func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version str // writeFilteredJSONObject writes the object obj holds, keeping the members // whose decoded key keep accepts. func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bool) error { + return writeFilteredJSONObjectWith(out, obj, keep, nil) +} + +// writeFilteredJSONObjectWith is writeFilteredJSONObject followed by extra +// string members. +func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { out.WriteByte('{') first := true err := forEachJSONMember(obj, func(m jsonMember) error { @@ -627,6 +668,17 @@ func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bo out.Write(obj[m.keyStart:m.valEnd]) return nil }) + for _, member := range extra { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(member[0]) + value, _ := json.Marshal(member[1]) + out.Write(key) + out.WriteByte(':') + out.Write(value) + } out.WriteByte('}') return err } @@ -761,6 +813,21 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) return } + // Pushed versions are listed without cooldown, so they are served without + // it too, even when the upstream published the same version recently. + local, err := h.proxy.GetLocalArtifact(r.Context(), "npm", packageName, version, filename) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to read pushed package") + return + } + if local != nil { + ServeArtifactRequest(w, r, local) + return + } + if h.proxy.hostedOnly("npm", packageName) { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() ([]byte, error) { upstreamURL := registryURL + "/" + url.PathEscape(packageName) diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go new file mode 100644 index 00000000..e8998c90 --- /dev/null +++ b/internal/handler/npm_hosted.go @@ -0,0 +1,195 @@ +package handler + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/Masterminds/semver/v3" +) + +// npmLocalEntry is a pushed version as a packument lists it: the version +// entry built from its package.json, and the time it was pushed. +type npmLocalEntry struct { + version string + entry json.RawMessage + time string +} + +// npmLocalEntries returns the pushed versions of a package as packument +// entries, or nil when there are none or hosted packages are disabled. +func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error) { + local, err := h.proxy.localVersions("npm", packageName) + if err != nil || len(local) == 0 { + return nil, err + } + entries := make([]npmLocalEntry, 0, len(local)) + for _, lv := range local { + var manifest map[string]any + if err := json.Unmarshal(lv.Manifest, &manifest); err != nil { + return nil, fmt.Errorf("decoding manifest of %s@%s: %w", packageName, lv.Version, err) + } + manifest["name"] = packageName + manifest["version"] = lv.Version + manifest["_id"] = packageName + "@" + lv.Version + manifest["dist"] = map[string]string{ + "tarball": h.proxyTarballURL(packageName, lv.Version, lv.Filename), + "integrity": lv.Integrity, + } + entry, err := json.Marshal(manifest) + if err != nil { + return nil, err + } + entries = append(entries, npmLocalEntry{version: lv.Version, entry: entry, time: lv.PushedAt.Format(time.RFC3339)}) + } + return entries, nil +} + +func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) { + for _, e := range local { + if e.version == version { + return e, true + } + } + return npmLocalEntry{}, false +} + +// npmMetadataRequest is a packument or version metadata request. +type npmMetadataRequest struct { + packageName string + specifier string + versionLookup bool + upstreamURL string + accept string + metadataCacheKey string +} + +// serveLocalMetadata answers the metadata requests that pushed versions +// decide, and reports whether it did: a request for a pushed version, any +// request for a package in a hosted-only scope, and a packument of a package +// with pushed versions. Everything else takes the usual upstream path. +func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, req npmMetadataRequest) bool { + local, err := h.npmLocalEntries(req.packageName) + if err != nil { + h.proxy.Logger.Error("failed to read pushed npm versions", "package", req.packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to read pushed versions") + return true + } + switch e, pushed := findNPMLocalEntry(local, req.specifier); { + case req.versionLookup && pushed: + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(e.entry) + case h.proxy.hostedOnly("npm", req.packageName): + h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local) + case len(local) > 0 && !req.versionLookup: + h.serveMetadataWithLocal(w, r, req.packageName, req.upstreamURL, req.accept, req.metadataCacheKey, local) + default: + return false + } + return true +} + +// serveMetadataWithLocal serves a packument with the pushed versions of the +// package added. It bypasses the rewrite cache, whose entries are keyed by the +// upstream document alone, so a push or delete shows up on the next request. +// When the upstream does not know the package, the packument holds only the +// pushed versions. +func (h *NPMHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string, local []npmLocalEntry) { + upstream := h.proxy.fetchMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + var out []byte + var err error + switch { + case errors.Is(upstream.err, ErrUpstreamNotFound): + out, err = localNPMPackument(packageName, local) + case upstream.err != nil: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", upstream.err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return + default: + out, _, err = h.rewriteMetadataWithLocal(packageName, upstream.body, local) + } + if err != nil { + if errors.Is(err, ErrVersionDenied) { + JSONError(w, http.StatusForbidden, err.Error()) + return + } + h.proxy.Logger.Error("failed to add pushed versions to npm metadata", "package", packageName, "error", err) + JSONError(w, http.StatusBadGateway, "failed to rewrite package metadata") + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) +} + +// serveHostedOnlyMetadata answers a metadata request for a package in a +// hosted-only scope from its pushed versions alone. +func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier string, versionLookup bool, metadataCacheKey string, local []npmLocalEntry) { + if len(local) == 0 { + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return + } + if versionLookup { + if specifier == "latest" { + specifier = latestNPMVersion(local) + } + e, ok := findNPMLocalEntry(local, specifier) + if !ok { + JSONError(w, http.StatusNotFound, ErrVersionNotFound.Error()) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(e.entry) + return + } + out, err := localNPMPackument(packageName, local) + if err != nil { + h.proxy.Logger.Error("failed to build npm metadata from pushed versions", "package", packageName, "error", err) + JSONError(w, http.StatusInternalServerError, "failed to build package metadata") + return + } + serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("npm", h.proxyURL, metadataCacheKey, out)) +} + +// localNPMPackument builds a packument from pushed versions alone. +func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error) { + versions := make(map[string]json.RawMessage, len(local)) + times := make(map[string]string, len(local)) + for _, e := range local { + versions[e.version] = e.entry + times[e.version] = e.time + } + return json.Marshal(map[string]any{ + "name": packageName, + "dist-tags": map[string]string{"latest": latestNPMVersion(local)}, + "versions": versions, + "time": times, + }) +} + +// latestNPMVersion returns the highest pushed version, or the last pushed +// when a version is not semver. +func latestNPMVersion(local []npmLocalEntry) string { + latest, latestTime := "", "" + var latestSemver *semver.Version + semverOnly := true + for _, e := range local { + if e.time > latestTime { + latest, latestTime = e.version, e.time + } + v, err := semver.NewVersion(e.version) + if err != nil { + semverOnly = false + continue + } + if latestSemver == nil || v.GreaterThan(latestSemver) { + latestSemver = v + } + } + if semverOnly && latestSemver != nil { + return latestSemver.Original() + } + return latest +} diff --git a/internal/handler/npm_hosted_test.go b/internal/handler/npm_hosted_test.go new file mode 100644 index 00000000..7102046c --- /dev/null +++ b/internal/handler/npm_hosted_test.go @@ -0,0 +1,351 @@ +package handler + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +const requestPackument = `{ + "name": "request", + "dist-tags": {"latest": "2.88.2"}, + "versions": { + "2.88.2": {"name": "request", "version": "2.88.2", "dependencies": {"qs": "~6.5.1"}, + "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-upstream2"}}, + "2.88.3": {"name": "request", "version": "2.88.3", "dependencies": {"old": "1.0.0"}, + "dist": {"tarball": "https://registry.npmjs.org/request/-/request-2.88.3.tgz", "integrity": "sha512-upstream3"}} + }, + "time": {"modified": "2020-02-11T00:00:00Z", "2.88.2": "2020-02-11T00:00:00Z", "2.88.3": "2020-02-12T00:00:00Z"} +}` + +type npmHostedFixture struct { + proxy *Proxy + handler http.Handler + upstreamRequests *int + fetcher *mockFetcher +} + +// newNPMHostedFixture serves packuments from upstream, keyed by request path, +// and answers 404 for anything else. +func newNPMHostedFixture(t *testing.T, upstream map[string]string) npmHostedFixture { + t.Helper() + upstreamRequests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamRequests++ + body, ok := upstream[r.URL.EscapedPath()] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HTTPClient = srv.Client() + return npmHostedFixture{proxy: proxy, handler: NewNPMHandler(proxy, "http://proxy.local", srv.URL).Routes(), + upstreamRequests: &upstreamRequests, fetcher: fetcher} +} + +func (f npmHostedFixture) push(t *testing.T, name, version, manifest string) HostedArtifact { + t.Helper() + target, err := HostedTargetFor("npm", name, version) + if err != nil { + t.Fatalf("HostedTargetFor: %v", err) + } + pushed, err := f.proxy.PushLocalArtifact(context.Background(), "npm", target, bytes.NewReader(npmTarball(manifest)), "") + if err != nil { + t.Fatalf("PushLocalArtifact: %v", err) + } + return pushed +} + +func (f npmHostedFixture) get(t *testing.T, path string) *httptest.ResponseRecorder { + t.Helper() + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +type npmTestPackument struct { + Name string `json:"name"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]npmTestVersion `json:"versions"` + Time map[string]string `json:"time"` + Extra map[string]json.RawMessage `json:"-"` +} + +type npmTestVersion struct { + Name string `json:"name"` + Version string `json:"version"` + ID string `json:"_id"` + Dependencies map[string]string `json:"dependencies"` + Dist struct { + Tarball string `json:"tarball"` + Integrity string `json:"integrity"` + } `json:"dist"` +} + +func decodePackument(t *testing.T, rec *httptest.ResponseRecorder) npmTestPackument { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body) + } + var doc npmTestPackument + if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding packument: %v\n%s", err, rec.Body) + } + return doc +} + +func TestNPMMetadataListsPushedVersion(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + pushed := f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) + + doc := decodePackument(t, f.get(t, "/request")) + + local, ok := doc.Versions["2.88.4"] + if !ok { + t.Fatalf("pushed version missing from versions: %v", doc.Versions) + } + if local.Name != "request" || local.Version != "2.88.4" || local.ID != "request@2.88.4" || local.Dependencies["qs"] != "~6.5.2" { + t.Errorf("pushed version entry = %+v", local) + } + if local.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { + t.Errorf("tarball = %q", local.Dist.Tarball) + } + if !strings.HasPrefix(local.Dist.Integrity, "sha512-") { + t.Errorf("integrity = %q, want the pushed archive's sha512", local.Dist.Integrity) + } + if doc.Time["2.88.4"] != pushed.PushedAt.Format(time.RFC3339) { + t.Errorf("time[2.88.4] = %q, want %q", doc.Time["2.88.4"], pushed.PushedAt.Format(time.RFC3339)) + } + if doc.Versions["2.88.2"].Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.2.tgz" { + t.Errorf("upstream version not rewritten: %+v", doc.Versions["2.88.2"]) + } + if doc.DistTags["latest"] != "2.88.2" { + t.Errorf("dist-tags.latest = %q, want upstream's 2.88.2", doc.DistTags["latest"]) + } +} + +func TestNPMMetadataPushedVersionReplacesUpstream(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3","dependencies":{"qs":"~6.5.2"}}`) + + rec := f.get(t, "/request") + if body := rec.Body.String(); strings.Contains(body, "sha512-upstream3") || strings.Count(body, `"2.88.3":{`) != 1 { + t.Fatalf("upstream entry for the pushed version still listed:\n%s", body) + } + local := decodePackument(t, rec).Versions["2.88.3"] + if local.Dependencies["qs"] != "~6.5.2" || local.Dependencies["old"] != "" { + t.Errorf("dependencies = %v, want the pushed package.json's", local.Dependencies) + } +} + +func TestNPMMetadataForPackageOnlyPushed(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.push(t, "@acme/private", "1.10.0", `{"name":"@acme/private","version":"1.10.0"}`) + f.push(t, "@acme/private", "1.9.0", `{"name":"@acme/private","version":"1.9.0"}`) + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if doc.Name != "@acme/private" || len(doc.Versions) != 2 { + t.Errorf("packument = %+v", doc) + } + if doc.DistTags["latest"] != "1.10.0" { + t.Errorf("dist-tags.latest = %q, want the highest pushed version 1.10.0", doc.DistTags["latest"]) + } + if doc.Versions["1.9.0"].Dist.Tarball != "http://proxy.local/npm/@acme%2Fprivate/-/private-1.9.0.tgz" { + t.Errorf("tarball = %q", doc.Versions["1.9.0"].Dist.Tarball) + } + + if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { + t.Errorf("unknown package status = %d, want 404", rec.Code) + } +} + +func TestNPMVersionMetadataForPushedVersion(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4","dependencies":{"qs":"~6.5.2"}}`) + + rec := f.get(t, "/request/2.88.4") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", rec.Code, rec.Body) + } + var version npmTestVersion + if err := json.Unmarshal(rec.Body.Bytes(), &version); err != nil { + t.Fatalf("decoding version: %v", err) + } + if version.Version != "2.88.4" || version.Dist.Tarball != "http://proxy.local/npm/request/-/request-2.88.4.tgz" { + t.Errorf("version document = %+v", version) + } +} + +func TestNPMMetadataFollowsPushAndDelete(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.proxy.CacheMetadata = true + f.proxy.MetadataTTL = time.Hour + f.proxy.SetMetadataRewriteCacheSize(1 << 20) + + before := f.get(t, "/request") + if before.Code != http.StatusOK { + t.Fatalf("status = %d", before.Code) + } + + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) + pushed := f.get(t, "/request") + if _, ok := decodePackument(t, pushed).Versions["2.88.4"]; !ok { + t.Fatal("pushed version not listed while the upstream document is cached") + } + if pushed.Header().Get("ETag") == before.Header().Get("ETag") { + t.Error("ETag unchanged after a push") + } + + if err := f.proxy.DeleteLocalVersion("npm", "request", "2.88.4"); err != nil { + t.Fatalf("DeleteLocalVersion: %v", err) + } + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { + t.Error("deleted version still listed") + } +} + +func TestNPMMetadataIgnoresPushesWhenHostedDisabled(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.4", `{"name":"request","version":"2.88.4"}`) + f.proxy.HostedEnabled = false + + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.4"]; ok { + t.Error("pushed version listed with hosted packages disabled") + } +} + +func TestNPMDownloadOfPushedPackageWithCooldown(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + + rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz") + if rec.Code != http.StatusOK { + t.Fatalf("download status = %d, body %s, want the pushed tarball", rec.Code, rec.Body) + } + if _, ok := decodePackument(t, f.get(t, "/@acme/private")).Versions["1.0.0"]; !ok { + t.Error("pushed version missing from the packument with cooldown enabled") + } +} + +func TestNPMDownloadOfPushedOverrideOfVersionInCooldown(t *testing.T) { + recent := time.Now().Add(-time.Hour).Format(time.RFC3339) + packument := `{"name":"request","dist-tags":{"latest":"2.88.2"}, + "versions":{ + "2.88.2":{"name":"request","version":"2.88.2","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.2.tgz"}}, + "2.88.3":{"name":"request","version":"2.88.3","dist":{"tarball":"https://registry.npmjs.org/request/-/request-2.88.3.tgz"}}}, + "time":{"2.88.2":"2020-02-11T00:00:00Z","2.88.3":"` + recent + `"}}` + f := newNPMHostedFixture(t, map[string]string{"/request": packument}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + + if _, ok := decodePackument(t, f.get(t, "/request")).Versions["2.88.3"]; !ok { + t.Fatal("pushed version missing from the packument") + } + rec := f.get(t, "/request/-/request-2.88.3.tgz") + if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(`{"name":"request","version":"2.88.3"}`)) { + t.Errorf("download status = %d, want 200 and the pushed tarball: the packument lists it, so the download must not be withheld", rec.Code) + } +} + +func TestNPMHostedOnlyScopeNeverAsksUpstream(t *testing.T) { + squatter := `{"name":"@acme/private","dist-tags":{"latest":"1.0.1"},"versions":{"1.0.1":{"name":"@acme/private","version":"1.0.1", + "dist":{"tarball":"https://registry.npmjs.org/@acme/private/-/private-1.0.1.tgz"}}}}` + f := newNPMHostedFixture(t, map[string]string{"/@acme%2fprivate": squatter, "/@acme%2Fprivate": squatter, "/request": requestPackument}) + f.proxy.HostedNPMScopes = []string{"@acme"} + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if len(doc.Versions) != 1 || doc.DistTags["latest"] != "1.0.0" { + t.Errorf("packument = %+v, want only the pushed 1.0.0", doc) + } + if rec := f.get(t, "/@acme/private/latest"); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"version":"1.0.0"`) { + t.Errorf("latest status = %d, body %s", rec.Code, rec.Body) + } + if rec := f.get(t, "/@acme/private/1.0.1"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed version status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/private/-/private-1.0.1.tgz"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed tarball status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/other"); rec.Code != http.StatusNotFound { + t.Errorf("unpushed package status = %d, want 404", rec.Code) + } + if rec := f.get(t, "/@acme/private/-/private-1.0.0.tgz"); rec.Code != http.StatusOK { + t.Errorf("pushed tarball status = %d, want 200", rec.Code) + } + if *f.upstreamRequests != 0 { + t.Errorf("hosted-only scope sent %d requests upstream", *f.upstreamRequests) + } + + if rec := f.get(t, "/request"); rec.Code != http.StatusOK || *f.upstreamRequests == 0 { + t.Errorf("package outside the scope status = %d after %d upstream requests, want it fetched upstream", rec.Code, *f.upstreamRequests) + } +} + +func TestNPMDownloadOfUnreadablePushedTarball(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") + + if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) + } +} + +func TestNPMMetadataLeavesOutDeniedPushedVersions(t *testing.T) { + f := newNPMHostedFixture(t, nil) + f.push(t, "@acme/private", "1.0.0", `{"name":"@acme/private","version":"1.0.0"}`) + f.push(t, "@acme/private", "1.1.0", `{"name":"@acme/private","version":"1.1.0"}`) + setTestDenylist(t, f.proxy, "pkg:npm/@acme/private@1.1.0") + + doc := decodePackument(t, f.get(t, "/@acme/private")) + if _, ok := doc.Versions["1.1.0"]; ok || len(doc.Versions) != 1 { + t.Errorf("versions = %v, want the denied 1.1.0 left out", doc.Versions) + } + if doc.DistTags["latest"] != "1.0.0" { + t.Errorf("dist-tags.latest = %q, want 1.0.0", doc.DistTags["latest"]) + } +} + +func TestNPMDownloadOfPushedTarballWhileStreamingArtifacts(t *testing.T) { + const manifest = `{"name":"request","version":"2.88.3"}` + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("upstream tarball")), ContentType: "application/gzip"} + f.push(t, "request", "2.88.3", manifest) + f.proxy.StreamArtifacts = true + + rec := f.get(t, "/request/-/request-2.88.3.tgz") + if rec.Code != http.StatusOK || rec.Body.String() != string(npmTarball(manifest)) { + t.Errorf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } + if f.fetcher.fetchCalled { + t.Error("a pushed tarball was fetched upstream while streaming artifacts") + } +} + +func TestNPMDownloadOfUnreadablePushedTarballWhileStreamingArtifacts(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/request": requestPackument}) + f.push(t, "request", "2.88.3", `{"name":"request","version":"2.88.3"}`) + removeStored(t, f.proxy, "pkg:npm/request@2.88.3", "request-2.88.3.tgz") + f.proxy.StreamArtifacts = true + + if rec := f.get(t, "/request/-/request-2.88.3.tgz"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("download status = %d, body %s, want 503 rather than the upstream tarball", rec.Code, rec.Body) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 7348bbe5..42273c3b 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -272,6 +272,17 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str return rewrittenMetadata{out, v}, err } +// uncachedRewrite gives a rewrite made outside the cache its validators: an +// ETag of the output itself and, as cachedRewrite does, a date that replicas +// agree on. +func (p *Proxy) uncachedRewrite(ecosystem, proxyURL, name string, out []byte) rewrittenMetadata { + v := metadataValidators{etag: metadataETag(out)} + if !p.cooldownEnabled() { + v.lastModified = p.metadataDate(ecosystem, proxyURL, name, v.etag) + } + return rewrittenMetadata{out, v} +} + func (p *Proxy) cooldownEnabled() bool { return p.Cooldown != nil && p.Cooldown.Enabled() } diff --git a/internal/server/errors.go b/internal/server/errors.go index d30bff92..7cbe20ed 100644 --- a/internal/server/errors.go +++ b/internal/server/errors.go @@ -8,11 +8,12 @@ import ( // Error codes returned in API error responses. These are stable identifiers // that clients can match on; the message text is for humans and may change. const ( - ErrCodeBadRequest = "BAD_REQUEST" - ErrCodeNotFound = "NOT_FOUND" - ErrCodeUpstream = "UPSTREAM_ERROR" - ErrCodeInternal = "INTERNAL_ERROR" - ErrCodeBusy = "BUSY" + ErrCodeBadRequest = "BAD_REQUEST" + ErrCodeNotFound = "NOT_FOUND" + ErrCodeUnauthorized = "UNAUTHORIZED" + ErrCodeUpstream = "UPSTREAM_ERROR" + ErrCodeInternal = "INTERNAL_ERROR" + ErrCodeBusy = "BUSY" ) // ErrorResponse is the JSON body returned for API errors. diff --git a/internal/server/hosted_api.go b/internal/server/hosted_api.go new file mode 100644 index 00000000..e9033d64 --- /dev/null +++ b/internal/server/hosted_api.go @@ -0,0 +1,194 @@ +package server + +import ( + "bufio" + "errors" + "io" + "log/slog" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +// HostedAPIHandler serves /api/hosted, where operators push archives that the +// proxy serves in place of, or in addition to, what the upstream has. +type HostedAPIHandler struct { + proxy *handler.Proxy + maxUploadBytes int64 + logger *slog.Logger +} + +// NewHostedAPIHandler creates the /api/hosted handler. +func NewHostedAPIHandler(proxy *handler.Proxy, maxUploadBytes int64, logger *slog.Logger) *HostedAPIHandler { + return &HostedAPIHandler{proxy: proxy, maxUploadBytes: maxUploadBytes, logger: logger} +} + +// HostedPushResponse describes a pushed archive. +type HostedPushResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + handler.HostedArtifact +} + +// HostedListResponse lists the archives pushed for a package. +type HostedListResponse struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Artifacts []handler.HostedArtifact `json:"artifacts"` +} + +// registerHostedRoutes mounts /api/hosted. Listing is open like the rest of +// the API; pushing and deleting require the bearer token. +func registerHostedRoutes(r chi.Router, h *HostedAPIHandler, token string) { + r.Get("/api/hosted/{ecosystem}/*", h.HandleList) + r.Group(func(w chi.Router) { + w.Use(requireBearerToken(token)) + w.Put("/api/hosted/{ecosystem}/*", h.HandlePush) + w.Delete("/api/hosted/{ecosystem}/*", h.HandleDelete) + }) +} + +// HandlePush handles PUT /api/hosted/{ecosystem}/{name}/{version} +// @Summary Push a package archive +// @Description Stores the request body as the archive for name@version. It is served in place of the upstream archive with the same version and is never evicted. +// @Tags hosted +// @Accept octet-stream +// @Produce json +// @Security BearerAuth +// @Param ecosystem path string true "Ecosystem (npm or composer)" +// @Param name path string true "Package name, including an npm scope or the Composer vendor" +// @Param version path string true "Version" +// @Success 201 {object} HostedPushResponse +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 403 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name}/{version} [put] +func (h *HostedAPIHandler) HandlePush(w http.ResponseWriter, r *http.Request) { + ecosystem, name, version, ok := hostedVersionPath(w, r) + if !ok { + return + } + target, err := handler.HostedTargetFor(ecosystem, name, version) + if err != nil { + badRequest(w, err.Error()) + return + } + + body := bufio.NewReader(http.MaxBytesReader(w, r.Body, h.maxUploadBytes)) + if _, err := body.Peek(1); errors.Is(err, io.EOF) { + badRequest(w, "request body is empty") + return + } + + pushed, err := h.proxy.PushLocalArtifact(r.Context(), ecosystem, target, body, r.Header.Get("Content-Type")) + if err != nil { + h.pushError(w, err) + return + } + h.logger.Info("pushed package archive", + "ecosystem", ecosystem, "name", target.Name, "version", target.Version, + "filename", target.Filename, "sha256", pushed.SHA256) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + writeJSON(w, HostedPushResponse{Ecosystem: ecosystem, Name: target.Name, HostedArtifact: pushed}) +} + +func (h *HostedAPIHandler) pushError(w http.ResponseWriter, err error) { + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "archive exceeds hosted.max_upload_size") + case errors.Is(err, handler.ErrHostedInvalidArchive), errors.Is(err, handler.ErrHostedManifestMismatch): + badRequest(w, err.Error()) + case errors.Is(err, handler.ErrVersionDenied), errors.Is(err, handler.ErrArtifactBlocked): + writeError(w, http.StatusForbidden, ErrCodeBadRequest, err.Error()) + default: + h.logger.Error("failed to push package archive", "error", err) + internalError(w, "failed to store archive") + } +} + +// HandleList handles GET /api/hosted/{ecosystem}/{name} +// @Summary List pushed package archives +// @Tags hosted +// @Produce json +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name" +// @Success 200 {object} HostedListResponse +// @Failure 400 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name} [get] +func (h *HostedAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { + ecosystem := chi.URLParam(r, "ecosystem") + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return + } + name := strings.Join(segments, "/") + if !handler.HostedSupported(ecosystem) { + badRequest(w, handler.ErrHostedUnsupported.Error()) + return + } + + listed, err := h.proxy.LocalArtifacts(ecosystem, name) + if err != nil { + h.logger.Error("failed to list pushed archives", "error", err) + internalError(w, "failed to list pushed archives") + return + } + writeJSON(w, HostedListResponse{Ecosystem: ecosystem, Name: name, Artifacts: listed}) +} + +// HandleDelete handles DELETE /api/hosted/{ecosystem}/{name}/{version} +// @Summary Delete a pushed version +// @Description Removes the archives pushed for name@version. If the upstream has the version, clients get the upstream archive again. +// @Tags hosted +// @Security BearerAuth +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name" +// @Param version path string true "Version" +// @Success 204 +// @Failure 401 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/hosted/{ecosystem}/{name}/{version} [delete] +func (h *HostedAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { + ecosystem, name, version, ok := hostedVersionPath(w, r) + if !ok { + return + } + err := h.proxy.DeleteLocalVersion(ecosystem, name, version) + switch { + case errors.Is(err, handler.ErrHostedVersionNotFound): + notFound(w, err.Error()) + case err != nil: + h.logger.Error("failed to delete pushed version", "error", err) + internalError(w, "failed to delete pushed version") + default: + h.logger.Info("deleted pushed version", "ecosystem", ecosystem, "name", name, "version", version) + w.WriteHeader(http.StatusNoContent) + } +} + +// minVersionPathSegments is one name segment plus the version. +const minVersionPathSegments = 2 + +// hostedVersionPath splits {name}/{version} from the wildcard. The version is +// always the last segment; the name may span several (npm scopes, Composer +// vendors). +func hostedVersionPath(w http.ResponseWriter, r *http.Request) (ecosystem, name, version string, ok bool) { + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return "", "", "", false + } + if len(segments) < minVersionPathSegments { + badRequest(w, "path must be /api/hosted/{ecosystem}/{name}/{version}") + return "", "", "", false + } + last := len(segments) - 1 + return chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last], true +} diff --git a/internal/server/hosted_api_test.go b/internal/server/hosted_api_test.go new file mode 100644 index 00000000..101ff318 --- /dev/null +++ b/internal/server/hosted_api_test.go @@ -0,0 +1,362 @@ +package server + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +const ( + testHostedToken = "push-token" + testHostedMaxUpload = 4096 +) + +type hostedTestServer struct { + router http.Handler + fetches *int + upstreamRequests *int +} + +// countingFetcher fails every fetch, so a test fails loudly if the proxy goes +// upstream for something it should serve from the pushed copy. +type countingFetcher struct{ calls *int } + +func (f countingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f countingFetcher) FetchWithHeaders(context.Context, string, http.Header) (*fetch.Artifact, error) { + *f.calls++ + return nil, fetch.ErrNotFound +} + +func (f countingFetcher) Head(context.Context, string) (int64, string, error) { + *f.calls++ + return 0, "", fetch.ErrNotFound +} + +func newHostedTestServer(t *testing.T) hostedTestServer { + t.Helper() + dir := t.TempDir() + db, err := database.Create(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("database.Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+filepath.Join(dir, "artifacts")) + if err != nil { + t.Fatalf("storage.OpenBucket: %v", err) + } + + calls := 0 + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(db, store, countingFetcher{calls: &calls}, fetch.NewResolver(), logger) + proxy.HostedEnabled = true + + upstreamRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamRequests++ + http.NotFound(w, nil) + })) + t.Cleanup(upstream.Close) + + r := chi.NewRouter() + registerHostedRoutes(r, NewHostedAPIHandler(proxy, testHostedMaxUpload, logger), testHostedToken) + r.Mount("/npm", http.StripPrefix("/npm", handler.NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes())) + composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + r.Mount("/composer", http.StripPrefix("/composer", composer.Routes())) + return hostedTestServer{router: r, fetches: &calls, upstreamRequests: &upstreamRequests} +} + +func (s hostedTestServer) do(t *testing.T, method, target, token, body string) *httptest.ResponseRecorder { + t.Helper() + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reader) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + s.router.ServeHTTP(rec, req) + return rec +} + +func TestHostedPushIsServedByTheEcosystemHandler(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, requestTarball) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + var pushed HostedPushResponse + if err := json.NewDecoder(rec.Body).Decode(&pushed); err != nil { + t.Fatalf("decoding PUT response: %v", err) + } + if pushed.Ecosystem != "npm" || pushed.Name != "request" || pushed.Version != "2.88.3" || + pushed.Filename != "request-2.88.3.tgz" || pushed.Size != int64(len(requestTarball)) { + t.Errorf("PUT response = %+v", pushed) + } + + rec = s.do(t, http.MethodGet, "/npm/request/-/request-2.88.3.tgz", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != requestTarball { + t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } + if *s.fetches != 0 { + t.Errorf("download contacted the upstream %d times", *s.fetches) + } +} + +func TestHostedPushIsServedByTheComposerHandler(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, monologZip) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec = s.do(t, http.MethodGet, "/composer/files/monolog/monolog/3.5.0/monolog-3.5.0.zip", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != monologZip { + t.Fatalf("download status = %d, body %q, want 200 and the pushed archive", rec.Code, rec.Body) + } + if *s.upstreamRequests != 0 || *s.fetches != 0 { + t.Errorf("download contacted the upstream (%d metadata requests, %d fetches)", *s.upstreamRequests, *s.fetches) + } +} + +func TestComposerDownloadOfUnpushedFileStillNeedsUpstreamMetadata(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/monolog/monolog/3.5.0", testHostedToken, monologZip); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/composer/files/monolog/monolog/3.5.0/other.zip", "", "") + if rec.Code != http.StatusNotFound { + t.Errorf("download of another filename status = %d, want 404 from the upstream lookup", rec.Code) + } + if *s.upstreamRequests == 0 { + t.Error("a filename that was not pushed was served without the upstream metadata lookup") + } +} + +func TestHostedPushScopedNpmPackage(t *testing.T) { + s := newHostedTestServer(t) + + rec := s.do(t, http.MethodPut, "/api/hosted/npm/@types/node/20.1.0", testHostedToken, nodeTarball) + if rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + rec = s.do(t, http.MethodGet, "/npm/@types/node/-/node-20.1.0.tgz", "", "") + if rec.Code != http.StatusOK || rec.Body.String() != nodeTarball { + t.Fatalf("download status = %d, body %q, want 200 and the pushed tarball", rec.Code, rec.Body) + } +} + +func TestHostedListAndDelete(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/npm/request/2.88.3", testHostedToken, requestTarball); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/api/hosted/npm/request", "", "") + if rec.Code != http.StatusOK { + t.Fatalf("GET status = %d, body %s", rec.Code, rec.Body) + } + var listed HostedListResponse + if err := json.NewDecoder(rec.Body).Decode(&listed); err != nil { + t.Fatalf("decoding GET response: %v", err) + } + if listed.Ecosystem != "npm" || listed.Name != "request" || len(listed.Artifacts) != 1 || listed.Artifacts[0].Version != "2.88.3" { + t.Errorf("GET response = %+v", listed) + } + + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", "", ""); rec.Code != http.StatusUnauthorized { + t.Errorf("DELETE without token status = %d, want 401", rec.Code) + } + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", testHostedToken, ""); rec.Code != http.StatusNoContent { + t.Errorf("DELETE status = %d, body %s", rec.Code, rec.Body) + } + if rec := s.do(t, http.MethodDelete, "/api/hosted/npm/request/2.88.3", testHostedToken, ""); rec.Code != http.StatusNotFound { + t.Errorf("second DELETE status = %d, want 404", rec.Code) + } +} + +func TestHostedPushRejections(t *testing.T) { + tests := []struct { + name string + target string + token string + body string + wantStatus int + }{ + {name: "no token", target: "/api/hosted/npm/request/2.88.3", body: requestTarball, wantStatus: http.StatusUnauthorized}, + {name: "wrong token", target: "/api/hosted/npm/request/2.88.3", token: "guess", body: requestTarball, wantStatus: http.StatusUnauthorized}, + {name: "unsupported ecosystem", target: "/api/hosted/cargo/serde/1.0.200", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "composer name without vendor", target: "/api/hosted/composer/monolog/3.5.0", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "missing version", target: "/api/hosted/npm/request", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "not an archive", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: "plain text", wantStatus: http.StatusBadRequest}, + {name: "manifest of another version", target: "/api/hosted/npm/request/2.88.4", token: testHostedToken, body: requestTarball, wantStatus: http.StatusBadRequest}, + {name: "empty body", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, wantStatus: http.StatusBadRequest}, + {name: "too large", target: "/api/hosted/npm/request/2.88.3", token: testHostedToken, body: strings.Repeat("x", testHostedMaxUpload+1), wantStatus: http.StatusRequestEntityTooLarge}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newHostedTestServer(t) + rec := s.do(t, http.MethodPut, tt.target, tt.token, tt.body) + if rec.Code != tt.wantStatus { + t.Fatalf("PUT status = %d, want %d, body %s", rec.Code, tt.wantStatus, rec.Body) + } + listed := s.do(t, http.MethodGet, "/api/hosted/npm/request", "", "") + if strings.Contains(listed.Body.String(), "2.88.3") { + t.Error("rejected push was stored") + } + }) + } +} + +func TestHostedListUnsupportedEcosystem(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodGet, "/api/hosted/maven/junit", "", ""); rec.Code != http.StatusBadRequest { + t.Errorf("GET status = %d, want 400", rec.Code) + } +} + +func startHostedServer(t *testing.T, hosted config.HostedConfig) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Hosted = hosted + upstream := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(upstream.Close) + cfg.Upstream.NPM = upstream.URL + cfg.Upstream.AllowLoopback = true + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + <-startErr + }) + return cfg.BaseURL +} + +func putArchive(t *testing.T, url, token string) int { + t.Helper() + return putBody(t, url, token, requestTarball) +} + +func putBody(t *testing.T, url, token, body string) int { + t.Helper() + req, err := http.NewRequest(http.MethodPut, url, strings.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("PUT %s: %v", url, err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +func TestHostedRoutesFollowConfig(t *testing.T) { + disabled := startHostedServer(t, config.HostedConfig{}) + if status := putArchive(t, disabled+"/api/hosted/npm/request/2.88.3", testHostedToken); status == http.StatusCreated { + t.Error("push accepted with hosted.enabled false") + } + + enabled := startHostedServer(t, config.HostedConfig{Enabled: true, Token: testHostedToken}) + if status := putArchive(t, enabled+"/api/hosted/npm/request/2.88.3", ""); status != http.StatusUnauthorized { + t.Errorf("push without token status = %d, want 401", status) + } + if status := putArchive(t, enabled+"/api/hosted/npm/request/2.88.3", testHostedToken); status != http.StatusCreated { + t.Errorf("push with token status = %d, want 201", status) + } +} + +func TestHostedNpmPackageIsListedByTheRunningServer(t *testing.T) { + base := startHostedServer(t, config.HostedConfig{Enabled: true, Token: testHostedToken}) + private := npmTarball(`{"name":"@acme/private","version":"1.0.0"}`) + if status := putBody(t, base+"/api/hosted/npm/@acme/private/1.0.0", testHostedToken, private); status != http.StatusCreated { + t.Fatalf("push status = %d, want 201", status) + } + + resp, err := http.Get(base + "/npm/@acme%2fprivate") + if err != nil { + t.Fatalf("GET packument: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"1.0.0"`) { + t.Errorf("packument status = %d, body %s", resp.StatusCode, body) + } +} + +func TestHostedComposerVersionIsInstallableFromMetadata(t *testing.T) { + s := newHostedTestServer(t) + if rec := s.do(t, http.MethodPut, "/api/hosted/composer/acme/private/1.0.0", testHostedToken, + composerZip(`{"name":"acme/private","require":{"php":">=8.1"}}`)); rec.Code != http.StatusCreated { + t.Fatalf("PUT status = %d, body %s", rec.Code, rec.Body) + } + + rec := s.do(t, http.MethodGet, "/composer/p2/acme/private.json", "", "") + if rec.Code != http.StatusOK { + t.Fatalf("metadata status = %d, body %s", rec.Code, rec.Body) + } + var doc struct { + Packages map[string][]struct { + Dist struct { + URL string `json:"url"` + } `json:"dist"` + } `json:"packages"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil { + t.Fatalf("decoding metadata: %v", err) + } + versions := doc.Packages["acme/private"] + if len(versions) != 1 { + t.Fatalf("metadata = %s", rec.Body) + } + distPath := strings.TrimPrefix(versions[0].Dist.URL, "http://proxy.test") + if rec := s.do(t, http.MethodGet, distPath, "", ""); rec.Code != http.StatusOK { + t.Errorf("download of %s status = %d", distPath, rec.Code) + } +} diff --git a/internal/server/hosted_archives_test.go b/internal/server/hosted_archives_test.go new file mode 100644 index 00000000..85150aa2 --- /dev/null +++ b/internal/server/hosted_archives_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "io" +) + +// npmTarball builds an npm package tarball holding manifest as +// package/package.json. Building an archive in memory cannot fail, so errors +// panic. +func npmTarball(manifest string) string { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + must(tw.WriteHeader(&tar.Header{Name: "package/package.json", Mode: 0o644, Size: int64(len(manifest))})) + _, err := io.WriteString(tw, manifest) + must(err) + must(tw.Close()) + must(gz.Close()) + return buf.String() +} + +// composerZip builds a Composer dist archive holding manifest as composer.json. +func composerZip(manifest string) string { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("composer.json") + must(err) + _, err = io.WriteString(w, manifest) + must(err) + must(zw.Close()) + return buf.String() +} + +func must(err error) { + if err != nil { + panic(err) + } +} + +var ( + requestTarball = npmTarball(`{"name":"request","version":"2.88.3","dependencies":{"qs":"~6.5.2"}}`) + nodeTarball = npmTarball(`{"name":"@types/node","version":"20.1.0"}`) + monologZip = composerZip(`{"name":"monolog/monolog","require":{"php":">=8.1","psr/log":"^3.0"}}`) +) diff --git a/internal/server/hosted_auth.go b/internal/server/hosted_auth.go new file mode 100644 index 00000000..b47a2971 --- /dev/null +++ b/internal/server/hosted_auth.go @@ -0,0 +1,37 @@ +package server + +import ( + "crypto/sha256" + "crypto/subtle" + "net/http" + "strings" +) + +const bearerScheme = "bearer " + +// requireBearerToken rejects requests that do not carry +// "Authorization: Bearer ". Both tokens are hashed before comparing so +// the check takes the same time whatever the length of the presented token. +func requireBearerToken(token string) func(http.Handler) http.Handler { + want := sha256.Sum256([]byte(token)) + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + presented, ok := bearerToken(r) + got := sha256.Sum256([]byte(presented)) + if !ok || subtle.ConstantTimeCompare(got[:], want[:]) != 1 { + w.Header().Set("WWW-Authenticate", `Bearer realm="proxy"`) + writeError(w, http.StatusUnauthorized, ErrCodeUnauthorized, "a valid bearer token is required") + return + } + next.ServeHTTP(w, r) + }) + } +} + +func bearerToken(r *http.Request) (string, bool) { + header := r.Header.Get("Authorization") + if len(header) <= len(bearerScheme) || !strings.EqualFold(header[:len(bearerScheme)], bearerScheme) { + return "", false + } + return header[len(bearerScheme):], true +} diff --git a/internal/server/hosted_auth_test.go b/internal/server/hosted_auth_test.go new file mode 100644 index 00000000..6a642b07 --- /dev/null +++ b/internal/server/hosted_auth_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRequireBearerToken(t *testing.T) { + tests := []struct { + name string + authorization string + wantStatus int + }{ + {name: "missing header", authorization: "", wantStatus: http.StatusUnauthorized}, + {name: "wrong token", authorization: "Bearer guess", wantStatus: http.StatusUnauthorized}, + {name: "token prefix only", authorization: "Bearer s3cre", wantStatus: http.StatusUnauthorized}, + {name: "basic scheme", authorization: "Basic czNjcmV0", wantStatus: http.StatusUnauthorized}, + {name: "empty bearer", authorization: "Bearer ", wantStatus: http.StatusUnauthorized}, + {name: "valid token", authorization: "Bearer s3cret", wantStatus: http.StatusNoContent}, + {name: "scheme is case-insensitive", authorization: "bearer s3cret", wantStatus: http.StatusNoContent}, + } + + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + handler := requireBearerToken("s3cret")(next) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/api/hosted/npm/request/2.88.3", nil) + if tt.authorization != "" { + req.Header.Set("Authorization", tt.authorization) + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != tt.wantStatus { + t.Fatalf("status = %d, want %d", rec.Code, tt.wantStatus) + } + if tt.wantStatus != http.StatusUnauthorized { + return + } + if got := rec.Header().Get("WWW-Authenticate"); got != `Bearer realm="proxy"` { + t.Errorf("WWW-Authenticate = %q, want Bearer realm=\"proxy\"", got) + } + var body ErrorResponse + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decoding error body: %v", err) + } + if body.Code != ErrCodeUnauthorized { + t.Errorf("error code = %q, want %q", body.Code, ErrCodeUnauthorized) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 055d19d4..89a60905 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -268,6 +268,9 @@ func (s *Server) serve(listener net.Listener) error { proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly + proxy.HostedEnabled = s.cfg.Hosted.Enabled + proxy.HostedNPMScopes = s.cfg.Hosted.NPMScopes + proxy.HostedComposerVendors = s.cfg.Hosted.ComposerVendors proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.DirectServe = s.cfg.Storage.DirectServe @@ -365,6 +368,12 @@ func (s *Server) serve(listener net.Listener) error { go jobStore.StartCleanup(bgCtx) } + // Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED) + if s.cfg.Hosted.Enabled { + hostedAPI := NewHostedAPIHandler(proxy, s.cfg.Hosted.MaxUploadBytes(), s.logger) + registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded()) + } + s.http = &http.Server{ Addr: s.cfg.Listen, Handler: r,