diff --git a/README.md b/README.md index b85dcd36..3aecba4f 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,16 @@ export GOPROXY=http://localhost:8080/go,direct Or in your shell profile for persistence. +Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at +`/go/sumdb/sum.golang.org/`, and the `go` command uses it without further +setup, so clients need no route of their own to `sum.golang.org`. If the +proxy loses `https://sum.golang.org`, it keeps answering until its cached +copy of the database's signed tree is older than `metadata_ttl`; after that +the `go` command contacts the database directly as before. Set +`upstream.go_sumdbs_offline` on networks where clients cannot do that. Other +checksum databases are listed under `upstream.go_sumdbs`, see +[Go checksum database](docs/configuration.md#go-checksum-database). + ### Homebrew Point Homebrew's JSON API and artifact domain at the proxy: diff --git a/docs/configuration.md b/docs/configuration.md index 89e91411..2ff1ab5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -237,6 +237,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | | `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` | +| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` | | `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` | | `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` | | `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` | @@ -495,6 +496,63 @@ When `upstream.arch` is empty, a single repository named `archlinux` pointing at that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not the `mirror.archlinuxarm.org` redirector. +### Go checksum database + +```yaml +upstream: + + # Named Go checksum databases, served at /go/sumdb/{name}/. + go_sumdbs: + sum.golang.org: "https://sum.golang.org" + + # Keep answering for a database from its cached tree, however old, when + # the database cannot be reached. For networks where clients have no + # route to the database of their own. Requires cache_metadata. + go_sumdbs_offline: false +``` + +The `go` command asks `GOPROXY` for the checksum database before contacting it +directly. The proxy answers for every name listed here and fetches lookups and +tiles from the database's URL, not from `upstream.go`. Unknown names get 404, +so the `go` command falls back to contacting that database itself. When +`upstream.go_sumdbs` is empty, `sum.golang.org` is served; configuring any +entry replaces that default. + +Once the proxy says it serves a database, the `go` command no longer contacts +that database directly. The proxy therefore only says so when it can fetch the +database's latest signed tree, or has a fresh one cached (fetched within +`metadata_ttl`). If it cannot, it answers 404 as for an unknown name, and +clients go to the database themselves, as they would without the proxy. The +check waits five seconds at most, and after a failed check the proxy waits a +minute before it tries the database again. + +Set `upstream.go_sumdbs_offline: true` when clients have no route to the +database of their own. The proxy then keeps answering for it from the cached +tree for as long as the outage lasts and serves cached lookups and tiles. +That does not mean every cached module still verifies: the `go` command +checks each lookup against tiles of the signed tree that came with it, the +database attaches a different tree to almost every lookup, and a tile no +client fetched through the proxy before cannot be served during the outage. +Lookups for modules that were never cached fail as well. Builds whose +`go.sum` already lists every module do not ask the database at all and are +not affected. This requires `cache_metadata`. + +With `cache_metadata` enabled, lookups, partial hash tiles and the latest +signed tree follow `metadata_ttl` and are served from cache when the database +is unreachable. Complete hash tiles never change and are not revalidated once +cached; a tile of the wrong size is refused and not stored. Data tiles are +always forwarded and never stored: the `go` command does not fetch them, only +mirrors and auditors do. +Without `cache_metadata` the requests are forwarded and nothing is stored. + +A 404 from the database keeps its text, which the `go` command prints to +explain why a version is missing. With `cache_metadata` enabled the text is +cut at 4 KiB; without it the response is passed on unchanged. + +If `upstream.go` points at a module proxy that serves the checksum database +itself, such as Athens, point the entry there instead, for example +`sum.golang.org: "https://athens.example/sumdb/sum.golang.org"`. + ### Debian archives ```yaml diff --git a/go.mod b/go.mod index af9984d2..052604e5 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 + golang.org/x/mod v0.41.0 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 google.golang.org/protobuf v1.36.12 @@ -117,7 +118,6 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/mod v0.41.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index b79a2463..080209db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -573,6 +573,21 @@ type UpstreamConfig struct { // Default: https://proxy.golang.org Go string `json:"go" yaml:"go"` + // GoSumDBs maps Go checksum database names to their base URLs, served at + // /go/sumdb/{name}/ so the go command reaches the database through + // GOPROXY. Unknown names get 404, which makes the go command contact the + // database directly. + // Default when empty: {"sum.golang.org": "https://sum.golang.org"}. + GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"` + + // GoSumDBsOffline keeps answering for a checksum database from its + // cached signed tree, however old, when the database cannot be reached. + // Set this when clients have no route to the database of their own; + // otherwise the proxy stops answering once the cached tree is older than + // metadata_ttl, and the go command contacts the database directly. + // Requires cache_metadata. Default: false. + GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"` + // Hex is the upstream Hex repository URL. // Default: https://repo.hex.pm Hex string `json:"hex" yaml:"hex"` @@ -773,6 +788,9 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } + if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil { + return err + } if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil { return err } @@ -813,6 +831,18 @@ func (u *UpstreamConfig) validateGenericArtifacts() error { return nil } +// validateUpstream checks the upstream block, including the one setting in +// it that depends on a top-level option. +func (c *Config) validateUpstream() error { + if err := c.Upstream.Validate(); err != nil { + return err + } + if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { + return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") + } + return nil +} + // CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern // so that it only matches a whole path, and checks it has the `version` and // `file` groups. @@ -1140,6 +1170,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") @@ -1315,7 +1346,7 @@ func (c *Config) validateComponents() error { return err } - if err := c.Upstream.Validate(); err != nil { + if err := c.validateUpstream(); err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9bee2dca..d7b5326c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1612,6 +1612,61 @@ func TestValidateNamedUpstreams(t *testing.T) { } } +func TestValidateGoSumDBs(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr bool + }{ + { + name: "valid Go checksum database", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "https://sum.golang.org"} + }, + }, + { + name: "Go checksum database name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org/x": "https://sum.golang.org"} + }, + wantErr: true, + }, + { + name: "Go checksum database URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "sum.golang.org"} + }, + wantErr: true, + }, + { + name: "Go checksum database offline mode requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.GoSumDBsOffline = true + }, + wantErr: true, + }, + { + name: "Go checksum database offline mode with metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = true + cfg.Upstream.GoSumDBsOffline = true + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + tt.modify(cfg) + err := cfg.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr) + } + }) + } +} + func TestValidateCacheArtifactsDisabled(t *testing.T) { tests := []struct { name string diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 0df443d7..2797d421 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -330,7 +330,6 @@ func TestGoHandler_UpstreamProxy(t *testing.T) { "/golang.org/x/text/@v/v0.14.0.info", "/golang.org/x/text/@v/v0.14.0.mod", "/golang.org/x/text/@latest", - "/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.14.0", } for _, path := range tests { diff --git a/internal/handler/go.go b/internal/handler/go.go index 353edbbc..a27c37be 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -28,9 +28,12 @@ func init() { // GoHandler handles Go module proxy protocol requests. type GoHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string + proxy *Proxy + upstreamURL string + proxyURL string + sumDBs map[string]string + sumDBOffline bool + sumDBProbes goSumDBProbes } // NewGoHandler creates a new Go module proxy handler. @@ -39,6 +42,7 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler { proxy: proxy, upstreamURL: goUpstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), + sumDBs: map[string]string{defaultGoSumDBName: defaultGoSumDBURL}, } } @@ -65,9 +69,9 @@ func (h *GoHandler) Routes() http.Handler { func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) { path := strings.TrimPrefix(r.URL.Path, "/") - // Sumdb requests - proxy through - if strings.HasPrefix(path, "sumdb/") { - h.proxyUpstream(w, r) + // Checksum database requests go to the database, not the module upstream + if rest, ok := strings.CutPrefix(path, "sumdb/"); ok { + h.handleSumDB(w, r, rest) return } @@ -141,11 +145,6 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// proxyUpstream forwards a request to proxy.golang.org without caching. -func (h *GoHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { - h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil) -} - // proxyCached forwards a request with metadata caching. func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey string) { h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*") diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go new file mode 100644 index 00000000..020a126e --- /dev/null +++ b/internal/handler/go_sumdb.go @@ -0,0 +1,429 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" + "golang.org/x/mod/sumdb/tlog" +) + +const ( + // defaultGoSumDBName is the checksum database the go command uses when + // GOSUMDB is not set. + defaultGoSumDBName = "sum.golang.org" + defaultGoSumDBURL = "https://sum.golang.org" + goEcosystem = "golang" + + // goSumDBProbeBackoff is how long supported keeps answering 404 after the + // proxy failed to reach a checksum database, before it tries again. + goSumDBProbeBackoff = time.Minute + // goSumDBProbeTimeout is how long supported waits for the database. The + // go command's own request has no timeout, so a database that drops + // packets must not hold every build for the full upstream timeout. + goSumDBProbeTimeout = 5 * time.Second +) + +// goSumDBProbes remembers when the proxy last failed to reach each checksum +// database, so a database that is unreachable does not make every go command +// wait for an upstream timeout before it falls back. +type goSumDBProbes struct { + // timeout overrides goSumDBProbeTimeout when set (tests). + timeout time.Duration + + mu sync.Mutex + failed map[string]time.Time + succeeded map[string]time.Time +} + +func (p *goSumDBProbes) probeTimeout() time.Duration { + if p.timeout > 0 { + return p.timeout + } + return goSumDBProbeTimeout +} + +func (p *goSumDBProbes) recentlyFailed(name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + failedAt, ok := p.failed[name] + return ok && time.Since(failedAt) < goSumDBProbeBackoff +} + +func (p *goSumDBProbes) record(name string, reachable bool) { + p.mu.Lock() + defer p.mu.Unlock() + if reachable { + delete(p.failed, name) + if p.succeeded == nil { + p.succeeded = make(map[string]time.Time) + } + p.succeeded[name] = time.Now() + return + } + if p.failed == nil { + p.failed = make(map[string]time.Time) + } + p.failed[name] = time.Now() +} + +// recordFailure records that the database did not answer a request started +// at start, unless a probe has reached it since: a probe that timed out, or +// a lookup that fell back to the cache, must not overwrite a newer success. +func (p *goSumDBProbes) recordFailure(name string, start time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + if !p.succeeded[name].Before(start) { + return + } + if p.failed == nil { + p.failed = make(map[string]time.Time) + } + p.failed[name] = time.Now() +} + +// SetSumDBs sets the checksum databases served at /sumdb/{name}/, mapping +// each database name to its base URL. An empty map keeps the default, +// sum.golang.org. offline makes a cached tree of any age enough to keep +// answering for a database the proxy cannot reach, for networks where +// clients have no route to the database of their own. +func (h *GoHandler) SetSumDBs(sumDBs map[string]string, offline bool) { + h.sumDBOffline = offline + if len(sumDBs) == 0 { + h.sumDBs = map[string]string{defaultGoSumDBName: defaultGoSumDBURL} + return + } + h.sumDBs = make(map[string]string, len(sumDBs)) + for name, sumDBURL := range sumDBs { + h.sumDBs[name] = strings.TrimSuffix(sumDBURL, "/") + } +} + +// handleSumDB serves checksum database requests that the go command sends +// through GOPROXY as /sumdb/{name}/{path}. The module proxy upstream does not +// serve them, so they are fetched from the configured database itself. +// Unknown names get 404, which makes the go command contact the database +// directly. +func (h *GoHandler) handleSumDB(w http.ResponseWriter, r *http.Request, rest string) { + name, path, _ := strings.Cut(rest, "/") + sumDBURL, ok := h.sumDBs[name] + if !ok { + http.NotFound(w, r) + return + } + if path == "supported" { + if h.sumDBReachable(r, name, sumDBURL) { + w.WriteHeader(http.StatusOK) + } else { + http.NotFound(w, r) + } + return + } + immutable, valid := parseGoSumDBPath(path) + if !valid { + http.NotFound(w, r) + return + } + // Only mirrors and auditors fetch data tiles, never the go command, and + // together they hold the whole log, so they are not stored. + if tile, err := tlog.ParseTilePath(path); err == nil && tile.L < 0 { + h.proxy.proxyMetadataStream(w, r, sumDBURL+"/"+path, "identity", "*/*") + return + } + + cacheKey := goSumDBCacheKey(name, sumDBURL, path) + check := goSumDBResponseCheck(path) + served, damaged := false, false + if immutable { + served, damaged = h.serveCachedSumDBTile(w, r, cacheKey, check) + } + if served { + return + } + h.proxySumDB(w, r, name, sumDBURL+"/"+path, cacheKey, check, damaged) +} + +// sumDBReachable reports whether the proxy can serve a checksum database: it +// can fetch the latest signed tree, or has it cached. Once supported answers +// 200 the go command uses only the proxy for that database and never falls +// back to contacting it directly, so a database the proxy cannot reach gets +// 404 instead. The probe goes through the same cache entry as a latest +// request: with cache_metadata it costs at most one upstream request per +// metadata TTL, without it each probe fetches latest. +func (h *GoHandler) sumDBReachable(r *http.Request, name, sumDBURL string) bool { + cacheKey := goSumDBCacheKey(name, sumDBURL, "latest") + treeCheck := goSumDBResponseCheck("latest") + if h.sumDBProbes.recentlyFailed(name) { + return h.cachedTreeServes(r.Context(), cacheKey, treeCheck) + } + // The fetch is shared with concurrent callers and runs detached from this + // request, so it is bounded by the upstream timeout, not by ours. Wait + // for it only up to the probe timeout; it finishes in the background, + // records its own outcome and fills the cache for the next probe. + start := time.Now() + done := make(chan error, 1) + go func() { + done <- h.probeSumDB(context.WithoutCancel(r.Context()), name, cacheKey, sumDBURL, treeCheck) + }() + var err error + select { + case err = <-done: + case <-time.After(h.sumDBProbes.probeTimeout()): + err = fmt.Errorf("no answer within %s", h.sumDBProbes.probeTimeout()) + h.sumDBProbes.recordFailure(name, start) + case <-r.Context().Done(): + // The client went away; that says nothing about the database. + return false + } + if err == nil { + return true + } + // The stale-cache fallback inside the fetch only runs once upstream has + // failed, which a silent database delays past the probe timeout. A tree + // cached earlier may still serve lookups, so look for it here. + if h.cachedTreeServes(r.Context(), cacheKey, treeCheck) { + return true + } + h.proxy.Logger.Info("checksum database unreachable, sending go clients to it directly", + "name", name, "url", sumDBURL, "error", err) + return false +} + +// probeSumDB fetches the database's latest tree and records whether the +// database answered. A tree still fresh in the cache is enough for supported +// but records nothing, since the database was not asked: counting it as an +// answer would keep lookups that fail during an outage from being +// remembered. A tree served because upstream failed is not an answer. +func (h *GoHandler) probeSumDB(ctx context.Context, name, cacheKey, sumDBURL string, check func([]byte) error) error { + if entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check); ok && h.treeFresh(entry) { + return nil + } + res := h.proxy.lookupMetadata(ctx, goEcosystem, cacheKey, sumDBURL+"/latest", "*/*", "identity", check) + err := res.err + if err == nil && res.stale { + err = errors.New("checksum database did not answer, tree served from cache") + } + h.sumDBProbes.record(name, err == nil) + return err +} + +// cachedTreeServes reports whether a cached tree lets the proxy answer for +// the database although the database did not: always with go_sumdbs_offline, +// otherwise only while the tree is fresh. +func (h *GoHandler) cachedTreeServes(ctx context.Context, cacheKey string, check func([]byte) error) bool { + entry, _, _, ok := h.readCachedSumDB(ctx, cacheKey, check) + return ok && (h.sumDBOffline || h.treeFresh(entry)) +} + +// treeFresh reports whether a cached tree is within the metadata TTL. +func (h *GoHandler) treeFresh(entry *database.MetadataCacheEntry) bool { + ttl := h.proxy.MetadataTTL + return ttl > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < ttl +} + +// readCachedSumDB reads a checksum database response from the metadata +// cache regardless of its age. It reports false when nothing usable is +// cached. +func (h *GoHandler) readCachedSumDB(ctx context.Context, cacheKey string, check func([]byte) error) (entry *database.MetadataCacheEntry, body []byte, contentType string, ok bool) { + p := h.proxy + if !p.CacheMetadata || p.DB == nil { + return nil, nil, "", false + } + entry, err := p.DB.GetMetadataCache(goEcosystem, cacheKey) + if err != nil || entry == nil { + return nil, nil, "", false + } + body, contentType, err = p.readCachedMetadata(ctx, entry, check) + if err != nil { + return nil, nil, "", false + } + return entry, body, contentType, true +} + +// parseGoSumDBPath reports whether path is a request the checksum database +// protocol defines and whether its response can never change. Only complete +// tiles are immutable; partial tiles, latest and lookups follow the metadata +// TTL. +func parseGoSumDBPath(path string) (immutable, valid bool) { + if path == "latest" { + return false, true + } + if mod, ok := strings.CutPrefix(path, "lookup/"); ok { + return false, validGoSumDBLookup(mod) + } + tile, err := tlog.ParseTilePath(path) + if err != nil { + return false, false + } + return tile.W == 1< 0: + w.Header().Set(headerContentType, "text/plain; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write(notFound.body) + case errors.Is(err, ErrUpstreamNotFound): + http.Error(w, "not found", http.StatusNotFound) + default: + p.Logger.Error("checksum database fetch failed", "url", upstreamURL, "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + } + return + } + + p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) +} + +// fetchSumDB fetches a checksum database response through the metadata +// cache. A cached copy that failed check is fetched again without +// revalidating it: past the TTL, a database that sends an ETag would answer +// 304 and hand back the damaged copy for good. +func (h *GoHandler) fetchSumDB(ctx context.Context, upstreamURL, cacheKey string, check func([]byte) error, damaged bool) metadataResult { + p := h.proxy + if damaged { + return p.fetchMetadataFromUpstream(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check, nil) + } + return p.lookupMetadata(ctx, goEcosystem, cacheKey, upstreamURL, "*/*", "identity", check) +} diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go new file mode 100644 index 00000000..6a0c47b4 --- /dev/null +++ b/internal/handler/go_sumdb_test.go @@ -0,0 +1,987 @@ +package handler + +import ( + "bytes" + "crypto/rand" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/mod/sumdb" + "golang.org/x/mod/sumdb/note" + "golang.org/x/mod/sumdb/tlog" +) + +const ( + testSumDBName = "sum.example" + // testSumDBTree looks like a signed tree as far as the proxy checks. + testSumDBTree = "go.sum database tree\n1\nhash\n\n— sum.example signature\n" + // testSumDBRecord looks like a lookup answer as far as the proxy checks. + testSumDBRecord = "1\ngolang.org/x/text v0.14.0 h1:hash=\ngolang.org/x/text v0.14.0/go.mod h1:hash=\n\n" + testSumDBTree +) + +// hashTile returns a hash tile body of the given width. +func hashTile(width int, fill string) string { + return strings.Repeat(fill, width*tlog.HashSize) +} + +// dataTile returns a data tile body holding width records, laid out the way +// sum.golang.org serves it: a blank line between records, none after the +// last. +func dataTile(width int) string { + records := make([]string, width) + for i := range records { + records[i] = fmt.Sprintf("example.com/m%d v1.0.0 h1:hash=\nexample.com/m%d v1.0.0/go.mod h1:hash=\n", i, i) + } + return strings.Join(records, "\n") +} + +// fakeSumDB is a checksum database upstream serving fixed files. Unknown +// paths get 404 with the body from notFound, or a default text. +type fakeSumDB struct { + server *httptest.Server + + mu sync.Mutex + files map[string]string + notFound map[string]string + hits map[string]int + down atomic.Bool +} + +func newFakeSumDB(t *testing.T, files map[string]string) *fakeSumDB { + t.Helper() + if files == nil { + files = map[string]string{} + } + db := &fakeSumDB{files: files, notFound: map[string]string{}, hits: map[string]int{}} + db.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + db.mu.Lock() + db.hits[r.URL.Path]++ + body, ok := db.files[r.URL.Path] + text, custom := db.notFound[r.URL.Path] + db.mu.Unlock() + if db.down.Load() { + http.Error(w, "unavailable", http.StatusInternalServerError) + return + } + if !ok { + if !custom { + text = "not found: " + r.URL.Path + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, text) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(db.server.Close) + return db +} + +func (db *fakeSumDB) setFile(path, body string) { + db.mu.Lock() + defer db.mu.Unlock() + db.files[path] = body +} + +func (db *fakeSumDB) setNotFound(path, text string) { + db.mu.Lock() + defer db.mu.Unlock() + db.notFound[path] = text +} + +func (db *fakeSumDB) requests(path string) int { + db.mu.Lock() + defer db.mu.Unlock() + return db.hits[path] +} + +func (db *fakeSumDB) totalRequests() int { + db.mu.Lock() + defer db.mu.Unlock() + total := 0 + for _, n := range db.hits { + total += n + } + return total +} + +func newSumDBTestHandler(t *testing.T, sumDBURL string, cacheMetadata bool, ttl time.Duration) *GoHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + proxy.CacheMetadata = cacheMetadata + proxy.MetadataTTL = ttl + h := NewGoHandler(proxy, "http://proxy.example") + h.SetSumDBs(map[string]string{testSumDBName: sumDBURL}, false) + return h +} + +func serveGoRequest(h *GoHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +func TestGoSumDB_DefaultsToSumGolangOrg(t *testing.T) { + h := NewGoHandler(nil, "http://proxy.example") + if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL { + t.Errorf("default checksum database = %q, want %q", got, defaultGoSumDBURL) + } + h.SetSumDBs(nil, false) + if got := h.sumDBs[defaultGoSumDBName]; got != defaultGoSumDBURL || len(h.sumDBs) != 1 { + t.Errorf("SetSumDBs(nil) = %v, want only the default", h.sumDBs) + } +} + +func TestGoSumDB_SupportedProbe(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for range 2 { + for target, want := range map[string]int{ + "/sumdb/sum.example/supported": http.StatusOK, + "/sumdb/sum.golang.org/supported": http.StatusNotFound, + "/sumdb/other.example/supported": http.StatusNotFound, + } { + if w := serveGoRequest(h, target); w.Code != want { + t.Errorf("%s: status = %d, want %d", target, w.Code, want) + } + } + } + if n := db.totalRequests(); n != 1 { + t.Errorf("upstream requests = %d, want 1: the probe fetches latest once, then uses the cache", n) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a probe served from the fresh cache was recorded as a failure") + } +} + +// TestGoSumDB_SupportedFallsBackWhenUnreachable covers the go command's only +// way back to the database: once supported answers 200 it never contacts the +// database directly, so a database the proxy cannot reach must get 404. +func TestGoSumDB_SupportedFallsBackWhenUnreachable(t *testing.T) { + for _, cacheMetadata := range []bool{true, false} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + db.down.Store(true) + h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) + + for i := range 2 { + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Fatalf("request %d: status = %d, want 404", i, w.Code) + } + } + if n := db.requests("/latest"); n != 1 { + t.Errorf("probes = %d, want 1: a failed probe is not repeated within %s", n, goSumDBProbeBackoff) + } + }) + } +} + +// TestGoSumDB_SupportedWithCachedTreeWhileUnreachable keeps the offline case: +// a proxy that has served the database before can still serve it from cache. +func TestGoSumDB_SupportedWithCachedTreeWhileUnreachable(t *testing.T) { + t.Run("fresh tree", func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("supported with a fresh cached tree: status = %d, want 200", w.Code) + } + }) + + // A stale tree only counts with go_sumdbs_offline. Without it, clients + // that can reach the database themselves should go there. + for _, offline := range []bool{true, false} { + t.Run(fmt.Sprintf("stale tree offline=%v", offline), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = offline + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.down.Store(true) + want := http.StatusNotFound + if offline { + want = http.StatusOK + } + for i := range 2 { // second probe hits the remembered failure + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != want { + t.Errorf("probe %d: status = %d, want %d", i, w.Code, want) + } + } + if n := db.requests("/latest"); n != 2 { + t.Errorf("upstream requests = %d, want 2: the failure is remembered", n) + } + }) + } +} + +// TestGoSumDB_SupportedGivesUpOnSilentDatabase: a database that accepts the +// connection but never answers (a firewall dropping packets) must not hold +// the go command, which waits for supported without a timeout of its own. +func TestGoSumDB_SupportedGivesUpOnSilentDatabase(t *testing.T) { + release := make(chan struct{}) + db := newFakeSumDB(t, nil) + t.Cleanup(func() { close(release) }) // runs before the server closes + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + h.sumDBProbes.timeout = 50 * time.Millisecond + + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() + select { + case w := <-answered: + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + case <-time.After(time.Second): + t.Fatal("supported still waiting for the database after a second") + } +} + +// TestGoSumDB_SupportedWithStaleTreeWhileSilent: the fetch only falls back +// to a stale cached tree once upstream has failed, which a silent database +// delays past the probe timeout. The proxy can still serve from cache, so +// supported must say so. +func TestGoSumDB_SupportedWithStaleTreeWhileSilent(t *testing.T) { + release := make(chan struct{}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + t.Cleanup(func() { close(release) }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + h.sumDBProbes.timeout = 50 * time.Millisecond + + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + + for i := range 2 { // second probe hits the remembered failure + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example/supported") }() + select { + case w := <-answered: + if w.Code != http.StatusOK { + t.Fatalf("probe %d: status = %d, want 200 from the cached tree", i, w.Code) + } + case <-time.After(time.Second): + t.Fatalf("probe %d: supported still waiting after a second", i) + } + } +} + +// TestGoSumDB_SlowProbeSuccessClearsFailure: a database that answers after +// the probe timeout gets a 404 the first time, but once its answer is in +// the cache, supported says yes without waiting for the next backoff. +func TestGoSumDB_SlowProbeSuccessClearsFailure(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + inner.ServeHTTP(w, r) + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + h.sumDBProbes.timeout = 10 * time.Millisecond + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Fatalf("first probe: status = %d, want 404", w.Code) + } + deadline := time.Now().Add(5 * time.Second) + for h.sumDBProbes.recentlyFailed(testSumDBName) { + if time.Now().After(deadline) { + t.Fatal("the late fetch never cleared the failure") + } + time.Sleep(10 * time.Millisecond) + } + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("after the late answer: status = %d, want 200", w.Code) + } + if n := db.requests("/latest"); n != 1 { + t.Errorf("upstream requests = %d, want 1: the second probe is a cache hit", n) + } +} + +// TestGoSumDB_StaleFallbackIsNotAnAnswer: the fetch reports no error when +// it falls back to the cached tree after the database failed. That must +// not count as the database answering, or the failure would be forgotten +// and every probe would wait for the timeout again. +func TestGoSumDB_StaleFallbackIsNotAnAnswer(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + h.sumDBProbes.timeout = 10 * time.Millisecond + + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(100 * time.Millisecond) + inner.ServeHTTP(w, r) + }) + db.down.Store(true) + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 from the cached tree", w.Code) + } + time.Sleep(300 * time.Millisecond) // the fetch has failed and fallen back by now + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("the stale fallback cleared the remembered failure") + } +} + +// TestGoSumDB_AnswerCountsWhenCacheWriteFails: an answer from the database +// is an answer even when it could not be cached afterwards, whether or not +// a stale tree was cached before. +func TestGoSumDB_AnswerCountsWhenCacheWriteFails(t *testing.T) { + for _, staleTree := range []bool{false, true} { + t.Run(fmt.Sprintf("stale tree cached=%v", staleTree), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if staleTree { + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusOK { + t.Fatalf("latest: status = %d, want 200", w.Code) + } + } + h.proxy.Storage.(*mockStorage).storeErr = errors.New("disk full") + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Errorf("status = %d, want 200: the database answered", w.Code) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("an answered probe was recorded as a failure") + } + }) + } +} + +// TestGoSumDB_OfflineServesCacheWithoutWaiting: in offline mode, once the +// database is known to be unreachable, stale lookups must come from the +// cache at once instead of waiting for the upstream timeout each time. +func TestGoSumDB_OfflineServesCacheWithoutWaiting(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + release := make(chan struct{}) + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) + t.Cleanup(func() { close(release) }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + h.sumDBProbes.timeout = 50 * time.Millisecond + + for _, path := range []string{"/latest", lookup} { + if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200", path, w.Code) + } + } + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-release + }) + + for _, path := range []string{"/supported", "/latest", lookup} { + answered := make(chan *httptest.ResponseRecorder, 1) + go func() { answered <- serveGoRequest(h, "/sumdb/sum.example"+path) }() + select { + case w := <-answered: + if w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200 from the cache", path, w.Code) + } + case <-time.After(time.Second): + t.Fatalf("%s: still waiting for the database after a second", path) + } + } +} + +// TestGoSumDB_StaleLookupRemembersFailure: the go command asks supported +// only once per process. A lookup that falls back to the cache must renew +// the remembered failure itself, or offline mode would wait for the upstream +// timeout on every lookup a minute into a long build. +func TestGoSumDB_StaleLookupRemembersFailure(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + h.sumDBOffline = true + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("online: status = %d, want 200", w.Code) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Fatal("an answered lookup was recorded as a failure") + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a lookup served from the cache after the database failed did not remember the failure") + } +} + +// TestGoSumDB_LateStaleLookupKeepsNewerSuccess: a lookup sent while the +// database was failing, which falls back to the cache only after a probe has +// reached the database again, must not mark it unreachable. +func TestGoSumDB_LateStaleLookupKeepsNewerSuccess(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree, lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + + var sentOnce sync.Once + sent := make(chan struct{}) + release := make(chan struct{}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == lookup { + sentOnce.Do(func() { close(sent) }) + <-release + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + inner.ServeHTTP(w, r) + }) + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- serveGoRequest(h, "/sumdb/sum.example"+lookup) }() + <-sent + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported: status = %d, want 200", w.Code) + } + close(release) + if w := <-done; w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("lookup: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + if h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a lookup that fell back after the probe succeeded marked the database unreachable") + } +} + +// TestGoSumDB_ShortcutNeedsOfflineMode: without go_sumdbs_offline, a +// remembered failure does not keep cached lookups from asking the database, +// which may well be back. +func TestGoSumDB_ShortcutNeedsOfflineMode(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + + h.sumDBProbes.record(testSumDBName, false) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup with a remembered failure: status = %d, want 200", w.Code) + } + if n := db.requests(lookup); n != 2 { + t.Errorf("database requests = %d, want 2: the cached lookup was served without asking", n) + } +} + +// TestGoSumDB_FreshTreeIsNotAnAnswer: a supported check answered from a +// fresh cached tree did not reach the database, so it must not cancel a +// failure that a lookup started earlier reports afterwards. +func TestGoSumDB_FreshTreeIsNotAnAnswer(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": testSumDBTree}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported: status = %d, want 200", w.Code) + } + + // Coarse clocks (Windows) can give the first probe's success and start + // the same time; start must come after it. + h.sumDBProbes.mu.Lock() + succeeded := h.sumDBProbes.succeeded[testSumDBName] + h.sumDBProbes.mu.Unlock() + start := time.Now() + for !start.After(succeeded) { + start = time.Now() + } + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusOK { + t.Fatalf("supported with a fresh cached tree: status = %d, want 200", w.Code) + } + h.sumDBProbes.recordFailure(testSumDBName, start) + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Error("a supported check answered from the cache counted as reaching the database") + } +} + +func TestGoSumDBProbes_TimeoutDoesNotOverwriteLaterSuccess(t *testing.T) { + var probes goSumDBProbes + start := time.Now() + probes.record(testSumDBName, true) + probes.recordFailure(testSumDBName, start) + if probes.recentlyFailed(testSumDBName) { + t.Error("a success recorded after the probe started was overwritten by its timeout") + } + probes.recordFailure(testSumDBName, time.Now().Add(time.Second)) + if !probes.recentlyFailed(testSumDBName) { + t.Error("a timeout with no later success was not recorded") + } +} + +// TestGoSumDB_SupportedRefusesBlockPage: egress proxies answer blocked hosts +// with a 200 page. That is not a reachable database, and must not be cached +// as its signed tree either. +func TestGoSumDB_SupportedRefusesBlockPage(t *testing.T) { + db := newFakeSumDB(t, map[string]string{"/latest": "access to this site is blocked"}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + if w := serveGoRequest(h, "/sumdb/sum.example/supported"); w.Code != http.StatusNotFound { + t.Errorf("supported: status = %d, want 404", w.Code) + } + if w := serveGoRequest(h, "/sumdb/sum.example/latest"); w.Code != http.StatusBadGateway { + t.Errorf("latest: status = %d, want 502 for a block page", w.Code) + } + + // The page was not cached: once the database answers, its tree is served. + db.setFile("/latest", testSumDBTree) + w := serveGoRequest(h, "/sumdb/sum.example/latest") + if w.Code != http.StatusOK || w.Body.String() != testSumDBTree { + t.Errorf("latest after recovery: status = %d, body = %q, want the signed tree", w.Code, w.Body.String()) + } +} + +// TestGoSumDB_LookupRefusesBlockPage: a block page in place of a lookup must +// not be cached, where it would replace the copy served during an outage. +// That includes a page that happens to start like a record. +func TestGoSumDB_LookupRefusesBlockPage(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + pages := map[string]string{ + "html": "access to this site is blocked", + "record without tree": "1\ngolang.org/x/text v0.14.0 h1:hash=\n\naccess to this site is blocked", + } + for name, page := range pages { + t.Run(name, func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("lookup: status = %d, want 200", w.Code) + } + db.setFile(lookup, page) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Errorf("behind a block page: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + + // The page did not replace the record: it is still there in an outage. + db.down.Store(true) + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Errorf("in an outage: status = %d, body = %q, want the cached record", w.Code, w.Body.String()) + } + }) + } +} + +func TestGoSumDB_RequestsGoToTheDatabase(t *testing.T) { + var moduleUpstreamRequests atomic.Int32 + moduleUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + moduleUpstreamRequests.Add(1) + http.NotFound(w, r) + })) + defer moduleUpstream.Close() + + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + h := NewGoHandlerWithUpstream(proxy, "http://proxy.example", moduleUpstream.URL) + h.SetSumDBs(map[string]string{testSumDBName: db.server.URL + "/"}, false) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("status = %d, body = %q; want 200 with the database's response", w.Code, w.Body.String()) + } + if n := db.requests(lookup); n != 1 { + t.Errorf("database requests for %s = %d, want 1", lookup, n) + } + if n := moduleUpstreamRequests.Load(); n != 0 { + t.Errorf("module upstream requests = %d, want 0", n) + } +} + +func TestGoSumDB_LookupCachedWithinTTL(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for i := range 2 { + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Body.String() != testSumDBRecord { + t.Fatalf("request %d: status = %d, body = %q", i, w.Code, w.Body.String()) + } + } + if n := db.requests(lookup); n != 1 { + t.Errorf("database requests = %d, want 1: the second lookup is a cache hit", n) + } +} + +func TestGoSumDB_CompleteTilesSkipRevalidation(t *testing.T) { + files := map[string]string{ + "/tile/8/0/000": hashTile(256, "a"), + "/tile/8/0/001.p/5": hashTile(5, "b"), + "/latest": testSumDBTree, + } + wantRequests := map[string]int{ + "/tile/8/0/000": 1, + "/tile/8/0/001.p/5": 2, + "/latest": 2, + } + db := newFakeSumDB(t, files) + // Every cached entry is past its TTL by the second request. + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + for range 2 { + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("%s: status = %d, body = %q, want %q", path, w.Code, w.Body.String(), want) + } + } + } + for path, want := range wantRequests { + if n := db.requests(path); n != want { + t.Errorf("%s: database requests = %d, want %d", path, n, want) + } + } +} + +// TestGoSumDB_DataTilesAreNotStored: the go command never fetches data +// tiles, and stored they would add up to the whole log. +func TestGoSumDB_DataTilesAreNotStored(t *testing.T) { + files := map[string]string{ + "/tile/8/data/000": dataTile(256), + "/tile/8/data/001.p/5": dataTile(5), + } + db := newFakeSumDB(t, files) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for range 2 { + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("%s: status = %d, body length %d, want the %d-byte tile", path, w.Code, w.Body.Len(), len(want)) + } + } + } + for path := range files { + if n := db.requests(path); n != 2 { + t.Errorf("%s: database requests = %d, want 2", path, n) + } + key := goSumDBCacheKey(testSumDBName, db.server.URL, strings.TrimPrefix(path, "/")) + if entry, _ := h.proxy.DB.GetMetadataCache(goEcosystem, key); entry != nil { + t.Errorf("%s: stored in the metadata cache", path) + } + } +} + +// TestGoSumDB_BadTilesAreNotCached guards complete tiles, which are never +// revalidated once cached: a response of the wrong size must not be stored. +func TestGoSumDB_BadTilesAreNotCached(t *testing.T) { + const tile = "/tile/8/0/000" + db := newFakeSumDB(t, map[string]string{ + tile: "please log in", + }) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusBadGateway { + t.Fatalf("wrong-size tile: status = %d, want 502", w.Code) + } + + want := hashTile(256, "a") + db.setFile(tile, want) + w := serveGoRequest(h, "/sumdb/sum.example"+tile) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("after upstream recovered: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) + } + if n := db.requests(tile); n != 2 { + t.Errorf("upstream requests = %d, want 2: the bad response was not cached", n) + } +} + +// TestGoSumDB_DamagedCachedTileIsFetchedAgain guards the check on the cached +// copy: a complete tile is never revalidated, so a damaged blob would +// otherwise be served for good. The database sends an ETag, so past the TTL +// a request that asks whether the tile changed would get 304 and the damaged +// copy back. +func TestGoSumDB_DamagedCachedTileIsFetchedAgain(t *testing.T) { + const tile = "/tile/8/0/000" + want := hashTile(256, "a") + for _, ttl := range []time.Duration{time.Hour, time.Nanosecond} { + t.Run(fmt.Sprintf("ttl=%s", ttl), func(t *testing.T) { + db := newFakeSumDB(t, map[string]string{tile: want}) + inner := db.server.Config.Handler + db.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"tile"`) + if r.Header.Get("If-None-Match") == `"tile"` { + w.WriteHeader(http.StatusNotModified) + return + } + inner.ServeHTTP(w, r) + }) + h := newSumDBTestHandler(t, db.server.URL, true, ttl) + + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + + store := h.proxy.Storage.(*mockStorage) + blob := metadataStoragePath(goEcosystem, goSumDBCacheKey(testSumDBName, db.server.URL, "tile/8/0/000")) + store.mu.Lock() + if _, ok := store.files[blob]; !ok { + store.mu.Unlock() + t.Fatalf("tile not stored at %s", blob) + } + store.files[blob] = []byte(want[:100]) + store.mu.Unlock() + + w := serveGoRequest(h, "/sumdb/sum.example"+tile) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Fatalf("after damage: status = %d, body length %d, want the %d-byte tile", w.Code, w.Body.Len(), len(want)) + } + if n := db.requests(tile); n != 2 { + t.Errorf("upstream requests = %d, want 2: the damaged copy was fetched again", n) + } + }) + } +} + +func TestGoSumDB_CompleteTilesNotStoredWithoutMetadataCache(t *testing.T) { + const tile = "/tile/8/0/000" + db := newFakeSumDB(t, map[string]string{tile: hashTile(256, "a")}) + h := newSumDBTestHandler(t, db.server.URL, false, time.Hour) + + for range 2 { + if w := serveGoRequest(h, "/sumdb/sum.example"+tile); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + } + if n := db.requests(tile); n != 2 { + t.Errorf("database requests = %d, want 2 with cache_metadata off", n) + } +} + +func TestGoSumDB_OfflineFallback(t *testing.T) { + files := map[string]string{ + "/lookup/golang.org/x/text@v0.14.0": testSumDBRecord, + "/latest": testSumDBTree, + "/tile/8/0/000": hashTile(256, "a"), + "/tile/8/0/001.p/5": hashTile(5, "b"), + } + db := newFakeSumDB(t, files) + h := newSumDBTestHandler(t, db.server.URL, true, time.Nanosecond) + + for path := range files { + if w := serveGoRequest(h, "/sumdb/sum.example"+path); w.Code != http.StatusOK { + t.Fatalf("%s online: status = %d, want 200", path, w.Code) + } + } + + db.down.Store(true) + for path, want := range files { + w := serveGoRequest(h, "/sumdb/sum.example"+path) + if w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("%s offline: status = %d, body = %q, want 200 with %q", path, w.Code, w.Body.String(), want) + } + } +} + +func TestGoSumDB_NotFoundRelaysUpstreamText(t *testing.T) { + const lookup = "/lookup/github.com/google/uuid@v9.9.9" + const reason = "not found: github.com/google/uuid@v9.9.9: invalid version: unknown revision v9.9.9" + + for _, cacheMetadata := range []bool{true, false} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + db := newFakeSumDB(t, nil) + db.setNotFound(lookup, reason) + h := newSumDBTestHandler(t, db.server.URL, cacheMetadata, time.Hour) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + if got := strings.TrimSpace(w.Body.String()); got != reason { + t.Errorf("body = %q, want the database's text %q", got, reason) + } + }) + } + + t.Run("long text is cut", func(t *testing.T) { + db := newFakeSumDB(t, nil) + db.setNotFound(lookup, strings.Repeat("x", upstreamNotFoundBodyLimit+1000)) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404", w.Code) + } + if got := w.Body.Len(); got != upstreamNotFoundBodyLimit { + t.Errorf("body length = %d, want %d", got, upstreamNotFoundBodyLimit) + } + }) +} + +func TestGoSumDB_InvalidPathsNeverReachUpstream(t *testing.T) { + db := newFakeSumDB(t, nil) + h := newSumDBTestHandler(t, db.server.URL, true, time.Hour) + + for _, target := range []string{ + "/sumdb/sum.example", + "/sumdb/sum.example/", + "/sumdb/sum.example/other", + "/sumdb/sum.example/latest/extra", + "/sumdb/sum.example/lookup/", + "/sumdb/sum.example/lookup/golang.org/x/text", + "/sumdb/sum.example/lookup/golang.org/x/text@", + "/sumdb/sum.example/lookup/golang.org/x/text@master", + "/sumdb/sum.example/lookup/github.com/Google/uuid@v1.6.0", + "/sumdb/sum.example/lookup/../../etc/passwd@v1.0.0", + "/sumdb/sum.example/tile/x/0/000", + "/sumdb/sum.example/tile/8/0/000.p/0", + "/sumdb/sum.example/tile/8/0/000.p/256", + "/sumdb/sum.example/tile/8/0/../../../latest", + "/sumdb/unknown.example/latest", + } { + if w := serveGoRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", target, w.Code) + } + } + if n := db.totalRequests(); n != 0 { + t.Errorf("upstream requests = %d, want 0", n) + } +} + +// sumDBClientOps lets the x/mod checksum database client read through the +// proxy, keeping its configuration and cache in memory. +type sumDBClientOps struct { + t *testing.T + remote string + key string + + mu sync.Mutex + config map[string][]byte + cache map[string][]byte +} + +func newSumDBClientOps(t *testing.T, remote, key string) *sumDBClientOps { + return &sumDBClientOps{t: t, remote: remote, key: key, config: map[string][]byte{}, cache: map[string][]byte{}} +} + +func (o *sumDBClientOps) ReadRemote(path string) ([]byte, error) { + resp, err := http.Get(o.remote + path) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: %s: %s", path, resp.Status, body) + } + return body, nil +} + +func (o *sumDBClientOps) ReadConfig(file string) ([]byte, error) { + if file == "key" { + return []byte(o.key), nil + } + o.mu.Lock() + defer o.mu.Unlock() + return o.config[file], nil +} + +func (o *sumDBClientOps) WriteConfig(file string, old, data []byte) error { + o.mu.Lock() + defer o.mu.Unlock() + if !bytes.Equal(o.config[file], old) { + return sumdb.ErrWriteConflict + } + o.config[file] = data + return nil +} + +func (o *sumDBClientOps) ReadCache(file string) ([]byte, error) { + o.mu.Lock() + defer o.mu.Unlock() + data, ok := o.cache[file] + if !ok { + return nil, errors.New("not cached") + } + return data, nil +} + +func (o *sumDBClientOps) WriteCache(file string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + o.cache[file] = data +} + +func (o *sumDBClientOps) Log(msg string) { o.t.Log(msg) } + +func (o *sumDBClientOps) SecurityError(msg string) { + o.t.Errorf("checksum database security error: %s", msg) +} + +// TestGoSumDB_ClientRoundTrip runs the go command's checksum database client +// against a real database implementation through the proxy, so signatures, +// record proofs and tile hashes are checked end to end. A second client then +// verifies the same lookup from the proxy's cache with the database down. +func TestGoSumDB_ClientRoundTrip(t *testing.T) { + signerKey, verifierKey, err := note.GenerateKey(rand.Reader, testSumDBName) + if err != nil { + t.Fatal(err) + } + gosum := func(path, version string) ([]byte, error) { + return fmt.Appendf(nil, "%s %s h1:Y29udGVudA==\n%s %s/go.mod h1:Z28ubW9k\n", path, version, path, version), nil + } + database := sumdb.NewServer(sumdb.NewTestServer(signerKey, gosum)) + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down.Load() { + http.Error(w, "unavailable", http.StatusInternalServerError) + return + } + database.ServeHTTP(w, r) + })) + defer upstream.Close() + + h := newSumDBTestHandler(t, upstream.URL, true, time.Hour) + proxy := httptest.NewServer(h.Routes()) + defer proxy.Close() + remote := proxy.URL + "/sumdb/" + testSumDBName + + client := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) + for _, m := range [][2]string{{"rsc.io/quote", "v1.5.2"}, {"golang.org/x/text", "v0.3.0"}, {"github.com/Google/UUID", "v1.6.0"}} { + lines, err := client.Lookup(m[0], m[1]) + if err != nil { + t.Fatalf("lookup %s@%s through the proxy: %v", m[0], m[1], err) + } + if want := m[0] + " " + m[1] + " h1:Y29udGVudA=="; len(lines) != 1 || lines[0] != want { + t.Errorf("lookup %s@%s = %q, want [%q]", m[0], m[1], lines, want) + } + } + + down.Store(true) + offline := sumdb.NewClient(newSumDBClientOps(t, remote, verifierKey)) + if _, err := offline.Lookup("rsc.io/quote", "v1.5.2"); err != nil { + t.Fatalf("lookup from the proxy cache with the database down: %v", err) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 776fa8cb..272beb58 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1101,6 +1101,20 @@ func JSONError(w http.ResponseWriter, status int, message string) { // ErrUpstreamNotFound indicates the upstream returned 404. var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) +// upstreamNotFoundBodyLimit caps the part of an upstream 404 body kept in +// upstreamNotFoundError. +const upstreamNotFoundBodyLimit = 4 << 10 + +// upstreamNotFoundError is ErrUpstreamNotFound with the start of the +// upstream's 404 body, for handlers that relay the reason to clients. +type upstreamNotFoundError struct { + body []byte +} + +func (e *upstreamNotFoundError) Error() string { return ErrUpstreamNotFound.Error() } + +func (e *upstreamNotFoundError) Unwrap() error { return ErrUpstreamNotFound } + // ErrArtifactBlocked indicates a pre-cache security scan blocked the artifact. var ErrArtifactBlocked = errors.New("artifact blocked by security scan") @@ -1283,7 +1297,7 @@ func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheK p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String}.describedBy(entry) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true}.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives @@ -1296,6 +1310,9 @@ type metadataResult struct { // empty otherwise. digest string err error + // stale is set when the body is the cached copy served because upstream + // failed. + stale bool } // describedBy returns res with the digest that entry, the cache row its body @@ -1461,7 +1478,8 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e } if resp.StatusCode == http.StatusNotFound { - return nil, ErrUpstreamNotFound + body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) + return nil, &upstreamNotFoundError{body: body} } if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) diff --git a/internal/server/go_sumdbs_test.go b/internal/server/go_sumdbs_test.go new file mode 100644 index 00000000..ff9dce29 --- /dev/null +++ b/internal/server/go_sumdbs_test.go @@ -0,0 +1,38 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func TestGoSumDBsReachGoHandler(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/lookup/golang.org/x/text@v0.14.0" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "lookup record") + })) + defer upstream.Close() + + cfg := config.Default() + cfg.BaseURL = "https://proxy.example" + cfg.Upstream.GoSumDBs = map[string]string{"sum.example": upstream.URL} + p := &handler.Proxy{HTTPClient: upstream.Client(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, p) + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/go/sumdb/sum.example/lookup/golang.org/x/text@v0.14.0", nil)) + if w.Code != http.StatusOK || w.Body.String() != "lookup record" { + t.Errorf("status = %d, body = %q; want the lookup from upstream.go_sumdbs", w.Code, w.Body.String()) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 8ee1c623..3ebc3e21 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -402,6 +402,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { ) gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) + goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, s.cfg.BaseURL,