Separate artifact and metadata cache metrics on /metrics and the analytics page

README.md

@@ -1351,10 +1351,13 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
 |--------|------|--------|-------------|
 | `proxy_requests_total` | counter | `ecosystem`, `status` | Proxy responses by package ecosystem and HTTP status |
 | `proxy_request_duration_seconds` | histogram | `ecosystem`, `status` | Proxy request duration |
-| `proxy_cache_hits_total` | counter | `ecosystem` | Cache hits |
-| `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses |
+| `proxy_cache_hits_total` | counter | `ecosystem`, `kind` | Cache hits. `kind` is `artifact` or `metadata`. |
+| `proxy_cache_misses_total` | counter | `ecosystem`, `kind` | Cache misses. `kind` is `artifact` or `metadata`; metadata misses are counted only while `cache_metadata` is on. |
 | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts |
 | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts |
+| `proxy_metadata_cache_served_bytes_total` | counter | `ecosystem` | Metadata bytes served from the metadata cache. |
+| `proxy_metadata_cache_size_bytes` | gauge | `ecosystem` | Size of cached metadata documents per ecosystem. |
+| `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. |
 | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) |
 | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream |
 | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures |
@@ -1414,11 +1417,17 @@ For history, trends and alerting, scrape `/metrics` with Prometheus. That is the
 
 The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way.
 
+#### Artifact and metadata caches
+
+The proxy keeps two caches. Artifacts (tarballs, wheels, crates and so on) live in storage and are what `proxy_cache_size_bytes`, `proxy_ecosystem_downloaded_bytes` and the other artifact figures describe. Registry metadata is cached separately when `cache_metadata` is on. It is stored persistently in the `metadata_cache` table rather than held in memory.
+
+The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keeps the two apart, so an artifact hit rate is `sum(rate(proxy_cache_hits_total{kind="artifact"}[5m]))` over the same expression with misses added. Metadata has its own size and entry gauges plus `proxy_metadata_cache_served_bytes_total`. That one is a process counter rather than a database total, because metadata hits are not recorded per row. On `/ui/analytics` metadata appears in its own card and in the Runtime card's metadata lookups, never mixed into the artifact figures. The card and the two gauges read one 60-second snapshot, like the artifact figures. The bundled Grafana dashboard's hit ratio covers artifacts only, while its hits and misses panel shows each kind as its own series.
+
 #### Three ecosystem label sets
 
 `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which.
 
-**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
+**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
 
 **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all.
 

deploy/grafana/git-pkgs-proxy.json

@@ -365,13 +365,13 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n/\n(\n  sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n  +\n  sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n)",
+          "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n/\n(\n  sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n  +\n  sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\", kind=\"artifact\"}[$__range]))\n)",
           "range": true,
           "instant": false,
           "refId": "A"
         }
       ],
-      "title": "Cache hit ratio",
+      "title": "Artifact cache hit ratio",
       "type": "stat"
     },
     {
@@ -981,23 +981,23 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))",
+          "expr": "sum by (kind) (rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))",
           "range": true,
           "instant": false,
           "refId": "A",
-          "legendFormat": "hits"
+          "legendFormat": "{{kind}} hits"
         },
         {
           "datasource": {
             "type": "prometheus",
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum(rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))",
+          "expr": "sum by (kind) (rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))",
           "range": true,
           "instant": false,
           "refId": "B",
-          "legendFormat": "misses"
+          "legendFormat": "{{kind}} misses"
         }
       ],
       "title": "Cache hits and misses",

internal/database/metadata_stats_test.go

@@ -0,0 +1,60 @@
+package database
+
+import (
+	"database/sql"
+	"testing"
+)
+
+func TestGetMetadataCacheStats(t *testing.T) {
+	db := newTestDB(t)
+
+	stats, err := db.GetMetadataCacheStats()
+	if err != nil {
+		t.Fatalf("GetMetadataCacheStats on an empty cache: %v", err)
+	}
+	if len(stats) != 0 {
+		t.Fatalf("empty cache returned %+v, want no rows", stats)
+	}
+
+	rows := []struct {
+		ecosystem, name string
+		size            sql.NullInt64
+	}{
+		{"npm", "lodash", sql.NullInt64{Int64: 300, Valid: true}},
+		{"npm", "react", sql.NullInt64{Int64: 200, Valid: true}},
+		// gem and rubygems are one ecosystem and must share a row.
+		{"gem", "rails", sql.NullInt64{Int64: 1000, Valid: true}},
+		{"rubygems", "rack", sql.NullInt64{Int64: 24, Valid: true}},
+		// A row written before sizes were recorded still counts as an entry.
+		{"pypi", "requests", sql.NullInt64{}},
+	}
+	for _, r := range rows {
+		if err := db.UpsertMetadataCache(&MetadataCacheEntry{
+			Ecosystem:   r.ecosystem,
+			Name:        r.name,
+			StoragePath: "_metadata/" + r.ecosystem + "/" + r.name,
+			Size:        r.size,
+		}); err != nil {
+			t.Fatalf("UpsertMetadataCache(%s/%s): %v", r.ecosystem, r.name, err)
+		}
+	}
+
+	stats, err = db.GetMetadataCacheStats()
+	if err != nil {
+		t.Fatalf("GetMetadataCacheStats: %v", err)
+	}
+
+	want := []MetadataCacheStats{
+		{Ecosystem: "rubygems", Entries: 2, Size: 1024},
+		{Ecosystem: "npm", Entries: 2, Size: 500},
+		{Ecosystem: "pypi", Entries: 1, Size: 0},
+	}
+	if len(stats) != len(want) {
+		t.Fatalf("got %+v, want %+v", stats, want)
+	}
+	for i := range want {
+		if stats[i] != want[i] {
+			t.Errorf("row %d = %+v, want %+v", i, stats[i], want[i])
+		}
+	}
+}

internal/database/queries.go

@@ -1123,6 +1123,62 @@ func (db *DB) CountCachedPackages(ecosystem string) (int64, error) {
 
 // Metadata cache queries
 
+// MetadataCacheStats is one ecosystem's share of the metadata cache.
+type MetadataCacheStats struct {
+	Ecosystem string
+	Entries   int64
+	Size      int64
+}
+
+// GetMetadataCacheStats returns the number and total size of cached metadata
+// documents per ecosystem, largest first. Ecosystem names are normalized so
+// aliases such as "gem" and "rubygems" share one row. Rows written before
+// sizes were recorded count as entries with no size.
+func (db *DB) GetMetadataCacheStats() ([]MetadataCacheStats, error) {
+	hasTable, err := db.HasTable("metadata_cache")
+	if err != nil || !hasTable {
+		return nil, err
+	}
+
+	rows, err := db.Query(`SELECT ecosystem, COUNT(*), COALESCE(SUM(size), 0) FROM metadata_cache GROUP BY ecosystem`)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = rows.Close() }()
+
+	byEcosystem := make(map[string]*MetadataCacheStats)
+	for rows.Next() {
+		var ecosystem string
+		var entries, size int64
+		if err := rows.Scan(&ecosystem, &entries, &size); err != nil {
+			return nil, err
+		}
+		key := purl.NormalizeEcosystem(ecosystem)
+		row, ok := byEcosystem[key]
+		if !ok {
+			row = &MetadataCacheStats{Ecosystem: key}
+			byEcosystem[key] = row
+		}
+		row.Entries += entries
+		row.Size += size
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	stats := make([]MetadataCacheStats, 0, len(byEcosystem))
+	for _, row := range byEcosystem {
+		stats = append(stats, *row)
+	}
+	sort.Slice(stats, func(i, j int) bool {
+		if stats[i].Size != stats[j].Size {
+			return stats[i].Size > stats[j].Size
+		}
+		return stats[i].Ecosystem < stats[j].Ecosystem
+	})
+	return stats, nil
+}
+
 func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, error) {
 	var entry MetadataCacheEntry
 	query := db.Rebind(`

internal/handler/arch.go

@@ -198,7 +198,7 @@ func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, rep
 	}
 	d := h.database(repository, upstreamURL, path)
 	if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil {
-		metrics.RecordCacheHit(archEcosystem)
+		metrics.RecordMetadataCacheHit(archEcosystem, len(hit.body))
 		h.writeMetadata(w, r, d.dbKey, *hit)
 		return
 	}
@@ -217,7 +217,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req
 	}
 	d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix))
 	if res, ok := h.cachedSignature(r.Context(), d); ok {
-		metrics.RecordCacheHit(archEcosystem)
+		metrics.RecordMetadataCacheHit(archEcosystem, len(res.body))
 		h.writeMetadata(w, r, d.sigKey, res)
 		return
 	}

internal/handler/go_sumdb.go

@@ -348,7 +348,7 @@ func (h *GoHandler) serveCachedSumDBTile(w http.ResponseWriter, r *http.Request,
 	if check != nil && check(body) != nil {
 		return false, true
 	}
-	metrics.RecordCacheHit(goEcosystem)
+	metrics.RecordMetadataCacheHit(goEcosystem, len(body))
 
 	if entry.ETag.Valid {
 		w.Header().Set(headerETag, entry.ETag.String)

internal/handler/gradle.go

@@ -106,11 +106,11 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 			return
 		}
 		if !exists {
-			metrics.RecordCacheMiss("gradle")
+			metrics.RecordArtifactCacheMiss("gradle")
 			http.NotFound(w, r)
 			return
 		}
-		metrics.RecordCacheHit("gradle")
+		metrics.RecordArtifactCacheHit("gradle")
 
 		sizeStart := time.Now()
 		size, err := h.proxy.Storage.Size(r.Context(), storagePath)
@@ -130,7 +130,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 	metrics.RecordStorageOperation("read", time.Since(readStart))
 	if err != nil {
 		if errors.Is(err, storage.ErrNotFound) {
-			metrics.RecordCacheMiss("gradle")
+			metrics.RecordArtifactCacheMiss("gradle")
 			http.NotFound(w, r)
 			return
 		}
@@ -140,7 +140,7 @@ func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http
 		return
 	}
 	defer func() { _ = reader.Close() }()
-	metrics.RecordCacheHit("gradle")
+	metrics.RecordArtifactCacheHit("gradle")
 
 	w.WriteHeader(http.StatusOK)
 	_, _ = io.Copy(w, reader)

internal/handler/gradle_test.go

@@ -237,8 +237,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) {
 	srv := httptest.NewServer(h.Routes())
 	defer srv.Close()
 
-	hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
-	missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
+	hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact))
+	missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact))
 
 	key := "metrics-key"
 	putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload"))
@@ -273,8 +273,8 @@ func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) {
 	}
 	_ = missResp.Body.Close()
 
-	hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
-	missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
+	hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle", metrics.CacheKindArtifact))
+	missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle", metrics.CacheKindArtifact))
 
 	if diff := hitsAfter - hitsBefore; diff != 2 {
 		t.Fatalf("cache hits delta = %.0f, want 2", diff)

internal/handler/handler.go

@@ -303,7 +303,7 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version
 	} else if cached != nil {
 		return cached, nil
 	}
-	metrics.RecordCacheMiss(ecosystem)
+	metrics.RecordArtifactCacheMiss(ecosystem)
 
 	key := artifactCoalesceKey(versionPURL, filename, "", "", "")
 	recheck := func() (artifacts.Artifact, string, bool) {
@@ -474,7 +474,7 @@ func rewriteSignedURLHost(signed, baseURL string) string {
 
 func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) {
 	_ = p.DB.RecordArtifactHit(versionPURL, filename)
-	metrics.RecordCacheHit(ecosystem)
+	metrics.RecordArtifactCacheHit(ecosystem)
 }
 
 func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) {
@@ -1266,7 +1266,7 @@ func (p *Proxy) lookupMetadata(ctx context.Context, ecosystem, cacheKey, upstrea
 
 	// Serve from cache if within TTL (skip upstream entirely)
 	if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil {
-		metrics.RecordCacheHit(ecosystem)
+		metrics.RecordMetadataCacheHit(ecosystem, len(hit.body))
 		return *hit
 	}
 	p.recordMetadataCacheMiss(ecosystem)
@@ -1477,7 +1477,7 @@ func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.Metadata
 
 func (p *Proxy) recordMetadataCacheMiss(ecosystem string) {
 	if p.CacheMetadata {
-		metrics.RecordCacheMiss(ecosystem)
+		metrics.RecordMetadataCacheMiss(ecosystem)
 	}
 }
 
@@ -1843,7 +1843,7 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco
 	} else if cached != nil {
 		return cached, nil
 	}
-	metrics.RecordCacheMiss(ecosystem)
+	metrics.RecordArtifactCacheMiss(ecosystem)
 	return p.coalescedFetchFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash, declared)
 }