Skip to content

Commit eb94686

Browse files
committed
cache: load remote fetches directly into the CAS by declared checksum
A remote fetch with a declared sha256 doesn't need the action cache: the checksum is itself a content address. Make the CAS hash-function aware — blobs live at cas/<function>/<xx>/<hash>, keyed by BLAKE3 by default — and add PutSHA256/SHA256Digest so remotely fetched content is stored and looked up directly under the checksum declared in the configuration, with the checksum re-verified on every read. The WASM plugin path drops its FetchPlugin action entirely; the action cache now backs only work whose outputs aren't derivable from its inputs, like query analysis. This mirrors Bazel, where downloads go through the checksum-addressed repository cache rather than the action cache. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MPgq3rwip76D554BktbqR4
1 parent 3c674f1 commit eb94686

6 files changed

Lines changed: 180 additions & 80 deletions

File tree

‎docs/reference/environment-variables.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,13 @@ Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-
2525

2626
The cache is designed after Bazel's local disk cache and has three parts:
2727

28-
- `cas/` — a content-addressable store holding blobs (WASM plugin binaries,
29-
query analysis results) keyed by the BLAKE3 hash of their contents.
28+
- `cas/` — a content-addressable store holding blobs keyed by the hash of
29+
their contents. Query analysis results are keyed by BLAKE3; remotely
30+
fetched WASM plugins are keyed by the sha256 checksum declared in the
31+
configuration file, so they are loaded directly by that address.
3032
- `ac/` — an action cache mapping the digest of a unit of cacheable work and
31-
its inputs (for example, fetching a plugin from a URL with an expected
32-
checksum, or analyzing a query against a schema) to the CAS digests of its
33-
outputs.
33+
its inputs (for example, analyzing a query against a schema) to the CAS
34+
digests of its outputs.
3435
- `wazero/` — compiled WASM machine code, managed by the
3536
[wazero](https://wazero.io) runtime in its own format.
3637

‎internal/cache/cache.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,20 @@
33
//
44
// It has two halves:
55
//
6-
// - A content-addressable store (CAS) holding blobs keyed by the BLAKE3
7-
// hash of their contents, laid out as cas/<xx>/<hash>.
6+
// - A content-addressable store (CAS) holding blobs keyed by the hash of
7+
// their contents — BLAKE3 by default — laid out as
8+
// cas/<function>/<xx>/<hash>.
89
// - An action cache (AC) mapping the digest of an action — a description of
910
// cacheable work and all of its inputs — to the digests of the outputs
1011
// that work produced, laid out as ac/<xx>/<hash>.
1112
//
12-
// Executing cached work therefore takes two steps, just like Bazel: hash the
13-
// action, look its digest up in the action cache, then fetch the referenced
14-
// output blobs from the CAS. Both the query analysis cache and the WASM
15-
// plugin cache are built on this pair.
13+
// Work whose output is not derivable from its inputs, like query analysis,
14+
// uses both halves, just like Bazel: hash the action, look its digest up in
15+
// the action cache, then fetch the referenced output blobs from the CAS.
16+
//
17+
// Remote fetches with a declared checksum, like WASM plugins, need no action
18+
// cache entry at all: the declared sha256 is itself a content address, so the
19+
// blob is stored and loaded directly from the CAS keyed by that checksum.
1620
package cache
1721

1822
import (

‎internal/cache/cache_test.go‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package cache
22

33
import (
4+
"crypto/sha256"
5+
"encoding/hex"
46
"errors"
57
"os"
6-
"path/filepath"
78
"testing"
89
)
910

@@ -63,7 +64,7 @@ func TestCASCorruptionEvicted(t *testing.T) {
6364
}
6465

6566
// Corrupt the blob on disk while keeping its size.
66-
path := filepath.Join(c.CAS.root, "cas", d.Hash[:2], d.Hash)
67+
path := c.CAS.path(d)
6768
if err := os.WriteFile(path, []byte("tampered contents"), 0644); err != nil {
6869
t.Fatal(err)
6970
}
@@ -86,7 +87,7 @@ func TestCASPutRepairsCorruptEntry(t *testing.T) {
8687

8788
// Corrupt the entry on disk with different-sized contents, then Put the
8889
// correct blob again: the entry must be repaired, not trusted.
89-
path := filepath.Join(c.CAS.root, "cas", d.Hash[:2], d.Hash)
90+
path := c.CAS.path(d)
9091
if err := os.WriteFile(path, append(blob, "tampered"...), 0644); err != nil {
9192
t.Fatal(err)
9293
}
@@ -102,6 +103,56 @@ func TestCASPutRepairsCorruptEntry(t *testing.T) {
102103
}
103104
}
104105

106+
func TestCASSHA256DirectLoad(t *testing.T) {
107+
c := testCache(t)
108+
blob := []byte("fetched plugin binary")
109+
s := sha256.Sum256(blob)
110+
declared := hex.EncodeToString(s[:])
111+
112+
// A declared checksum whose content was never fetched is a miss.
113+
if _, err := c.CAS.Get(SHA256Digest(declared)); !errors.Is(err, ErrNotFound) {
114+
t.Errorf("want ErrNotFound before Put, got %v", err)
115+
}
116+
117+
d, err := c.CAS.PutSHA256(blob)
118+
if err != nil {
119+
t.Fatal(err)
120+
}
121+
if d.Hash != declared {
122+
t.Errorf("stored under %s, want declared checksum %s", d.Hash, declared)
123+
}
124+
125+
// Later runs load the blob using only the checksum from the config file,
126+
// with no size and no action cache entry.
127+
got, err := c.CAS.Get(SHA256Digest(declared))
128+
if err != nil {
129+
t.Fatal(err)
130+
}
131+
if string(got) != string(blob) {
132+
t.Errorf("got %q, want %q", got, blob)
133+
}
134+
135+
// sha256- and blake3-keyed entries live in separate namespaces.
136+
if c.CAS.Contains(DigestOf(blob)) {
137+
t.Error("blob stored by sha256 must not be visible under its blake3 digest")
138+
}
139+
}
140+
141+
func TestCASSHA256CorruptionEvicted(t *testing.T) {
142+
c := testCache(t)
143+
blob := []byte("plugin contents")
144+
d, err := c.CAS.PutSHA256(blob)
145+
if err != nil {
146+
t.Fatal(err)
147+
}
148+
if err := os.WriteFile(c.CAS.path(d), []byte("tampered!contents"), 0644); err != nil {
149+
t.Fatal(err)
150+
}
151+
if _, err := c.CAS.Get(SHA256Digest(d.Hash)); !errors.Is(err, ErrNotFound) {
152+
t.Errorf("want ErrNotFound for corrupt blob, got %v", err)
153+
}
154+
}
155+
105156
func TestActionDigestFraming(t *testing.T) {
106157
a := NewAction("Test").AddInput("query", []byte("ab")).AddInput("schema", []byte("c"))
107158
b := NewAction("Test").AddInput("query", []byte("a")).AddInput("schema", []byte("bc"))
@@ -157,7 +208,7 @@ func TestActionCacheMissingOutputIsMiss(t *testing.T) {
157208
}
158209

159210
// Simulate the blob being garbage collected out from under the entry.
160-
if err := os.Remove(filepath.Join(c.CAS.root, "cas", out.Hash[:2], out.Hash)); err != nil {
211+
if err := os.Remove(c.CAS.path(out)); err != nil {
161212
t.Fatal(err)
162213
}
163214

‎internal/cache/cas.go‎

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,16 @@ import (
1010
// ErrNotFound is returned when a blob or action result is not in the cache.
1111
var ErrNotFound = errors.New("cache: not found")
1212

13-
// CAS is an on-disk content-addressable store keyed by BLAKE3, modeled on
14-
// Bazel's disk cache. Blobs live at cas/<xx>/<hash> under the cache root,
15-
// where <xx> is the first two hex characters of the hash. Because a blob's
16-
// name is derived from its contents, entries never change once written:
17-
// writers race benignly and readers can detect corruption by re-hashing.
13+
// CAS is an on-disk content-addressable store modeled on Bazel's disk cache.
14+
// Blobs live at cas/<function>/<xx>/<hash> under the cache root, where <xx>
15+
// is the first two hex characters of the hash. Because a blob's name is
16+
// derived from its contents, entries never change once written: writers race
17+
// benignly and readers can detect corruption by re-hashing.
18+
//
19+
// Blobs are keyed by BLAKE3 by default. Content with an externally declared
20+
// SHA-256 checksum (remotely fetched plugins) is stored keyed by that
21+
// checksum instead — see PutSHA256 — so fetches need no action cache entry:
22+
// the declared checksum is the address.
1823
type CAS struct {
1924
root string
2025
tmp string
@@ -29,14 +34,25 @@ func newCAS(root string) (*CAS, error) {
2934
}
3035

3136
func (c *CAS) path(d Digest) string {
32-
return filepath.Join(c.root, "cas", d.Hash[:2], d.Hash)
37+
return filepath.Join(c.root, "cas", string(d.function()), d.Hash[:2], d.Hash)
3338
}
3439

35-
// Put stores a blob and returns its digest. Writing is atomic: the blob is
36-
// staged in a temp file and renamed into place, so concurrent sqlc processes
37-
// never observe partial entries.
40+
// Put stores a blob keyed by its BLAKE3 hash and returns its digest.
3841
func (c *CAS) Put(data []byte) (Digest, error) {
39-
d := DigestOf(data)
42+
return c.put(DigestOf(data), data)
43+
}
44+
45+
// PutSHA256 stores a blob keyed by its SHA-256 hash and returns its digest.
46+
// Use this for content addressed by a checksum declared outside the cache,
47+
// so later runs can load it with a bare SHA256Digest lookup.
48+
func (c *CAS) PutSHA256(data []byte) (Digest, error) {
49+
hash, _ := sum(SHA256, data)
50+
return c.put(Digest{Function: SHA256, Hash: hash, SizeBytes: int64(len(data))}, data)
51+
}
52+
53+
// put writes a blob atomically: it is staged in a temp file and renamed into
54+
// place, so concurrent sqlc processes never observe partial entries.
55+
func (c *CAS) put(d Digest, data []byte) (Digest, error) {
4056
path := c.path(d)
4157
// Skip the write only when an entry of the right size already exists; a
4258
// wrong-sized entry is corrupt and is atomically replaced by the rename
@@ -65,9 +81,10 @@ func (c *CAS) Put(data []byte) (Digest, error) {
6581
return d, nil
6682
}
6783

68-
// Get returns the blob for a digest. Contents are re-hashed before being
69-
// returned; a corrupt entry is evicted and reported as ErrNotFound so callers
70-
// simply re-run the action that produced it.
84+
// Get returns the blob for a digest. Contents are re-hashed with the
85+
// digest's hash function before being returned; a corrupt entry is evicted
86+
// and reported as ErrNotFound so callers simply redo the work that produced
87+
// it.
7188
func (c *CAS) Get(d Digest) ([]byte, error) {
7289
if !d.valid() {
7390
return nil, ErrNotFound
@@ -80,19 +97,23 @@ func (c *CAS) Get(d Digest) ([]byte, error) {
8097
}
8198
return nil, fmt.Errorf("cache: %w", err)
8299
}
83-
if DigestOf(data) != d {
100+
if hash, _ := sum(d.function(), data); hash != d.Hash {
84101
os.Remove(path)
85102
return nil, ErrNotFound
86103
}
87104
return data, nil
88105
}
89106

90-
// Contains reports whether a blob with the given digest and size is present.
91-
// It does not verify contents; Get performs full verification.
107+
// Contains reports whether a blob with the given digest is present, checking
108+
// size when the digest carries one. It does not verify contents; Get performs
109+
// full verification.
92110
func (c *CAS) Contains(d Digest) bool {
93111
if !d.valid() {
94112
return false
95113
}
96114
fi, err := os.Stat(c.path(d))
97-
return err == nil && fi.Size() == d.SizeBytes
115+
if err != nil {
116+
return false
117+
}
118+
return d.SizeBytes < 0 || fi.Size() == d.SizeBytes
98119
}

‎internal/cache/digest.go‎

Lines changed: 62 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,95 @@
11
package cache
22

33
import (
4+
"crypto/sha256"
45
"encoding/binary"
56
"encoding/hex"
67
"fmt"
78

89
"lukechampine.com/blake3"
910
)
1011

11-
// Digest identifies a blob by its BLAKE3 hash and size, mirroring the Digest
12-
// message from Bazel's remote execution API. The size is stored alongside the
13-
// hash so that entries can be validated without reading blob contents.
12+
// HashFunc identifies the hash function that keys a blob in the CAS.
13+
//
14+
// BLAKE3 is the native digest function. SHA256 exists so that content with an
15+
// externally declared SHA-256 checksum — like a WASM plugin's checksum in
16+
// sqlc's configuration — can be addressed directly by that checksum without
17+
// any translation table.
18+
type HashFunc string
19+
20+
const (
21+
BLAKE3 HashFunc = "blake3"
22+
SHA256 HashFunc = "sha256"
23+
)
24+
25+
func sum(fn HashFunc, data []byte) (string, bool) {
26+
switch fn {
27+
case BLAKE3:
28+
s := blake3.Sum256(data)
29+
return hex.EncodeToString(s[:]), true
30+
case SHA256:
31+
s := sha256.Sum256(data)
32+
return hex.EncodeToString(s[:]), true
33+
}
34+
return "", false
35+
}
36+
37+
// Digest identifies a blob by hash function, hash, and size, mirroring the
38+
// Digest message from Bazel's remote execution API. The size is stored
39+
// alongside the hash so that entries can be validated without reading blob
40+
// contents; a negative size means the size is unknown, as with a checksum
41+
// declared in a configuration file.
1442
type Digest struct {
15-
// Hash is the lowercase hex-encoded BLAKE3-256 hash of the blob.
43+
// Function is the hash function; an empty value means BLAKE3.
44+
Function HashFunc `json:"function,omitempty"`
45+
// Hash is the lowercase hex-encoded 256-bit hash of the blob.
1646
Hash string `json:"hash"`
17-
// SizeBytes is the length of the blob in bytes.
47+
// SizeBytes is the length of the blob in bytes, or negative if unknown.
1848
SizeBytes int64 `json:"size_bytes"`
1949
}
2050

2151
func (d Digest) String() string {
22-
return fmt.Sprintf("blake3:%s/%d", d.Hash, d.SizeBytes)
52+
return fmt.Sprintf("%s:%s/%d", d.function(), d.Hash, d.SizeBytes)
53+
}
54+
55+
func (d Digest) function() HashFunc {
56+
if d.Function == "" {
57+
return BLAKE3
58+
}
59+
return d.Function
2360
}
2461

2562
func (d Digest) valid() bool {
26-
if len(d.Hash) != 64 || d.SizeBytes < 0 {
63+
if _, ok := sum(d.function(), nil); !ok {
64+
return false
65+
}
66+
if len(d.Hash) != 64 {
2767
return false
2868
}
2969
_, err := hex.DecodeString(d.Hash)
3070
return err == nil
3171
}
3272

33-
// DigestOf returns the Digest of a blob.
73+
// DigestOf returns the BLAKE3 Digest of a blob.
3474
func DigestOf(data []byte) Digest {
35-
sum := blake3.Sum256(data)
75+
hash, _ := sum(BLAKE3, data)
3676
return Digest{
37-
Hash: hex.EncodeToString(sum[:]),
77+
Hash: hash,
3878
SizeBytes: int64(len(data)),
3979
}
4080
}
4181

82+
// SHA256Digest returns a Digest referencing a blob by a declared SHA-256
83+
// checksum whose size is not known, suitable for looking up remotely fetched
84+
// content in the CAS.
85+
func SHA256Digest(hexhash string) Digest {
86+
return Digest{
87+
Function: SHA256,
88+
Hash: hexhash,
89+
SizeBytes: -1,
90+
}
91+
}
92+
4293
// An Action describes a unit of cacheable work, playing the role of Bazel's
4394
// Action message: a mnemonic naming the kind of work plus the complete set of
4495
// inputs that determine its outputs. Two actions with the same digest are
@@ -52,7 +103,7 @@ type Action struct {
52103
}
53104

54105
// NewAction starts building an action key for the given mnemonic, e.g.
55-
// "QueryAnalysis" or "FetchPlugin".
106+
// "QueryAnalysis".
56107
func NewAction(mnemonic string) *Action {
57108
a := &Action{hasher: blake3.New(32, nil)}
58109
a.write([]byte(mnemonic))

0 commit comments

Comments
 (0)