Skip to content

Commit 5bdddfe

Browse files
committed
cache: use SHA-256 as the only digest function
The cache doesn't hold anything big enough for hash speed to matter, so drop BLAKE3 (and the lukechampine.com/blake3 dependency) and key everything — CAS blobs, action digests — by SHA-256. The CAS layout flattens back to cas/<xx>/<hash>, and a remotely fetched plugin's address is now literally the checksum declared in the configuration file. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MPgq3rwip76D554BktbqR4
1 parent eb94686 commit 5bdddfe

8 files changed

Lines changed: 37 additions & 117 deletions

File tree

‎docs/reference/environment-variables.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-
2525

2626
The cache is designed after Bazel's local disk cache and has three parts:
2727

28-
- `cas/` — a content-addressable store holding blobs keyed by the hash of
29-
their contents. Query analysis results are keyed by BLAKE3; remotely
30-
fetched WASM plugins are keyed by the sha256 checksum declared in the
31-
configuration file, so they are loaded directly by that address.
28+
- `cas/` — a content-addressable store holding blobs (query analysis
29+
results, WASM plugin binaries) keyed by the SHA-256 hash of their
30+
contents. A remotely fetched plugin's address is exactly the checksum
31+
declared in the configuration file, so it is loaded directly by that
32+
address.
3233
- `ac/` — an action cache mapping the digest of a unit of cacheable work and
3334
its inputs (for example, analyzing a query against a schema) to the CAS
3435
digests of its outputs.

‎go.mod‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ require (
3030
google.golang.org/grpc v1.83.0
3131
google.golang.org/protobuf v1.36.11
3232
gopkg.in/yaml.v3 v3.0.1
33-
lukechampine.com/blake3 v1.4.1
3433
modernc.org/sqlite v1.55.0
3534
)
3635

@@ -44,7 +43,6 @@ require (
4443
github.com/jackc/pgpassfile v1.0.0 // indirect
4544
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
4645
github.com/jackc/puddle/v2 v2.2.2 // indirect
47-
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
4846
github.com/kr/text v0.2.0 // indirect
4947
github.com/mattn/go-isatty v0.0.20 // indirect
5048
github.com/ncruces/go-sqlite3-wasm/v3 v3.2.35303 // indirect

‎go.sum‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
4747
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
4848
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
4949
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
50-
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
51-
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
5250
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
5351
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
5452
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -151,8 +149,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
151149
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
152150
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
153151
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
154-
lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg=
155-
lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo=
156152
modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc=
157153
modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
158154
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=

‎internal/cache/cache.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
//
44
// It has two halves:
55
//
6-
// - A content-addressable store (CAS) holding blobs keyed by the hash of
7-
// their contents — BLAKE3 by default — laid out as
8-
// cas/<function>/<xx>/<hash>.
6+
// - A content-addressable store (CAS) holding blobs keyed by the SHA-256
7+
// hash of their contents, laid out as cas/<xx>/<hash>.
98
// - An action cache (AC) mapping the digest of an action — a description of
109
// cacheable work and all of its inputs — to the digests of the outputs
1110
// that work produced, laid out as ac/<xx>/<hash>.

‎internal/cache/cache_test.go‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func TestCASPutRepairsCorruptEntry(t *testing.T) {
103103
}
104104
}
105105

106-
func TestCASSHA256DirectLoad(t *testing.T) {
106+
func TestCASDeclaredChecksumLoad(t *testing.T) {
107107
c := testCache(t)
108108
blob := []byte("fetched plugin binary")
109109
s := sha256.Sum256(blob)
@@ -114,7 +114,7 @@ func TestCASSHA256DirectLoad(t *testing.T) {
114114
t.Errorf("want ErrNotFound before Put, got %v", err)
115115
}
116116

117-
d, err := c.CAS.PutSHA256(blob)
117+
d, err := c.CAS.Put(blob)
118118
if err != nil {
119119
t.Fatal(err)
120120
}
@@ -131,26 +131,6 @@ func TestCASSHA256DirectLoad(t *testing.T) {
131131
if string(got) != string(blob) {
132132
t.Errorf("got %q, want %q", got, blob)
133133
}
134-
135-
// sha256- and blake3-keyed entries live in separate namespaces.
136-
if c.CAS.Contains(DigestOf(blob)) {
137-
t.Error("blob stored by sha256 must not be visible under its blake3 digest")
138-
}
139-
}
140-
141-
func TestCASSHA256CorruptionEvicted(t *testing.T) {
142-
c := testCache(t)
143-
blob := []byte("plugin contents")
144-
d, err := c.CAS.PutSHA256(blob)
145-
if err != nil {
146-
t.Fatal(err)
147-
}
148-
if err := os.WriteFile(c.CAS.path(d), []byte("tampered!contents"), 0644); err != nil {
149-
t.Fatal(err)
150-
}
151-
if _, err := c.CAS.Get(SHA256Digest(d.Hash)); !errors.Is(err, ErrNotFound) {
152-
t.Errorf("want ErrNotFound for corrupt blob, got %v", err)
153-
}
154134
}
155135

156136
func TestActionDigestFraming(t *testing.T) {

‎internal/cache/cas.go‎

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,15 @@ import (
1010
// ErrNotFound is returned when a blob or action result is not in the cache.
1111
var ErrNotFound = errors.New("cache: not found")
1212

13-
// CAS is an on-disk content-addressable store modeled on Bazel's disk cache.
14-
// Blobs live at cas/<function>/<xx>/<hash> under the cache root, where <xx>
15-
// is the first two hex characters of the hash. Because a blob's name is
16-
// derived from its contents, entries never change once written: writers race
17-
// benignly and readers can detect corruption by re-hashing.
13+
// CAS is an on-disk content-addressable store keyed by SHA-256, modeled on
14+
// Bazel's disk cache. Blobs live at cas/<xx>/<hash> under the cache root,
15+
// where <xx> is the first two hex characters of the hash. Because a blob's
16+
// name is derived from its contents, entries never change once written:
17+
// writers race benignly and readers can detect corruption by re-hashing.
1818
//
19-
// Blobs are keyed by BLAKE3 by default. Content with an externally declared
20-
// SHA-256 checksum (remotely fetched plugins) is stored keyed by that
21-
// checksum instead — see PutSHA256 — so fetches need no action cache entry:
22-
// the declared checksum is the address.
19+
// Content with an externally declared checksum (remotely fetched plugins)
20+
// needs no action cache entry: the declared sha256 is the address, so it is
21+
// stored and loaded directly — see SHA256Digest.
2322
type CAS struct {
2423
root string
2524
tmp string
@@ -34,25 +33,14 @@ func newCAS(root string) (*CAS, error) {
3433
}
3534

3635
func (c *CAS) path(d Digest) string {
37-
return filepath.Join(c.root, "cas", string(d.function()), d.Hash[:2], d.Hash)
36+
return filepath.Join(c.root, "cas", d.Hash[:2], d.Hash)
3837
}
3938

40-
// Put stores a blob keyed by its BLAKE3 hash and returns its digest.
39+
// Put stores a blob and returns its digest. Writing is atomic: the blob is
40+
// staged in a temp file and renamed into place, so concurrent sqlc processes
41+
// never observe partial entries.
4142
func (c *CAS) Put(data []byte) (Digest, error) {
42-
return c.put(DigestOf(data), data)
43-
}
44-
45-
// PutSHA256 stores a blob keyed by its SHA-256 hash and returns its digest.
46-
// Use this for content addressed by a checksum declared outside the cache,
47-
// so later runs can load it with a bare SHA256Digest lookup.
48-
func (c *CAS) PutSHA256(data []byte) (Digest, error) {
49-
hash, _ := sum(SHA256, data)
50-
return c.put(Digest{Function: SHA256, Hash: hash, SizeBytes: int64(len(data))}, data)
51-
}
52-
53-
// put writes a blob atomically: it is staged in a temp file and renamed into
54-
// place, so concurrent sqlc processes never observe partial entries.
55-
func (c *CAS) put(d Digest, data []byte) (Digest, error) {
43+
d := DigestOf(data)
5644
path := c.path(d)
5745
// Skip the write only when an entry of the right size already exists; a
5846
// wrong-sized entry is corrupt and is atomically replaced by the rename
@@ -81,10 +69,9 @@ func (c *CAS) put(d Digest, data []byte) (Digest, error) {
8169
return d, nil
8270
}
8371

84-
// Get returns the blob for a digest. Contents are re-hashed with the
85-
// digest's hash function before being returned; a corrupt entry is evicted
86-
// and reported as ErrNotFound so callers simply redo the work that produced
87-
// it.
72+
// Get returns the blob for a digest. Contents are re-hashed before being
73+
// returned; a corrupt entry is evicted and reported as ErrNotFound so
74+
// callers simply redo the work that produced it.
8875
func (c *CAS) Get(d Digest) ([]byte, error) {
8976
if !d.valid() {
9077
return nil, ErrNotFound
@@ -97,7 +84,7 @@ func (c *CAS) Get(d Digest) ([]byte, error) {
9784
}
9885
return nil, fmt.Errorf("cache: %w", err)
9986
}
100-
if hash, _ := sum(d.function(), data); hash != d.Hash {
87+
if DigestOf(data).Hash != d.Hash {
10188
os.Remove(path)
10289
return nil, ErrNotFound
10390
}

‎internal/cache/digest.go‎

Lines changed: 10 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -5,76 +5,38 @@ import (
55
"encoding/binary"
66
"encoding/hex"
77
"fmt"
8-
9-
"lukechampine.com/blake3"
10-
)
11-
12-
// HashFunc identifies the hash function that keys a blob in the CAS.
13-
//
14-
// BLAKE3 is the native digest function. SHA256 exists so that content with an
15-
// externally declared SHA-256 checksum — like a WASM plugin's checksum in
16-
// sqlc's configuration — can be addressed directly by that checksum without
17-
// any translation table.
18-
type HashFunc string
19-
20-
const (
21-
BLAKE3 HashFunc = "blake3"
22-
SHA256 HashFunc = "sha256"
8+
"hash"
239
)
2410

25-
func sum(fn HashFunc, data []byte) (string, bool) {
26-
switch fn {
27-
case BLAKE3:
28-
s := blake3.Sum256(data)
29-
return hex.EncodeToString(s[:]), true
30-
case SHA256:
31-
s := sha256.Sum256(data)
32-
return hex.EncodeToString(s[:]), true
33-
}
34-
return "", false
35-
}
36-
37-
// Digest identifies a blob by hash function, hash, and size, mirroring the
11+
// Digest identifies a blob by its SHA-256 hash and size, mirroring the
3812
// Digest message from Bazel's remote execution API. The size is stored
3913
// alongside the hash so that entries can be validated without reading blob
4014
// contents; a negative size means the size is unknown, as with a checksum
4115
// declared in a configuration file.
4216
type Digest struct {
43-
// Function is the hash function; an empty value means BLAKE3.
44-
Function HashFunc `json:"function,omitempty"`
45-
// Hash is the lowercase hex-encoded 256-bit hash of the blob.
17+
// Hash is the lowercase hex-encoded SHA-256 hash of the blob.
4618
Hash string `json:"hash"`
4719
// SizeBytes is the length of the blob in bytes, or negative if unknown.
4820
SizeBytes int64 `json:"size_bytes"`
4921
}
5022

5123
func (d Digest) String() string {
52-
return fmt.Sprintf("%s:%s/%d", d.function(), d.Hash, d.SizeBytes)
53-
}
54-
55-
func (d Digest) function() HashFunc {
56-
if d.Function == "" {
57-
return BLAKE3
58-
}
59-
return d.Function
24+
return fmt.Sprintf("sha256:%s/%d", d.Hash, d.SizeBytes)
6025
}
6126

6227
func (d Digest) valid() bool {
63-
if _, ok := sum(d.function(), nil); !ok {
64-
return false
65-
}
6628
if len(d.Hash) != 64 {
6729
return false
6830
}
6931
_, err := hex.DecodeString(d.Hash)
7032
return err == nil
7133
}
7234

73-
// DigestOf returns the BLAKE3 Digest of a blob.
35+
// DigestOf returns the Digest of a blob.
7436
func DigestOf(data []byte) Digest {
75-
hash, _ := sum(BLAKE3, data)
37+
sum := sha256.Sum256(data)
7638
return Digest{
77-
Hash: hash,
39+
Hash: hex.EncodeToString(sum[:]),
7840
SizeBytes: int64(len(data)),
7941
}
8042
}
@@ -84,7 +46,6 @@ func DigestOf(data []byte) Digest {
8446
// content in the CAS.
8547
func SHA256Digest(hexhash string) Digest {
8648
return Digest{
87-
Function: SHA256,
8849
Hash: hexhash,
8950
SizeBytes: -1,
9051
}
@@ -99,13 +60,13 @@ func SHA256Digest(hexhash string) Digest {
9960
// boundary between inputs is unambiguous ("ab"+"c" hashes differently from
10061
// "a"+"bc").
10162
type Action struct {
102-
hasher *blake3.Hasher
63+
hasher hash.Hash
10364
}
10465

10566
// NewAction starts building an action key for the given mnemonic, e.g.
10667
// "QueryAnalysis".
10768
func NewAction(mnemonic string) *Action {
108-
a := &Action{hasher: blake3.New(32, nil)}
69+
a := &Action{hasher: sha256.New()}
10970
a.write([]byte(mnemonic))
11071
return a
11172
}
@@ -120,10 +81,8 @@ func (a *Action) AddInput(name string, data []byte) *Action {
12081

12182
// Digest returns the action's digest, used as the action cache key.
12283
func (a *Action) Digest() Digest {
123-
var sum [32]byte
124-
a.hasher.Sum(sum[:0])
12584
return Digest{
126-
Hash: hex.EncodeToString(sum[:]),
85+
Hash: hex.EncodeToString(a.hasher.Sum(nil)),
12786
SizeBytes: 0,
12887
}
12988
}

‎internal/ext/wasm/wasm.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (r *Runner) loadAndCompileWASM(ctx context.Context, store *cache.Cache, exp
127127
if expected != actual {
128128
return nil, fmt.Errorf("invalid checksum: expected %s, got %s", expected, actual)
129129
}
130-
if _, err := store.CAS.PutSHA256(wmod); err != nil {
130+
if _, err := store.CAS.Put(wmod); err != nil {
131131
return nil, fmt.Errorf("cache wasm: %w", err)
132132
}
133133
} else if err != nil {

0 commit comments

Comments
 (0)