Skip to content

Commit 961e6b0

Browse files
committed
core: cache the catalog a dialect and a schema produce
Building the catalog the analysis core works against means seeding a dialect — thousands of rows, since PostgreSQL's functions and system catalogs alone run to five figures — and then parsing the user's schema and applying its DDL. Both halves produce the same database every time, so the finished database is serialized into the CAS and deserialized back on the next run. The dialect and the schema are one action key, not two lookups: together they are what the catalog is. A hit restores the database and never parses the schema at all. The dialect data is embedded in the sqlc binary, whose digest is already an input to every action, so the dialect only has to be named; the schema is hashed in the order it is applied, since DDL order decides what the catalog holds. Nothing is cached unless the whole schema made it in, so a syntax error is reported the same way on every run. A cache that cannot be opened, read or written is not an error — a corrupt blob, a missing one and a read-only cache directory all fall back to building the catalog. Per analyze run, catalog setup and schema load included: PostgreSQL 112ms -> 21ms, MySQL 31ms -> 16ms, SQLite 27ms -> 15ms. The endtoend suite runs in half the time on a warm cache. Co-Authored-By: Claude <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011MnoUabwBWW9gaEn2Nj7eG
1 parent 76901be commit 961e6b0

6 files changed

Lines changed: 312 additions & 48 deletions

File tree

‎internal/compiler/compile.go‎

Lines changed: 85 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"path/filepath"
1010
"strings"
1111

12+
"github.com/sqlc-dev/sqlc/internal/core"
1213
coreschema "github.com/sqlc-dev/sqlc/internal/core/schema"
1314
"github.com/sqlc-dev/sqlc/internal/migrations"
1415
"github.com/sqlc-dev/sqlc/internal/multierr"
@@ -28,31 +29,64 @@ type Parser interface {
2829
IsReservedKeyword(string) bool
2930
}
3031

32+
// schemaFile is a schema file's contents, after the rewrites that are applied
33+
// before any of it is parsed.
34+
type schemaFile struct {
35+
name string
36+
contents string
37+
}
38+
3139
func (c *Compiler) parseCatalog(schemas []string) error {
32-
files, err := sqlpath.Glob(schemas)
40+
paths, err := sqlpath.Glob(schemas)
3341
if err != nil {
3442
return err
3543
}
44+
3645
merr := multierr.New()
37-
for _, filename := range files {
38-
blob, err := os.ReadFile(filename)
46+
files := make([]schemaFile, 0, len(paths))
47+
for _, path := range paths {
48+
blob, err := os.ReadFile(path)
3949
if err != nil {
40-
merr.Add(filename, "", 0, err)
50+
merr.Add(path, "", 0, err)
4151
continue
4252
}
4353
contents := migrations.RemoveRollbackStatements(string(blob))
4454
contents = migrations.RemovePsqlMetaCommands(contents)
55+
files = append(files, schemaFile{name: path, contents: contents})
4556
c.schema = append(c.schema, contents)
57+
}
58+
59+
if c.coreAnalysis {
60+
// A schema file that could not be read is not part of the schema the
61+
// action was keyed on, so there is no catalog to restore and nothing
62+
// further to report.
63+
if len(merr.Errs()) > 0 {
64+
return merr
65+
}
66+
if err := c.parseCatalogCore(files, merr); err != nil {
67+
return err
68+
}
69+
} else {
70+
c.parseCatalogLegacy(files, merr)
71+
}
4672

73+
if len(merr.Errs()) > 0 {
74+
return merr
75+
}
76+
return nil
77+
}
78+
79+
func (c *Compiler) parseCatalogLegacy(files []schemaFile, merr *multierr.Error) {
80+
for _, file := range files {
4781
// In database-only mode, we parse the schema to validate syntax
4882
// but don't update the catalog - the database will be the source of truth
49-
stmts, err := c.parser.Parse(strings.NewReader(contents))
83+
stmts, err := c.parser.Parse(strings.NewReader(file.contents))
5084
if err != nil {
5185
// A schema file and a query file are often the same file, so a
5286
// query's sqlc syntax can fail here. Look for an explanation
5387
// before reporting the syntax error it caused.
54-
if reported := addSyntaxErrors(merr, filename, contents, preprocess.File(c.conf.Engine, contents)); !reported {
55-
merr.Add(filename, contents, 0, err)
88+
if reported := addSyntaxErrors(merr, file.name, file.contents, preprocess.File(c.conf.Engine, file.contents)); !reported {
89+
merr.Add(file.name, file.contents, 0, err)
5690
}
5791
continue
5892
}
@@ -62,26 +96,57 @@ func (c *Compiler) parseCatalog(schemas []string) error {
6296
continue
6397
}
6498

65-
if c.coreCatalog != nil {
66-
for i := range stmts {
67-
if err := coreschema.Apply(c.coreCatalog, stmts[i].Raw); err != nil {
68-
merr.Add(filename, contents, stmts[i].Pos(), err)
69-
continue
70-
}
71-
}
72-
continue
73-
}
74-
7599
for i := range stmts {
76100
if err := c.catalog.Update(stmts[i], c); err != nil {
77-
merr.Add(filename, contents, stmts[i].Pos(), err)
101+
merr.Add(file.name, file.contents, stmts[i].Pos(), err)
78102
continue
79103
}
80104
}
81105
}
82-
if len(merr.Errs()) > 0 {
83-
return merr
106+
}
107+
108+
// parseCatalogCore builds the catalog the analysis core works against: the
109+
// dialect's seed with the schema's DDL applied. The two are one cached action,
110+
// so a run that has built this catalog before restores it and never parses the
111+
// schema at all.
112+
func (c *Compiler) parseCatalogCore(files []schemaFile, merr *multierr.Error) error {
113+
contents := make([]string, 0, len(files))
114+
for _, file := range files {
115+
contents = append(contents, file.contents)
116+
}
117+
118+
cat, err := core.NewCached(string(c.conf.Engine), contents, func(cat *core.Catalog) error {
119+
for _, file := range files {
120+
stmts, err := c.parser.Parse(strings.NewReader(file.contents))
121+
if err != nil {
122+
// A schema file and a query file are often the same file, so a
123+
// query's sqlc syntax can fail here. Look for an explanation
124+
// before reporting the syntax error it caused.
125+
if reported := addSyntaxErrors(merr, file.name, file.contents, preprocess.File(c.conf.Engine, file.contents)); !reported {
126+
merr.Add(file.name, file.contents, 0, err)
127+
}
128+
continue
129+
}
130+
for i := range stmts {
131+
if err := coreschema.Apply(cat, stmts[i].Raw); err != nil {
132+
merr.Add(file.name, file.contents, stmts[i].Pos(), err)
133+
continue
134+
}
135+
}
136+
}
137+
// A catalog the whole schema did not make it into is not the catalog
138+
// this action describes, so it is not one to keep.
139+
if len(merr.Errs()) > 0 {
140+
return merr
141+
}
142+
return nil
143+
}, c.coreDialect)
144+
145+
if cat == nil {
146+
return fmt.Errorf("%s: init catalog: %w", c.conf.Engine, err)
84147
}
148+
// Whatever apply reported is already in merr, which the caller returns.
149+
c.coreCatalog = cat
85150
return nil
86151
}
87152

‎internal/compiler/engine.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,12 @@ type Compiler struct {
3232

3333
coreCatalog *core.Catalog
3434

35-
// coreAnalysis routes every engine through the core catalog and analyzer.
35+
// coreAnalysis routes every engine through the core catalog and analyzer,
36+
// and coreDialect seeds the catalog it does so with. The catalog itself is
37+
// not built until the schema is loaded, because the two are cached
38+
// together.
3639
coreAnalysis bool
40+
coreDialect core.Option
3741

3842
schema []string
3943

@@ -178,11 +182,7 @@ func (c *Compiler) initCore() error {
178182
default:
179183
return fmt.Errorf("unknown engine: %s", c.conf.Engine)
180184
}
181-
cat, err := core.New(dialect)
182-
if err != nil {
183-
return fmt.Errorf("%s: init catalog: %w", c.conf.Engine, err)
184-
}
185-
c.coreCatalog = cat
185+
c.coreDialect = dialect
186186
return nil
187187
}
188188

‎internal/core/cache.go‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
package core
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"errors"
7+
"fmt"
8+
"log/slog"
9+
10+
"github.com/sqlc-dev/sqlc/internal/cache"
11+
"github.com/sqlc-dev/sqlc/internal/core/catalogdb"
12+
)
13+
14+
// The name of the sole output blob a CoreCatalog action produces.
15+
const catalogOutput = "catalog.db"
16+
17+
// serializer is the driver's serialization API, which the SQLite driver
18+
// implements on its connections.
19+
type serializer interface {
20+
Serialize() ([]byte, error)
21+
Deserialize([]byte) error
22+
}
23+
24+
// NewCached returns the catalog a dialect and a schema produce, restored from
25+
// the cache when a run has produced it before.
26+
//
27+
// Producing one means seeding the dialect — thousands of rows, since
28+
// PostgreSQL's functions and system catalogs alone run to five figures — and
29+
// then parsing the schema and applying its DDL. The result is the same every
30+
// time, so the finished database is serialized into the cache and
31+
// deserialized back on the next run, turning all of that work into one read.
32+
// It is a single action: the dialect and the schema together are what the
33+
// catalog is, so they are one key, and a hit means apply is never called.
34+
//
35+
// The dialect data is embedded in the sqlc binary, whose digest is an input to
36+
// every action, so the dialect only has to be named. The schema is hashed in
37+
// the order it is applied, since DDL order decides what the catalog holds.
38+
//
39+
// A catalog whose apply fails is returned but not cached, so that a caller can
40+
// report the failure against the catalog it got. A nil catalog means the
41+
// catalog could not be created at all. A cache that cannot be opened, read or
42+
// written is not an error: the catalog is simply built the long way.
43+
func NewCached(dialect string, schema []string, apply func(*Catalog) error, opts ...Option) (*Catalog, error) {
44+
store, action := openAction(dialect, schema)
45+
if store != nil {
46+
defer store.Close()
47+
if cat, err := restore(store, action); err == nil {
48+
return cat, nil
49+
} else if !errors.Is(err, cache.ErrNotFound) {
50+
slog.Debug("restoring the catalog from the cache failed", "err", err)
51+
}
52+
}
53+
54+
cat, err := New(opts...)
55+
if err != nil {
56+
return nil, err
57+
}
58+
if err := apply(cat); err != nil {
59+
return cat, err
60+
}
61+
if store != nil {
62+
if err := save(store, action, cat); err != nil {
63+
slog.Debug("saving the catalog to the cache failed", "err", err)
64+
}
65+
}
66+
return cat, nil
67+
}
68+
69+
// openAction opens the cache and digests the action that produces a catalog. A
70+
// nil store means the cache is unavailable and the catalog has to be built.
71+
func openAction(dialect string, schema []string) (*cache.Cache, cache.Digest) {
72+
store, err := cache.Open()
73+
if err != nil {
74+
slog.Debug("opening the cache failed", "err", err)
75+
return nil, cache.Digest{}
76+
}
77+
action := store.NewAction("CoreCatalog").AddInput("dialect", []byte(dialect))
78+
for _, s := range schema {
79+
action.AddInput("schema", []byte(s))
80+
}
81+
return store, action.Digest()
82+
}
83+
84+
// restore opens the catalog a previous run cached.
85+
func restore(store *cache.Cache, action cache.Digest) (*Catalog, error) {
86+
result, err := store.Actions.Get(action)
87+
if err != nil {
88+
return nil, err
89+
}
90+
blob, err := store.CAS.Get(result.Outputs[catalogOutput])
91+
if err != nil {
92+
return nil, err
93+
}
94+
95+
db, err := openDB()
96+
if err != nil {
97+
return nil, err
98+
}
99+
if err := deserialize(db, blob); err != nil {
100+
db.Close()
101+
return nil, err
102+
}
103+
104+
stmts := newStmtCache(db)
105+
cat := &Catalog{db: db, stmts: stmts, q: catalogdb.New(stmts)}
106+
107+
// The seeds recorded the dialect on the catalog as they ran. A restored
108+
// catalog reads it back from the database instead.
109+
row, err := cat.q.SeededDialect(context.Background())
110+
if err != nil {
111+
cat.Close()
112+
return nil, fmt.Errorf("core: restored catalog has no dialect: %w", err)
113+
}
114+
cat.dialectOID = row.Oid
115+
return cat, nil
116+
}
117+
118+
// save stores a finished catalog for the next run.
119+
func save(store *cache.Cache, action cache.Digest, cat *Catalog) error {
120+
blob, err := serialize(cat.db)
121+
if err != nil {
122+
return err
123+
}
124+
digest, err := store.CAS.Put(blob)
125+
if err != nil {
126+
return err
127+
}
128+
return store.Actions.Put(action, &cache.ActionResult{
129+
Outputs: map[string]cache.Digest{catalogOutput: digest},
130+
})
131+
}
132+
133+
// serialize returns the bytes a database would be written to disk as.
134+
func serialize(db *sql.DB) ([]byte, error) {
135+
var blob []byte
136+
err := withRawConn(db, func(s serializer) error {
137+
var err error
138+
blob, err = s.Serialize()
139+
return err
140+
})
141+
if err != nil {
142+
return nil, fmt.Errorf("core: serialize catalog: %w", err)
143+
}
144+
return blob, nil
145+
}
146+
147+
// deserialize loads a serialized database into an open one. The pool is
148+
// pinned to a single connection, so the connection this replaces the contents
149+
// of is the only one the catalog will ever use.
150+
func deserialize(db *sql.DB, blob []byte) error {
151+
if err := withRawConn(db, func(s serializer) error {
152+
return s.Deserialize(blob)
153+
}); err != nil {
154+
return fmt.Errorf("core: deserialize catalog: %w", err)
155+
}
156+
return nil
157+
}
158+
159+
func withRawConn(db *sql.DB, fn func(serializer) error) error {
160+
conn, err := db.Conn(context.Background())
161+
if err != nil {
162+
return err
163+
}
164+
defer conn.Close()
165+
return conn.Raw(func(driverConn any) error {
166+
s, ok := driverConn.(serializer)
167+
if !ok {
168+
return fmt.Errorf("driver does not serialize databases")
169+
}
170+
return fn(s)
171+
})
172+
}

0 commit comments

Comments
 (0)