Skip to content

Commit b31f7ce

Browse files
committed
core: rewrite sqlc.arg() to native bind parameters before parsing
sqlc.arg(), sqlc.narg(), sqlc.slice(), sqlc.embed() and @name are sqlc syntax, not SQL, but they used to survive all the way through every engine's parser. The pipeline only worked because each converter happened to round-trip them as an ast.FuncCall with Func.Schema == "sqlc", and that knowledge was scattered across named.IsParamFunc, three validators, two AST rewriters and the PostgreSQL engine. Add internal/sql/preprocess, which rewrites the query text to native SQL before any engine parser runs. A dialect-parameterized lexer skips comments, string literals, quoted identifiers and dollar-quoted strings, then replaces each construct with the engine's placeholder ($1, ?, ?1 or @name) and records what it replaced: the parameter set, embed spans, slice spans, placeholder numbers and an offset map back to the original text. The compiler consumes that side table instead of walking the AST: - parameter numbering comes from the preprocessor in source order, and is applied to the parsed AST by placeholder location, so an engine that converts nodes out of order no longer reorders a query's params - embeds are matched by location rather than node identity, which tells a rewritten "t.*" apart from one the user wrote - placeholder-style validation (mixed styles, numbering gaps) and sqlc.* validation move into the preprocessor, where they can report the span of the actual call Deleted: internal/sql/rewrite, named.IsParamFunc/IsParamSign, validate.SqlcFunctions, validate.In, validate.ParamRef, and the sqlc.arg awareness in the PostgreSQL engine. A missed astutils container node can no longer silently drop a parameter or an embed. Engine changes are limited to what the removal exposed: GoogleSQL now converts @name to a ParamRef like every other engine, and ClickHouse tags binary/unary operators as A_Expr_Kind_OP and seeds comparison and arithmetic operators so the analysis core can type a WHERE clause. With those in place sqlc.arg/narg work for ClickHouse with no sqlc-specific engine code at all. Error messages for invalid sqlc.* calls now name the offending text rather than an AST type, and point at the call instead of the statement. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015BSM9xGoosq58r1Mz6B7YP
1 parent b82c64f commit b31f7ce

158 files changed

Lines changed: 1897 additions & 857 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎internal/compiler/analyze.go‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/sqlc-dev/sqlc/internal/source"
99
"github.com/sqlc-dev/sqlc/internal/sql/ast"
1010
"github.com/sqlc-dev/sqlc/internal/sql/named"
11-
"github.com/sqlc-dev/sqlc/internal/sql/rewrite"
11+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
1212
"github.com/sqlc-dev/sqlc/internal/sql/validate"
1313
)
1414

@@ -114,15 +114,15 @@ func combineAnalysis(prev *analysis, a *analyzer.Analysis) *analysis {
114114
return prev
115115
}
116116

117-
func (c *Compiler) analyzeQuery(raw *ast.RawStmt, query string) (*analysis, error) {
118-
return c._analyzeQuery(raw, query, true)
117+
func (c *Compiler) analyzeQuery(raw *ast.RawStmt, query string, pre *preprocess.Statement) (*analysis, error) {
118+
return c._analyzeQuery(raw, query, pre, true)
119119
}
120120

121-
func (c *Compiler) inferQuery(raw *ast.RawStmt, query string) (*analysis, error) {
122-
return c._analyzeQuery(raw, query, false)
121+
func (c *Compiler) inferQuery(raw *ast.RawStmt, query string, pre *preprocess.Statement) (*analysis, error) {
122+
return c._analyzeQuery(raw, query, pre, false)
123123
}
124124

125-
func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool) (*analysis, error) {
125+
func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, pre *preprocess.Statement, failfast bool) (*analysis, error) {
126126
errors := make([]error, 0)
127127
check := func(err error) error {
128128
if failfast {
@@ -134,12 +134,12 @@ func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool)
134134
return nil
135135
}
136136

137-
numbers, dollar, err := validate.ParamRef(raw)
138-
if err := check(err); err != nil {
137+
dollar := pre.Dollar
138+
if err := check(pre.ParamErr); err != nil {
139139
return nil, err
140140
}
141141

142-
raw, namedParams, edits := rewrite.NamedParameters(c.conf.Engine, raw, numbers, dollar)
142+
namedParams := pre.Params
143143

144144
var table *ast.TableName
145145
switch n := raw.Stmt.(type) {
@@ -158,7 +158,7 @@ func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool)
158158
return nil, err
159159
}
160160

161-
if err := check(validate.In(c.catalog, raw)); err != nil {
161+
if err := check(validate.Slice(raw, pre.Slices)); err != nil {
162162
return nil, err
163163
}
164164
rvs := rangeVars(raw.Stmt)
@@ -175,7 +175,7 @@ func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool)
175175
} else {
176176
sort.Slice(refs, func(i, j int) bool { return refs[i].ref.Number < refs[j].ref.Number })
177177
}
178-
raw, embeds := rewrite.Embeds(raw)
178+
embeds := pre.Embeds
179179
qc, err := c.buildQueryCatalog(c.catalog, raw.Stmt, embeds)
180180
if err := check(err); err != nil {
181181
return nil, err
@@ -190,11 +190,10 @@ func (c *Compiler) _analyzeQuery(raw *ast.RawStmt, query string, failfast bool)
190190
return nil, err
191191
}
192192

193-
expandEdits, err := c.expand(qc, raw)
193+
edits, err := c.expand(qc, raw)
194194
if check(err); err != nil {
195195
return nil, err
196196
}
197-
edits = append(edits, expandEdits...)
198197
expanded, err := source.Mutate(query, edits)
199198
if err != nil {
200199
return nil, err

‎internal/compiler/compile.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/sqlc-dev/sqlc/internal/rpc"
1717
"github.com/sqlc-dev/sqlc/internal/source"
1818
"github.com/sqlc-dev/sqlc/internal/sql/ast"
19+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
1920
"github.com/sqlc-dev/sqlc/internal/sql/sqlerr"
2021
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
2122
)
@@ -103,19 +104,34 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) {
103104
continue
104105
}
105106
src := string(blob)
106-
stmts, err := c.parser.Parse(strings.NewReader(src))
107+
108+
// sqlc syntax is not SQL. Rewrite it to native placeholders before the
109+
// engine parser ever sees the query, so parsers only handle SQL.
110+
pp, err := preprocess.File(c.conf.Engine, src)
111+
if err != nil {
112+
merr.Add(filename, src, 0, err)
113+
continue
114+
}
115+
116+
stmts, err := c.parser.Parse(strings.NewReader(pp.Text))
107117
if err != nil {
108118
merr.Add(filename, src, 0, err)
109119
continue
110120
}
111121
for _, stmt := range stmts {
112-
query, err := c.parseQuery(stmt.Raw, src, o)
122+
query, err := c.parseQuery(stmt.Raw, pp, o)
113123
if err != nil {
114124
var e *sqlerr.Error
115125
loc := stmt.Raw.Pos()
116126
if errors.As(err, &e) && e.Location != 0 {
117127
loc = e.Location
118128
}
129+
// Locations are reported against the rewritten query; map them
130+
// back so errors point at what the user wrote.
131+
loc = pp.Origin(loc)
132+
if e != nil && e.Location != 0 {
133+
e.Location = loc
134+
}
119135
merr.Add(filename, src, loc, err)
120136
// If this rpc unauthenticated error bubbles up, then all future parsing/analysis will fail
121137
if errors.Is(err, rpc.ErrUnauthenticated) {
@@ -130,7 +146,7 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) {
130146
queryName := query.Metadata.Name
131147
if queryName != "" {
132148
if _, exists := set[queryName]; exists {
133-
merr.Add(filename, src, stmt.Raw.Pos(), fmt.Errorf("duplicate query name: %s", queryName))
149+
merr.Add(filename, src, pp.Origin(stmt.Raw.Pos()), fmt.Errorf("duplicate query name: %s", queryName))
134150
continue
135151
}
136152
set[queryName] = struct{}{}

‎internal/compiler/expand.go‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -166,34 +166,27 @@ func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node)
166166
}
167167
}
168168

169-
var oldString string
170-
var oldFunc func(string) int
171-
172-
// use the sqlc.embed string instead
173-
if embed, ok := qc.embeds.Find(ref); ok {
174-
oldString = embed.Orig()
175-
} else {
176-
oldFunc = func(s string) int {
177-
length := 0
178-
for i, o := range old {
179-
if hasSeparator := i > 0; hasSeparator {
180-
length++
181-
}
182-
if strings.HasPrefix(s[length:], o) {
183-
length += len(o)
184-
} else if quoted := c.quote(o); strings.HasPrefix(s[length:], quoted) {
185-
length += len(quoted)
186-
} else {
187-
length += len(o)
188-
}
169+
// An embed was rewritten to "table.*" in the query text, so it is
170+
// measured the same way as a star reference the user wrote.
171+
oldFunc := func(s string) int {
172+
length := 0
173+
for i, o := range old {
174+
if hasSeparator := i > 0; hasSeparator {
175+
length++
176+
}
177+
if strings.HasPrefix(s[length:], o) {
178+
length += len(o)
179+
} else if quoted := c.quote(o); strings.HasPrefix(s[length:], quoted) {
180+
length += len(quoted)
181+
} else {
182+
length += len(o)
189183
}
190-
return length
191184
}
185+
return length
192186
}
193187

194188
edits = append(edits, source.Edit{
195189
Location: res.Location - raw.StmtLocation,
196-
Old: oldString,
197190
OldFunc: oldFunc,
198191
New: strings.Join(cols, ", "),
199192
})

‎internal/compiler/output_columns.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ func (c *Compiler) outputColumns(qc *QueryCatalog, node ast.Node) ([]*Column, er
258258
if hasStarRef(n) {
259259

260260
// add a column with a reference to an embedded table
261-
if embed, ok := qc.embeds.Find(n); ok {
261+
if embed, ok := qc.embeds.Find(n.Location, res.Location); ok {
262262
cols = append(cols, &Column{
263263
Name: embed.Table.Name,
264264
EmbedTable: embed.Table,

‎internal/compiler/parse.go‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,32 @@ import (
1212
"github.com/sqlc-dev/sqlc/internal/source"
1313
"github.com/sqlc-dev/sqlc/internal/sql/ast"
1414
"github.com/sqlc-dev/sqlc/internal/sql/astutils"
15+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
1516
"github.com/sqlc-dev/sqlc/internal/sql/validate"
1617
"github.com/sqlc-dev/sqlc/internal/sqlcdebug"
1718
)
1819

1920
var debugDumpAST = sqlcdebug.New("dumpast")
2021

21-
func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query, error) {
22+
func (c *Compiler) parseQuery(stmt ast.Node, pp *preprocess.Result, o opts.Parser) (*Query, error) {
23+
raw, ok := stmt.(*ast.RawStmt)
24+
if !ok {
25+
return nil, errors.New("node is not a statement")
26+
}
27+
28+
// The preprocessor already replaced every sqlc.* construct with native SQL
29+
// and recorded what it replaced.
30+
pre := pp.Statement(raw.StmtLocation)
31+
if pre.Err != nil {
32+
return nil, pre.Err
33+
}
34+
35+
// Engines number bind parameters in whatever order they convert the AST.
36+
// Restore the numbering the preprocessor assigned in source order.
37+
renumberParams(raw, pre.Numbers)
38+
2239
if c.coreCatalog != nil {
23-
return c.parseQueryCore(stmt, src)
40+
return c.parseQueryCore(raw, pp.Text, pre)
2441
}
2542

2643
ctx := context.Background()
@@ -29,18 +46,7 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
2946
debug.Dump(stmt)
3047
}
3148

32-
// validate sqlc-specific syntax
33-
if err := validate.SqlcFunctions(stmt); err != nil {
34-
return nil, err
35-
}
36-
37-
// rewrite queries to remove sqlc.* functions
38-
39-
raw, ok := stmt.(*ast.RawStmt)
40-
if !ok {
41-
return nil, errors.New("node is not a statement")
42-
}
43-
rawSQL, err := source.Pluck(src, raw.StmtLocation, raw.StmtLen)
49+
rawSQL, err := source.Pluck(pp.Text, raw.StmtLocation, raw.StmtLen)
4450
if err != nil {
4551
return nil, err
4652
}
@@ -128,7 +134,7 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
128134
Query: expandedQuery,
129135
}
130136
} else if c.analyzer != nil {
131-
inference, _ := c.inferQuery(raw, rawSQL)
137+
inference, _ := c.inferQuery(raw, rawSQL, pre)
132138
if inference == nil {
133139
inference = &analysis{}
134140
}
@@ -156,7 +162,7 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
156162
// FOOTGUN: combineAnalysis mutates inference
157163
anlys = combineAnalysis(inference, result)
158164
} else {
159-
anlys, err = c.analyzeQuery(raw, rawSQL)
165+
anlys, err = c.analyzeQuery(raw, rawSQL, pre)
160166
if err != nil {
161167
return nil, err
162168
}
@@ -188,6 +194,23 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query,
188194
}, nil
189195
}
190196

197+
// renumberParams applies the parameter numbers the preprocessor assigned,
198+
// matching each node by the location of its placeholder in the query text.
199+
func renumberParams(raw *ast.RawStmt, numbers map[int]int) {
200+
if len(numbers) == 0 {
201+
return
202+
}
203+
astutils.Walk(astutils.VisitorFunc(func(node ast.Node) {
204+
ref, ok := node.(*ast.ParamRef)
205+
if !ok {
206+
return
207+
}
208+
if n, ok := numbers[ref.Location]; ok {
209+
ref.Number = n
210+
}
211+
}), raw)
212+
}
213+
191214
func rangeVars(root ast.Node) []*ast.RangeVar {
192215
var vars []*ast.RangeVar
193216
find := astutils.VisitorFunc(func(node ast.Node) {

‎internal/compiler/parse_core.go‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,11 @@ import (
1010
"github.com/sqlc-dev/sqlc/internal/source"
1111
"github.com/sqlc-dev/sqlc/internal/sql/ast"
1212
"github.com/sqlc-dev/sqlc/internal/sql/named"
13-
"github.com/sqlc-dev/sqlc/internal/sql/rewrite"
13+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
1414
"github.com/sqlc-dev/sqlc/internal/sql/validate"
1515
)
1616

17-
func (c *Compiler) parseQueryCore(stmt ast.Node, src string) (*Query, error) {
18-
raw, ok := stmt.(*ast.RawStmt)
19-
if !ok {
20-
return nil, errors.New("node is not a statement")
21-
}
17+
func (c *Compiler) parseQueryCore(raw *ast.RawStmt, src string, pre *preprocess.Statement) (*Query, error) {
2218
rawSQL, err := source.Pluck(src, raw.StmtLocation, raw.StmtLen)
2319
if err != nil {
2420
return nil, err
@@ -48,21 +44,17 @@ func (c *Compiler) parseQueryCore(stmt ast.Node, src string) (*Query, error) {
4844
return nil, err
4945
}
5046

51-
numbers, dollar, err := validate.ParamRef(raw)
52-
if err != nil {
53-
return nil, err
54-
}
55-
rewritten, namedParams, edits := rewrite.NamedParameters(c.conf.Engine, raw, numbers, dollar)
56-
expanded, err := source.Mutate(rawSQL, edits)
57-
if err != nil {
58-
return nil, err
47+
if pre.ParamErr != nil {
48+
return nil, pre.ParamErr
5949
}
50+
namedParams := pre.Params
51+
expanded := rawSQL
6052

6153
var cols []*Column
6254
var params []Parameter
63-
switch rewritten.Stmt.(type) {
55+
switch raw.Stmt.(type) {
6456
case *ast.SelectStmt, *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:
65-
res, err := coreanalyzer.Prepare(c.coreCatalog, rewritten)
57+
res, err := coreanalyzer.Prepare(c.coreCatalog, raw)
6658
if err != nil {
6759
return nil, err
6860
}
@@ -123,12 +115,17 @@ func coreParamColumn(p core.Parameter, params *named.ParamSet) *Column {
123115
col.Table = &ast.TableName{Schema: p.Source.Schema, Name: p.Source.Table}
124116
col.OriginalName = p.Source.Column
125117
}
126-
if col.Name == "" {
127-
if name, ok := params.NameFor(p.Number); ok && name != "" {
128-
col.Name = name
129-
} else if p.Source != nil {
130-
col.Name = p.Source.Column
131-
}
118+
if col.Name == "" && p.Source != nil {
119+
col.Name = p.Source.Column
120+
}
121+
// Merge in what the user asked for: sqlc.narg() makes the parameter
122+
// nullable and sqlc.slice() marks it as a slice, whichever way the
123+
// analyzer typed it.
124+
if param, isNamed := params.FetchMerge(p.Number, named.NewInferredParam(col.Name, p.NotNull)); isNamed {
125+
col.Name = param.Name()
126+
col.NotNull = param.NotNull()
127+
col.IsSqlcSlice = param.IsSqlcSlice()
128+
col.IsNamedParam = true
132129
}
133130
return col
134131
}

‎internal/compiler/query_catalog.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,16 @@ import (
55

66
"github.com/sqlc-dev/sqlc/internal/sql/ast"
77
"github.com/sqlc-dev/sqlc/internal/sql/catalog"
8-
"github.com/sqlc-dev/sqlc/internal/sql/rewrite"
8+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
99
)
1010

1111
type QueryCatalog struct {
1212
catalog *catalog.Catalog
1313
ctes map[string]*Table
14-
embeds rewrite.EmbedSet
14+
embeds preprocess.EmbedSet
1515
}
1616

17-
func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embeds rewrite.EmbedSet) (*QueryCatalog, error) {
17+
func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embeds preprocess.EmbedSet) (*QueryCatalog, error) {
1818
var with *ast.WithClause
1919
switch n := node.(type) {
2020
case *ast.DeleteStmt:

‎internal/compiler/resolve.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/sqlc-dev/sqlc/internal/sql/astutils"
1010
"github.com/sqlc-dev/sqlc/internal/sql/catalog"
1111
"github.com/sqlc-dev/sqlc/internal/sql/named"
12-
"github.com/sqlc-dev/sqlc/internal/sql/rewrite"
12+
"github.com/sqlc-dev/sqlc/internal/sql/preprocess"
1313
"github.com/sqlc-dev/sqlc/internal/sql/sqlerr"
1414
)
1515

@@ -21,7 +21,7 @@ func dataType(n *ast.TypeName) string {
2121
}
2222
}
2323

24-
func (comp *Compiler) resolveCatalogRefs(qc *QueryCatalog, rvs []*ast.RangeVar, args []paramRef, params *named.ParamSet, embeds rewrite.EmbedSet) ([]Parameter, error) {
24+
func (comp *Compiler) resolveCatalogRefs(qc *QueryCatalog, rvs []*ast.RangeVar, args []paramRef, params *named.ParamSet, embeds preprocess.EmbedSet) ([]Parameter, error) {
2525
c := comp.catalog
2626

2727
aliasMap := map[string]*ast.TableName{}

0 commit comments

Comments
 (0)