Skip to content

Commit 4f101f1

Browse files
committed
ast: omit absent fields from JSON output
Most of what parse printed was absent fields. On a four-query file, 25% of the keys were null and another 9% were empty containers, so reading an AST meant scanning past clauses the statement never had. Mark every pointer, slice, interface and map field of the node structs with json:",omitempty". Scalars keep no such tag on purpose: StmtLocation is 0 for the first statement in a file and LIMIT 0 parses to an Ival of 0, so omitting zero-valued scalars would lose what the parser found rather than what it did not. Field names are unchanged. Beyond nil fields this also drops empty lists, which is a small normalization: the dolphin converter builds Items as an empty slice where the postgresql one leaves it nil, so the same SQL printed different JSON per engine. Both now print as a bare tagged List. This depends on the type tags. Omitting a nil Items turns an empty List into "{}", which without a tag would be indistinguishable from A_Star and TODO. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T9LCt1mwdY3mE14x3Zz5Vw
1 parent 8c81481 commit 4f101f1

245 files changed

Lines changed: 755 additions & 864 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/howto/parse.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,24 @@ clause would all print as `{}`.
8787
A `tag` of `TODO` marks a clause the dialect's converter does not translate
8888
yet. It means the clause was parsed but is not represented in the AST, not that
8989
the clause was absent from the query.
90+
91+
## Absent fields
92+
93+
A field the statement does not use is left out rather than printed as `null`.
94+
An `A_Const` carrying an integer reports only what it has:
95+
96+
```json
97+
"Val": {
98+
"tag": "A_Const",
99+
"Val": {
100+
"tag": "Integer",
101+
"Ival": 1
102+
},
103+
"Location": 30
104+
}
105+
```
106+
107+
Only absent fields — and empty lists, which engines construct differently for
108+
the same SQL — are omitted. A zero keeps its place, because zero is a value the
109+
parser can find: `StmtLocation` is 0 for the first statement in a file, and
110+
`LIMIT 0` parses to an `Ival` of 0.

‎internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt‎

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,15 @@
2828
"Stmt": {
2929
"tag": "SelectStmt",
3030
"DistinctClause": {
31-
"tag": "List",
32-
"Items": null
31+
"tag": "List"
3332
},
34-
"IntoClause": null,
3533
"TargetList": {
3634
"tag": "List",
3735
"Items": [
3836
{
3937
"tag": "ResTarget",
40-
"Name": null,
4138
"Indirection": {
42-
"tag": "List",
43-
"Items": null
39+
"tag": "List"
4440
},
4541
"Val": {
4642
"tag": "ColumnRef",
@@ -65,12 +61,9 @@
6561
"Items": [
6662
{
6763
"tag": "RangeVar",
68-
"Catalogname": null,
69-
"Schemaname": null,
7064
"Relname": "authors",
7165
"Inh": true,
7266
"Relpersistence": 112,
73-
"Alias": null,
7467
"Location": 45
7568
}
7669
]
@@ -110,23 +103,19 @@
110103
"Location": 62
111104
},
112105
"GroupClause": {
113-
"tag": "List",
114-
"Items": null
106+
"tag": "List"
115107
},
116108
"HavingClause": {
117109
"tag": "TODO"
118110
},
119111
"WindowClause": {
120-
"tag": "List",
121-
"Items": null
112+
"tag": "List"
122113
},
123114
"ValuesLists": {
124-
"tag": "List",
125-
"Items": null
115+
"tag": "List"
126116
},
127117
"SortClause": {
128-
"tag": "List",
129-
"Items": null
118+
"tag": "List"
130119
},
131120
"LimitOffset": {
132121
"tag": "TODO"
@@ -135,14 +124,10 @@
135124
"tag": "TODO"
136125
},
137126
"LockingClause": {
138-
"tag": "List",
139-
"Items": null
127+
"tag": "List"
140128
},
141-
"WithClause": null,
142129
"Op": 0,
143-
"All": false,
144-
"Larg": null,
145-
"Rarg": null
130+
"All": false
146131
},
147132
"StmtLocation": 0,
148133
"StmtLen": 66

‎internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,11 @@
66
"tag": "RawStmt",
77
"Stmt": {
88
"tag": "SelectStmt",
9-
"DistinctClause": null,
10-
"IntoClause": null,
119
"TargetList": {
1210
"tag": "List",
1311
"Items": [
1412
{
1513
"tag": "ResTarget",
16-
"Name": null,
17-
"Indirection": null,
1814
"Val": {
1915
"tag": "A_Const",
2016
"Val": {
@@ -27,21 +23,8 @@
2723
}
2824
]
2925
},
30-
"FromClause": null,
31-
"WhereClause": null,
32-
"GroupClause": null,
33-
"HavingClause": null,
34-
"WindowClause": null,
35-
"ValuesLists": null,
36-
"SortClause": null,
37-
"LimitOffset": null,
38-
"LimitCount": null,
39-
"LockingClause": null,
40-
"WithClause": null,
4126
"Op": 0,
42-
"All": false,
43-
"Larg": null,
44-
"Rarg": null
27+
"All": false
4528
},
4629
"StmtLocation": 0,
4730
"StmtLen": 32

‎internal/endtoend/testdata/parse_basic/duckdb/stdout.txt‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,11 @@
66
"tag": "RawStmt",
77
"Stmt": {
88
"tag": "SelectStmt",
9-
"DistinctClause": null,
10-
"IntoClause": null,
119
"TargetList": {
1210
"tag": "List",
1311
"Items": [
1412
{
1513
"tag": "ResTarget",
16-
"Name": null,
17-
"Indirection": null,
1814
"Val": {
1915
"tag": "A_Const",
2016
"Val": {
@@ -27,21 +23,8 @@
2723
}
2824
]
2925
},
30-
"FromClause": null,
31-
"WhereClause": null,
32-
"GroupClause": null,
33-
"HavingClause": null,
34-
"WindowClause": null,
35-
"ValuesLists": null,
36-
"SortClause": null,
37-
"LimitOffset": null,
38-
"LimitCount": null,
39-
"LockingClause": null,
40-
"WithClause": null,
4126
"Op": 0,
42-
"All": false,
43-
"Larg": null,
44-
"Rarg": null
27+
"All": false
4528
},
4629
"StmtLocation": 0,
4730
"StmtLen": 33

‎internal/endtoend/testdata/parse_basic/googlesql/stdout.txt‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,11 @@
66
"tag": "RawStmt",
77
"Stmt": {
88
"tag": "SelectStmt",
9-
"DistinctClause": null,
10-
"IntoClause": null,
119
"TargetList": {
1210
"tag": "List",
1311
"Items": [
1412
{
1513
"tag": "ResTarget",
16-
"Name": null,
17-
"Indirection": null,
1814
"Val": {
1915
"tag": "A_Const",
2016
"Val": {
@@ -27,21 +23,8 @@
2723
}
2824
]
2925
},
30-
"FromClause": null,
31-
"WhereClause": null,
32-
"GroupClause": null,
33-
"HavingClause": null,
34-
"WindowClause": null,
35-
"ValuesLists": null,
36-
"SortClause": null,
37-
"LimitOffset": null,
38-
"LimitCount": null,
39-
"LockingClause": null,
40-
"WithClause": null,
4126
"Op": 0,
42-
"All": false,
43-
"Larg": null,
44-
"Rarg": null
27+
"All": false
4528
},
4629
"StmtLocation": 0,
4730
"StmtLen": 31

‎internal/endtoend/testdata/parse_basic/mssql/stdout.txt‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,11 @@
66
"tag": "RawStmt",
77
"Stmt": {
88
"tag": "SelectStmt",
9-
"DistinctClause": null,
10-
"IntoClause": null,
119
"TargetList": {
1210
"tag": "List",
1311
"Items": [
1412
{
1513
"tag": "ResTarget",
16-
"Name": null,
17-
"Indirection": null,
1814
"Val": {
1915
"tag": "A_Const",
2016
"Val": {
@@ -27,21 +23,8 @@
2723
}
2824
]
2925
},
30-
"FromClause": null,
31-
"WhereClause": null,
32-
"GroupClause": null,
33-
"HavingClause": null,
34-
"WindowClause": null,
35-
"ValuesLists": null,
36-
"SortClause": null,
37-
"LimitOffset": null,
38-
"LimitCount": null,
39-
"LockingClause": null,
40-
"WithClause": null,
4126
"Op": 0,
42-
"All": false,
43-
"Larg": null,
44-
"Rarg": null
27+
"All": false
4528
},
4629
"StmtLocation": 0,
4730
"StmtLen": 32

‎internal/endtoend/testdata/parse_basic/mysql/stdout.txt‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,11 @@
66
"tag": "RawStmt",
77
"Stmt": {
88
"tag": "SelectStmt",
9-
"DistinctClause": null,
10-
"IntoClause": null,
119
"TargetList": {
1210
"tag": "List",
1311
"Items": [
1412
{
1513
"tag": "ResTarget",
16-
"Name": null,
17-
"Indirection": null,
1814
"Val": {
1915
"tag": "A_Const",
2016
"Val": {
@@ -28,29 +24,16 @@
2824
]
2925
},
3026
"FromClause": {
31-
"tag": "List",
32-
"Items": null
27+
"tag": "List"
3328
},
34-
"WhereClause": null,
3529
"GroupClause": {
36-
"tag": "List",
37-
"Items": null
30+
"tag": "List"
3831
},
39-
"HavingClause": null,
4032
"WindowClause": {
41-
"tag": "List",
42-
"Items": []
33+
"tag": "List"
4334
},
44-
"ValuesLists": null,
45-
"SortClause": null,
46-
"LimitOffset": null,
47-
"LimitCount": null,
48-
"LockingClause": null,
49-
"WithClause": null,
5035
"Op": 0,
51-
"All": false,
52-
"Larg": null,
53-
"Rarg": null
36+
"All": false
5437
},
5538
"StmtLocation": 0,
5639
"StmtLen": 31

‎internal/endtoend/testdata/parse_basic/postgresql/stdout.txt‎

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,15 @@
77
"Stmt": {
88
"tag": "SelectStmt",
99
"DistinctClause": {
10-
"tag": "List",
11-
"Items": null
10+
"tag": "List"
1211
},
13-
"IntoClause": null,
1412
"TargetList": {
1513
"tag": "List",
1614
"Items": [
1715
{
1816
"tag": "ResTarget",
19-
"Name": null,
2017
"Indirection": {
21-
"tag": "List",
22-
"Items": null
18+
"tag": "List"
2319
},
2420
"Val": {
2521
"tag": "A_Const",
@@ -34,30 +30,25 @@
3430
]
3531
},
3632
"FromClause": {
37-
"tag": "List",
38-
"Items": null
33+
"tag": "List"
3934
},
4035
"WhereClause": {
4136
"tag": "TODO"
4237
},
4338
"GroupClause": {
44-
"tag": "List",
45-
"Items": null
39+
"tag": "List"
4640
},
4741
"HavingClause": {
4842
"tag": "TODO"
4943
},
5044
"WindowClause": {
51-
"tag": "List",
52-
"Items": null
45+
"tag": "List"
5346
},
5447
"ValuesLists": {
55-
"tag": "List",
56-
"Items": null
48+
"tag": "List"
5749
},
5850
"SortClause": {
59-
"tag": "List",
60-
"Items": null
51+
"tag": "List"
6152
},
6253
"LimitOffset": {
6354
"tag": "TODO"
@@ -66,14 +57,10 @@
6657
"tag": "TODO"
6758
},
6859
"LockingClause": {
69-
"tag": "List",
70-
"Items": null
60+
"tag": "List"
7161
},
72-
"WithClause": null,
7362
"Op": 0,
74-
"All": false,
75-
"Larg": null,
76-
"Rarg": null
63+
"All": false
7764
},
7865
"StmtLocation": 0,
7966
"StmtLen": 31

0 commit comments

Comments
 (0)