Skip to content

Commit b6aa124

Browse files
Cover more CREATE VECTOR INDEX dialect variants
Extend the shared `CREATE VECTOR INDEX` parsing to the trailers used by the other dialects, and add per-dialect tests: - `STORING(...)` covering-column clause (BigQuery); new `storing` field on `CreateIndex`, rendered after `INCLUDE`. - Accept a `WITH (...)` options clause on a vector index in every dialect (SQL Server `WITH (METRIC = ..., TYPE = ..., MAXDOP = ...)`), not only the dialects that enable it for a plain `CREATE INDEX`. Tests: - `tests/sqlparser_common.rs` — the generic core plus the shared trailers (`INCLUDE`, `STORING`, `WITH`, `USING`, expression targets) across all dialects. - `tests/sqlparser_bigquery.rs` — `OPTIONS(...)` with index_type / distance_type / JSON tuning keys, and `STORING(...)`. - `tests/sqlparser_mssql.rs` — bracket-quoted names with `WITH (...)`. - `tests/sqlparser_mysql.rs` — TiDB's distance-function target with `USING`. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 265c469 commit b6aa124

9 files changed

Lines changed: 72 additions & 4 deletions

File tree

src/ast/ddl.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2843,6 +2843,8 @@ pub struct CreateIndex {
28432843
pub if_not_exists: bool,
28442844
/// INCLUDE clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
28452845
pub include: Vec<Ident>,
2846+
/// `STORING(...)` clause (covering columns on a `CREATE VECTOR INDEX`)
2847+
pub storing: Vec<Ident>,
28462848
/// NULLS DISTINCT / NOT DISTINCT clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
28472849
pub nulls_distinct: Option<bool>,
28482850
/// WITH clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
@@ -2893,6 +2895,9 @@ impl fmt::Display for CreateIndex {
28932895
if !self.include.is_empty() {
28942896
write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?;
28952897
}
2898+
if !self.storing.is_empty() {
2899+
write!(f, " STORING({})", display_comma_separated(&self.storing))?;
2900+
}
28962901
if let Some(value) = self.nulls_distinct {
28972902
if value {
28982903
write!(f, " NULLS DISTINCT")?;

src/ast/spans.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -706,6 +706,7 @@ impl Spanned for CreateIndex {
706706
r#async: _, // bool
707707
if_not_exists: _, // bool
708708
include,
709+
storing,
709710
nulls_distinct: _, // bool
710711
with,
711712
options,
@@ -720,6 +721,7 @@ impl Spanned for CreateIndex {
720721
.chain(core::iter::once(table_name.span()))
721722
.chain(columns.iter().map(|i| i.column.span()))
722723
.chain(include.iter().map(|i| i.span))
724+
.chain(storing.iter().map(|i| i.span))
723725
.chain(with.iter().map(|i| i.span()))
724726
.chain(options.iter().map(|i| i.span()))
725727
.chain(predicate.iter().map(|i| i.span()))

src/keywords.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,7 @@ define_keywords!(
10081008
STORAGE_INTEGRATION,
10091009
STORAGE_SERIALIZATION_POLICY,
10101010
STORED,
1011+
STORING,
10111012
STRAIGHT_JOIN,
10121013
STREAM,
10131014
STRICT,

src/parser/mod.rs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8300,6 +8300,16 @@ impl<'a> Parser<'a> {
83008300
vec![]
83018301
};
83028302

8303+
// `STORING(...)` covering columns (e.g. `CREATE VECTOR INDEX`).
8304+
let storing = if self.parse_keyword(Keyword::STORING) {
8305+
self.expect_token(&Token::LParen)?;
8306+
let columns = self.parse_comma_separated(|p| p.parse_identifier())?;
8307+
self.expect_token(&Token::RParen)?;
8308+
columns
8309+
} else {
8310+
vec![]
8311+
};
8312+
83038313
let nulls_distinct = if self.parse_keyword(Keyword::NULLS) {
83048314
let not = self.parse_keyword(Keyword::NOT);
83058315
self.expect_keyword_is(Keyword::DISTINCT)?;
@@ -8308,7 +8318,9 @@ impl<'a> Parser<'a> {
83088318
None
83098319
};
83108320

8311-
let with = if self.dialect.supports_create_index_with_clause()
8321+
// A vector index accepts a `WITH (...)` options clause in every dialect
8322+
// (e.g. SQL Server `WITH (METRIC = ..., TYPE = ...)`).
8323+
let with = if (self.dialect.supports_create_index_with_clause() || vector)
83128324
&& self.parse_keyword(Keyword::WITH)
83138325
{
83148326
self.expect_token(&Token::LParen)?;
@@ -8356,6 +8368,7 @@ impl<'a> Parser<'a> {
83568368
r#async,
83578369
if_not_exists,
83588370
include,
8371+
storing,
83598372
nulls_distinct,
83608373
with,
83618374
options,

tests/sqlparser_bigquery.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2950,3 +2950,15 @@ fn test_create_snapshot_table() {
29502950
"CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')",
29512951
);
29522952
}
2953+
2954+
#[test]
2955+
fn parse_bigquery_create_vector_index() {
2956+
// BigQuery's form: an `OPTIONS(...)` clause with its index_type / distance_type
2957+
// / JSON tuning keys, and a `STORING(...)` covering-column list.
2958+
bigquery().verified_stmt(
2959+
"CREATE VECTOR INDEX my_index ON my_dataset.my_table(embedding) OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ivf_options = '{\"num_lists\": 2500}')",
2960+
);
2961+
bigquery().verified_stmt(
2962+
"CREATE OR REPLACE VECTOR INDEX my_index ON my_dataset.my_table(embedding) STORING(type, creation_time) OPTIONS(index_type = 'TREE_AH', distance_type = 'EUCLIDEAN')",
2963+
);
2964+
}

tests/sqlparser_common.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9731,6 +9731,7 @@ fn test_create_index_with_using_function() {
97319731
vector: _,
97329732
or_replace: _,
97339733
options: _,
9734+
storing: _,
97349735
}) => {
97359736
assert_eq!("idx_name", name.to_string());
97369737
assert_eq!("test", table_name.to_string());
@@ -9791,6 +9792,7 @@ fn test_create_index_with_with_clause() {
97919792
vector: _,
97929793
or_replace: _,
97939794
options: _,
9795+
storing: _,
97949796
}) => {
97959797
pretty_assertions::assert_eq!("title_idx", name.to_string());
97969798
pretty_assertions::assert_eq!("films", table_name.to_string());
@@ -9893,11 +9895,15 @@ fn parse_create_vector_index() {
98939895
other => panic!("expected CreateIndex, got {other:?}"),
98949896
}
98959897

9896-
// The bare core, an `INCLUDE` covering-column trailer, and an expression
9897-
// target all round-trip.
9898+
// The bare core plus the shared trailers all round-trip across dialects: an
9899+
// expression target, `INCLUDE` / `STORING` covering columns, a `WITH`
9900+
// options clause, and a trailing `USING <method>`.
98989901
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)");
9899-
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)");
99009902
verified_stmt("CREATE VECTOR INDEX emb ON t(VEC_COSINE_DISTANCE(embedding))");
9903+
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)");
9904+
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) STORING(a, b)");
9905+
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) WITH (metric = 'cosine')");
9906+
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) USING HNSW");
99019907
}
99029908

99039909
#[test]

tests/sqlparser_mssql.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2925,3 +2925,12 @@ fn parse_mssql_money_constants() {
29252925
expr_from_projection(only(&select.projection)),
29262926
);
29272927
}
2928+
2929+
#[test]
2930+
fn parse_mssql_create_vector_index() {
2931+
// SQL Server's form: bracket-quoted names and a `WITH (...)` options clause
2932+
// (`METRIC` / `TYPE` / `MAXDOP`).
2933+
ms().verified_stmt(
2934+
"CREATE VECTOR INDEX vec_idx ON [dbo].[articles]([title_vector]) WITH (METRIC = 'cosine', TYPE = 'DiskANN', MAXDOP = 8)",
2935+
);
2936+
}

tests/sqlparser_mysql.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4946,3 +4946,12 @@ fn parse_adjacent_string_literal_concatenation() {
49464946
fn parse_group_by_with_rollup() {
49474947
mysql().verified_stmt("SELECT * FROM tbl GROUP BY col1, col2 WITH ROLLUP");
49484948
}
4949+
4950+
#[test]
4951+
fn parse_mysql_create_vector_index() {
4952+
// TiDB's form: the vector column is wrapped in a distance function and the
4953+
// algorithm is named with a trailing `USING`.
4954+
mysql().verified_stmt(
4955+
"CREATE VECTOR INDEX idx_cos ON tidb_vectors((VEC_COSINE_DISTANCE(embedding))) USING HNSW",
4956+
);
4957+
}

tests/sqlparser_postgres.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2996,6 +2996,7 @@ fn parse_create_index() {
29962996
vector: _,
29972997
or_replace: _,
29982998
options: _,
2999+
storing: _,
29993000
}) => {
30003001
assert_eq_vec(&["my_index"], &name);
30013002
assert_eq_vec(&["my_table"], &table_name);
@@ -3036,6 +3037,7 @@ fn parse_create_anonymous_index() {
30363037
vector: _,
30373038
or_replace: _,
30383039
options: _,
3040+
storing: _,
30393041
}) => {
30403042
assert_eq!(None, name);
30413043
assert_eq_vec(&["my_table"], &table_name);
@@ -3159,6 +3161,7 @@ fn parse_create_indices_with_operator_classes() {
31593161
vector: _,
31603162
or_replace: _,
31613163
options: _,
3164+
storing: _,
31623165
}) => {
31633166
assert_eq_vec(&["the_index_name"], &name);
31643167
assert_eq_vec(&["users"], &table_name);
@@ -3191,6 +3194,7 @@ fn parse_create_indices_with_operator_classes() {
31913194
vector: _,
31923195
or_replace: _,
31933196
options: _,
3197+
storing: _,
31943198
}) => {
31953199
assert_eq_vec(&["the_index_name"], &name);
31963200
assert_eq_vec(&["users"], &table_name);
@@ -3278,6 +3282,7 @@ fn parse_create_bloom() {
32783282
vector: _,
32793283
or_replace: _,
32803284
options: _,
3285+
storing: _,
32813286
}) => {
32823287
assert_eq_vec(&["bloomidx"], &name);
32833288
assert_eq_vec(&["tbloom"], &table_name);
@@ -3338,6 +3343,7 @@ fn parse_create_brin() {
33383343
vector: _,
33393344
or_replace: _,
33403345
options: _,
3346+
storing: _,
33413347
}) => {
33423348
assert_eq_vec(&["brin_sensor_data_recorded_at"], &name);
33433349
assert_eq_vec(&["sensor_data"], &table_name);
@@ -3409,6 +3415,7 @@ fn parse_create_index_concurrently() {
34093415
vector: _,
34103416
or_replace: _,
34113417
options: _,
3418+
storing: _,
34123419
}) => {
34133420
assert_eq_vec(&["my_index"], &name);
34143421
assert_eq_vec(&["my_table"], &table_name);
@@ -3449,6 +3456,7 @@ fn parse_create_index_with_predicate() {
34493456
vector: _,
34503457
or_replace: _,
34513458
options: _,
3459+
storing: _,
34523460
}) => {
34533461
assert_eq_vec(&["my_index"], &name);
34543462
assert_eq_vec(&["my_table"], &table_name);
@@ -3489,6 +3497,7 @@ fn parse_create_index_with_include() {
34893497
vector: _,
34903498
or_replace: _,
34913499
options: _,
3500+
storing: _,
34923501
}) => {
34933502
assert_eq_vec(&["my_index"], &name);
34943503
assert_eq_vec(&["my_table"], &table_name);
@@ -3529,6 +3538,7 @@ fn parse_create_index_with_nulls_distinct() {
35293538
vector: _,
35303539
or_replace: _,
35313540
options: _,
3541+
storing: _,
35323542
}) => {
35333543
assert_eq_vec(&["my_index"], &name);
35343544
assert_eq_vec(&["my_table"], &table_name);
@@ -3567,6 +3577,7 @@ fn parse_create_index_with_nulls_distinct() {
35673577
vector: _,
35683578
or_replace: _,
35693579
options: _,
3580+
storing: _,
35703581
}) => {
35713582
assert_eq_vec(&["my_index"], &name);
35723583
assert_eq_vec(&["my_table"], &table_name);

0 commit comments

Comments
 (0)