From 854b26007ceb48910fd022d0942a816e843c1cc2 Mon Sep 17 00:00:00 2001 From: Wendell Smith Date: Fri, 24 Jul 2026 13:01:09 -0400 Subject: [PATCH 1/2] feat: support explicit Project outputs --- API.md | 6 +- GRAMMAR.md | 13 ++- README.md | 4 +- example-plans/basic.substrait | 2 +- example-plans/simple.substrait | 6 +- examples/advanced_usage.rs | 2 +- examples/basic_usage.rs | 2 +- src/cli.rs | 8 +- src/lib.rs | 6 +- src/parser/expression_grammar.pest | 14 ++- src/parser/extensions.rs | 2 +- src/parser/relations.rs | 161 ++++++++++++++++++++++++----- src/parser/structural.rs | 14 +-- src/textify/plan.rs | 2 +- src/textify/rels.rs | 56 +++++++--- tests/adv_extension_roundtrip.rs | 6 +- tests/literal_roundtrip.rs | 26 ++--- tests/verbose_roundtrip.rs | 28 +++++ 18 files changed, 269 insertions(+), 89 deletions(-) diff --git a/API.md b/API.md index 61a67592..34d0c4fa 100644 --- a/API.md +++ b/API.md @@ -36,7 +36,7 @@ Functions: ## 10 @ 1: add === Plan -Project[$0, $1, add($0, $1):i32?] +Project[_ => $0, $1, add($0, $1):i32?] Read[table1 => col1:i32?, col2:i32?] "#; @@ -59,7 +59,7 @@ use substrait_explain::{parse, format_with_options, OutputOptions, Visibility}; let plan = parse(r#" === Plan -Project[$0, 42, 54:i16] +Project[_ => $0, 42, 54:i16] Read[data => name:string?, num:i64] "#).unwrap(); @@ -244,7 +244,7 @@ Functions: === Plan Root[result] Aggregate[$0 => $0, sum($1):i32?, count($1):i64] - Project[$0, add($1, $2):i32?] + Project[_ => $0, add($1, $2):i32?] Read[table1 => category:string, col1:i32?, col2:i32?] ``` diff --git a/GRAMMAR.md b/GRAMMAR.md index e57e3fbb..b0f10c90 100644 --- a/GRAMMAR.md +++ b/GRAMMAR.md @@ -996,11 +996,18 @@ Root[result] #### Syntax -`"Project" "[" (expression ("," expression)*)? "]"` +```text +project_relation := "Project" "[" "_" project_output? "]" +project_output := "=>" project_values + / "+>" project_additions ("|>" emit_mapping)? +``` #### Components -- `expression` - field reference, function call, or literal (see Expressions section) +- The leading `_` is Project's required empty parameter list. +- `project_values` is a comma-separated list of input references and expressions. In compact `=>` syntax, it describes Project's final output. +- `project_additions` is a comma-separated list of expressions appended to Project's inherited input fields. Use `_` when it is empty. +- `emit_mapping` references the complete direct output: inherited input fields first, followed by `project_additions`. #### Example @@ -1010,7 +1017,7 @@ Root[result] # let plan_text = r#" === Plan Root[result] - Project[$1, 42] // project field 1 and literal 42 + Project[_ => $1, 42] // project field 1 and literal 42 Read[data => a:i64, b:string] # "#; # diff --git a/README.md b/README.md index e323b6d9..19a4b761 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Functions: === Plan Root[revenue] Filter[gt($2, 100) => $0, $1] - Project[$0, $1, multiply($0, $1)] + Project[_ => $0, $1, multiply($0, $1)] Read[orders => quantity:i32?, price:fp64?] ``` @@ -157,7 +157,7 @@ Functions: === Plan Root[revenue] Filter[gt($2, 100) => $0, $1] - Project[$0, $1, multiply($0, $1)] + Project[_ => $0, $1, multiply($0, $1)] Read[orders => quantity:i32?, price:fp64?] "#; diff --git a/example-plans/basic.substrait b/example-plans/basic.substrait index 70662e37..09577b58 100644 --- a/example-plans/basic.substrait +++ b/example-plans/basic.substrait @@ -1,4 +1,4 @@ === Plan Root[result] - Project[$0, $1] + Project[_ => $0, $1] Read[data => a:i64, b:string] diff --git a/example-plans/simple.substrait b/example-plans/simple.substrait index 8b4a6a81..0d650428 100644 --- a/example-plans/simple.substrait +++ b/example-plans/simple.substrait @@ -13,7 +13,7 @@ Functions: === Plan Root[c, d] Filter[eq($0, 0):boolean => $0, $1] - Project[$1, abs($0):fp64?] + Project[_ => $1, abs($0):fp64?] Read[schema.table => a:fp64?, b:i64] Root[c, d, e] @@ -21,7 +21,7 @@ Root[c, d, e] Read[schema.table => name:string?, num:fp64?, id:i64] Root[name, num] - Project[$1, coalesce($1, $2):fp64?] + Project[_ => $1, coalesce($1, $2):fp64?] Read[schema.table => name:string?, num:fp64?, other_num:fp64?, id:i64] Root[name, parent, sum, count] @@ -35,7 +35,7 @@ Root[name, num, id] Read[schema.table => name:string?, num:fp64?, id:i64] Root[result] - Project[$0, $1, $2] + Project[_ => $0, $1, $2] Filter[eq($2, 100):boolean => $0, $1, $2] Read[orders => customer:string, amount:fp64, order_id:i64] diff --git a/examples/advanced_usage.rs b/examples/advanced_usage.rs index 105ff961..8a728914 100644 --- a/examples/advanced_usage.rs +++ b/examples/advanced_usage.rs @@ -42,7 +42,7 @@ Functions: === Plan Root[revenue] Filter[gt($2, 100):boolean => $0, $1] - Project[$0, $1, multiply($0, $1):fp64?] + Project[_ => $0, $1, multiply($0, $1):fp64?] Read[orders => quantity:i32?, price:fp64?] "#; diff --git a/examples/basic_usage.rs b/examples/basic_usage.rs index a898d5c3..3776fd1e 100644 --- a/examples/basic_usage.rs +++ b/examples/basic_usage.rs @@ -18,7 +18,7 @@ Functions: === Plan Root[revenue] Filter[gt($2, 100):boolean => $0, $1] - Project[$0, $1, multiply($0, $1):fp64?] + Project[_ => $0, $1, multiply($0, $1):fp64?] Read[orders => quantity:i32?, price:fp64?] "#; diff --git a/src/cli.rs b/src/cli.rs index ba4998c8..788dc37f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -484,7 +484,7 @@ mod tests { const BASIC_PLAN: &str = r#"=== Plan Root[result] - Project[$0, $1] + Project[_ => $0, $1] Read[data => a:i64, b:string] "#; @@ -497,7 +497,7 @@ Functions: === Plan Root[result] Filter[gt($2, 100):boolean => $0, $1, $2] - Project[$0, $1, $2] + Project[_ => $0, $1, $2] Read[data => a:i64, b:string, c:i32] "#; @@ -523,7 +523,7 @@ Root[result] let output_content = String::from_utf8(output).unwrap(); assert!(output_content.contains("=== Plan")); assert!(output_content.contains("Root[result]")); - assert!(output_content.contains("Project[$0, $1]")); + assert!(output_content.contains("Project[_]")); assert!(output_content.contains("Read[data => a:i64, b:string]")); } @@ -644,7 +644,7 @@ Root[result] let output_content = String::from_utf8(output).unwrap(); assert!(output_content.contains("=== Plan")); assert!(output_content.contains("Root[result]")); - assert!(output_content.contains("Project[$0, $1]")); + assert!(output_content.contains("Project[_]")); assert!(output_content.contains("Read[data => a:i64, b:string]")); } diff --git a/src/lib.rs b/src/lib.rs index 9d1ddf72..8ed77cd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,7 +47,7 @@ use textify::plan::PlanWriter; /// let plan_text = r#" /// === Plan /// Root[c, d] -/// Project[$1, 42] +/// Project[_ => $1, 42] /// Read[schema.table => a:i64, b:string?] /// "#; /// @@ -108,7 +108,7 @@ pub fn parse_with_registry( /// let plan: Plan = parse(r#" /// === Plan /// Root[result] -/// Project[$0, $1] +/// Project[_ => $0, $1] /// Read[data => a:i64, b:string] /// "#).unwrap(); /// @@ -142,7 +142,7 @@ pub fn format(plan: &Plan) -> (String, Vec) { /// let plan = parse(r#" /// === Plan /// Root[result] -/// Project[$0, 42] +/// Project[_ => $0, 42] /// Read[data => a:i64] /// "#).unwrap(); /// diff --git a/src/parser/expression_grammar.pest b/src/parser/expression_grammar.pest index e2fe68dc..36257eef 100644 --- a/src/parser/expression_grammar.pest +++ b/src/parser/expression_grammar.pest @@ -300,9 +300,17 @@ virtual_row = { "(" ~ sp ~ expression_list? ~ sp ~ ")" } filter_relation = { "Filter" ~ "[" ~ expression ~ (sp ~ reference_output)? ~ "]" } -project_relation = { "Project" ~ "[" ~ project_argument_list ~ "]" } -project_argument_list = { (project_argument ~ (sp ~ "," ~ sp ~ project_argument)*)? } -project_argument = { reference | expression } +// Project has no relation parameters, so `_` marks the empty parameter list. +// Compact output combines input references and added expressions; explicit +// output declares only the expressions appended to the inherited input fields. +project_relation = { "Project" ~ "[" ~ empty ~ (sp ~ project_output)? ~ "]" } +project_output = { implicit_project_output | explicit_project_output } +implicit_project_output = { "=>" ~ sp ~ project_argument_list } +explicit_project_output = { ("+>" ~ sp ~ project_additions ~ (sp ~ explicit_emit)?) | explicit_emit } +project_argument_list = { empty | (project_argument ~ (sp ~ "," ~ sp ~ project_argument)*) } +project_argument = { reference | expression } +project_additions = { project_expression_list } +project_expression_list = { empty | expression_list } // Aggregate relation: groups by specified fields and applies aggregate functions // Format: Aggregate[group_by_fields => output_items] diff --git a/src/parser/extensions.rs b/src/parser/extensions.rs index 9d08a2b9..e37e6c4a 100644 --- a/src/parser/extensions.rs +++ b/src/parser/extensions.rs @@ -737,7 +737,7 @@ Types: # 11 @ 1: u!point === Plan Root[result] - Project[$0] + Project[_ => $0] Read[data => p:point#11]"; let plan = Parser::parse(plan_text).unwrap(); let (text, errors) = format(&plan); diff --git a/src/parser/relations.rs b/src/parser/relations.rs index 0d22f096..73b719af 100644 --- a/src/parser/relations.rs +++ b/src/parser/relations.rs @@ -738,6 +738,46 @@ impl RelationParsePair for FilterRel { } } +fn parse_project_argument_list( + extensions: &SimpleExtensions, + pair: Pair, + input_field_count: usize, +) -> Result<(Vec, OutputMapping), MessageParseError> { + assert_eq!(pair.as_rule(), Rule::project_argument_list); + + let mut expressions = Vec::new(); + let mut mapping = Vec::new(); + for argument in pair.into_inner() { + let argument = unwrap_single_pair(argument); + match argument.as_rule() { + Rule::empty => {} + Rule::reference => mapping.push(FieldIndex::parse_pair(argument).0), + Rule::expression => { + expressions.push(Expression::parse_pair(extensions, argument)?); + mapping.push(input_field_count as i32 + expressions.len() as i32 - 1); + } + other => unreachable!("Unexpected project argument rule: {other:?}"), + } + } + Ok((expressions, OutputMapping(mapping))) +} + +fn parse_project_expression_list( + extensions: &SimpleExtensions, + pair: Pair, +) -> Result, MessageParseError> { + assert_eq!(pair.as_rule(), Rule::project_expression_list); + let list = unwrap_single_pair(pair); + if list.as_rule() == Rule::empty { + return Ok(Vec::new()); + } + assert_eq!(list.as_rule(), Rule::expression_list); + list.into_inner() + .filter(|pair| pair.as_rule() == Rule::expression) + .map(|pair| Expression::parse_pair(extensions, pair)) + .collect() +} + impl RelationParsePair for ProjectRel { fn rule() -> Rule { Rule::project_relation @@ -763,30 +803,54 @@ impl RelationParsePair for ProjectRel { assert_eq!(pair.as_rule(), Self::rule()); let input = expect_one_child(Self::message(), &pair, input_children)?; - let arguments_pair = unwrap_single_pair(pair); - - let mut expressions = Vec::new(); - let mut output_mapping = Vec::new(); + let mut iter = RuleIter::from(pair.into_inner()); + iter.pop(Rule::empty); + let output = iter.try_pop(Rule::project_output); + iter.done(); - for arg in arguments_pair.into_inner() { - let inner_arg = unwrap_single_pair(arg); - match inner_arg.as_rule() { - Rule::reference => { - let field_index = FieldIndex::parse_pair(inner_arg); - output_mapping.push(field_index.0); - } - Rule::expression => { - let expr = Expression::parse_pair(extensions, inner_arg)?; - expressions.push(expr); - // Index into the combined schema: [input fields][computed expressions]. - output_mapping.push(input_field_count as i32 + (expressions.len() as i32 - 1)); + let (expressions, emit, output_count) = match output { + None => (Vec::new(), EmitKind::Direct(Direct {}), input_field_count), + Some(output) => { + let output = unwrap_single_pair(output); + match output.as_rule() { + Rule::implicit_project_output => { + let arguments = unwrap_single_pair(output); + let (expressions, mapping) = + parse_project_argument_list(extensions, arguments, input_field_count)?; + let (emit, output_count) = + mapping.into_compact_emit(input_field_count + expressions.len()); + (expressions, emit, output_count) + } + Rule::explicit_project_output => { + let mut output_iter = RuleIter::from(output.into_inner()); + let expressions = output_iter + .try_pop(Rule::project_additions) + .map(|additions| { + parse_project_expression_list( + extensions, + unwrap_single_pair(additions), + ) + }) + .transpose()? + .unwrap_or_default(); + let mapping = output_iter + .try_pop(Rule::explicit_emit) + .map(|emit| OutputMapping::parse_pair(unwrap_single_pair(emit))); + output_iter.done(); + + let direct_count = input_field_count + expressions.len(); + match mapping { + Some(mapping) => { + let (emit, output_count) = mapping.into_explicit_emit(); + (expressions, emit, output_count) + } + None => (expressions, EmitKind::Direct(Direct {}), direct_count), + } + } + other => unreachable!("Unexpected project output rule: {other:?}"), } - _ => panic!("Unexpected inner argument rule: {:?}", inner_arg.as_rule()), } - } - - let direct_count = input_field_count + expressions.len(); - let (emit, output_count) = OutputMapping(output_mapping).into_compact_emit(direct_count); + }; let common = RelCommon { emit_kind: Some(emit), ..Default::default() @@ -1365,14 +1429,13 @@ impl RelationParsePair for JoinRel { Expression::parse_pair(extensions, expression_pair).map(Box::new) }) .transpose()?; - let references_pair = iter.pop(Rule::reference_list); + let references = iter.parse_next::(); iter.done(); // TODO: For semi/anti joins, the direct output width differs from // left+right — `input_field_count` would misclassify the emit as Direct. // Revisit when those join types are supported. - let (emit, output_count) = - OutputMapping::parse_pair(references_pair).into_compact_emit(input_field_count); + let (emit, output_count) = references.into_compact_emit(input_field_count); let common = RelCommon { emit_kind: Some(emit), ..Default::default() @@ -1802,7 +1865,7 @@ mod tests { let extensions = SimpleExtensions::default(); let project = ProjectRel::parse_pair_with_context( &extensions, - parse_exact(Rule::project_relation, "Project[$0, $1, 42]"), + parse_exact(Rule::project_relation, "Project[_ => $0, $1, 42]"), vec![example_read_relation().into_rel(None)], 3, ) @@ -1821,12 +1884,58 @@ mod tests { assert_eq!(emit, &[0, 1, 3]); } + #[test] + fn test_project_relation_requires_empty_parameter_list() { + for valid in [ + "Project[_]", + "Project[_ => _]", + "Project[_ +> _]", + "Project[_ +> 42]", + "Project[_ +> 42 |> $0]", + "Project[_ |> $0]", + ] { + assert!( + ExpressionParser::parse(Rule::project_relation, valid).is_ok(), + "expected Project form to parse: {valid}" + ); + } + + for invalid in ["Project[]", "Project[$0]", "Project[42]", "Project[=> $0]"] { + assert!( + ExpressionParser::parse(Rule::project_relation, invalid).is_err(), + "expected Project form to fail: {invalid}" + ); + } + } + + #[test] + fn test_parse_project_relation_explicit_identity_emit() { + let extensions = SimpleExtensions::default(); + let (project, output_count) = ProjectRel::parse_pair_with_context( + &extensions, + parse_exact(Rule::project_relation, "Project[_ +> 42 |> $0, $1, $2, $3]"), + vec![example_read_relation().into_rel(None)], + 3, + ) + .unwrap(); + + assert_eq!(project.expressions.len(), 1); + assert_eq!(output_count, 4); + assert!(matches!( + project + .common + .as_ref() + .and_then(|common| common.emit_kind.as_ref()), + Some(EmitKind::Emit(emit)) if emit.output_mapping == [0, 1, 2, 3], + )); + } + #[test] fn test_parse_project_relation_complex() { let extensions = SimpleExtensions::default(); let project = ProjectRel::parse_pair_with_context( &extensions, - parse_exact(Rule::project_relation, "Project[42, $0, 100, $2, $1]"), + parse_exact(Rule::project_relation, "Project[_ => 42, $0, 100, $2, $1]"), vec![example_read_relation().into_rel(None)], 5, // Assume 5 input fields ) diff --git a/src/parser/structural.rs b/src/parser/structural.rs index 596be5f0..b35aecc7 100644 --- a/src/parser/structural.rs +++ b/src/parser/structural.rs @@ -793,7 +793,7 @@ impl<'a> RelationParser<'a> { /// let plan_text = r#" /// === Plan /// Root[c, d] -/// Project[$1, 42] +/// Project[_ => $1, 42] /// Read[schema.table => a:i64, b:string?] /// "#; /// @@ -844,7 +844,7 @@ impl<'a> RelationParser<'a> { /// /// The parser supports all standard Substrait relations: /// - `Read[table => columns]` - Read from a table -/// - `Project[expressions]` - Project columns/expressions +/// - `Project[_ => expressions]` - Project columns/expressions /// - `Filter[condition => columns]` - Filter rows /// - `Root[columns]` - Root relation with output columns /// - And more... @@ -867,7 +867,7 @@ impl<'a> RelationParser<'a> { /// ## 10 @ 1: my_custom_function /// === Plan /// Root[result] -/// Project[my_custom_function($0, $1):i32] +/// Project[_ => my_custom_function($0, $1):i32] /// Read[table => col1:i32, col2:i32] /// "#; /// @@ -1350,7 +1350,7 @@ Type Variations: fn test_parse_relation_tree() { // Example plan with a Project, a Filter, and a Read, nested by indentation let plan = r#"=== Plan -Project[$0, $1, 42, 84] +Project[_ => $0, $1, 42, 84] Filter[$2 => $0, $1] Read[my.table => a:i32, b:string?, c:boolean] "#; @@ -1403,7 +1403,7 @@ Project[$0, $1, 42, 84] // Test a plan with a Root relation let plan = r#"=== Plan Root[result] - Project[$0, $1] + Project[_ => $0, $1] Read[my.table => a:i32, b:string?] "#; let mut parser = Parser::default(); @@ -1453,7 +1453,7 @@ Root[result] // Test a plan with a Root relation with no names let plan = r#"=== Plan Root[_] - Project[$0, $1] + Project[_ => $0, $1] Read[my.table => a:i32, b:string?] "#; let mut parser = Parser::default(); @@ -1495,7 +1495,7 @@ Type Variations: # 30 @ 2: VarX === Plan -Project[$0, $1, 42, 84] +Project[_ => $0, $1, 42, 84] Filter[$2 => $0, $1] Read[my.table => a:i32, b:string?, c:boolean] "#; diff --git a/src/textify/plan.rs b/src/textify/plan.rs index 36d66f07..12e599a4 100644 --- a/src/textify/plan.rs +++ b/src/textify/plan.rs @@ -257,7 +257,7 @@ Functions: # 10 @ 1: add === Plan -Project[$0, $1, add($0, $1):i64] +Project[_ => $0, $1, add($0, $1):i64] Read[table1 => col1:i32?, col2:i32?] "# .trim_start(); diff --git a/src/textify/rels.rs b/src/textify/rels.rs index f06a9870..14959737 100644 --- a/src/textify/rels.rs +++ b/src/textify/rels.rs @@ -266,6 +266,7 @@ enum OutputClause<'a> { impl<'a> OutputClause<'a> { fn new( direct_output: &'a [Value<'a>], + additions: &'a [Value<'a>], emit: Option<&'a EmitKind>, syntax: OutputSyntax, presence: OutputClausePresence, @@ -279,20 +280,26 @@ impl<'a> OutputClause<'a> { (OutputSyntax::Compact, OutputClausePresence::Required, _) => { Some(Self::Compact(Emitted::new(direct_output, emit))) } - (OutputSyntax::Compact, OutputClausePresence::WhenNeeded, Some(_)) => { - Some(Self::Compact(Emitted::new(direct_output, emit))) + (OutputSyntax::Compact, OutputClausePresence::WhenNeeded, mapping) => { + (!additions.is_empty() || mapping.is_some()) + .then(|| Self::Compact(Emitted::new(direct_output, emit))) } - (OutputSyntax::Compact, OutputClausePresence::WhenNeeded, None) => None, (OutputSyntax::Explicit, OutputClausePresence::Required, mapping) => { Some(Self::Explicit(ExplicitOutput::Additions { - additions: OutputAdditions(direct_output), + additions: OutputAdditions(additions), mapping, })) } - (OutputSyntax::Explicit, OutputClausePresence::WhenNeeded, Some(mapping)) => { - Some(Self::Explicit(ExplicitOutput::Mapping(mapping))) + (OutputSyntax::Explicit, OutputClausePresence::WhenNeeded, mapping) => { + match (additions.is_empty(), mapping) { + (false, mapping) => Some(Self::Explicit(ExplicitOutput::Additions { + additions: OutputAdditions(additions), + mapping, + })), + (true, Some(mapping)) => Some(Self::Explicit(ExplicitOutput::Mapping(mapping))), + (true, None) => None, + } } - (OutputSyntax::Explicit, OutputClausePresence::WhenNeeded, None) => None, } } } @@ -352,9 +359,11 @@ pub struct Relation<'a> { /// - `Some(RelationArgs::Rows { .. })` prints one row per line, followed by /// any named arguments, one per line. pub arguments: Option>, - /// The columns emitted by this relation, pre-emit - the 'direct' column - /// output. + /// The relation's complete direct output, before any emit mapping. pub columns: Vec>, + /// The first index of direct-output additions. The preceding fields are + /// inherited or otherwise form the relation's base direct output. + output_additions_start: usize, /// The emit kind, if any. If none, use the columns directly. pub emit: Option<&'a EmitKind>, /// How an output clause is rendered when one is present. @@ -434,6 +443,7 @@ impl Relation<'_> { } let output = OutputClause::new( &self.columns, + &self.columns[self.output_additions_start..], self.emit, self.output_syntax, self.output_clause_presence, @@ -449,6 +459,7 @@ impl Relation<'_> { let args = ctx.display(args); match OutputClause::new( &self.columns, + &self.columns[self.output_additions_start..], self.emit, self.output_syntax, self.output_clause_presence, @@ -505,6 +516,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Read"), arguments: Some(RelationArgs::inline(vec![table_name], vec![])), columns, + output_additions_start: 0, emit, output_syntax, output_clause_presence: OutputClausePresence::Required, @@ -549,6 +561,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Read:Virtual"), arguments: Some(arguments), columns, + output_additions_start: 0, emit, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -569,6 +582,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Read:Extension"), arguments: None, columns, + output_additions_start: 0, emit, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -590,6 +604,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Read"), arguments: Some(RelationArgs::inline(vec![Value::Missing(err)], vec![])), columns, + output_additions_start: 0, emit, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -657,13 +672,16 @@ impl<'a> Relation<'a> { let positional = vec![condition]; let arguments = Some(RelationArgs::inline(positional, vec![])); let emit = get_emit(rel.common.as_ref()); - let (children, columns) = Relation::convert_children(vec![rel.input.as_deref()], ctx); - let columns = (0..columns).map(|i| Value::Reference(i as i32)).collect(); + let (children, input_columns) = Relation::convert_children(vec![rel.input.as_deref()], ctx); + let columns = (0..input_columns) + .map(|i| Value::Reference(i as i32)) + .collect(); Relation { name: Cow::Borrowed("Filter"), arguments, columns, + output_additions_start: input_columns, emit, output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), output_clause_presence: OutputClausePresence::WhenNeeded, @@ -684,11 +702,12 @@ impl<'a> Relation<'a> { Relation { name: Cow::Borrowed("Project"), - arguments: None, + arguments: Some(RelationArgs::inline(vec![], vec![])), columns, + output_additions_start: input_columns, emit: get_emit(rel.common.as_ref()), - output_syntax: OutputSyntax::Compact, - output_clause_presence: OutputClausePresence::Required, + output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), + output_clause_presence: OutputClausePresence::WhenNeeded, addenda: AddendumLines::from_advanced_extension(ctx, rel.advanced_extension.as_ref()), children, } @@ -719,6 +738,7 @@ impl<'a> Relation<'a> { name: Cow::Owned(format!("{token}")), arguments: None, columns: vec![], + output_additions_start: 0, emit: None, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -788,6 +808,7 @@ impl<'a> Relation<'a> { name: Cow::Owned(format!("{}:{}", ext_type, name)), arguments: Some(RelationArgs::inline(positional, named)), columns, + output_additions_start: 0, emit: None, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -808,6 +829,7 @@ impl<'a> Relation<'a> { None::<&str>, error.to_string(), ))], + output_additions_start: 0, emit: None, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -903,6 +925,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Aggregate"), arguments, columns: all_outputs, + output_additions_start: 0, emit, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -1006,6 +1029,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Sort"), arguments, columns: col_values, + output_additions_start: input_columns, emit, output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), output_clause_presence: OutputClausePresence::WhenNeeded, @@ -1060,6 +1084,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Fetch"), arguments: Some(RelationArgs::inline(vec![], named_args)), columns, + output_additions_start: input_columns, emit, output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), output_clause_presence: OutputClausePresence::WhenNeeded, @@ -1174,6 +1199,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Join"), arguments, columns, + output_additions_start: 0, emit, output_syntax: OutputSyntax::Compact, output_clause_presence: OutputClausePresence::Required, @@ -1206,6 +1232,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Set"), arguments, columns, + output_additions_start: width, emit, output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), output_clause_presence: OutputClausePresence::WhenNeeded, @@ -1228,6 +1255,7 @@ impl<'a> Relation<'a> { name: Cow::Borrowed("Cross"), arguments: Some(RelationArgs::inline(vec![], vec![])), columns, + output_additions_start: total_columns, emit: get_emit(rel.common.as_ref()), output_syntax: OutputSyntax::from_show_emit(ctx.options().show_emit), output_clause_presence: OutputClausePresence::WhenNeeded, diff --git a/tests/adv_extension_roundtrip.rs b/tests/adv_extension_roundtrip.rs index 07a3775c..2fb61d20 100644 --- a/tests/adv_extension_roundtrip.rs +++ b/tests/adv_extension_roundtrip.rs @@ -567,7 +567,7 @@ mod extension_child_fixture { } } -/// `Project[$0, $1, 42]` over an `ExtensionLeaf:TwoColumnScan` with 2 output columns. +/// `Project[_ => $0, $1, 42]` over an `ExtensionLeaf:TwoColumnScan` with 2 output columns. /// /// Without the `get_input_field_count` fix, the literal `42` would be given emit /// index 0 (wrapping around) instead of 2, producing `[0, 1, 0]` in the proto. @@ -585,7 +585,7 @@ fn test_project_over_extension_leaf_emit_mapping() { let plan_text = r#"=== Plan Root[result] - Project[$0, $1, 42] + Project[_ => $0, $1, 42] ExtensionLeaf:TwoColumnScan[_ => col0:i64, col1:i32]"#; let parser = Parser::new().with_extension_registry(registry.clone()); @@ -634,7 +634,7 @@ fn test_enhancement_on_project_roundtrip() { let registry = make_registry(); let plan_text = r#"=== Plan Root[col] - Project[$0] + Project[_] + Enh:PartitionHint[&HASH] Read[my.table => col:i64]"#; let parser = Parser::new().with_extension_registry(registry.clone()); diff --git a/tests/literal_roundtrip.rs b/tests/literal_roundtrip.rs index 9b398aba..7e96d02e 100644 --- a/tests/literal_roundtrip.rs +++ b/tests/literal_roundtrip.rs @@ -10,7 +10,7 @@ fn test_float_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[3.14, -2.5, 1] + Project[_ => 3.14, -2.5, 1] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -21,7 +21,7 @@ fn test_boolean_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[true, false] + Project[_ => true, false] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -32,7 +32,7 @@ fn test_mixed_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[42, 3.14, true, 'hello'] + Project[_ => 42, 3.14, true, 'hello'] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -43,7 +43,7 @@ fn test_negative_literals_roundtrip() { let plan = r#" === Plan Root[result] - Project[-42, -3.14, false] + Project[_ => -42, -3.14, false] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -54,7 +54,7 @@ fn test_date_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project['2023-12-25':date] + Project[_ => '2023-12-25':date] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -65,7 +65,7 @@ fn test_time_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project['14:30:45.000000':time] + Project[_ => '14:30:45.000000':time] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -76,7 +76,7 @@ fn test_timestamp_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project['2023-01-01T12:00:00.000000':timestamp] + Project[_ => '2023-01-01T12:00:00.000000':timestamp] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -87,7 +87,7 @@ fn test_nullable_integer_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[78:i32?, 42:i64?] + Project[_ => 78:i32?, 42:i64?] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -98,7 +98,7 @@ fn test_nullable_boolean_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[true:boolean?, false:boolean?] + Project[_ => true:boolean?, false:boolean?] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -109,7 +109,7 @@ fn test_nullable_float_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[3.14:fp64?, 2.5:fp32?] + Project[_ => 3.14:fp64?, 2.5:fp32?] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -120,7 +120,7 @@ fn test_null_literal_roundtrip() { let plan = r#" === Plan Root[result] - Project[null:i64?, null:string?, null:date?] + Project[_ => null:i64?, null:string?, null:date?] Read[data => a:i64] "#; roundtrip_plan(plan); @@ -131,7 +131,7 @@ fn test_bare_null_literal_requires_type() { let plan = r#" === Plan Root[result] - Project[null] + Project[_ => null] Read[data => a:i64] "#; @@ -150,7 +150,7 @@ fn test_if_then_expression_roundtrip() { === Plan Root[statusq] Fetch[limit=10, offset=0 => _] - Project[if_then(true -> $0, false -> $1, _ -> $2)] + Project[_ => if_then(true -> $0, false -> $1, _ -> $2)] Read[events.logs => status:string?] "#; roundtrip_plan(plan); diff --git a/tests/verbose_roundtrip.rs b/tests/verbose_roundtrip.rs index d0f2746a..44435071 100644 --- a/tests/verbose_roundtrip.rs +++ b/tests/verbose_roundtrip.rs @@ -27,6 +27,34 @@ Root[_] assert_roundtrip_verbose(compact, verbose); } +#[test] +fn test_project_reordered_output_styles() { + let compact = r#"=== Plan +Root[added, original] + Project[_ => 2:i32, $0] + Read[table => original:i32]"#; + let verbose = r#"=== Plan +Root[added, original] + Project[_ +> 2:i32 |> $1, $0] + Read[table => original:i32]"#; + + assert_roundtrip_verbose(compact, verbose); +} + +#[test] +fn test_project_mapping_without_additions_output_styles() { + let compact = r#"=== Plan +Root[b, a] + Project[_ => $1, $0] + Read[table => a:i32, b:string]"#; + let verbose = r#"=== Plan +Root[b, a] + Project[_ |> $1, $0] + Read[table => a:i32, b:string]"#; + + assert_roundtrip_verbose(compact, verbose); +} + #[test] fn test_sort_partial_reordered_output_styles() { let compact = r#"=== Plan From 9e827e032fb21fc783b3367ffd21e52b0178b90f Mon Sep 17 00:00:00 2001 From: Wendell Smith Date: Tue, 28 Jul 2026 16:47:25 -0400 Subject: [PATCH 2/2] test: migrate Project fixtures to explicit output syntax --- GRAMMAR.md | 20 ++++++------ tests/plan_roundtrip.rs | 68 ++++++++++++++++++++--------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/GRAMMAR.md b/GRAMMAR.md index b0f10c90..fef5baa0 100644 --- a/GRAMMAR.md +++ b/GRAMMAR.md @@ -81,7 +81,7 @@ Functions: === Plan Root[result] - Project[$0, $1, add($0, $1):i64] + Project[_ => $0, $1, add($0, $1):i64] Read[orders => quantity:i32?, price:i64] # "#; # @@ -178,7 +178,7 @@ Functions: === Plan Root[result] // Level 0 (no indentation) - Project[$0, $1] // Level 1 (2 spaces) + Project[_ => $0, $1] // Level 1 (2 spaces) Filter[gt($0, 10):boolean => $0] // Level 2 (4 spaces) Read[data => a:i64] // Level 3 (6 spaces) # "#; @@ -298,7 +298,7 @@ From [official Substrait grammar](https://raw.githubusercontent.com/substrait-io let plan_text = r#" === Plan Root[result] - Project[$0, $1, $2, $3] + Project[_ => $0, $1, $2, $3] Read[data => int_field:i64, string_field:string?, created_at:timestamp?, user_id:uuid] "#; # @@ -360,7 +360,7 @@ use substrait_explain::Parser; let plan_text = r#" === Plan Root[result] - Project[$0, $1, $2] + Project[_ => $0, $1, $2] Read[data => list_field:list, map_field:map, struct_field:struct] "#; @@ -400,7 +400,7 @@ Functions: === Plan Root[result] - Project[$0, $1, $2] + Project[_ => $0, $1, $2] Read[data => point_field:point#8@1?, custom_field:custom_type#9, prefixed_field:u!custom_type] # "#; # @@ -437,7 +437,7 @@ Currently, only references to fields in the Relations' input are supported. # let plan_text = r#" === Plan Root[result] - Project[$0, $1, $42] + Project[_ => $0, $1, $42] Read[data => field0:i64, field1:string, field42:boolean] # "#; # @@ -523,7 +523,7 @@ An IfThen expression is a conditional function or logical operator that evaluate === Plan Root[status] Fetch[limit=10, offset=0 => $0] - Project[if_then(true -> $0, false -> $1, _ -> $2)] + Project[_ => if_then(true -> $0, false -> $1, _ -> $2)] Read[events.logs => status:string?] # "#; # @@ -689,7 +689,7 @@ form a 1-element tuple: `(x,)`. For 2+ elements the trailing comma is optional: # let plan_text = r#" === Plan Root[c, d] // root with output columns c and d - Project[$0, $1] + Project[_ => $0, $1] Read[data => a:i64, b:string] # "#; # @@ -984,7 +984,7 @@ Functions: === Plan Root[result] Filter[gt($2, 100):boolean => $0, $1, $2] - Project[$0, $1, $2] + Project[_ => $0, $1, $2] Read[data => a:i64, b:string, c:i32] # "#; # @@ -1447,7 +1447,7 @@ Functions: Root[customer_revenue] Aggregate[$0, $1 => $0, $1, sum($3):i64] Filter[gt($3, 100):boolean => $0, $1, $2, $3] - Project[$0, $1, $2, multiply($4, $5):i64] + Project[_ => $0, $1, $2, multiply($4, $5):i64] Join[&Inner, eq($0, $3):boolean => $0, $1, $2, $3, $4, $5] Read[users => id:i64, name:string, region:string] Read[orders => user_id:i64, quantity:i32, price:i64] diff --git a/tests/plan_roundtrip.rs b/tests/plan_roundtrip.rs index d77fd203..fcf47028 100644 --- a/tests/plan_roundtrip.rs +++ b/tests/plan_roundtrip.rs @@ -15,7 +15,7 @@ fn test_simple_plan_roundtrip() { let plan = r#"=== Plan Root[c, d] Filter[$0] - Project[$1, 42] + Project[_ => $1, 42] Read[my.table => a:i32, b:string?, c:boolean]"#; roundtrip_plan(plan); @@ -77,7 +77,7 @@ Functions: === Plan Root[result] - Project[$0, $1, add($0, $1):i32?] + Project[_ => $0, $1, add($0, $1):i32?] Read[table1 => col1:i32?, col2:i32?]"#; roundtrip_plan(plan); @@ -96,7 +96,7 @@ Functions: === Plan Root[name, parent, sum, count] - Project[$0, $3, $1, $2] + Project[_ => $0, $3, $1, $2] Read[schema.table => name:string?, parent:string?, sum:fp64?, count:fp64?]"#; roundtrip_plan(plan); @@ -112,7 +112,7 @@ Functions: === Plan Root[name, num] - Project[$1, coalesce($1, $2):fp64?] + Project[_ => $1, coalesce($1, $2):fp64?] Read[schema.table => name:string?, num:fp64?, other_num:fp64?, id:i64]"#; let verbose_plan = r#"=== Extensions @@ -123,7 +123,7 @@ Functions: === Plan Root[name, num] - Project[$1, coalesce#10($1, $2):fp64?] + Project[_ => $1, coalesce#10($1, $2):fp64?] Read[schema.table => name:string?, num:fp64?, other_num:fp64?, id:i64]"#; assert_roundtrip_canonical(simple_plan, verbose_plan); @@ -133,7 +133,7 @@ Root[name, num] fn test_multiple_relations_roundtrip() { let plan = r#"=== Plan Root[name, num, id] - Project[$0, $1, $2] + Project[_] Read[t1 => name:string?, num:fp64, id:i64] Root[name, num, id] @@ -147,7 +147,7 @@ Root[name, num, id] fn test_plan_with_fetch_and_sort_roundtrip() { let plan = r#"=== Plan Root[name, num, id] - Project[$0, $1, $2] + Project[_] Read[schema.table => name:string?, num:fp64?, id:i64]"#; roundtrip_plan(plan); @@ -157,7 +157,7 @@ Root[name, num, id] fn test_plan_with_extension_leaf_roundtrip() { let plan = r#"=== Plan Root[name, some_value] - Project[$0, $1] + Project[_] Read[schema.table => name:string?, some_value:fp64?]"#; roundtrip_plan(plan); @@ -276,7 +276,7 @@ Functions: === Plan Root[n] - Project[count():i64] + Project[_ => count():i64] Read[events => n:i64]"#; roundtrip_plan(plan); } @@ -293,7 +293,7 @@ Functions: === Plan Root[result] - Project[$0, $1, add($0, $1):i64] + Project[_ => $0, $1, add($0, $1):i64] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan); } @@ -311,7 +311,7 @@ Functions: === Plan Root[result] - Project[add($0, $1)] + Project[_ => add($0, $1)] Read[t => a:i64, b:i64]"#; assert!(Parser::parse(plan).is_err()); @@ -331,7 +331,7 @@ Functions: === Plan Root[result] - Project[$0, $1, add#1($0, $1):i64, add:($0, $1):i64] + Project[_ => $0, $1, add#1($0, $1):i64, add:($0, $1):i64] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan); } @@ -352,7 +352,7 @@ Functions: === Plan Root[result] - Project[$0, $1, add#1($0, $1):i64, add:#2($0, $1):i64, add:#3($0, $1):i64] + Project[_ => $0, $1, add#1($0, $1):i64, add:#2($0, $1):i64, add:#3($0, $1):i64] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan); } @@ -370,7 +370,7 @@ Types: === Plan Root[result] - Project[$0] + Project[_] Read[data => doc:json]"#; // Plan authored with u! prefix on the type reference round-trips to the canonical form. @@ -382,7 +382,7 @@ Types: === Plan Root[result] - Project[$0] + Project[_ => $0] Read[data => doc:u!json]"#; assert_roundtrip_canonical(canonical, with_prefix); @@ -400,7 +400,7 @@ Functions: === Plan Root[result] - Project[json_extract_path($0, $1):string] + Project[_ => json_extract_path($0, $1):string] Read[data => doc:string, path:string]"#; // Explicit compound name in the call site resolves to the same canonical output. @@ -412,7 +412,7 @@ Functions: === Plan Root[result] - Project[json_extract_path:u!json_str($0, $1):string] + Project[_ => json_extract_path:u!json_str($0, $1):string] Read[data => doc:string, path:string]"#; roundtrip_plan(canonical); @@ -431,7 +431,7 @@ Types: === Plan Root[result] - Project[($0)::json] + Project[_ => ($0)::json] Read[data => doc:string]"#; let with_prefix = r#"=== Extensions @@ -442,7 +442,7 @@ Types: === Plan Root[result] - Project[($0)::u!json] + Project[_ => ($0)::u!json] Read[data => doc:string]"#; roundtrip_plan(canonical); @@ -761,7 +761,7 @@ Functions: === Plan Root[result] - Project[$0, equal:any_any($0, $1):boolean, equal:str_str($0, $1):boolean] + Project[_ => $0, equal:any_any($0, $1):boolean, equal:str_str($0, $1):boolean] Read[t => a:i64, b:i64, c:string]"#; // Verbose output always shows signatures and anchors for all functions, @@ -775,7 +775,7 @@ Functions: === Plan Root[result] - Project[$0, equal:any_any#1($0, $1):boolean, equal:str_str#2($0, $1):boolean] + Project[_ => $0, equal:any_any#1($0, $1):boolean, equal:str_str#2($0, $1):boolean] Read[t => a:i64, b:i64, c:string]"#; roundtrip_plan(simple); @@ -797,7 +797,7 @@ Functions: === Plan Root[result] - Project[$0, add($0, $1):i64] + Project[_ => $0, add($0, $1):i64] Read[t => a:i64, b:i64]"#; // Explicit compound name in input resolves to the same canonical output @@ -809,7 +809,7 @@ Functions: === Plan Root[result] - Project[$0, add:i64_i64($0, $1):i64] + Project[_ => $0, add:i64_i64($0, $1):i64] Read[t => a:i64, b:i64]"#; // Verbose output always shows the full compound name and anchor @@ -821,7 +821,7 @@ Functions: === Plan Root[result] - Project[$0, add:i64_i64#1($0, $1):i64] + Project[_ => $0, add:i64_i64#1($0, $1):i64] Read[t => a:i64, b:i64]"#; assert_roundtrip_canonical(compact, compound); @@ -843,7 +843,7 @@ Functions: === Plan Root[result] - Project[$0, equal:any_any($0, $1):boolean, equal:str_str($0, $2):boolean, like($0, $2):boolean] + Project[_ => $0, equal:any_any($0, $1):boolean, equal:str_str($0, $2):boolean, like($0, $2):boolean] Read[t => id:i64, score:i64, name:string]"#; roundtrip_plan(simple); @@ -863,7 +863,7 @@ Functions: === Plan Root[result] - Project[$0, equal:any_any#1($0, $1):boolean, equal:any_any#2($0, $1):boolean] + Project[_ => $0, equal:any_any#1($0, $1):boolean, equal:any_any#2($0, $1):boolean] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan); @@ -1025,7 +1025,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan); @@ -1045,7 +1045,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Filter[gt($2, 0):boolean => $0, $1] Read[t => a:i64, b:i64, c:i64]"#; @@ -1064,7 +1064,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Sort[($0, &AscNullsFirst)] Read[t => a:i64, b:i64]"#; @@ -1083,7 +1083,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Fetch[limit=10, offset=0] Read[t => a:i64, b:i64]"#; @@ -1104,7 +1104,7 @@ Functions: === Plan Root[sum] - Project[add($1, $3):i64] + Project[_ => add($1, $3):i64] Join[&Inner, eq($0, $2):boolean => $0, $1, $2, $3] Read[users => id:i64, age:i64] Read[orders => user_id:i64, amount:i64]"#; @@ -1126,7 +1126,7 @@ Functions: === Plan Root[total] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Aggregate[$0 => $0, sum($1):i64] Read[t => category:i64, amount:i64]"#; @@ -1144,7 +1144,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Read:Virtual[(1, 2), (3, 4) => a:i64, b:i64]"#; roundtrip_plan(plan); @@ -1183,7 +1183,7 @@ Functions: === Plan Root[sum] - Project[add($0, $1):i64] + Project[_ => add($0, $1):i64] Read[t => a:i64, b:i64]"#; roundtrip_plan(plan);