From 1a1cd7c684d16fe0d5e389f04db9508111f97e56 Mon Sep 17 00:00:00 2001 From: Wendell Smith Date: Tue, 2 Jun 2026 15:41:08 -0400 Subject: [PATCH] refactor: simplify literal parsing and textification --- src/parser/expressions.rs | 324 +------------------------------- src/parser/literals.rs | 373 +++++++++++++++++++++++++++++++++++++ src/parser/mod.rs | 1 + src/textify/expressions.rs | 231 +---------------------- src/textify/literals.rs | 250 +++++++++++++++++++++++++ src/textify/mod.rs | 1 + 6 files changed, 629 insertions(+), 551 deletions(-) create mode 100644 src/parser/literals.rs create mode 100644 src/textify/literals.rs diff --git a/src/parser/expressions.rs b/src/parser/expressions.rs index ff3b153b..2f5ac842 100644 --- a/src/parser/expressions.rs +++ b/src/parser/expressions.rs @@ -1,14 +1,11 @@ -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; use substrait::proto::aggregate_rel::Measure; use substrait::proto::expression::field_reference::{ReferenceType, RootReference, RootType}; use substrait::proto::expression::if_then::IfClause; -use substrait::proto::expression::literal::LiteralType; use substrait::proto::expression::{ Cast, FieldReference, IfThen, Literal, ReferenceSegment, RexType, ScalarFunction, cast, reference_segment, }; use substrait::proto::function_argument::ArgType; -use substrait::proto::r#type::{Fp64, I64, Kind, Nullability}; use substrait::proto::{AggregateFunction, Expression, FunctionArgument, Type}; use super::types::get_and_validate_anchor; @@ -76,326 +73,6 @@ impl ParsePair for FieldReference { } } -fn to_int_literal( - value: pest::iterators::Pair, - typ: Option, -) -> Result { - assert_eq!(value.as_rule(), Rule::integer); - let parsed_value: i64 = value.as_str().parse().unwrap(); - - const DEFAULT_KIND: Kind = Kind::I64(I64 { - type_variation_reference: 0, - nullability: Nullability::Required as i32, - }); - - // If no type is provided, we assume i64, Nullability::Required. - let kind = typ.and_then(|t| t.kind).unwrap_or(DEFAULT_KIND); - - let (lit, nullability, tvar) = match &kind { - // If no type is provided, we assume i64, Nullability::Required. - Kind::I8(i) => ( - LiteralType::I8(parsed_value as i32), - i.nullability, - i.type_variation_reference, - ), - Kind::I16(i) => ( - LiteralType::I16(parsed_value as i32), - i.nullability, - i.type_variation_reference, - ), - Kind::I32(i) => ( - LiteralType::I32(parsed_value as i32), - i.nullability, - i.type_variation_reference, - ), - Kind::I64(i) => ( - LiteralType::I64(parsed_value), - i.nullability, - i.type_variation_reference, - ), - k => { - return Err(MessageParseError::invalid( - "int_literal_type", - value.as_span(), - format!("Invalid type for integer literal: {k:?}"), - )); - } - }; - - Ok(Literal { - literal_type: Some(lit), - nullable: nullability != Nullability::Required as i32, - type_variation_reference: tvar, - }) -} - -fn to_float_literal( - value: pest::iterators::Pair, - typ: Option, -) -> Result { - assert_eq!(value.as_rule(), Rule::float); - let parsed_value: f64 = value.as_str().parse().unwrap(); - - const DEFAULT_KIND: Kind = Kind::Fp64(Fp64 { - type_variation_reference: 0, - nullability: Nullability::Required as i32, - }); - - // If no type is provided, we assume fp64, Nullability::Required. - let kind = typ.and_then(|t| t.kind).unwrap_or(DEFAULT_KIND); - - let (lit, nullability, tvar) = match &kind { - Kind::Fp32(f) => ( - LiteralType::Fp32(parsed_value as f32), - f.nullability, - f.type_variation_reference, - ), - Kind::Fp64(f) => ( - LiteralType::Fp64(parsed_value), - f.nullability, - f.type_variation_reference, - ), - k => { - return Err(MessageParseError::invalid( - "float_literal_type", - value.as_span(), - format!("Invalid type for float literal: {k:?}"), - )); - } - }; - - Ok(Literal { - literal_type: Some(lit), - nullable: nullability != Nullability::Required as i32, - type_variation_reference: tvar, - }) -} - -fn to_boolean_literal( - value: pest::iterators::Pair, - typ: Option, -) -> Result { - assert_eq!(value.as_rule(), Rule::boolean); - let parsed_value: bool = value.as_str().parse().unwrap(); - - let (nullable, tvar) = match typ.and_then(|t| t.kind) { - Some(Kind::Bool(b)) => ( - b.nullability != Nullability::Required as i32, - b.type_variation_reference, - ), - None => (false, 0), - Some(k) => { - return Err(MessageParseError::invalid( - "bool_literal_type", - value.as_span(), - format!("Invalid type for boolean literal: {k:?}"), - )); - } - }; - - Ok(Literal { - literal_type: Some(LiteralType::Boolean(parsed_value)), - nullable, - type_variation_reference: tvar, - }) -} - -fn to_string_literal( - value: pest::iterators::Pair, - typ: Option, -) -> Result { - assert_eq!(value.as_rule(), Rule::string_literal); - let string_value = unescape_string(value.clone()); - - // If no type is provided, default to string - let Some(typ) = typ else { - return Ok(Literal { - literal_type: Some(LiteralType::String(string_value)), - nullable: false, - type_variation_reference: 0, - }); - }; - - let Some(kind) = typ.kind else { - return Ok(Literal { - literal_type: Some(LiteralType::String(string_value)), - nullable: false, - type_variation_reference: 0, - }); - }; - - match &kind { - Kind::Date(d) => { - // Parse date in ISO 8601 format: YYYY-MM-DD - let date_days = parse_date_to_days(&string_value, value.as_span())?; - Ok(Literal { - literal_type: Some(LiteralType::Date(date_days)), - nullable: d.nullability != Nullability::Required as i32, - type_variation_reference: d.type_variation_reference, - }) - } - #[allow(deprecated)] - Kind::Time(t) => { - // Parse time in ISO 8601 format: HH:MM:SS[.fff] - let time_microseconds = parse_time_to_microseconds(&string_value, value.as_span())?; - Ok(Literal { - literal_type: Some(LiteralType::Time(time_microseconds)), - nullable: t.nullability != Nullability::Required as i32, - type_variation_reference: t.type_variation_reference, - }) - } - #[allow(deprecated)] - Kind::Timestamp(ts) => { - // Parse timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SS[.fff] or YYYY-MM-DD HH:MM:SS[.fff] - let timestamp_microseconds = - parse_timestamp_to_microseconds(&string_value, value.as_span())?; - Ok(Literal { - literal_type: Some(LiteralType::Timestamp(timestamp_microseconds)), - nullable: ts.nullability != Nullability::Required as i32, - type_variation_reference: ts.type_variation_reference, - }) - } - _ => { - // For other types, treat as string - Ok(Literal { - literal_type: Some(LiteralType::String(string_value)), - nullable: false, - type_variation_reference: 0, - }) - } - } -} - -fn to_null_literal( - value: pest::iterators::Pair, - typ: Option, -) -> Result { - assert_eq!(value.as_rule(), Rule::null); - let typ = typ.ok_or_else(|| { - MessageParseError::invalid( - "null_literal_type", - value.as_span(), - "Null literals require an explicit type annotation, e.g. null:i64?", - ) - })?; - - Ok(Literal { - literal_type: Some(LiteralType::Null(typ)), - nullable: false, - type_variation_reference: 0, - }) -} - -/// Parse a date string using chrono to days since Unix epoch -fn parse_date_to_days(date_str: &str, span: pest::Span) -> Result { - // Try multiple date formats for flexibility - let formats = ["%Y-%m-%d", "%Y/%m/%d"]; - - for format in &formats { - if let Ok(date) = NaiveDate::parse_from_str(date_str, format) { - // Calculate days since Unix epoch (1970-01-01) - let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); - let days = date.signed_duration_since(epoch).num_days(); - return Ok(days as i32); - } - } - - Err(MessageParseError::invalid( - "date_parse_format", - span, - format!("Invalid date format: '{date_str}'. Expected YYYY-MM-DD or YYYY/MM/DD"), - )) -} - -/// Parse a time string using chrono to microseconds since midnight -fn parse_time_to_microseconds(time_str: &str, span: pest::Span) -> Result { - // Try multiple time formats for flexibility - let formats = ["%H:%M:%S%.f", "%H:%M:%S"]; - - for format in &formats { - if let Ok(time) = NaiveTime::parse_from_str(time_str, format) { - // Convert to microseconds since midnight - let midnight = NaiveTime::from_hms_opt(0, 0, 0).unwrap(); - let duration = time.signed_duration_since(midnight); - return Ok(duration.num_microseconds().unwrap_or(0)); - } - } - - Err(MessageParseError::invalid( - "time_parse_format", - span, - format!("Invalid time format: '{time_str}'. Expected HH:MM:SS or HH:MM:SS.fff"), - )) -} - -/// Parse a timestamp string using chrono to microseconds since Unix epoch -fn parse_timestamp_to_microseconds( - timestamp_str: &str, - span: pest::Span, -) -> Result { - // Try multiple timestamp formats for flexibility - let formats = [ - "%Y-%m-%dT%H:%M:%S%.f", // ISO 8601 with T and fractional seconds - "%Y-%m-%dT%H:%M:%S", // ISO 8601 with T - "%Y-%m-%d %H:%M:%S%.f", // Space separator with fractional seconds - "%Y-%m-%d %H:%M:%S", // Space separator - "%Y/%m/%dT%H:%M:%S%.f", // Alternative date format with T - "%Y/%m/%dT%H:%M:%S", // Alternative date format with T - "%Y/%m/%d %H:%M:%S%.f", // Alternative date format with space - "%Y/%m/%d %H:%M:%S", // Alternative date format with space - ]; - - for format in &formats { - if let Ok(datetime) = NaiveDateTime::parse_from_str(timestamp_str, format) { - // Calculate microseconds since Unix epoch (1970-01-01 00:00:00) - let epoch = DateTime::from_timestamp(0, 0).unwrap().naive_utc(); - let duration = datetime.signed_duration_since(epoch); - return Ok(duration.num_microseconds().unwrap_or(0)); - } - } - - Err(MessageParseError::invalid( - "timestamp_parse_format", - span, - format!( - "Invalid timestamp format: '{timestamp_str}'. Expected YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD HH:MM:SS" - ), - )) -} - -impl ScopedParsePair for Literal { - fn rule() -> Rule { - Rule::literal - } - - fn message() -> &'static str { - "Literal" - } - - fn parse_pair( - extensions: &SimpleExtensions, - pair: pest::iterators::Pair, - ) -> Result { - assert_eq!(pair.as_rule(), Self::rule()); - let mut pairs = pair.into_inner(); - let value = pairs.next().unwrap(); // First item is always the value - let typ = pairs.next(); // Second item is optional type - assert!(pairs.next().is_none()); - let typ = match typ { - Some(t) => Some(Type::parse_pair(extensions, t)?), - None => None, - }; - match value.as_rule() { - Rule::integer => to_int_literal(value, typ), - Rule::float => to_float_literal(value, typ), - Rule::boolean => to_boolean_literal(value, typ), - Rule::string_literal => to_string_literal(value, typ), - Rule::null => to_null_literal(value, typ), - _ => unreachable!("Literal unexpected rule: {:?}", value.as_rule()), - } - } -} - impl ScopedParsePair for ScalarFunction { fn rule() -> Rule { Rule::function_call @@ -691,6 +368,7 @@ impl ScopedParsePair for Measure { #[cfg(test)] mod tests { use pest::Parser as PestParser; + use substrait::proto::expression::literal::LiteralType; use super::*; use crate::parser::ExpressionParser; diff --git a/src/parser/literals.rs b/src/parser/literals.rs new file mode 100644 index 00000000..625439a6 --- /dev/null +++ b/src/parser/literals.rs @@ -0,0 +1,373 @@ +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; +use substrait::proto::Type; +use substrait::proto::expression::Literal; +use substrait::proto::expression::literal::LiteralType; +use substrait::proto::r#type::{Kind, Nullability}; + +use super::{MessageParseError, Rule, ScopedParsePair, unescape_string}; +use crate::extensions::SimpleExtensions; + +#[derive(Debug)] +enum LiteralSyntax<'i> { + Integer { + source: &'i str, + span: pest::Span<'i>, + }, + Float { + source: &'i str, + span: pest::Span<'i>, + }, + Boolean { + value: bool, + span: pest::Span<'i>, + }, + String { + value: String, + span: pest::Span<'i>, + }, + Null { + span: pest::Span<'i>, + }, +} + +impl<'i> LiteralSyntax<'i> { + fn from_pair(pair: pest::iterators::Pair<'i, Rule>) -> Self { + match pair.as_rule() { + Rule::integer => Self::Integer { + source: pair.as_str(), + span: pair.as_span(), + }, + Rule::float => Self::Float { + source: pair.as_str(), + span: pair.as_span(), + }, + Rule::boolean => Self::Boolean { + value: pair.as_str().parse().unwrap(), + span: pair.as_span(), + }, + Rule::string_literal => Self::String { + value: unescape_string(pair.clone()), + span: pair.as_span(), + }, + Rule::null => Self::Null { + span: pair.as_span(), + }, + _ => unreachable!("Literal unexpected rule: {:?}", pair.as_rule()), + } + } + + fn default_target(&self) -> LiteralTarget { + match self { + Self::Integer { .. } => LiteralTarget::I64(LiteralAttrs::required()), + Self::Float { .. } => LiteralTarget::Fp64(LiteralAttrs::required()), + Self::Boolean { .. } => LiteralTarget::Boolean(LiteralAttrs::required()), + Self::String { .. } => LiteralTarget::String(LiteralAttrs::required()), + Self::Null { span } => { + unreachable!( + "null literal without type should be rejected before this point: {span:?}" + ) + } + } + } +} + +#[derive(Debug, Clone, Copy)] +struct LiteralAttrs { + nullable: bool, + type_variation_reference: u32, +} + +impl LiteralAttrs { + fn required() -> Self { + Self { + nullable: false, + type_variation_reference: 0, + } + } + + fn new(nullability: i32, type_variation_reference: u32) -> Self { + Self { + nullable: nullability != Nullability::Required as i32, + type_variation_reference, + } + } +} + +#[derive(Debug)] +enum LiteralTarget { + Boolean(LiteralAttrs), + I8(LiteralAttrs), + I16(LiteralAttrs), + I32(LiteralAttrs), + I64(LiteralAttrs), + Fp32(LiteralAttrs), + Fp64(LiteralAttrs), + String(LiteralAttrs), + Date(LiteralAttrs), + TimeMicros(LiteralAttrs), + TimestampMicros(LiteralAttrs), + Null(Type), + UnsupportedStringFallback, +} + +impl LiteralTarget { + fn from_type<'i>( + typ: Option, + syntax: &LiteralSyntax<'i>, + ) -> Result { + if let LiteralSyntax::Null { span } = syntax { + let Some(typ) = typ else { + return Err(MessageParseError::invalid( + "null_literal_type", + *span, + "Null literals require an explicit type annotation, e.g. null:i64?", + )); + }; + return Ok(Self::Null(typ)); + } + let Some(kind) = typ.and_then(|t| t.kind) else { + return Ok(syntax.default_target()); + }; + Ok(Self::from_kind(kind)) + } + + fn from_kind(kind: Kind) -> Self { + match kind { + Kind::Bool(k) => { + Self::Boolean(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + Kind::I8(k) => Self::I8(LiteralAttrs::new(k.nullability, k.type_variation_reference)), + Kind::I16(k) => Self::I16(LiteralAttrs::new(k.nullability, k.type_variation_reference)), + Kind::I32(k) => Self::I32(LiteralAttrs::new(k.nullability, k.type_variation_reference)), + Kind::I64(k) => Self::I64(LiteralAttrs::new(k.nullability, k.type_variation_reference)), + Kind::Fp32(k) => { + Self::Fp32(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + Kind::Fp64(k) => { + Self::Fp64(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + Kind::String(k) => { + Self::String(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + Kind::Date(k) => { + Self::Date(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + #[allow(deprecated)] + Kind::Time(k) => { + Self::TimeMicros(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + #[allow(deprecated)] + Kind::Timestamp(k) => { + Self::TimestampMicros(LiteralAttrs::new(k.nullability, k.type_variation_reference)) + } + other => { + if matches!( + other, + Kind::Bool(_) + | Kind::I8(_) + | Kind::I16(_) + | Kind::I32(_) + | Kind::I64(_) + | Kind::Fp32(_) + | Kind::Fp64(_) + ) { + unreachable!("handled scalar literal target") + } + Self::UnsupportedStringFallback + } + } + } + + fn parse<'i>(self, syntax: LiteralSyntax<'i>) -> Result { + match (syntax, self) { + (LiteralSyntax::Integer { source, span }, target) => target.parse_integer(source, span), + (LiteralSyntax::Float { source, span }, target) => target.parse_float(source, span), + (LiteralSyntax::Boolean { value, span }, target) => target.parse_boolean(value, span), + (LiteralSyntax::String { value, span }, target) => target.parse_string(value, span), + (LiteralSyntax::Null { span }, target) => target.parse_null(span), + } + } + + fn parse_integer(self, source: &str, span: pest::Span) -> Result { + let value: i64 = source.parse().unwrap(); + match self { + Self::I8(attrs) => Ok(literal(LiteralType::I8(value as i32), attrs)), + Self::I16(attrs) => Ok(literal(LiteralType::I16(value as i32), attrs)), + Self::I32(attrs) => Ok(literal(LiteralType::I32(value as i32), attrs)), + Self::I64(attrs) => Ok(literal(LiteralType::I64(value), attrs)), + other => Err(invalid_literal( + span, + format!("Invalid type for integer literal: {other:?}"), + )), + } + } + + fn parse_float(self, source: &str, span: pest::Span) -> Result { + let value: f64 = source.parse().unwrap(); + match self { + Self::Fp32(attrs) => Ok(literal(LiteralType::Fp32(value as f32), attrs)), + Self::Fp64(attrs) => Ok(literal(LiteralType::Fp64(value), attrs)), + other => Err(invalid_literal( + span, + format!("Invalid type for float literal: {other:?}"), + )), + } + } + + fn parse_boolean(self, value: bool, span: pest::Span) -> Result { + match self { + Self::Boolean(attrs) => Ok(literal(LiteralType::Boolean(value), attrs)), + other => Err(invalid_literal( + span, + format!("Invalid type for boolean literal: {other:?}"), + )), + } + } + + #[allow(deprecated)] + fn parse_string(self, value: String, span: pest::Span) -> Result { + match self { + Self::String(attrs) => Ok(literal(LiteralType::String(value), attrs)), + Self::Date(attrs) => Ok(literal( + LiteralType::Date(parse_date_to_days(&value, span)?), + attrs, + )), + Self::TimeMicros(attrs) => Ok(literal( + LiteralType::Time(parse_time_to_microseconds(&value, span)?), + attrs, + )), + Self::TimestampMicros(attrs) => Ok(literal( + LiteralType::Timestamp(parse_timestamp_to_microseconds(&value, span)?), + attrs, + )), + Self::UnsupportedStringFallback => Ok(literal( + LiteralType::String(value), + LiteralAttrs::required(), + )), + other => Err(invalid_literal( + span, + format!("Invalid type for string literal: {other:?}"), + )), + } + } + + fn parse_null(self, span: pest::Span) -> Result { + match self { + Self::Null(typ) => Ok(Literal { + literal_type: Some(LiteralType::Null(typ)), + nullable: false, + type_variation_reference: 0, + }), + other => Err(invalid_literal( + span, + format!("Invalid type for null literal: {other:?}"), + )), + } + } +} + +fn literal(literal_type: LiteralType, attrs: LiteralAttrs) -> Literal { + Literal { + literal_type: Some(literal_type), + nullable: attrs.nullable, + type_variation_reference: attrs.type_variation_reference, + } +} + +fn invalid_literal(span: pest::Span, message: impl ToString) -> MessageParseError { + MessageParseError::invalid("literal", span, message) +} + +fn parse_date_to_days(date_str: &str, span: pest::Span) -> Result { + let formats = ["%Y-%m-%d", "%Y/%m/%d"]; + + for format in &formats { + if let Ok(date) = NaiveDate::parse_from_str(date_str, format) { + let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); + let days = date.signed_duration_since(epoch).num_days(); + return Ok(days as i32); + } + } + + Err(invalid_literal( + span, + format!("Invalid date format: '{date_str}'. Expected YYYY-MM-DD or YYYY/MM/DD"), + )) +} + +fn parse_time_to_microseconds(time_str: &str, span: pest::Span) -> Result { + let formats = ["%H:%M:%S%.f", "%H:%M:%S"]; + + for format in &formats { + if let Ok(time) = NaiveTime::parse_from_str(time_str, format) { + let midnight = NaiveTime::from_hms_opt(0, 0, 0).unwrap(); + let duration = time.signed_duration_since(midnight); + return Ok(duration.num_microseconds().unwrap_or(0)); + } + } + + Err(invalid_literal( + span, + format!("Invalid time format: '{time_str}'. Expected HH:MM:SS or HH:MM:SS.fff"), + )) +} + +fn parse_timestamp_to_microseconds( + timestamp_str: &str, + span: pest::Span, +) -> Result { + let formats = [ + "%Y-%m-%dT%H:%M:%S%.f", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S%.f", + "%Y-%m-%d %H:%M:%S", + "%Y/%m/%dT%H:%M:%S%.f", + "%Y/%m/%dT%H:%M:%S", + "%Y/%m/%d %H:%M:%S%.f", + "%Y/%m/%d %H:%M:%S", + ]; + + for format in &formats { + if let Ok(datetime) = NaiveDateTime::parse_from_str(timestamp_str, format) { + let epoch = DateTime::from_timestamp(0, 0).unwrap().naive_utc(); + let duration = datetime.signed_duration_since(epoch); + return Ok(duration.num_microseconds().unwrap_or(0)); + } + } + + Err(invalid_literal( + span, + format!( + "Invalid timestamp format: '{timestamp_str}'. Expected YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD HH:MM:SS" + ), + )) +} + +impl ScopedParsePair for Literal { + fn rule() -> Rule { + Rule::literal + } + + fn message() -> &'static str { + "Literal" + } + + fn parse_pair( + extensions: &SimpleExtensions, + pair: pest::iterators::Pair, + ) -> Result { + assert_eq!(pair.as_rule(), Self::rule()); + let mut pairs = pair.into_inner(); + let value = pairs.next().unwrap(); + let typ = pairs.next(); + assert!(pairs.next().is_none()); + + let syntax = LiteralSyntax::from_pair(value); + let typ = match typ { + Some(t) => Some(Type::parse_pair(extensions, t)?), + None => None, + }; + let target = LiteralTarget::from_type(typ, &syntax)?; + target.parse(syntax) + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b5dbe9c8..fa4b2b66 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod common; pub(crate) mod errors; pub(crate) mod expressions; pub(crate) mod extensions; +pub(crate) mod literals; pub(crate) mod relations; pub(crate) mod structural; pub(crate) mod types; diff --git a/src/textify/expressions.rs b/src/textify/expressions.rs index 7d9aaa67..f9213974 100644 --- a/src/textify/expressions.rs +++ b/src/textify/expressions.rs @@ -1,9 +1,7 @@ use std::fmt::{self}; -use chrono::{DateTime, NaiveDate}; use expr::RexType; use substrait::proto::expression::field_reference::{ReferenceType, RootReference, RootType}; -use substrait::proto::expression::literal::LiteralType; use substrait::proto::expression::{ Cast, FieldReference, IfThen, ReferenceSegment, ScalarFunction, cast, reference_segment, }; @@ -12,9 +10,9 @@ use substrait::proto::{ AggregateFunction, Expression, FunctionArgument, FunctionOption, expression as expr, }; -use super::{PlanError, Scope, Textify, Visibility}; +use super::{PlanError, Scope, Textify}; use crate::extensions::simple::ExtensionKind; -use crate::textify::types::{Name, NamedAnchor, OutputType, escaped}; +use crate::textify::types::{Name, NamedAnchor, OutputType}; // …(…) for function call // […] for variant @@ -28,236 +26,12 @@ use crate::textify::types::{Name, NamedAnchor, OutputType, escaped}; // …:… for specifying type // &… for enum -pub fn textify_binary(items: &[u8], ctx: &S, w: &mut W) -> fmt::Result { - if ctx.options().show_literal_binaries { - write!(w, "0x")?; - for &n in items { - write!(w, "{n:02x}")?; - } - } else { - write!(w, "{{binary}}")?; - } - Ok(()) -} - -/// Write an error token for a literal type that hasn't been implemented yet. -fn unimplemented_literal( - variant: &'static str, - ctx: &S, - w: &mut W, -) -> fmt::Result { - write!( - w, - "{}", - ctx.failure(PlanError::unimplemented( - "LiteralType", - Some(variant), - format!("{variant} literal textification not implemented"), - )) - ) -} - /// Write an enum value. Enums are written as `&`, if the string is /// a valid identifier; otherwise, they are written as `&''`. pub fn textify_enum(s: &str, _ctx: &S, w: &mut W) -> fmt::Result { write!(w, "&{}", Name(s)) } -/// Convert days since Unix epoch to date string -fn days_to_date_string(days: i32) -> String { - let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); - let date = epoch + chrono::Duration::days(days as i64); - date.format("%Y-%m-%d").to_string() -} - -/// Convert microseconds since midnight to time string -fn microseconds_to_time_string(microseconds: i64) -> String { - let total_seconds = microseconds / 1_000_000; - let remaining_microseconds = microseconds % 1_000_000; - - let hours = total_seconds / 3600; - let minutes = (total_seconds % 3600) / 60; - let seconds = total_seconds % 60; - - if remaining_microseconds == 0 { - format!("{hours:02}:{minutes:02}:{seconds:02}") - } else { - // Convert microseconds to fractional seconds - let fractional = remaining_microseconds as f64 / 1_000_000.0; - format!("{hours:02}:{minutes:02}:{seconds:02}{fractional:.6}") - .trim_end_matches('0') - .trim_end_matches('.') - .to_string() - } -} - -/// Convert microseconds since Unix epoch to timestamp string -fn microseconds_to_timestamp_string(microseconds: i64) -> String { - let epoch = DateTime::from_timestamp(0, 0).unwrap().naive_utc(); - let duration = chrono::Duration::microseconds(microseconds); - let datetime = epoch + duration; - - // Format with fractional seconds, then clean up trailing zeros - let formatted = datetime.format("%Y-%m-%dT%H:%M:%S%.f").to_string(); - - // If there are fractional seconds, trim trailing zeros and dot if needed - if formatted.contains('.') { - formatted - .trim_end_matches('0') - .trim_end_matches('.') - .to_string() - } else { - formatted - } -} - -/// Write just the value portion of a literal, with no type suffix or -/// nullability marker. -/// -/// For unimplemented types, writes an error token via `ctx.failure()`. -fn write_literal_value( - lit: &LiteralType, - ctx: &S, - w: &mut W, -) -> fmt::Result { - match lit { - LiteralType::Boolean(b) => write!(w, "{b}"), - LiteralType::I8(i) | LiteralType::I16(i) | LiteralType::I32(i) => write!(w, "{i}"), - LiteralType::I64(i) => write!(w, "{i}"), - LiteralType::Fp32(f) => write!(w, "{f}"), - LiteralType::Fp64(f) => write!(w, "{f}"), - LiteralType::String(s) => write!(w, "'{}'", s.escape_debug()), - LiteralType::Binary(items) => textify_binary(items, ctx, w), - LiteralType::Date(days) => { - write!(w, "'{}'", escaped(&days_to_date_string(*days))) - } - #[allow(deprecated)] - LiteralType::Time(microseconds) => { - write!( - w, - "'{}'", - escaped(µseconds_to_time_string(*microseconds)) - ) - } - #[allow(deprecated)] - LiteralType::Timestamp(microseconds) => { - write!( - w, - "'{}'", - escaped(µseconds_to_timestamp_string(*microseconds)) - ) - } - LiteralType::IntervalYearToMonth(_) => unimplemented_literal("IntervalYearToMonth", ctx, w), - LiteralType::IntervalDayToSecond(_) => unimplemented_literal("IntervalDayToSecond", ctx, w), - LiteralType::IntervalCompound(_) => unimplemented_literal("IntervalCompound", ctx, w), - LiteralType::FixedChar(_) => unimplemented_literal("FixedChar", ctx, w), - LiteralType::VarChar(_) => unimplemented_literal("VarChar", ctx, w), - LiteralType::FixedBinary(_) => unimplemented_literal("FixedBinary", ctx, w), - LiteralType::Decimal(_) => unimplemented_literal("Decimal", ctx, w), - LiteralType::PrecisionTime(_) => unimplemented_literal("PrecisionTime", ctx, w), - LiteralType::PrecisionTimestamp(_) => unimplemented_literal("PrecisionTimestamp", ctx, w), - LiteralType::PrecisionTimestampTz(_) => { - unimplemented_literal("PrecisionTimestampTz", ctx, w) - } - LiteralType::Struct(_) => unimplemented_literal("Struct", ctx, w), - LiteralType::Map(_) => unimplemented_literal("Map", ctx, w), - #[allow(deprecated)] - LiteralType::TimestampTz(_) => unimplemented_literal("TimestampTz", ctx, w), - LiteralType::Uuid(_) => unimplemented_literal("Uuid", ctx, w), - LiteralType::Null(_) => write!(w, "null"), - LiteralType::List(_) => unimplemented_literal("List", ctx, w), - LiteralType::EmptyList(_) => unimplemented_literal("EmptyList", ctx, w), - LiteralType::EmptyMap(_) => unimplemented_literal("EmptyMap", ctx, w), - LiteralType::UserDefined(_) => unimplemented_literal("UserDefined", ctx, w), - } -} - -/// The type suffix for a literal (e.g., `"i32"`, `"fp64"`, `"date"`). -/// -/// Returns `None` for unimplemented types whose [`write_literal_value`] already -/// emitted an error token. -fn literal_type_suffix(lit: &LiteralType) -> Option<&'static str> { - match lit { - LiteralType::Boolean(_) => Some("boolean"), - LiteralType::I8(_) => Some("i8"), - LiteralType::I16(_) => Some("i16"), - LiteralType::I32(_) => Some("i32"), - LiteralType::I64(_) => Some("i64"), - LiteralType::Fp32(_) => Some("fp32"), - LiteralType::Fp64(_) => Some("fp64"), - LiteralType::String(_) => Some("string"), - LiteralType::Binary(_) => Some("binary"), - LiteralType::Date(_) => Some("date"), - #[allow(deprecated)] - LiteralType::Time(_) => Some("time"), - #[allow(deprecated)] - LiteralType::Timestamp(_) => Some("timestamp"), - _ => None, - } -} - -/// Whether this type is the default interpretation for its value syntax. -/// -/// Each literal value syntax has a default type that the parser assumes when -/// no explicit type suffix is present: -/// - `true`/`false` → `boolean` -/// - bare integers (`42`) → `i64` -/// - bare floats (`3.19`) → `fp64` -/// - single-quoted strings (`'hello'`) → `string` -/// - hex literals (`0x...`) → `binary` -/// -/// Non-default types (e.g., `i32`, `fp32`, `date`) always need an explicit -/// suffix to distinguish them from the default. -fn is_default_for_syntax(lit: &LiteralType) -> bool { - matches!( - lit, - LiteralType::Boolean(_) - | LiteralType::String(_) - | LiteralType::Binary(_) - | LiteralType::I64(_) - | LiteralType::Fp64(_) - ) -} - -impl Textify for expr::Literal { - fn name() -> &'static str { - "Literal" - } - - fn textify(&self, ctx: &S, w: &mut W) -> fmt::Result { - let Some(lit) = self.literal_type.as_ref() else { - return write!( - w, - "{}", - ctx.failure(PlanError::invalid( - "Literal", - Some("literal_type"), - "missing literal_type", - )) - ); - }; - write_literal_value(lit, ctx, w)?; - let show_suffix = match ctx.options().literal_types { - Visibility::Never => false, - Visibility::Always => true, - Visibility::Required => self.nullable || !is_default_for_syntax(lit), - }; - if let LiteralType::Null(typ) = lit { - write!(w, ":{}", ctx.expect(Some(typ)))?; - return Ok(()); - } - if show_suffix { - if let Some(suffix) = literal_type_suffix(lit) { - write!(w, ":{suffix}")?; - } - if self.nullable { - write!(w, "?")?; - } - } - Ok(()) - } -} - pub struct Reference(pub i32); impl fmt::Display for Reference { @@ -671,6 +445,7 @@ impl Textify for AggregateFunction { #[cfg(test)] mod tests { use substrait::proto::Type; + use substrait::proto::expression::literal::LiteralType; use substrait::proto::expression::{cast, if_then}; use substrait::proto::r#type::{Boolean, I16, I32, I64, Kind, Nullability, UserDefined}; diff --git a/src/textify/literals.rs b/src/textify/literals.rs new file mode 100644 index 00000000..61ca08b8 --- /dev/null +++ b/src/textify/literals.rs @@ -0,0 +1,250 @@ +use std::fmt; + +use chrono::{DateTime, NaiveDate}; +use substrait::proto::expression::literal::LiteralType; +use substrait::proto::r#type::{self as ptype, Kind, Nullability}; +use substrait::proto::{Type, expression as expr}; + +use super::{PlanError, Scope, Textify, Visibility}; +use crate::textify::types::escaped; + +pub fn textify_binary(items: &[u8], ctx: &S, w: &mut W) -> fmt::Result { + if ctx.options().show_literal_binaries { + write!(w, "0x")?; + for &n in items { + write!(w, "{n:02x}")?; + } + } else { + write!(w, "{{binary}}")?; + } + Ok(()) +} + +fn unimplemented_literal( + variant: &'static str, + ctx: &S, + w: &mut W, +) -> fmt::Result { + write!( + w, + "{}", + ctx.failure(PlanError::unimplemented( + "LiteralType", + Some(variant), + format!("{variant} literal textification not implemented"), + )) + ) +} + +fn days_to_date_string(days: i32) -> String { + let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); + let date = epoch + chrono::Duration::days(days as i64); + date.format("%Y-%m-%d").to_string() +} + +fn microseconds_to_time_string(microseconds: i64) -> String { + let total_seconds = microseconds / 1_000_000; + let remaining_micros = microseconds % 1_000_000; + let hours = total_seconds / 3600; + let minutes = (total_seconds % 3600) / 60; + let seconds = total_seconds % 60; + if remaining_micros == 0 { + format!("{hours:02}:{minutes:02}:{seconds:02}") + } else { + let fraction = format!("{remaining_micros:06}") + .trim_end_matches('0') + .to_string(); + format!("{hours:02}:{minutes:02}:{seconds:02}.{fraction}") + } +} + +fn microseconds_to_timestamp_string(microseconds: i64) -> String { + let epoch = DateTime::from_timestamp(0, 0).unwrap().naive_utc(); + let duration = chrono::Duration::microseconds(microseconds); + let datetime = epoch + duration; + let formatted = datetime.format("%Y-%m-%dT%H:%M:%S%.f").to_string(); + if formatted.contains('.') { + formatted + .trim_end_matches('0') + .trim_end_matches('.') + .to_string() + } else { + formatted + } +} + +fn write_literal_value( + lit: &LiteralType, + ctx: &S, + w: &mut W, +) -> fmt::Result { + match lit { + LiteralType::Boolean(b) => write!(w, "{b}"), + LiteralType::I8(i) | LiteralType::I16(i) | LiteralType::I32(i) => write!(w, "{i}"), + LiteralType::I64(i) => write!(w, "{i}"), + LiteralType::Fp32(f) => write!(w, "{f}"), + LiteralType::Fp64(f) => write!(w, "{f}"), + LiteralType::String(s) => write!(w, "'{}'", s.escape_debug()), + LiteralType::Binary(items) => textify_binary(items, ctx, w), + LiteralType::Date(days) => write!(w, "'{}'", escaped(&days_to_date_string(*days))), + #[allow(deprecated)] + LiteralType::Time(microseconds) => { + write!( + w, + "'{}'", + escaped(µseconds_to_time_string(*microseconds)) + ) + } + #[allow(deprecated)] + LiteralType::Timestamp(microseconds) => { + write!( + w, + "'{}'", + escaped(µseconds_to_timestamp_string(*microseconds)) + ) + } + LiteralType::IntervalYearToMonth(_) => unimplemented_literal("IntervalYearToMonth", ctx, w), + LiteralType::IntervalDayToSecond(_) => unimplemented_literal("IntervalDayToSecond", ctx, w), + LiteralType::IntervalCompound(_) => unimplemented_literal("IntervalCompound", ctx, w), + LiteralType::FixedChar(_) => unimplemented_literal("FixedChar", ctx, w), + LiteralType::VarChar(_) => unimplemented_literal("VarChar", ctx, w), + LiteralType::FixedBinary(_) => unimplemented_literal("FixedBinary", ctx, w), + LiteralType::Decimal(_) => unimplemented_literal("Decimal", ctx, w), + LiteralType::PrecisionTime(_) => unimplemented_literal("PrecisionTime", ctx, w), + LiteralType::PrecisionTimestamp(_) => unimplemented_literal("PrecisionTimestamp", ctx, w), + LiteralType::PrecisionTimestampTz(_) => { + unimplemented_literal("PrecisionTimestampTz", ctx, w) + } + LiteralType::Struct(_) => unimplemented_literal("Struct", ctx, w), + LiteralType::Map(_) => unimplemented_literal("Map", ctx, w), + #[allow(deprecated)] + LiteralType::TimestampTz(_) => unimplemented_literal("TimestampTz", ctx, w), + LiteralType::Uuid(_) => unimplemented_literal("Uuid", ctx, w), + LiteralType::Null(_) => write!(w, "null"), + LiteralType::List(_) => unimplemented_literal("List", ctx, w), + LiteralType::EmptyList(_) => unimplemented_literal("EmptyList", ctx, w), + LiteralType::EmptyMap(_) => unimplemented_literal("EmptyMap", ctx, w), + LiteralType::UserDefined(_) => unimplemented_literal("UserDefined", ctx, w), + } +} + +fn literal_type(literal: &expr::Literal) -> Option { + let lit = literal.literal_type.as_ref()?; + let nullability = if literal.nullable { + Nullability::Nullable as i32 + } else { + Nullability::Required as i32 + }; + let type_variation_reference = literal.type_variation_reference; + let kind = match lit { + LiteralType::Boolean(_) => Kind::Bool(ptype::Boolean { + nullability, + type_variation_reference, + }), + LiteralType::I8(_) => Kind::I8(ptype::I8 { + nullability, + type_variation_reference, + }), + LiteralType::I16(_) => Kind::I16(ptype::I16 { + nullability, + type_variation_reference, + }), + LiteralType::I32(_) => Kind::I32(ptype::I32 { + nullability, + type_variation_reference, + }), + LiteralType::I64(_) => Kind::I64(ptype::I64 { + nullability, + type_variation_reference, + }), + LiteralType::Fp32(_) => Kind::Fp32(ptype::Fp32 { + nullability, + type_variation_reference, + }), + LiteralType::Fp64(_) => Kind::Fp64(ptype::Fp64 { + nullability, + type_variation_reference, + }), + LiteralType::String(_) => Kind::String(ptype::String { + nullability, + type_variation_reference, + }), + LiteralType::Binary(_) => Kind::Binary(ptype::Binary { + nullability, + type_variation_reference, + }), + LiteralType::Date(_) => Kind::Date(ptype::Date { + nullability, + type_variation_reference, + }), + #[allow(deprecated)] + LiteralType::Time(_) => Kind::Time(ptype::Time { + nullability, + type_variation_reference, + }), + #[allow(deprecated)] + LiteralType::Timestamp(_) => Kind::Timestamp(ptype::Timestamp { + nullability, + type_variation_reference, + }), + _ => return None, + }; + Some(Type { kind: Some(kind) }) +} + +fn is_default_for_syntax(lit: &LiteralType) -> bool { + matches!( + lit, + LiteralType::Boolean(_) + | LiteralType::String(_) + | LiteralType::Binary(_) + | LiteralType::I64(_) + | LiteralType::Fp64(_) + ) +} + +pub(crate) fn textify_literal( + literal: &expr::Literal, + ctx: &S, + w: &mut W, +) -> fmt::Result { + let Some(lit) = literal.literal_type.as_ref() else { + return write!( + w, + "{}", + ctx.failure(PlanError::invalid( + "Literal", + Some("literal_type"), + "missing literal_type", + )) + ); + }; + write_literal_value(lit, ctx, w)?; + if let LiteralType::Null(typ) = lit { + write!(w, ":{}", ctx.display(typ))?; + return Ok(()); + } + let show_suffix = match ctx.options().literal_types { + Visibility::Never => false, + Visibility::Always => true, + Visibility::Required => literal.nullable || !is_default_for_syntax(lit), + }; + if show_suffix { + if let Some(typ) = literal_type(literal) { + write!(w, ":{}", ctx.display(&typ))?; + } else if literal.nullable { + write!(w, "?")?; + } + } + Ok(()) +} + +impl Textify for expr::Literal { + fn name() -> &'static str { + "Literal" + } + + fn textify(&self, ctx: &S, w: &mut W) -> fmt::Result { + textify_literal(self, ctx, w) + } +} diff --git a/src/textify/mod.rs b/src/textify/mod.rs index af541def..6d335b41 100644 --- a/src/textify/mod.rs +++ b/src/textify/mod.rs @@ -4,6 +4,7 @@ mod addenda; pub(crate) mod expressions; pub(crate) mod extensions; pub(crate) mod foundation; +pub(crate) mod literals; pub(crate) mod plan; pub(crate) mod rels; pub(crate) mod types;