Skip to content

refactor: simplify literal parsing and textification #151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wackywendell wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

refactor: simplify literal parsing and textification #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
324 changes: 1 addition & 323 deletions src/parser/expressions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime};
use substrait::proto::aggregate_rel::Measure;
use substrait::proto::expression::field_reference::{ReferenceType, RootReference, RootType};
use substrait::proto::expression::if_then::IfClause;
use substrait::proto::expression::literal::LiteralType;
use substrait::proto::expression::{
Cast, FieldReference, IfThen, Literal, ReferenceSegment, RexType, ScalarFunction, cast,
reference_segment,
};
use substrait::proto::function_argument::ArgType;
use substrait::proto::r#type::{Fp64, I64, Kind, Nullability};
use substrait::proto::{AggregateFunction, Expression, FunctionArgument, Type};

use super::types::get_and_validate_anchor;
Expand Down Expand Up @@ -76,326 +73,6 @@ impl ParsePair for FieldReference {
}
}

fn to_int_literal(
value: pest::iterators::Pair<Rule>,
typ: Option<Type>,
) -> Result<Literal, MessageParseError> {
assert_eq!(value.as_rule(), Rule::integer);
let parsed_value: i64 = value.as_str().parse().unwrap();

const DEFAULT_KIND: Kind = Kind::I64(I64 {
type_variation_reference: 0,
nullability: Nullability::Required as i32,
});

// If no type is provided, we assume i64, Nullability::Required.
let kind = typ.and_then(|t| t.kind).unwrap_or(DEFAULT_KIND);

let (lit, nullability, tvar) = match &kind {
// If no type is provided, we assume i64, Nullability::Required.
Kind::I8(i) => (
LiteralType::I8(parsed_value as i32),
i.nullability,
i.type_variation_reference,
),
Kind::I16(i) => (
LiteralType::I16(parsed_value as i32),
i.nullability,
i.type_variation_reference,
),
Kind::I32(i) => (
LiteralType::I32(parsed_value as i32),
i.nullability,
i.type_variation_reference,
),
Kind::I64(i) => (
LiteralType::I64(parsed_value),
i.nullability,
i.type_variation_reference,
),
k => {
return Err(MessageParseError::invalid(
"int_literal_type",
value.as_span(),
format!("Invalid type for integer literal: {k:?}"),
));
}
};

Ok(Literal {
literal_type: Some(lit),
nullable: nullability != Nullability::Required as i32,
type_variation_reference: tvar,
})
}

fn to_float_literal(
value: pest::iterators::Pair<Rule>,
typ: Option<Type>,
) -> Result<Literal, MessageParseError> {
assert_eq!(value.as_rule(), Rule::float);
let parsed_value: f64 = value.as_str().parse().unwrap();

const DEFAULT_KIND: Kind = Kind::Fp64(Fp64 {
type_variation_reference: 0,
nullability: Nullability::Required as i32,
});

// If no type is provided, we assume fp64, Nullability::Required.
let kind = typ.and_then(|t| t.kind).unwrap_or(DEFAULT_KIND);

let (lit, nullability, tvar) = match &kind {
Kind::Fp32(f) => (
LiteralType::Fp32(parsed_value as f32),
f.nullability,
f.type_variation_reference,
),
Kind::Fp64(f) => (
LiteralType::Fp64(parsed_value),
f.nullability,
f.type_variation_reference,
),
k => {
return Err(MessageParseError::invalid(
"float_literal_type",
value.as_span(),
format!("Invalid type for float literal: {k:?}"),
));
}
};

Ok(Literal {
literal_type: Some(lit),
nullable: nullability != Nullability::Required as i32,
type_variation_reference: tvar,
})
}

fn to_boolean_literal(
value: pest::iterators::Pair<Rule>,
typ: Option<Type>,
) -> Result<Literal, MessageParseError> {
assert_eq!(value.as_rule(), Rule::boolean);
let parsed_value: bool = value.as_str().parse().unwrap();

let (nullable, tvar) = match typ.and_then(|t| t.kind) {
Some(Kind::Bool(b)) => (
b.nullability != Nullability::Required as i32,
b.type_variation_reference,
),
None => (false, 0),
Some(k) => {
return Err(MessageParseError::invalid(
"bool_literal_type",
value.as_span(),
format!("Invalid type for boolean literal: {k:?}"),
));
}
};

Ok(Literal {
literal_type: Some(LiteralType::Boolean(parsed_value)),
nullable,
type_variation_reference: tvar,
})
}

fn to_string_literal(
value: pest::iterators::Pair<Rule>,
typ: Option<Type>,
) -> Result<Literal, MessageParseError> {
assert_eq!(value.as_rule(), Rule::string_literal);
let string_value = unescape_string(value.clone());

// If no type is provided, default to string
let Some(typ) = typ else {
return Ok(Literal {
literal_type: Some(LiteralType::String(string_value)),
nullable: false,
type_variation_reference: 0,
});
};

let Some(kind) = typ.kind else {
return Ok(Literal {
literal_type: Some(LiteralType::String(string_value)),
nullable: false,
type_variation_reference: 0,
});
};

match &kind {
Kind::Date(d) => {
// Parse date in ISO 8601 format: YYYY-MM-DD
let date_days = parse_date_to_days(&string_value, value.as_span())?;
Ok(Literal {
literal_type: Some(LiteralType::Date(date_days)),
nullable: d.nullability != Nullability::Required as i32,
type_variation_reference: d.type_variation_reference,
})
}
#[allow(deprecated)]
Kind::Time(t) => {
// Parse time in ISO 8601 format: HH:MM:SS[.fff]
let time_microseconds = parse_time_to_microseconds(&string_value, value.as_span())?;
Ok(Literal {
literal_type: Some(LiteralType::Time(time_microseconds)),
nullable: t.nullability != Nullability::Required as i32,
type_variation_reference: t.type_variation_reference,
})
}
#[allow(deprecated)]
Kind::Timestamp(ts) => {
// Parse timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SS[.fff] or YYYY-MM-DD HH:MM:SS[.fff]
let timestamp_microseconds =
parse_timestamp_to_microseconds(&string_value, value.as_span())?;
Ok(Literal {
literal_type: Some(LiteralType::Timestamp(timestamp_microseconds)),
nullable: ts.nullability != Nullability::Required as i32,
type_variation_reference: ts.type_variation_reference,
})
}
_ => {
// For other types, treat as string
Ok(Literal {
literal_type: Some(LiteralType::String(string_value)),
nullable: false,
type_variation_reference: 0,
})
}
}
}

fn to_null_literal(
value: pest::iterators::Pair<Rule>,
typ: Option<Type>,
) -> Result<Literal, MessageParseError> {
assert_eq!(value.as_rule(), Rule::null);
let typ = typ.ok_or_else(|| {
MessageParseError::invalid(
"null_literal_type",
value.as_span(),
"Null literals require an explicit type annotation, e.g. null:i64?",
)
})?;

Ok(Literal {
literal_type: Some(LiteralType::Null(typ)),
nullable: false,
type_variation_reference: 0,
})
}

/// Parse a date string using chrono to days since Unix epoch
fn parse_date_to_days(date_str: &str, span: pest::Span) -> Result<i32, MessageParseError> {
// Try multiple date formats for flexibility
let formats = ["%Y-%m-%d", "%Y/%m/%d"];

for format in &formats {
if let Ok(date) = NaiveDate::parse_from_str(date_str, format) {
// Calculate days since Unix epoch (1970-01-01)
let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
let days = date.signed_duration_since(epoch).num_days();
return Ok(days as i32);
}
}

Err(MessageParseError::invalid(
"date_parse_format",
span,
format!("Invalid date format: '{date_str}'. Expected YYYY-MM-DD or YYYY/MM/DD"),
))
}

/// Parse a time string using chrono to microseconds since midnight
fn parse_time_to_microseconds(time_str: &str, span: pest::Span) -> Result<i64, MessageParseError> {
// Try multiple time formats for flexibility
let formats = ["%H:%M:%S%.f", "%H:%M:%S"];

for format in &formats {
if let Ok(time) = NaiveTime::parse_from_str(time_str, format) {
// Convert to microseconds since midnight
let midnight = NaiveTime::from_hms_opt(0, 0, 0).unwrap();
let duration = time.signed_duration_since(midnight);
return Ok(duration.num_microseconds().unwrap_or(0));
}
}

Err(MessageParseError::invalid(
"time_parse_format",
span,
format!("Invalid time format: '{time_str}'. Expected HH:MM:SS or HH:MM:SS.fff"),
))
}

/// Parse a timestamp string using chrono to microseconds since Unix epoch
fn parse_timestamp_to_microseconds(
timestamp_str: &str,
span: pest::Span,
) -> Result<i64, MessageParseError> {
// Try multiple timestamp formats for flexibility
let formats = [
"%Y-%m-%dT%H:%M:%S%.f", // ISO 8601 with T and fractional seconds
"%Y-%m-%dT%H:%M:%S", // ISO 8601 with T
"%Y-%m-%d %H:%M:%S%.f", // Space separator with fractional seconds
"%Y-%m-%d %H:%M:%S", // Space separator
"%Y/%m/%dT%H:%M:%S%.f", // Alternative date format with T
"%Y/%m/%dT%H:%M:%S", // Alternative date format with T
"%Y/%m/%d %H:%M:%S%.f", // Alternative date format with space
"%Y/%m/%d %H:%M:%S", // Alternative date format with space
];

for format in &formats {
if let Ok(datetime) = NaiveDateTime::parse_from_str(timestamp_str, format) {
// Calculate microseconds since Unix epoch (1970-01-01 00:00:00)
let epoch = DateTime::from_timestamp(0, 0).unwrap().naive_utc();
let duration = datetime.signed_duration_since(epoch);
return Ok(duration.num_microseconds().unwrap_or(0));
}
}

Err(MessageParseError::invalid(
"timestamp_parse_format",
span,
format!(
"Invalid timestamp format: '{timestamp_str}'. Expected YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD HH:MM:SS"
),
))
}

impl ScopedParsePair for Literal {
fn rule() -> Rule {
Rule::literal
}

fn message() -> &'static str {
"Literal"
}

fn parse_pair(
extensions: &SimpleExtensions,
pair: pest::iterators::Pair<Rule>,
) -> Result<Self, MessageParseError> {
assert_eq!(pair.as_rule(), Self::rule());
let mut pairs = pair.into_inner();
let value = pairs.next().unwrap(); // First item is always the value
let typ = pairs.next(); // Second item is optional type
assert!(pairs.next().is_none());
let typ = match typ {
Some(t) => Some(Type::parse_pair(extensions, t)?),
None => None,
};
match value.as_rule() {
Rule::integer => to_int_literal(value, typ),
Rule::float => to_float_literal(value, typ),
Rule::boolean => to_boolean_literal(value, typ),
Rule::string_literal => to_string_literal(value, typ),
Rule::null => to_null_literal(value, typ),
_ => unreachable!("Literal unexpected rule: {:?}", value.as_rule()),
}
}
}

impl ScopedParsePair for ScalarFunction {
fn rule() -> Rule {
Rule::function_call
Expand Down Expand Up @@ -691,6 +368,7 @@ impl ScopedParsePair for Measure {
#[cfg(test)]
mod tests {
use pest::Parser as PestParser;
use substrait::proto::expression::literal::LiteralType;

use super::*;
use crate::parser::ExpressionParser;
Expand Down
Loading