diff --git a/crates/sqllogictest/test_files/README.md b/crates/sqllogictest/test_files/README.md index f06180e1..980b3008 100644 --- a/crates/sqllogictest/test_files/README.md +++ b/crates/sqllogictest/test_files/README.md @@ -69,6 +69,31 @@ SELECT expm1(0::INT); SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a); ``` +## Simplify and evaluate modes + +`cargo test -p datafusion-spark-sqllogictest` runs every file twice: + +- `simplify` uses the default optimizer, so a call whose arguments are all literals, + such as `SELECT expm1(0::INT)`, is constant-folded while planning. This calls the + function's `simplify` (if it has one) and evaluates it on a one-row batch. +- `evaluate` replaces the `simplify_expressions` optimizer rule, so a call that reaches + the physical plan runs through `invoke_with_args` during execution, as it does in + engines such as Comet that build physical expressions directly. DataFusion functions + that only work as a rewrite, such as `arrow_cast`, are still rewritten. + +`evaluate` mode only changes the optimizer. SQL planning still runs first and can +evaluate or simplify some expressions before the optimizer sees them, for example the +arguments of a table function. A call in a projection or filter over rows, such as +`SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a)`, is the reliable way to test +`invoke_with_args` directly. + +Every function must work in both modes: `invoke_with_args` is required, and `simplify` +is an optional optimization. Pass `--mode=simplify` or `--mode=evaluate` to run one mode, +for example `cargo test -p datafusion-spark-sqllogictest -- math/expm1 --mode=evaluate`. + +A record that cannot run in `evaluate` mode yet is marked `skipif evaluate`, with a +comment that links the issue. Remove the marks when the issue is fixed. + ## Checking the expected results against Spark The expected results in these files come from DataFusion. To compare them with diff --git a/crates/sqllogictest/test_files/conditional/if.slt b/crates/sqllogictest/test_files/conditional/if.slt index 3c53de7e..3fa57433 100644 --- a/crates/sqllogictest/test_files/conditional/if.slt +++ b/crates/sqllogictest/test_files/conditional/if.slt @@ -2,6 +2,9 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `if` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/305 is fixed. + # This file was originally created by a porting script from: # https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function # This file is part of the implementation of the datafusion-spark function library. @@ -11,23 +14,27 @@ ## Basic IF function tests # Test basic true condition +skipif evaluate query T SELECT if(true, 'yes', 'no'); ---- yes # Test basic false condition +skipif evaluate query T SELECT if(false, 'yes', 'no'); ---- no # Test with comparison operators +skipif evaluate query T SELECT if(1 < 2, 'a', 'b'); ---- a +skipif evaluate query T SELECT if(1 > 2, 'a', 'b'); ---- @@ -37,17 +44,20 @@ b ## Numeric type tests # Test with integers +skipif evaluate query I SELECT if(true, 10, 20); ---- 10 +skipif evaluate query I SELECT if(false, 10, 20); ---- 20 # Test with different integer types +skipif evaluate query I SELECT if(true, 100, 200); ---- @@ -56,11 +66,13 @@ SELECT if(true, 100, 200); ## Float type tests # Test with floating point numbers +skipif evaluate query R SELECT if(true, 1.5, 2.5); ---- 1.5 +skipif evaluate query R SELECT if(false, 1.5, 2.5); ---- @@ -69,11 +81,13 @@ SELECT if(false, 1.5, 2.5); ## String type tests # Test with different string values +skipif evaluate query T SELECT if(true, 'hello', 'world'); ---- hello +skipif evaluate query T SELECT if(false, 'hello', 'world'); ---- @@ -82,29 +96,34 @@ world ## NULL handling tests # Test with NULL condition +skipif evaluate query T SELECT if(NULL, 'yes', 'no'); ---- no +skipif evaluate query T SELECT if(NOT NULL, 'yes', 'no'); ---- no # Test with NULL true value +skipif evaluate query T SELECT if(true, NULL, 'no'); ---- NULL # Test with NULL false value +skipif evaluate query T SELECT if(false, 'yes', NULL); ---- NULL # Test with all NULL +skipif evaluate query ? SELECT if(true, NULL, NULL); ---- @@ -114,23 +133,27 @@ NULL # Test integer to float coercion # xfail spark: Spark types the unsuffixed literals as INT/DECIMAL, DataFusion as BIGINT/DOUBLE (https://github.com/apache/datafusion-spark/issues/97) +skipif evaluate query R SELECT if(true, 10, 20.5); ---- 10 +skipif evaluate query R SELECT if(false, 10, 20.5); ---- 20.5 # Test float to integer coercion +skipif evaluate query R SELECT if(true, 10.5, 20); ---- 10.5 # xfail spark: Spark types the unsuffixed literals as INT/DECIMAL, DataFusion as BIGINT/DOUBLE (https://github.com/apache/datafusion-spark/issues/97) +skipif evaluate query R SELECT if(false, 10.5, 20); ---- @@ -143,12 +166,14 @@ SELECT if(1, 10.5, 20); statement error Utf8 is not a boolean or null SELECT if('x', 10.5, 20); +skipif evaluate query IR SELECT v, IF(v < 0, 10/0, 1) FROM (VALUES (1), (2)) t(v) ---- 1 1 2 1 +skipif evaluate query R SELECT IF(true, 1 / 1, 1 / 0); ---- diff --git a/crates/sqllogictest/test_files/datetime/add_months.slt b/crates/sqllogictest/test_files/datetime/add_months.slt index 8d2f1e8f..03ae8bcc 100644 --- a/crates/sqllogictest/test_files/datetime/add_months.slt +++ b/crates/sqllogictest/test_files/datetime/add_months.slt @@ -2,44 +2,55 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `add_months` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/312 is fixed. + +skipif evaluate query D SELECT add_months('2016-07-30'::date, 1::int); ---- 2016-08-30 +skipif evaluate query D SELECT add_months('2016-07-30'::date, 0::int); ---- 2016-07-30 +skipif evaluate query D SELECT add_months('2016-07-30'::date, 10000::int); ---- 2849-11-30 # Test integer overflow +skipif evaluate query error SELECT add_months('2016-07-30'::date, 2147483647::int); ---- DataFusion error: Arrow error: Compute error: Date arithmetic overflow: 17012 + 2147483647 months +skipif evaluate query D SELECT add_months('2016-07-30'::date, -5::int); ---- 2016-02-29 # Test with NULL values +skipif evaluate query D SELECT add_months(NULL::date, 1::int); ---- NULL +skipif evaluate query D SELECT add_months('2016-07-30'::date, NULL::int); ---- NULL +skipif evaluate query D SELECT add_months(NULL::date, NULL::int); ---- diff --git a/crates/sqllogictest/test_files/datetime/date_diff.slt b/crates/sqllogictest/test_files/datetime/date_diff.slt index 48d843b9..b4af7adc 100644 --- a/crates/sqllogictest/test_files/datetime/date_diff.slt +++ b/crates/sqllogictest/test_files/datetime/date_diff.slt @@ -2,44 +2,55 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `date_diff` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/307 is fixed. + # date input +skipif evaluate query I SELECT date_diff('2009-07-30'::date, '2009-07-31'::date); ---- -1 +skipif evaluate query I SELECT date_diff('2009-07-31'::date, '2009-07-30'::date); ---- 1 +skipif evaluate query I SELECT date_diff('2009-07-31'::string, '2009-07-30'::date); ---- 1 +skipif evaluate query I SELECT date_diff('2009-07-31'::timestamp, '2009-07-30'::date); ---- 1 # Date64 input +skipif evaluate query I SELECT date_diff(arrow_cast('2009-07-31', 'Date64'), arrow_cast('2009-07-30', 'Date64')); ---- 1 +skipif evaluate query I SELECT date_diff(arrow_cast('2009-07-30', 'Date64'), arrow_cast('2009-07-31', 'Date64')); ---- -1 # Mixed Date32 and Date64 input +skipif evaluate query I SELECT date_diff('2009-07-31'::date, arrow_cast('2009-07-30', 'Date64')); ---- 1 +skipif evaluate query I SELECT date_diff(arrow_cast('2009-07-31', 'Date64'), '2009-07-30'::date); ---- @@ -47,60 +58,71 @@ SELECT date_diff(arrow_cast('2009-07-31', 'Date64'), '2009-07-30'::date); # Same date returns 0 +skipif evaluate query I SELECT date_diff('2009-07-30'::date, '2009-07-30'::date); ---- 0 # Large difference +skipif evaluate query I SELECT date_diff('2020-01-01'::date, '1970-01-01'::date); ---- 18262 # timestamp input +skipif evaluate query I SELECT date_diff('2009-07-30 12:34:56'::timestamp, '2009-07-31 23:45:01'::timestamp); ---- -1 +skipif evaluate query I SELECT date_diff('2009-07-31 23:45:01'::timestamp, '2009-07-30 12:34:56'::timestamp); ---- 1 +skipif evaluate query I SELECT date_diff('2009-07-31 23:45:01'::string, '2009-07-30 12:34:56'::timestamp); ---- 1 # string input +skipif evaluate query I SELECT date_diff('2009-07-30', '2009-07-31'); ---- -1 +skipif evaluate query I SELECT date_diff('2009-07-31', '2009-07-30'); ---- 1 # NULL handling +skipif evaluate query I SELECT date_diff(NULL::date, '2009-07-30'::date); ---- NULL +skipif evaluate query I SELECT date_diff('2009-07-31'::date, NULL::date); ---- NULL +skipif evaluate query I SELECT date_diff(NULL::date, NULL::date); ---- NULL +skipif evaluate query I SELECT date_diff(column1, column2) FROM VALUES @@ -118,11 +140,13 @@ NULL # Alias datediff +skipif evaluate query I SELECT datediff('2009-07-30'::date, '2009-07-31'::date); ---- -1 +skipif evaluate query I SELECT datediff(column1, column2) FROM VALUES diff --git a/crates/sqllogictest/test_files/datetime/date_from_unix_date.slt b/crates/sqllogictest/test_files/datetime/date_from_unix_date.slt index 3dcb2519..41b5361a 100644 --- a/crates/sqllogictest/test_files/datetime/date_from_unix_date.slt +++ b/crates/sqllogictest/test_files/datetime/date_from_unix_date.slt @@ -31,7 +31,10 @@ SELECT date_from_unix_date(1), date_from_unix_date(18993), date_from_unix_date(- ---- 1970-01-02 2022-01-01 1900-01-01 0001-01-01 9999-12-31 -# Any INT is a valid date, even where it is too large to print +# Any INT is a valid date, even where it is too large to print. `unix_date` has no +# `invoke_with_args` yet, so this is skipped in evaluate mode +# (https://github.com/apache/datafusion-spark/issues/306). +skipif evaluate query BB SELECT unix_date(date_from_unix_date(2147483647)) = 2147483647, unix_date(date_from_unix_date(-2147483648)) = -2147483648; @@ -118,6 +121,9 @@ SELECT arrow_typeof(date_from_unix_date(0)); Date32 # Inverse of unix_date +# `unix_date` has no `invoke_with_args` yet, so this is skipped in evaluate mode +# (https://github.com/apache/datafusion-spark/issues/306). +skipif evaluate query B SELECT unix_date(date_from_unix_date(12345)) = 12345; ---- @@ -144,6 +150,9 @@ NULL 1970-01-21 1970-10-28 +# `unix_date` has no `invoke_with_args` yet, so this is skipped in evaluate mode +# (https://github.com/apache/datafusion-spark/issues/306). +skipif evaluate query B SELECT unix_date(date_from_unix_date(column1)) = column1 FROM (VALUES (CAST(-2147483648 AS INT)), (2147483647::INT)) AS t(column1); diff --git a/crates/sqllogictest/test_files/datetime/date_part.slt b/crates/sqllogictest/test_files/datetime/date_part.slt index d62e0319..644487da 100644 --- a/crates/sqllogictest/test_files/datetime/date_part.slt +++ b/crates/sqllogictest/test_files/datetime/date_part.slt @@ -45,7 +45,9 @@ SELECT date_part('QTR'::string, '2000-01-01'::date); ---- 1 -# Checked when the query is planned +# The error comes from planning in simplify mode and from execution in evaluate mode, +# so the message differs. +skipif evaluate query error SELECT date_part('QUARTERS'::string, '2000-01-01'::date); ---- @@ -135,7 +137,9 @@ SELECT date_part('DOY'::string, '2000-01-01'::date); ---- 1 -# Checked when the query is planned +# The error comes from planning in simplify mode and from execution in evaluate mode, +# so the message differs. +skipif evaluate query error SELECT date_part('DAYOFYEAR'::string, '2000-01-01'::date); ---- @@ -247,7 +251,9 @@ SELECT date_part(NULL::string, '2000-01-01'::date); NULL # Invalid part -# Checked when the query is planned +# The error comes from planning in simplify mode and from execution in evaluate mode, +# so the message differs. +skipif evaluate query error SELECT date_part('test'::string, '2000-01-01'::date); ---- @@ -308,7 +314,9 @@ DESCRIBE SELECT date_part('SECOND'::string, ts) AS s FROM (VALUES (timestamp '20 ---- s Int32 YES -# A field from a column is rejected when the query is planned, like Spark +# A field from a column is rejected when the query is planned, like Spark. Evaluate +# mode rejects it at execution instead, so the message differs. +skipif evaluate query error SELECT date_part(column1, column2) FROM VALUES ('year', '2000-01-01'::date); ---- @@ -317,7 +325,8 @@ caused by Error during planning: First argument of `DATE_PART` must be a constant string -# Also for an empty input +# Also for an empty input. Evaluate mode runs nothing, so it does not fail. +skipif evaluate query error SELECT date_part(column1, column2) FROM VALUES ('year', '2000-01-01'::date) WHERE false; ---- diff --git a/crates/sqllogictest/test_files/datetime/date_trunc.slt b/crates/sqllogictest/test_files/datetime/date_trunc.slt index f853d864..05867b79 100644 --- a/crates/sqllogictest/test_files/datetime/date_trunc.slt +++ b/crates/sqllogictest/test_files/datetime/date_trunc.slt @@ -2,97 +2,117 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `date_trunc` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/309 is fixed. + # YEAR - truncate to first date of year, time zeroed +skipif evaluate query P SELECT date_trunc('YEAR', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-01-01T00:00:00 +skipif evaluate query P SELECT date_trunc('YYYY', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-01-01T00:00:00 +skipif evaluate query P SELECT date_trunc('YY', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-01-01T00:00:00 # QUARTER - truncate to first date of quarter, time zeroed +skipif evaluate query P SELECT date_trunc('QUARTER', '2015-05-05T09:32:05.123456'::timestamp); ---- 2015-04-01T00:00:00 # MONTH - truncate to first date of month, time zeroed +skipif evaluate query P SELECT date_trunc('MONTH', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-01T00:00:00 +skipif evaluate query P SELECT date_trunc('MM', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-01T00:00:00 +skipif evaluate query P SELECT date_trunc('MON', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-01T00:00:00 # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query P SELECT date_trunc('MONTHS', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-01T00:00:00 # WEEK - truncate to Monday of the week, time zeroed +skipif evaluate query P SELECT date_trunc('WEEK', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-02T00:00:00 # DAY - zero out time part +skipif evaluate query P SELECT date_trunc('DAY', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T00:00:00 +skipif evaluate query P SELECT date_trunc('DD', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T00:00:00 # HOUR - zero out minute and second with fraction +skipif evaluate query P SELECT date_trunc('HOUR', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T09:00:00 # MINUTE - zero out second with fraction +skipif evaluate query P SELECT date_trunc('MINUTE', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T09:32:00 # SECOND - zero out fraction +skipif evaluate query P SELECT date_trunc('SECOND', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T09:32:05 # MILLISECOND - zero out microseconds +skipif evaluate query P SELECT date_trunc('MILLISECOND', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T09:32:05.123 # MICROSECOND - everything remains +skipif evaluate query P SELECT date_trunc('MICROSECOND', '2015-03-05T09:32:05.123456'::timestamp); ---- 2015-03-05T09:32:05.123456 +skipif evaluate query P SELECT date_trunc('YEAR', column1) FROM VALUES @@ -107,6 +127,7 @@ FROM VALUES NULL # String input +skipif evaluate query P SELECT date_trunc('YEAR', '2015-03-05T09:32:05.123456'); ---- @@ -114,9 +135,11 @@ SELECT date_trunc('YEAR', '2015-03-05T09:32:05.123456'); # Null handling # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: First argument of `DATE_TRUNC` must be non-null scalar Utf8 SELECT date_trunc(NULL, '2015-03-05T09:32:05.123456'); +skipif evaluate query P SELECT date_trunc('YEAR', NULL::timestamp); ---- @@ -124,17 +147,20 @@ NULL # incorrect format # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query error DataFusion error: Execution error: Unsupported date_trunc granularity: 'test'. Supported granularities are: microsecond, millisecond, second, minute, hour, day, week, month, quarter, year SELECT date_trunc('test', '2015-03-05T09:32:05.123456'); # Timezone handling - Spark-compatible behavior # Spark converts timestamps to session timezone before truncating for coarse granularities +skipif evaluate query P SELECT date_trunc('DAY', arrow_cast(timestamp '2024-07-15T03:30:00', 'Timestamp(Microsecond, Some("UTC"))')); ---- 2024-07-15T00:00:00Z +skipif evaluate query P SELECT date_trunc('DAY', arrow_cast(timestamp '2024-07-15T03:30:00', 'Timestamp(Microsecond, None)')); ---- @@ -145,11 +171,13 @@ SET datafusion.execution.time_zone = 'America/New_York'; # This timestamp is 03:30 UTC = 23:30 EDT (previous day) on July 14 # With session timezone, truncation happens in America/New_York timezone +skipif evaluate query P SELECT date_trunc('DAY', arrow_cast(timestamp '2024-07-15T03:30:00', 'Timestamp(Microsecond, Some("UTC"))')); ---- 2024-07-14T00:00:00Z +skipif evaluate query P SELECT date_trunc('DAY', arrow_cast(timestamp '2024-07-15T03:30:00', 'Timestamp(Microsecond, None)')); ---- diff --git a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt index 2a77ad0a..be19b359 100644 --- a/crates/sqllogictest/test_files/datetime/make_dt_interval.slt +++ b/crates/sqllogictest/test_files/datetime/make_dt_interval.slt @@ -143,6 +143,9 @@ SELECT make_dt_interval(1491308, 0, 0, 0); ---- 1491308 days 0 hours 0 mins 0.000000 secs +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(1491309, 0, 0, 0); ---- @@ -164,6 +167,9 @@ SELECT make_dt_interval(106751991, 0, 0, 0); 106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(106751992, 0, 0, 0); ---- @@ -175,6 +181,9 @@ SELECT make_dt_interval(-106751991, 0, 0, 0); -106751991 days 0 hours 0 mins 0.000000 secs # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(-106751992, 0, 0, 0); ---- @@ -182,12 +191,18 @@ NULL # Mixed-sign cancellation checks (Spark raises INTERVAL_ARITHMETIC_OVERFLOW, returns NULL pending #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(106751992, -24, 0, 0); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(106751991, 5, -60, 0); ---- @@ -206,12 +221,18 @@ SELECT make_dt_interval(-106751991, -4, 0, -54.775808); # One microsecond beyond boundaries returns NULL (overflow tracked in #20) # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(106751991, 4, 0, 54.775808); ---- NULL # xfail spark: Spark raises INTERVAL_ARITHMETIC_OVERFLOW where make_dt_interval returns NULL (https://github.com/apache/datafusion-spark/issues/20) +# Without constant folding this fails because the result is declared non-nullable +# (https://github.com/apache/datafusion-spark/issues/315). +skipif evaluate query ? SELECT make_dt_interval(-106751991, -4, 0, -54.775809); ---- diff --git a/crates/sqllogictest/test_files/datetime/trunc.slt b/crates/sqllogictest/test_files/datetime/trunc.slt index d2f5843d..ce6aaba8 100644 --- a/crates/sqllogictest/test_files/datetime/trunc.slt +++ b/crates/sqllogictest/test_files/datetime/trunc.slt @@ -2,62 +2,76 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `trunc` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/311 is fixed. + # YEAR - truncate to first date of year +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'YEAR'::string); ---- 2009-01-01 +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'YYYY'::string); ---- 2009-01-01 +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'YY'::string); ---- 2009-01-01 # QUARTER - truncate to first date of quarter +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'QUARTER'::string); ---- 2009-01-01 # MONTH - truncate to first date of month +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'MONTH'::string); ---- 2009-02-01 +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'MM'::string); ---- 2009-02-01 +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'MON'::string); ---- 2009-02-01 # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'MONTHS'::string); ---- 2009-02-01 # WEEK - truncate to Monday of the week +skipif evaluate query D SELECT trunc('2009-02-12'::date, 'WEEK'::string); ---- 2009-02-09 # string input +skipif evaluate query D SELECT trunc('2009-02-12'::string, 'YEAR'::string); ---- 2009-01-01 +skipif evaluate query D SELECT trunc(column1, 'YEAR'::string) FROM VALUES @@ -72,16 +86,19 @@ FROM VALUES NULL # Null handling +skipif evaluate query D SELECT trunc(NULL::date, 'YEAR'::string); ---- NULL # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: Second argument of `TRUNC` must be non-null scalar Utf8 SELECT trunc('2009-02-12'::date, NULL::string); # incorrect format # xfail spark: Spark returns NULL for an unsupported or NULL trunc/date_trunc format (https://github.com/apache/datafusion-spark/issues/40) +skipif evaluate query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nError during planning: The format argument of `TRUNC` must represent a year, month, day, week, or quarter. SELECT trunc('2009-02-12'::date, 'test'::string); diff --git a/crates/sqllogictest/test_files/datetime/unix.slt b/crates/sqllogictest/test_files/datetime/unix.slt index 0c93db63..ec476282 100644 --- a/crates/sqllogictest/test_files/datetime/unix.slt +++ b/crates/sqllogictest/test_files/datetime/unix.slt @@ -4,22 +4,28 @@ # Unix Date tests +# `unix_date` has no `invoke_with_args` yet, so these are skipped in evaluate mode +# (https://github.com/apache/datafusion-spark/issues/306). +skipif evaluate query I SELECT unix_date('1970-01-02'::date); ---- 1 +skipif evaluate query I SELECT unix_date('1900-01-02'::date); ---- -25566 +skipif evaluate query I SELECT unix_date(arrow_cast('1970-01-02', 'Date64')); ---- 1 +skipif evaluate query I SELECT unix_date(NULL::date); ---- diff --git a/crates/sqllogictest/test_files/math/bround.slt b/crates/sqllogictest/test_files/math/bround.slt index 6327eb3f..daade2f4 100644 --- a/crates/sqllogictest/test_files/math/bround.slt +++ b/crates/sqllogictest/test_files/math/bround.slt @@ -10,6 +10,12 @@ ## Original Query: SELECT bround(2.5, 0); ## PySpark 3.5.5 Result: {'bround(2.5, 0)': Decimal('2'), 'typeof(bround(2.5, 0))': 'decimal(2,0)', 'typeof(2.5)': 'decimal(2,1)', 'typeof(0)': 'int'} +# The scale is a constant expression, not a literal. The planner computes the decimal +# result type from literal arguments only, so without constant folding the result keeps +# the input scale. This is a limitation of planning in DataFusion, not of +# `invoke_with_args`: a caller that builds the physical expression, such as Comet, passes +# the result type itself. +skipif evaluate query RT SELECT bround(2.5::decimal(2,1), 0::int), arrow_typeof(bround(2.5::decimal(2,1), 0::int)); ---- @@ -145,12 +151,24 @@ SELECT bround(column1, 1) FROM VALUES (0.25::decimal(3,2)), (0.35::decimal(3,2)) NULL # A constant scale that is not a bare literal +# The scale is a constant expression, not a literal. The planner computes the decimal +# result type from literal arguments only, so without constant folding the result keeps +# the input scale. This is a limitation of planning in DataFusion, not of +# `invoke_with_args`: a caller that builds the physical expression, such as Comet, passes +# the result type itself. +skipif evaluate query RRR SELECT bround(2.25::decimal(3,2), 1::bigint) AS a, bround(2.25::decimal(3,2), 1 + 0) AS b, bround(2.25::decimal(3,2), CAST(1 AS INT)) AS c; ---- 2.2 2.2 2.2 # The result type needs the scale, so it relies on the optimizer folding it first +# The scale is a constant expression, not a literal. The planner computes the decimal +# result type from literal arguments only, so without constant folding the result keeps +# the input scale. This is a limitation of planning in DataFusion, not of +# `invoke_with_args`: a caller that builds the physical expression, such as Comet, passes +# the result type itself. +skipif evaluate query TTTT SELECT arrow_typeof(bround(2.25::decimal(3,2), 1::bigint)) AS a, arrow_typeof(bround(2.25::decimal(3,2), 1 + 0)) AS b, arrow_typeof(bround(2.25::decimal(3,2), CAST(1 AS INT))) AS c, arrow_typeof(bround(column1, 1::bigint)) AS d @@ -189,6 +207,12 @@ SELECT bround(arrow_cast(arrow_cast(25, 'Int32'), 'Dictionary(Int32, Int32)'), - 20 Int32 # A cast scale on a decimal column, used in a filter +# The scale is a constant expression, not a literal. The planner computes the decimal +# result type from literal arguments only, so without constant folding the result keeps +# the input scale. This is a limitation of planning in DataFusion, not of +# `invoke_with_args`: a caller that builds the physical expression, such as Comet, passes +# the result type itself. +skipif evaluate query R SELECT r FROM (SELECT bround(column1, 1::bigint) AS r FROM VALUES (2.25::decimal(3,2)), (2.35::decimal(3,2))) WHERE r > 2.3; ---- diff --git a/crates/sqllogictest/test_files/math/positive.slt b/crates/sqllogictest/test_files/math/positive.slt index fb5e674c..e0bc216a 100644 --- a/crates/sqllogictest/test_files/math/positive.slt +++ b/crates/sqllogictest/test_files/math/positive.slt @@ -130,6 +130,8 @@ SELECT positive(arrow_cast('-1.5', 'Dictionary(Int32, Utf8)')), arrow_typeof(pos statement ok set datafusion.explain.logical_plan_only = true; +# The plan is only rewritten when simplify runs. +onlyif simplify query TT EXPLAIN SELECT positive(column1) FROM VALUES (1); ---- @@ -137,6 +139,8 @@ logical_plan 01)Projection: column1 AS positive(column1) 02)--Values: (Int64(1)) +# The plan is only rewritten when simplify runs. +onlyif simplify query TT EXPLAIN SELECT positive(column1) FROM VALUES ('-1.5'); ---- diff --git a/crates/sqllogictest/test_files/string/base64.slt b/crates/sqllogictest/test_files/string/base64.slt index e2abef40..8d2b96a3 100644 --- a/crates/sqllogictest/test_files/string/base64.slt +++ b/crates/sqllogictest/test_files/string/base64.slt @@ -2,36 +2,46 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# `base64` has no `invoke_with_args` yet, so the records that call it are marked +# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/313 is fixed. + +skipif evaluate query T SELECT base64('Spark SQL'::string); ---- U3BhcmsgU1FM +skipif evaluate query T SELECT base64('Spark SQ'::string); ---- U3BhcmsgU1E= +skipif evaluate query T SELECT base64('Spark S'::string); ---- U3BhcmsgUw== +skipif evaluate query T SELECT base64('Spark SQL'::bytea); ---- U3BhcmsgU1FM +skipif evaluate query T SELECT base64(NULL::string); ---- NULL +skipif evaluate query T SELECT base64(NULL::bytea); ---- NULL +skipif evaluate query T SELECT base64(column1) FROM VALUES diff --git a/crates/sqllogictest/test_files/string/concat_ws.slt b/crates/sqllogictest/test_files/string/concat_ws.slt index 17def584..5f244c24 100644 --- a/crates/sqllogictest/test_files/string/concat_ws.slt +++ b/crates/sqllogictest/test_files/string/concat_ws.slt @@ -135,6 +135,8 @@ FROM VALUES ('a', 'b'), ('c', CAST(NULL AS STRING)), (CAST(NULL AS STRING), 'd') result Utf8 NO ## Scalar-only arguments over multiple rows (broadcast test) +# Without constant folding this panics (https://github.com/apache/datafusion-spark/issues/314). +skipif evaluate query T SELECT concat_ws(',', 'a', 'b') AS result FROM VALUES (1), (2), (3) AS t(x); ---- diff --git a/crates/sqllogictest/tests/sqllogictests.rs b/crates/sqllogictest/tests/sqllogictests.rs index da38b2a1..0c2d9dbe 100644 --- a/crates/sqllogictest/tests/sqllogictests.rs +++ b/crates/sqllogictest/tests/sqllogictests.rs @@ -5,17 +5,46 @@ //! Runs the `.slt` files under `test_files` against a DataFusion session with the Spark //! functions registered and the Spark SQL dialect enabled. //! -//! Usage: `cargo test -p datafusion-spark-sqllogictest -- [FILTER]`, where `FILTER` is an -//! optional substring of the test file path. Pass `--complete` to rewrite the expected results -//! in the matching files with the actual results instead of checking them. +//! Each file runs twice, once in each [`Mode`]: +//! +//! - `simplify`: the default optimizer rules. Expressions whose arguments are all +//! literals are constant-folded during planning, which calls a function's `simplify` +//! (if it has one) and then evaluates it against a one-row batch. +//! - `evaluate`: the same rules without `simplify_expressions`, so every expression, +//! including `SELECT f(1)`, is planned as written and run through `invoke_with_args` +//! during execution. This is the path engines such as Comet use, since they build +//! physical expressions directly and never call `simplify`. DataFusion's own +//! functions that can only run as a rewrite, such as `arrow_cast`, are still +//! rewritten (see [`RewriteCoreFunctions`]). +//! +//! A record that cannot run in `evaluate` mode yet uses `skipif evaluate`, with a comment +//! that links the issue. Prefer it over `onlyif simplify`, because the PySpark runner +//! skips every `onlyif` record that is not for `spark`. +//! +//! Usage: `cargo test -p datafusion-spark-sqllogictest -- [FILTER]... [--mode=MODE]`, where +//! each `FILTER` is a substring of the test file path and `MODE` is `simplify` or +//! `evaluate` (both by default). Pass `--complete` to rewrite the expected results in the +//! matching files with the actual results instead of checking them. It runs only in +//! `simplify` mode, so each file is rewritten once. mod engine; use std::path::{Path, PathBuf}; use std::process::ExitCode; +use std::sync::Arc; use datafusion::common::config::Dialect; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::{DFSchema, Result}; use datafusion::execution::SessionStateBuilder; +use datafusion::logical_expr::expr::ScalarFunction; +use datafusion::logical_expr::expr_rewriter::NamePreserver; +use datafusion::logical_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use datafusion::logical_expr::utils::merge_schema; +use datafusion::logical_expr::{Expr, LogicalPlan}; +use datafusion::optimizer::optimizer::ApplyOrder; +use datafusion::optimizer::simplify_expressions::SimplifyExpressions; +use datafusion::optimizer::{Optimizer, OptimizerConfig, OptimizerRule}; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_spark::SessionStateBuilderSpark; use datafusion_sqllogictest::{df_value_validator, value_normalizer}; @@ -34,20 +63,122 @@ fn collect_slt_files(dir: &Path, files: &mut Vec) -> std::io::Result<() Ok(()) } -fn new_context() -> SessionContext { +/// How expressions in a test file reach the function implementations. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Mode { + /// Default optimizer rules, so literal-only expressions are constant-folded. + Simplify, + /// No `simplify_expressions` rule, so every expression runs through `invoke_with_args`. + Evaluate, +} + +impl Mode { + const ALL: [Mode; 2] = [Mode::Simplify, Mode::Evaluate]; + + /// The `.slt` label for `onlyif` and `skipif`. + fn label(self) -> &'static str { + match self { + Mode::Simplify => "simplify", + Mode::Evaluate => "evaluate", + } + } +} + +/// DataFusion functions whose `invoke_with_args` always errors because they are meant +/// to be rewritten by `simplify` (for example `arrow_cast` becomes `CAST`). The tests +/// use them to build typed inputs, so [`Mode::Evaluate`] still rewrites them. +const SIMPLIFY_ONLY_CORE_FUNCTIONS: [&str; 5] = [ + "arrow_cast", + "arrow_try_cast", + "cast_to_type", + "try_cast_to_type", + "nvl2", +]; + +/// Applies `simplify` to [`SIMPLIFY_ONLY_CORE_FUNCTIONS`] only, leaving every other +/// expression, including literal-only calls of the Spark functions, as written. +#[derive(Debug)] +struct RewriteCoreFunctions; + +impl OptimizerRule for RewriteCoreFunctions { + fn name(&self) -> &str { + "rewrite_core_functions" + } + + fn apply_order(&self) -> Option { + Some(ApplyOrder::BottomUp) + } + + fn rewrite( + &self, + plan: LogicalPlan, + config: &dyn OptimizerConfig, + ) -> Result> { + let schema = match &plan { + LogicalPlan::TableScan(scan) => DFSchema::try_from_qualified_schema( + scan.table_name.clone(), + &scan.source.schema(), + )?, + _ => merge_schema(&plan.inputs()), + }; + let info = SimplifyContext::builder() + .with_schema(Arc::new(schema)) + .with_config_options(config.options()) + .with_query_execution_start_time(config.query_execution_start_time()) + .build(); + let name_preserver = NamePreserver::new(&plan); + plan.map_expressions(|expr| { + let name = name_preserver.save(&expr); + let expr = expr.transform_up(|expr| match expr { + Expr::ScalarFunction(f) + if SIMPLIFY_ONLY_CORE_FUNCTIONS.contains(&f.func.name()) => + { + match f.func.simplify(f.args, &info)? { + ExprSimplifyResult::Simplified(expr) => { + Ok(Transformed::yes(expr)) + } + ExprSimplifyResult::Original(args) => Ok(Transformed::no( + Expr::ScalarFunction(ScalarFunction::new_udf(f.func, args)), + )), + } + } + _ => Ok(Transformed::no(expr)), + })?; + Ok(expr.update_data(|expr| name.restore(expr))) + }) + } +} + +fn new_context(mode: Mode) -> SessionContext { let mut config = SessionConfig::new().with_target_partitions(4); config.options_mut().sql_parser.dialect = Dialect::Spark; - let state = SessionStateBuilder::new() + let mut builder = SessionStateBuilder::new() .with_config(config) .with_default_features() - .with_spark_features() - .build(); - SessionContext::new_with_state(state) + .with_spark_features(); + if mode == Mode::Evaluate { + let simplify = SimplifyExpressions::new().name().to_string(); + let rules = Optimizer::new() + .rules + .into_iter() + .map(|rule| { + if rule.name() == simplify { + Arc::new(RewriteCoreFunctions) as _ + } else { + rule + } + }) + .collect(); + builder = builder.with_optimizer_rules(rules); + } + SessionContext::new_with_state(builder.build()) } -async fn run_file(path: PathBuf, complete: bool) -> Result<(), String> { - let mut runner = - sqllogictest::Runner::new(|| async { Ok(SparkDataFusion::new(new_context())) }); +async fn run_file(path: PathBuf, mode: Mode, complete: bool) -> Result<(), String> { + let mut runner = sqllogictest::Runner::new(|| async { + Ok(SparkDataFusion::new(new_context(mode))) + }); + runner.add_label(mode.label()); if complete { return runner .update_test_file( @@ -71,10 +202,22 @@ async fn run_file(path: PathBuf, complete: bool) -> Result<(), String> { #[tokio::main] async fn main() -> ExitCode { - let complete = std::env::args().any(|arg| arg == "--complete"); - // Ignore other flags such as `--nocapture` that `cargo test` may pass through. - let filters: Vec = std::env::args() - .skip(1) + let args: Vec = std::env::args().skip(1).collect(); + let complete = args.iter().any(|arg| arg == "--complete"); + let modes: Vec = match args.iter().find_map(|arg| arg.strip_prefix("--mode=")) { + None if complete => vec![Mode::Simplify], + None => Mode::ALL.to_vec(), + Some(name) => match Mode::ALL.into_iter().find(|m| m.label() == name) { + Some(mode) => vec![mode], + None => { + eprintln!("unknown mode {name:?}, expected `simplify` or `evaluate`"); + return ExitCode::FAILURE; + } + }, + }; + // Ignore flags such as `--nocapture` that `cargo test` may pass through. + let filters: Vec = args + .into_iter() .filter(|arg| !arg.starts_with('-')) .collect(); @@ -92,21 +235,27 @@ async fn main() -> ExitCode { { continue; } - handles.push((relative.clone(), tokio::spawn(run_file(path, complete)))); + for &mode in &modes { + let task = tokio::spawn(run_file(path.clone(), mode, complete)); + handles.push((relative.clone(), mode, task)); + } } let total = handles.len(); let mut failed = 0; - for (relative, handle) in handles { - match handle.await.expect("test task panicked") { - Ok(()) => println!("PASS {}", relative.display()), + for (relative, mode, handle) in handles { + let name = format!("[{}] {}", mode.label(), relative.display()); + // A panic in one file (for example a function indexing past the end of an + // array) is reported as that file failing, so the other results still print. + match handle.await.map_err(|e| e.to_string()).and_then(|r| r) { + Ok(()) => println!("PASS {name}"), Err(e) => { failed += 1; - println!("FAIL {}\n{e}\n", relative.display()); + println!("FAIL {name}\n{e}\n"); } } } - println!("{} of {total} test files passed", total - failed); + println!("{} of {total} test runs passed", total - failed); if failed == 0 { ExitCode::SUCCESS } else {