Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions crates/sqllogictest/test_files/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,31 @@ SELECT expm1(0::INT);
SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a);
```

## Simplify and evaluate modes

`cargo test -p datafusion-spark-sqllogictest` runs every file twice:

- `simplify` uses the default optimizer, so a call whose arguments are all literals,
such as `SELECT expm1(0::INT)`, is constant-folded while planning. This calls the
function's `simplify` (if it has one) and evaluates it on a one-row batch.
- `evaluate` replaces the `simplify_expressions` optimizer rule, so a call that reaches
the physical plan runs through `invoke_with_args` during execution, as it does in
engines such as Comet that build physical expressions directly. DataFusion functions
that only work as a rewrite, such as `arrow_cast`, are still rewritten.

`evaluate` mode only changes the optimizer. SQL planning still runs first and can
evaluate or simplify some expressions before the optimizer sees them, for example the
arguments of a table function. A call in a projection or filter over rows, such as
`SELECT expm1(a) FROM (VALUES (0::INT), (1::INT)) AS t(a)`, is the reliable way to test
`invoke_with_args` directly.

Every function must work in both modes: `invoke_with_args` is required, and `simplify`
is an optional optimization. Pass `--mode=simplify` or `--mode=evaluate` to run one mode,
for example `cargo test -p datafusion-spark-sqllogictest -- math/expm1 --mode=evaluate`.

A record that cannot run in `evaluate` mode yet is marked `skipif evaluate`, with a
comment that links the issue. Remove the marks when the issue is fixed.

## Checking the expected results against Spark

The expected results in these files come from DataFusion. To compare them with
Expand Down
25 changes: 25 additions & 0 deletions crates/sqllogictest/test_files/conditional/if.slt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# `if` has no `invoke_with_args` yet, so the records that call it are marked
# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/305 is fixed.

# This file was originally created by a porting script from:
# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
# This file is part of the implementation of the datafusion-spark function library.
Expand All @@ -11,23 +14,27 @@
## Basic IF function tests

# Test basic true condition
skipif evaluate
query T
SELECT if(true, 'yes', 'no');
----
yes

# Test basic false condition
skipif evaluate
query T
SELECT if(false, 'yes', 'no');
----
no

# Test with comparison operators
skipif evaluate
query T
SELECT if(1 < 2, 'a', 'b');
----
a

skipif evaluate
query T
SELECT if(1 > 2, 'a', 'b');
----
Expand All @@ -37,17 +44,20 @@ b
## Numeric type tests

# Test with integers
skipif evaluate
query I
SELECT if(true, 10, 20);
----
10

skipif evaluate
query I
SELECT if(false, 10, 20);
----
20

# Test with different integer types
skipif evaluate
query I
SELECT if(true, 100, 200);
----
Expand All @@ -56,11 +66,13 @@ SELECT if(true, 100, 200);
## Float type tests

# Test with floating point numbers
skipif evaluate
query R
SELECT if(true, 1.5, 2.5);
----
1.5

skipif evaluate
query R
SELECT if(false, 1.5, 2.5);
----
Expand All @@ -69,11 +81,13 @@ SELECT if(false, 1.5, 2.5);
## String type tests

# Test with different string values
skipif evaluate
query T
SELECT if(true, 'hello', 'world');
----
hello

skipif evaluate
query T
SELECT if(false, 'hello', 'world');
----
Expand All @@ -82,29 +96,34 @@ world
## NULL handling tests

# Test with NULL condition
skipif evaluate
query T
SELECT if(NULL, 'yes', 'no');
----
no

skipif evaluate
query T
SELECT if(NOT NULL, 'yes', 'no');
----
no

# Test with NULL true value
skipif evaluate
query T
SELECT if(true, NULL, 'no');
----
NULL

# Test with NULL false value
skipif evaluate
query T
SELECT if(false, 'yes', NULL);
----
NULL

# Test with all NULL
skipif evaluate
query ?
SELECT if(true, NULL, NULL);
----
Expand All @@ -114,23 +133,27 @@ NULL

# Test integer to float coercion
# xfail spark: Spark types the unsuffixed literals as INT/DECIMAL, DataFusion as BIGINT/DOUBLE (https://github.com/apache/datafusion-spark/issues/97)
skipif evaluate
query R
SELECT if(true, 10, 20.5);
----
10

skipif evaluate
query R
SELECT if(false, 10, 20.5);
----
20.5

# Test float to integer coercion
skipif evaluate
query R
SELECT if(true, 10.5, 20);
----
10.5

# xfail spark: Spark types the unsuffixed literals as INT/DECIMAL, DataFusion as BIGINT/DOUBLE (https://github.com/apache/datafusion-spark/issues/97)
skipif evaluate
query R
SELECT if(false, 10.5, 20);
----
Expand All @@ -143,12 +166,14 @@ SELECT if(1, 10.5, 20);
statement error Utf8 is not a boolean or null
SELECT if('x', 10.5, 20);

skipif evaluate
query IR
SELECT v, IF(v < 0, 10/0, 1) FROM (VALUES (1), (2)) t(v)
----
1 1
2 1

skipif evaluate
query R
SELECT IF(true, 1 / 1, 1 / 0);
----
Expand Down
11 changes: 11 additions & 0 deletions crates/sqllogictest/test_files/datetime/add_months.slt
Original file line number Diff line number Diff line change
Expand Up @@ -2,44 +2,55 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# `add_months` has no `invoke_with_args` yet, so the records that call it are marked
# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/312 is fixed.

skipif evaluate
query D
SELECT add_months('2016-07-30'::date, 1::int);
----
2016-08-30

skipif evaluate
query D
SELECT add_months('2016-07-30'::date, 0::int);
----
2016-07-30

skipif evaluate
query D
SELECT add_months('2016-07-30'::date, 10000::int);
----
2849-11-30

# Test integer overflow
skipif evaluate
query error
SELECT add_months('2016-07-30'::date, 2147483647::int);
----
DataFusion error: Arrow error: Compute error: Date arithmetic overflow: 17012 + 2147483647 months


skipif evaluate
query D
SELECT add_months('2016-07-30'::date, -5::int);
----
2016-02-29

# Test with NULL values
skipif evaluate
query D
SELECT add_months(NULL::date, 1::int);
----
NULL

skipif evaluate
query D
SELECT add_months('2016-07-30'::date, NULL::int);
----
NULL

skipif evaluate
query D
SELECT add_months(NULL::date, NULL::int);
----
Expand Down
24 changes: 24 additions & 0 deletions crates/sqllogictest/test_files/datetime/date_diff.slt
Original file line number Diff line number Diff line change
Expand Up @@ -2,105 +2,127 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# `date_diff` has no `invoke_with_args` yet, so the records that call it are marked
# `skipif evaluate` until https://github.com/apache/datafusion-spark/issues/307 is fixed.

# date input
skipif evaluate
query I
SELECT date_diff('2009-07-30'::date, '2009-07-31'::date);
----
-1

skipif evaluate
query I
SELECT date_diff('2009-07-31'::date, '2009-07-30'::date);
----
1

skipif evaluate
query I
SELECT date_diff('2009-07-31'::string, '2009-07-30'::date);
----
1

skipif evaluate
query I
SELECT date_diff('2009-07-31'::timestamp, '2009-07-30'::date);
----
1

# Date64 input
skipif evaluate
query I
SELECT date_diff(arrow_cast('2009-07-31', 'Date64'), arrow_cast('2009-07-30', 'Date64'));
----
1

skipif evaluate
query I
SELECT date_diff(arrow_cast('2009-07-30', 'Date64'), arrow_cast('2009-07-31', 'Date64'));
----
-1

# Mixed Date32 and Date64 input
skipif evaluate
query I
SELECT date_diff('2009-07-31'::date, arrow_cast('2009-07-30', 'Date64'));
----
1

skipif evaluate
query I
SELECT date_diff(arrow_cast('2009-07-31', 'Date64'), '2009-07-30'::date);
----
1


# Same date returns 0
skipif evaluate
query I
SELECT date_diff('2009-07-30'::date, '2009-07-30'::date);
----
0

# Large difference
skipif evaluate
query I
SELECT date_diff('2020-01-01'::date, '1970-01-01'::date);
----
18262

# timestamp input
skipif evaluate
query I
SELECT date_diff('2009-07-30 12:34:56'::timestamp, '2009-07-31 23:45:01'::timestamp);
----
-1

skipif evaluate
query I
SELECT date_diff('2009-07-31 23:45:01'::timestamp, '2009-07-30 12:34:56'::timestamp);
----
1

skipif evaluate
query I
SELECT date_diff('2009-07-31 23:45:01'::string, '2009-07-30 12:34:56'::timestamp);
----
1

# string input
skipif evaluate
query I
SELECT date_diff('2009-07-30', '2009-07-31');
----
-1

skipif evaluate
query I
SELECT date_diff('2009-07-31', '2009-07-30');
----
1

# NULL handling
skipif evaluate
query I
SELECT date_diff(NULL::date, '2009-07-30'::date);
----
NULL

skipif evaluate
query I
SELECT date_diff('2009-07-31'::date, NULL::date);
----
NULL

skipif evaluate
query I
SELECT date_diff(NULL::date, NULL::date);
----
NULL

skipif evaluate
query I
SELECT date_diff(column1, column2)
FROM VALUES
Expand All @@ -118,11 +140,13 @@ NULL


# Alias datediff
skipif evaluate
query I
SELECT datediff('2009-07-30'::date, '2009-07-31'::date);
----
-1

skipif evaluate
query I
SELECT datediff(column1, column2)
FROM VALUES
Expand Down
Loading
Loading