Skip to content

test: cover lpad and rpad routing configurations #5895

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rich7420 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

test: cover lpad and rpad routing configurations #5895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81fa972
test: cover lpad and rpad routing configurations
rich7420 Sep 11, 2026
c161509
test: move padding routing coverage into SQL fixtures
rich7420 Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions spark/src/test/resources/sql-tests/expressions/string/string_lpad.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,39 +15,47 @@
-- specific language governing permissions and limitations
-- under the License.

-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true

statement
CREATE TABLE test_lpad(s string, len int, pad string) USING parquet

statement
INSERT INTO test_lpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x'), ('hi', NULL, 'x'), (NULL, NULL, 'x'), ('hi', 5, NULL), ('hi', 5, ''), ('', 3, ''), ('hi', -100, 'x'), (NULL, NULL, NULL), ('"hi', 7, '"x'), ('café', 7, '中文'), ('é', 5, '🙂')
INSERT INTO test_lpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), ('', 0, 'x'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x'), ('hi', NULL, 'x'), (NULL, NULL, 'x'), ('hi', 5, NULL), ('hi', 5, ''), ('', 3, ''), ('hi', -100, 'x'), (NULL, NULL, NULL), ('"hi', 7, '"x'), ('café', 7, '中文'), ('é', 5, '🙂')

-- Column padding runs through the codegen dispatcher (issue #5579).
query
query expect_dispatch(lpad)
SELECT lpad(s, len, pad) FROM test_lpad

query
query expect_dispatch(lpad)
SELECT lpad(s, 5, pad) FROM test_lpad

query
query expect_dispatch(lpad)
SELECT lpad('hi', len, pad) FROM test_lpad

query
query expect_dispatch(lpad)
SELECT lpad('hi', len, 'xy') FROM test_lpad

query
query expect_dispatch(lpad)
SELECT lpad('hi', len) FROM test_lpad

query
query expect_native(lpad)
SELECT lpad(s, len) FROM test_lpad

-- column + column + literal
query
query expect_native(lpad)
SELECT lpad(s, len, 'x') FROM test_lpad

query expect_native(lpad)
SELECT lpad(s, len, 'xy') FROM test_lpad

-- column + literal + literal
query
query expect_native(lpad)
SELECT lpad(s, 5, 'x') FROM test_lpad

-- literal + literal + literal
query
query expect_dispatch(lpad)
SELECT lpad('hi', 5, 'x'), lpad('hello', 3, 'x'), lpad('', 3, 'a'), lpad(NULL, 5, 'x')

query expect_dispatch(lpad)
SELECT lpad('hi', 5, 'xy') FROM test_lpad
15 changes: 9 additions & 6 deletions spark/src/test/resources/sql-tests/expressions/string/string_lpad_fallback.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,23 @@ statement
CREATE TABLE test_lpad_fallback(s string, len int, pad string) USING parquet

statement
INSERT INTO test_lpad_fallback VALUES ('hi', 5, 'xy'), ('hello', 3, 'x'), (NULL, NULL, NULL)
INSERT INTO test_lpad_fallback VALUES ('hi', 5, 'xy'), ('hello', 3, 'x'), ('', 3, 'a'), ('', 0, 'x'), ('hi', 5, ''), (NULL, 5, 'x'), ('hi', NULL, 'x'), ('hi', 5, NULL), (NULL, NULL, NULL)

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(lpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT lpad(s, len, pad) FROM test_lpad_fallback

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(lpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT lpad('hi', len, 'xy') FROM test_lpad_fallback

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(lpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT lpad('hi', 5, 'xy')

query expect_fallback(lpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT lpad('hi', 5, 'xy') FROM test_lpad_fallback

-- The native argument shapes do not require the dispatcher.
query
query expect_native(lpad)
SELECT lpad(s, len, 'xy') FROM test_lpad_fallback

query
query expect_native(lpad)
SELECT lpad(s, len) FROM test_lpad_fallback
28 changes: 18 additions & 10 deletions spark/src/test/resources/sql-tests/expressions/string/string_rpad.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,39 +15,47 @@
-- specific language governing permissions and limitations
-- under the License.

-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true

statement
CREATE TABLE test_rpad(s string, len int, pad string) USING parquet

statement
INSERT INTO test_rpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x'), ('hi', NULL, 'x'), (NULL, NULL, 'x'), ('hi', 5, NULL), ('hi', 5, ''), ('', 3, ''), ('hi', -100, 'x'), (NULL, NULL, NULL), ('"hi', 7, '"x'), ('café', 7, '中文'), ('é', 5, '🙂')
INSERT INTO test_rpad VALUES ('hi', 5, 'x'), ('hello', 3, 'x'), ('hi', 5, 'xy'), ('', 3, 'a'), ('', 0, 'x'), (NULL, 5, 'x'), ('hi', 0, 'x'), ('hi', -1, 'x'), ('hi', NULL, 'x'), (NULL, NULL, 'x'), ('hi', 5, NULL), ('hi', 5, ''), ('', 3, ''), ('hi', -100, 'x'), (NULL, NULL, NULL), ('"hi', 7, '"x'), ('café', 7, '中文'), ('é', 5, '🙂')

-- Column padding runs through the codegen dispatcher (issue #5579).
query
query expect_dispatch(rpad)
SELECT rpad(s, len, pad) FROM test_rpad

query
query expect_dispatch(rpad)
SELECT rpad(s, 5, pad) FROM test_rpad

query
query expect_dispatch(rpad)
SELECT rpad('hi', len, pad) FROM test_rpad

query
query expect_dispatch(rpad)
SELECT rpad('hi', len, 'xy') FROM test_rpad

query
query expect_dispatch(rpad)
SELECT rpad('hi', len) FROM test_rpad

query
query expect_native(rpad)
SELECT rpad(s, len) FROM test_rpad

-- column + column + literal
query
query expect_native(rpad)
SELECT rpad(s, len, 'x') FROM test_rpad

query expect_native(rpad)
SELECT rpad(s, len, 'xy') FROM test_rpad

-- column + literal + literal
query
query expect_native(rpad)
SELECT rpad(s, 5, 'x') FROM test_rpad

-- literal + literal + literal
query
query expect_dispatch(rpad)
SELECT rpad('hi', 5, 'x'), rpad('hello', 3, 'x'), rpad('', 3, 'a'), rpad(NULL, 5, 'x')

query expect_dispatch(rpad)
SELECT rpad('hi', 5, 'xy') FROM test_rpad
15 changes: 9 additions & 6 deletions spark/src/test/resources/sql-tests/expressions/string/string_rpad_fallback.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,23 @@ statement
CREATE TABLE test_rpad_fallback(s string, len int, pad string) USING parquet

statement
INSERT INTO test_rpad_fallback VALUES ('hi', 5, 'xy'), ('hello', 3, 'x'), (NULL, NULL, NULL)
INSERT INTO test_rpad_fallback VALUES ('hi', 5, 'xy'), ('hello', 3, 'x'), ('', 3, 'a'), ('', 0, 'x'), ('hi', 5, ''), (NULL, 5, 'x'), ('hi', NULL, 'x'), ('hi', 5, NULL), (NULL, NULL, NULL)

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(rpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT rpad(s, len, pad) FROM test_rpad_fallback

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(rpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT rpad('hi', len, 'xy') FROM test_rpad_fallback

query expect_fallback(spark.comet.exec.scalaUDF.codegen.enabled)
query expect_fallback(rpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT rpad('hi', 5, 'xy')

query expect_fallback(rpad: spark.comet.exec.scalaUDF.codegen.enabled=false)
SELECT rpad('hi', 5, 'xy') FROM test_rpad_fallback

-- The native argument shapes do not require the dispatcher.
query
query expect_native(rpad)
SELECT rpad(s, len, 'xy') FROM test_rpad_fallback

query
query expect_native(rpad)
SELECT rpad(s, len) FROM test_rpad_fallback
47 changes: 0 additions & 47 deletions spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,53 +46,6 @@ class CometStringExpressionSuite extends CometTestBase with CometCodegenAssertio
testStringPadding("rpad")
}

for ((function, expressionName) <- Seq("lpad" -> "StringLPad", "rpad" -> "StringRPad")) {
test(s"$function dispatches unsupported argument shapes (issue #5579)") {
val data: Seq[(String, Option[Int], String)] = Seq(
("hi", Some(5), "xy"),
("hello", Some(3), "x"),
("", Some(3), "a"),
("hi", Some(5), ""),
(null, Some(5), "x"),
("hi", None, "x"),
("hi", Some(5), null),
(null, None, null))
withParquetTable(data, "tbl") {
withSQLConf(
SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
"org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
for (allowIncompatible <- Seq("false", "true")) {
withSQLConf(
CometConf.getExprAllowIncompatConfigKey(expressionName) -> allowIncompatible) {
for (query <- Seq(
s"SELECT $function(_1, _2, _3) FROM tbl",
s"SELECT $function('hi', _2, 'xy') FROM tbl",
s"SELECT $function('hi', 5, 'xy') FROM tbl")) {
assertCodegenRan {
checkSparkAnswerAndOperator(query)
}
}
}
}
}
}
}

test(s"$function keeps supported argument shapes native") {
withParquetTable(Seq(("hi", 5), ("hello", 3), ("", 0)), "tbl") {
for (query <- Seq(
s"SELECT $function(_1, _2) FROM tbl",
s"SELECT $function(_1, _2, 'xy') FROM tbl")) {
CometScalaUDFCodegen.resetStats()
checkSparkAnswerAndOperator(query)
assert(
CometScalaUDFCodegen.stats().totalLookups == 0,
s"expected native execution for $query")
}
}
}
}

test("lpad/rpad with NULL length") {
// FuzzDataGenerator never generates NULL integers (#5389), so build the rows explicitly.
// Spark's StringLPad/StringRPad are null-intolerant: a NULL length yields a NULL row.
Expand Down
Loading