From 7d9c57c3f506b35bfd62d0f11a34f72c96a0d44a Mon Sep 17 00:00:00 2001 From: JITENDRA Date: Mon, 20 Jul 2026 16:32:44 +0530 Subject: [PATCH 1/3] fix: support Dictionary arrays in regex_match_dyn (#23709) --- .../src/expressions/binary/kernels.rs | 17 +++++++++++------ .../test_files/regexp/regexp_match.slt | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index fca824c14bee..6b6e8b9ae661 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -31,7 +31,7 @@ use datafusion_common::{exec_err, internal_err, plan_err}; use std::sync::Arc; -/// Downcasts $LEFT and $RIGHT to $ARRAY_TYPE and then calls $KERNEL($LEFT, $RIGHT) + macro_rules! call_kernel { ($LEFT:expr, $RIGHT:expr, $KERNEL:expr, $ARRAY_TYPE:ident) => {{ let left = $LEFT.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap(); @@ -41,8 +41,6 @@ macro_rules! call_kernel { }}; } -/// Creates a $FUNC(left: ArrayRef, right: ArrayRef) that -/// downcasts left / right to the appropriate integral type and calls the kernel macro_rules! create_left_integral_dyn_kernel { ($FUNC:ident, $KERNEL:ident) => { pub(crate) fn $FUNC(left: ArrayRef, right: ArrayRef) -> Result { @@ -87,7 +85,7 @@ create_left_integral_dyn_kernel!(bitwise_and_dyn, bitwise_and); create_left_integral_dyn_kernel!(bitwise_shift_right_dyn, bitwise_shift_right); create_left_integral_dyn_kernel!(bitwise_shift_left_dyn, bitwise_shift_left); -/// Downcasts $LEFT as $ARRAY_TYPE and $RIGHT as TYPE and calls $KERNEL($LEFT, $RIGHT) + macro_rules! call_scalar_kernel { ($LEFT:expr, $RIGHT:expr, $KERNEL:ident, $ARRAY_TYPE:ident, $TYPE:ty) => {{ let len = $LEFT.len(); @@ -103,8 +101,6 @@ macro_rules! call_scalar_kernel { }}; } -/// Creates a $FUNC(left: ArrayRef, right: ScalarValue) that -/// downcasts left / right to the appropriate integral type and calls the kernel macro_rules! create_left_integral_dyn_scalar_kernel { ($FUNC:ident, $KERNEL:ident) => { pub(crate) fn $FUNC( @@ -213,6 +209,15 @@ pub(crate) fn regex_match_dyn( DataType::LargeUtf8 => { regexp_is_match_flag!(left, right, LargeStringArray, not_match, flag) } + DataType::Dictionary(_, _) => { + let dict = left.as_any_dictionary(); + let unpacked_left = arrow::compute::kernels::take::take( + dict.values().as_ref(), + dict.keys(), + None, + )?; + regex_match_dyn(&unpacked_left, right, not_match, flag) + } other => internal_err!( "Data type {} not supported for regex_match_dyn on string array", other diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt index e79af4774aa2..afb3897a8466 100644 --- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt +++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt @@ -199,3 +199,22 @@ query B select null !~* 'abc'; ---- NULL + + +# Test for Issue 23709: regex_match_dyn with Dictionary encoded patterns +statement ok +CREATE TABLE dict_regex_t AS SELECT * FROM (VALUES ('user auth failed')) v(s); + +statement ok +CREATE TABLE dict_regex_p AS SELECT * FROM (VALUES ('(auth|login)')) v(pat); + +query B +SELECT arrow_cast(dict_regex_t.s, 'Dictionary(Int32, Utf8)') SIMILAR TO dict_regex_p.pat FROM dict_regex_t CROSS JOIN dict_regex_p; +---- +true + +statement ok +DROP TABLE dict_regex_t; + +statement ok +DROP TABLE dict_regex_p; \ No newline at end of file From 1b82685bf418f7c7e9fa45271eab7c554ce60d22 Mon Sep 17 00:00:00 2001 From: JITENDRA Date: Tue, 21 Jul 2026 09:13:48 +0530 Subject: [PATCH 2/3] address reviewer feedback: add unit test, restore docs, fix imports --- .../src/expressions/binary/kernels.rs | 35 +++++++++++++++---- .../test_files/regexp/regexp_match.slt | 5 +-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index 6b6e8b9ae661..195c783cd033 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -25,13 +25,14 @@ use arrow::compute::kernels::bitwise::{ }; use arrow::compute::kernels::boolean::not; use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scalar}; +use arrow::compute::kernels::take::take; use arrow::datatypes::DataType; use datafusion_common::{Result, ScalarValue}; use datafusion_common::{exec_err, internal_err, plan_err}; use std::sync::Arc; - +/// Downcasts $LEFT and $RIGHT to $ARRAY_TYPE and then calls $KERNEL($LEFT, $RIGHT) macro_rules! call_kernel { ($LEFT:expr, $RIGHT:expr, $KERNEL:expr, $ARRAY_TYPE:ident) => {{ let left = $LEFT.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap(); @@ -41,6 +42,8 @@ macro_rules! call_kernel { }}; } +/// Creates a $FUNC(left: ArrayRef, right: ArrayRef) that +/// downcasts left / right to the appropriate integral type and calls the kernel macro_rules! create_left_integral_dyn_kernel { ($FUNC:ident, $KERNEL:ident) => { pub(crate) fn $FUNC(left: ArrayRef, right: ArrayRef) -> Result { @@ -85,7 +88,7 @@ create_left_integral_dyn_kernel!(bitwise_and_dyn, bitwise_and); create_left_integral_dyn_kernel!(bitwise_shift_right_dyn, bitwise_shift_right); create_left_integral_dyn_kernel!(bitwise_shift_left_dyn, bitwise_shift_left); - +/// Downcasts $LEFT as $ARRAY_TYPE and $RIGHT as TYPE and calls $KERNEL($LEFT, $RIGHT) macro_rules! call_scalar_kernel { ($LEFT:expr, $RIGHT:expr, $KERNEL:ident, $ARRAY_TYPE:ident, $TYPE:ty) => {{ let len = $LEFT.len(); @@ -101,6 +104,8 @@ macro_rules! call_scalar_kernel { }}; } +/// Creates a $FUNC(left: ArrayRef, right: ScalarValue) that +/// downcasts left / right to the appropriate integral type and calls the kernel macro_rules! create_left_integral_dyn_scalar_kernel { ($FUNC:ident, $KERNEL:ident) => { pub(crate) fn $FUNC( @@ -211,11 +216,7 @@ pub(crate) fn regex_match_dyn( } DataType::Dictionary(_, _) => { let dict = left.as_any_dictionary(); - let unpacked_left = arrow::compute::kernels::take::take( - dict.values().as_ref(), - dict.keys(), - None, - )?; + let unpacked_left = take(dict.values().as_ref(), dict.keys(), None)?; regex_match_dyn(&unpacked_left, right, not_match, flag) } other => internal_err!( @@ -303,3 +304,23 @@ pub(crate) fn regex_match_dyn_scalar( }; Some(result) } + +#[test] +fn test_regex_match_dyn_dictionary() -> Result<()> { + let values = StringArray::from(vec![Some("user auth failed"), Some("anonymous")]); + let keys = Int32Array::from(vec![Some(0), Some(1), None, Some(0)]); + let left: ArrayRef = Arc::new(DictionaryArray::new(keys, Arc::new(values))); + let right: ArrayRef = Arc::new(StringArray::from(vec![ + Some("(auth|login)"), + Some("^anon"), + Some("x"), + Some("nope"), + ])); + + let result = regex_match_dyn(&left, &right, false, false)?; + assert_eq!( + result.as_any().downcast_ref::().unwrap(), + &BooleanArray::from(vec![Some(true), Some(true), None, Some(false)]) + ); + Ok(()) +} diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt index afb3897a8466..19140f150a60 100644 --- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt +++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt @@ -201,7 +201,7 @@ select null !~* 'abc'; NULL -# Test for Issue 23709: regex_match_dyn with Dictionary encoded patterns +# Test for Issue 23709: regex_match_dyn with Dictionary encoded value side statement ok CREATE TABLE dict_regex_t AS SELECT * FROM (VALUES ('user auth failed')) v(s); @@ -217,4 +217,5 @@ statement ok DROP TABLE dict_regex_t; statement ok -DROP TABLE dict_regex_p; \ No newline at end of file +DROP TABLE dict_regex_p; + From b77ad245d9dcf30f7f048c482ebbb7043c77d289 Mon Sep 17 00:00:00 2001 From: JITENDRA Date: Tue, 21 Jul 2026 12:50:39 +0530 Subject: [PATCH 3/3] refactor: use public take API per copilot suggestion --- datafusion/physical-expr/src/expressions/binary/kernels.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index 195c783cd033..d27e95f5fb90 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -25,7 +25,7 @@ use arrow::compute::kernels::bitwise::{ }; use arrow::compute::kernels::boolean::not; use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scalar}; -use arrow::compute::kernels::take::take; +use arrow::compute::take; use arrow::datatypes::DataType; use datafusion_common::{Result, ScalarValue}; use datafusion_common::{exec_err, internal_err, plan_err};