diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 2eb76efa3a7f..2452bb651d08 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -32,6 +32,10 @@ provide_find_module(LLVMAlt "Gandiva") if(ARROW_WITH_ZSTD AND "${zstd_SOURCE}" STREQUAL "SYSTEM") provide_find_module(zstdAlt "Gandiva") endif() + +# ICU is required for Unicode normalization (normalize_string) +find_package(ICU COMPONENTS uc) + provide_cmake_module(GandivaAddBitcode "Gandiva") # Set the path where the bitcode file generated, see precompiled/CMakeLists.txt @@ -91,6 +95,7 @@ set(SRC_FILES selection_vector.cc tree_expr_builder.cc to_date_holder.cc + parse_timestamp_holder.cc random_generator_holder.cc ${GANDIVA_PRECOMPILED_CC_PATH}) @@ -121,6 +126,16 @@ if(ARROW_WITH_UTF8PROC) list(APPEND GANDIVA_STATIC_LINK_LIBS utf8proc::utf8proc) endif() +if(ICU_FOUND) + if(TARGET ICU::uc) + list(APPEND GANDIVA_SHARED_PRIVATE_LINK_LIBS ICU::uc) + list(APPEND GANDIVA_STATIC_LINK_LIBS ICU::uc) + else() + list(APPEND GANDIVA_SHARED_PRIVATE_LINK_LIBS ${ICU_UC_LIBRARIES}) + list(APPEND GANDIVA_STATIC_LINK_LIBS ${ICU_UC_LIBRARIES}) + endif() +endif() + if(ARROW_GANDIVA_STATIC_LIBSTDCPP AND (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX )) list(APPEND GANDIVA_STATIC_LINK_LIBS -static-libstdc++ -static-libgcc) @@ -170,6 +185,10 @@ add_arrow_lib(gandiva foreach(LIB_TARGET ${GANDIVA_LIBRARIES}) target_compile_definitions(${LIB_TARGET} PRIVATE GANDIVA_EXPORTING) + if(ICU_FOUND) + target_compile_definitions(${LIB_TARGET} PRIVATE GANDIVA_WITH_ICU) + target_include_directories(${LIB_TARGET} PRIVATE ${ICU_INCLUDE_DIRS}) + endif() endforeach() if(ARROW_BUILD_STATIC AND WIN32) @@ -263,6 +282,7 @@ add_gandiva_test(internals-test selection_vector_test.cc lru_cache_test.cc to_date_holder_test.cc + parse_timestamp_holder_test.cc simple_arena_test.cc regex_functions_holder_test.cc decimal_type_util_test.cc diff --git a/cpp/src/gandiva/array_ops.cc b/cpp/src/gandiva/array_ops.cc index 64548bf09abb..40988a57c9f4 100644 --- a/cpp/src/gandiva/array_ops.cc +++ b/cpp/src/gandiva/array_ops.cc @@ -109,13 +109,68 @@ bool array_contains_template(const Type* entry_buf, return true; } } - //If there is null in the input and the item is not found the result is null. + // If there is null in the input and the item is not found, the result is null. if (found_null_in_data) { *valid_row = false; } return false; } +template +int32_t array_length_template(const Type* entry_buf, int32_t entry_len, + const int32_t* entry_validity, bool combined_row_validity, + int64_t loop_var, int64_t validity_index_var, + bool* valid_row) { + (void)entry_buf; + (void)entry_validity; + (void)loop_var; + (void)validity_index_var; + + if (!combined_row_validity) { + *valid_row = false; + return 0; + } + *valid_row = true; + return entry_len; +} + +template +int32_t array_position_template(const Type* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, Type search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + if (!combined_row_validity || !search_data_valid) { + *valid_row = false; + return 0; + } + *valid_row = true; + + const int32_t* entry_validityAdjusted = entry_validity - (loop_var); + int64_t validityBitIndex = validity_index_var - entry_len; + + bool found_null_in_data = false; + for (int i = 0; i < entry_len; i++) { + if (!arrow::bit_util::GetBit(reinterpret_cast(entry_validityAdjusted), + validityBitIndex + i)) { + found_null_in_data = true; + continue; + } + + Type entry_item = *(entry_buf + i); + if (entry_item == search_data) { + // Position is 1-indexed. + return i + 1; + } + } + + // If there is null in the input and the item is not found, the result is null. + if (found_null_in_data) { + *valid_row = false; + } + return 0; +} + extern "C" { bool array_int32_contains_int32(int64_t context_ptr, const int32_t* entry_buf, @@ -158,6 +213,85 @@ bool array_float64_contains_float64(int64_t context_ptr, const double* entry_buf loop_var, validity_index_var, valid_row); } +int32_t array_int32_length(int64_t context_ptr, const int32_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_length_template(entry_buf, entry_len, entry_validity, + combined_row_validity, loop_var, + validity_index_var, valid_row); +} + +int32_t array_int64_length(int64_t context_ptr, const int64_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_length_template(entry_buf, entry_len, entry_validity, + combined_row_validity, loop_var, + validity_index_var, valid_row); +} + +int32_t array_float32_length(int64_t context_ptr, const float* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_length_template(entry_buf, entry_len, entry_validity, + combined_row_validity, loop_var, + validity_index_var, valid_row); +} + +int32_t array_float64_length(int64_t context_ptr, const double* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_length_template(entry_buf, entry_len, entry_validity, + combined_row_validity, loop_var, + validity_index_var, valid_row); +} + +int32_t array_int32_position_int32(int64_t context_ptr, const int32_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int32_t search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_position_template(entry_buf, entry_len, entry_validity, + combined_row_validity, search_data, + search_data_valid, loop_var, + validity_index_var, valid_row); +} + +int32_t array_int64_position_int64(int64_t context_ptr, const int64_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_position_template(entry_buf, entry_len, entry_validity, + combined_row_validity, search_data, + search_data_valid, loop_var, + validity_index_var, valid_row); +} + +int32_t array_float32_position_float32(int64_t context_ptr, const float* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, float search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_position_template(entry_buf, entry_len, entry_validity, + combined_row_validity, search_data, + search_data_valid, loop_var, + validity_index_var, valid_row); +} + +int32_t array_float64_position_float64(int64_t context_ptr, const double* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, double search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_row) { + return array_position_template(entry_buf, entry_len, entry_validity, + combined_row_validity, search_data, + search_data_valid, loop_var, + validity_index_var, valid_row); +} int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf, @@ -278,6 +412,125 @@ arrow::Status ExportedArrayFunctions::AddMappings(Engine* engine) const { engine->AddGlobalMappingForFunc("array_float64_contains_float64", types->i1_type() /*return_type*/, args, reinterpret_cast(array_float64_contains_float64)); + + // array_length / cardinality + args = {types->i64_type(), // int64_t execution_context + types->i64_ptr_type(), // int32_t* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + + engine->AddGlobalMappingForFunc("array_int32_length", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_int32_length)); + + args = {types->i64_type(), // int64_t execution_context + types->i64_ptr_type(), // int64_t* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + + engine->AddGlobalMappingForFunc("array_int64_length", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_int64_length)); + + args = {types->i64_type(), // int64_t execution_context + types->float_ptr_type(), // float* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + + engine->AddGlobalMappingForFunc("array_float32_length", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_float32_length)); + + args = {types->i64_type(), // int64_t execution_context + types->double_ptr_type(), // double* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + + engine->AddGlobalMappingForFunc("array_float64_length", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_float64_length)); + + // array_position + args = {types->i64_type(), // int64_t execution_context + types->i64_ptr_type(), // int32_t* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i32_type(), // search value + types->i1_type(), // search value validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + engine->AddGlobalMappingForFunc("array_int32_position_int32", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_int32_position_int32)); + + args = {types->i64_type(), // int64_t execution_context + types->i64_ptr_type(), // int64_t* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->i64_type(), // search value + types->i1_type(), // search value validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + engine->AddGlobalMappingForFunc("array_int64_position_int64", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_int64_position_int64)); + + args = {types->i64_type(), // int64_t execution_context + types->float_ptr_type(), // float* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->float_type(), // search value + types->i1_type(), // search value validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + engine->AddGlobalMappingForFunc("array_float32_position_float32", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_float32_position_float32)); + + args = {types->i64_type(), // int64_t execution_context + types->double_ptr_type(), // double* data ptr + types->i32_type(), // int32_t data length + types->i32_ptr_type(), // input validity buffer + types->i1_type(), // bool input row validity + types->double_type(), // search value + types->i1_type(), // search value validity + types->i64_type(), // in loop var + types->i64_type(), // in validity_index_var + types->i1_ptr_type() // output validity + }; + engine->AddGlobalMappingForFunc("array_float64_position_float64", types->i32_type() /*return_type*/, + args, + reinterpret_cast(array_float64_position_float64)); + //Array remove. args = {types->i64_type(), // int64_t execution_context types->i32_ptr_type(), // int8_t* input data ptr diff --git a/cpp/src/gandiva/array_ops.h b/cpp/src/gandiva/array_ops.h index c0de72a39472..481eababe015 100644 --- a/cpp/src/gandiva/array_ops.h +++ b/cpp/src/gandiva/array_ops.h @@ -55,6 +55,60 @@ bool array_float64_contains_float64(int64_t context_ptr, const double* entry_buf int64_t loop_var, int64_t validity_index_var, bool* valid_buf); +// array_length / cardinality +GANDIVA_EXPORT +int32_t array_int32_length(int64_t context_ptr, const int32_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_int64_length(int64_t context_ptr, const int64_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_float32_length(int64_t context_ptr, const float* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_float64_length(int64_t context_ptr, const double* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +// array_position +GANDIVA_EXPORT +int32_t array_int32_position_int32(int64_t context_ptr, const int32_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int32_t search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_int64_position_int64(int64_t context_ptr, const int64_t* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, int64_t search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_float32_position_float32(int64_t context_ptr, const float* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, float search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + +GANDIVA_EXPORT +int32_t array_float64_position_float64(int64_t context_ptr, const double* entry_buf, + int32_t entry_len, const int32_t* entry_validity, + bool combined_row_validity, double search_data, + bool search_data_valid, int64_t loop_var, + int64_t validity_index_var, bool* valid_buf); + GANDIVA_EXPORT int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf, int32_t entry_len, const int32_t* entry_validity, bool combined_row_validity, diff --git a/cpp/src/gandiva/function_holder_maker_registry.cc b/cpp/src/gandiva/function_holder_maker_registry.cc index 2d9657489670..b403aaf96bdf 100644 --- a/cpp/src/gandiva/function_holder_maker_registry.cc +++ b/cpp/src/gandiva/function_holder_maker_registry.cc @@ -25,6 +25,7 @@ #include "gandiva/random_generator_holder.h" #include "gandiva/regex_functions_holder.h" #include "gandiva/to_date_holder.h" +#include "gandiva/parse_timestamp_holder.h" namespace gandiva { @@ -60,6 +61,7 @@ FunctionHolderMakerRegistry::MakerMap FunctionHolderMakerRegistry::DefaultHolder {"like", HolderMaker}, {"ilike", HolderMaker}, {"to_date", HolderMaker}, + {"parse_timestamp", HolderMaker}, {"random", HolderMaker}, {"rand", HolderMaker}, {"regexp_replace", HolderMaker}, diff --git a/cpp/src/gandiva/function_registry_arithmetic.cc b/cpp/src/gandiva/function_registry_arithmetic.cc index b2579eee0740..c99c0da65513 100644 --- a/cpp/src/gandiva/function_registry_arithmetic.cc +++ b/cpp/src/gandiva/function_registry_arithmetic.cc @@ -226,7 +226,15 @@ std::vector GetArithmeticFunctionRegistry() { // binary representation of integer values UNARY_UNSAFE_NULL_IF_NULL(bin, {}, int32, utf8), - UNARY_UNSAFE_NULL_IF_NULL(bin, {}, int64, utf8)}; + UNARY_UNSAFE_NULL_IF_NULL(bin, {}, int64, utf8), + + // bit shift operations + BINARY_SAFE_NULL_IF_NULL(lshift, {}, int32, int32, int32), + BINARY_SAFE_NULL_IF_NULL(lshift, {}, int64, int64, int64), + BINARY_SAFE_NULL_IF_NULL(rshift, {}, int32, int32, int32), + BINARY_SAFE_NULL_IF_NULL(rshift, {}, int64, int64, int64), + BINARY_SAFE_NULL_IF_NULL(rshiftunsigned, {}, int32, int32, int32), + BINARY_SAFE_NULL_IF_NULL(rshiftunsigned, {}, int64, int64, int64)}; return arithmetic_fn_registry_; } diff --git a/cpp/src/gandiva/function_registry_array.cc b/cpp/src/gandiva/function_registry_array.cc index 893ba6e3d2b0..353321120388 100644 --- a/cpp/src/gandiva/function_registry_array.cc +++ b/cpp/src/gandiva/function_registry_array.cc @@ -35,6 +35,47 @@ std::vector GetArrayFunctionRegistry() { boolean(), kResultNullInternal, "array_float64_contains_float64", NativeFunction::kNeedsContext), + // array_length / cardinality + NativeFunction("array_length", {}, DataTypeVector{list(int32())}, int32(), + kResultNullInternal, "array_int32_length", + NativeFunction::kNeedsContext), + NativeFunction("array_length", {}, DataTypeVector{list(int64())}, int32(), + kResultNullInternal, "array_int64_length", + NativeFunction::kNeedsContext), + NativeFunction("array_length", {}, DataTypeVector{list(float32())}, int32(), + kResultNullInternal, "array_float32_length", + NativeFunction::kNeedsContext), + NativeFunction("array_length", {}, DataTypeVector{list(float64())}, int32(), + kResultNullInternal, "array_float64_length", + NativeFunction::kNeedsContext), + + NativeFunction("cardinality", {}, DataTypeVector{list(int32())}, int32(), + kResultNullInternal, "array_int32_length", + NativeFunction::kNeedsContext), + NativeFunction("cardinality", {}, DataTypeVector{list(int64())}, int32(), + kResultNullInternal, "array_int64_length", + NativeFunction::kNeedsContext), + NativeFunction("cardinality", {}, DataTypeVector{list(float32())}, int32(), + kResultNullInternal, "array_float32_length", + NativeFunction::kNeedsContext), + NativeFunction("cardinality", {}, DataTypeVector{list(float64())}, int32(), + kResultNullInternal, "array_float64_length", + NativeFunction::kNeedsContext), + + // array_position + NativeFunction("array_position", {}, DataTypeVector{list(int32()), int32()}, + int32(), kResultNullInternal, "array_int32_position_int32", + NativeFunction::kNeedsContext), + NativeFunction("array_position", {}, DataTypeVector{list(int64()), int64()}, + int32(), kResultNullInternal, "array_int64_position_int64", + NativeFunction::kNeedsContext), + NativeFunction("array_position", {}, DataTypeVector{list(float32()), float32()}, + int32(), kResultNullInternal, "array_float32_position_float32", + NativeFunction::kNeedsContext), + NativeFunction("array_position", {}, DataTypeVector{list(float64()), float64()}, + int32(), kResultNullInternal, "array_float64_position_float64", + NativeFunction::kNeedsContext), + NativeFunction("array_remove", {}, DataTypeVector{list(int32()), int32()}, list(int32()), kResultNullInternal, "array_int32_remove", NativeFunction::kNeedsContext), diff --git a/cpp/src/gandiva/function_registry_datetime.cc b/cpp/src/gandiva/function_registry_datetime.cc index a2fe75c80f3f..51001d73dc87 100644 --- a/cpp/src/gandiva/function_registry_datetime.cc +++ b/cpp/src/gandiva/function_registry_datetime.cc @@ -26,7 +26,8 @@ namespace gandiva { DATE_TYPES(INNER, name##Decade, {}), DATE_TYPES(INNER, name##Year, {"year"}), \ DATE_TYPES(INNER, name##Quarter, ({"quarter"})), \ DATE_TYPES(INNER, name##Month, {"month"}), \ - DATE_TYPES(INNER, name##Week, ({"weekofyear", "yearweek"})), \ + DATE_TYPES(INNER, name##Week, ({"weekofyear"})), \ + DATE_TYPES(INNER, name##Yearweek, ({"yearweek"})), \ DATE_TYPES(INNER, name##Day, ({"day", "dayofmonth"})), \ DATE_TYPES(INNER, name##Hour, {"hour"}), \ DATE_TYPES(INNER, name##Minute, {"minute"}), \ @@ -51,6 +52,15 @@ std::vector GetDateTimeFunctionRegistry() { static std::vector date_time_fn_registry_ = { UNARY_SAFE_NULL_NEVER_BOOL(isnull, {}, day_time_interval), UNARY_SAFE_NULL_NEVER_BOOL(isnull, {}, month_interval), + // Non-deterministic current time functions (evaluated at execution time). + NativeFunction("current_date", {}, DataTypeVector{}, date64(), kResultNullNever, + "current_date"), + + NativeFunction("current_time", {}, DataTypeVector{}, time32(), kResultNullNever, + "current_time"), + + NativeFunction("current_timestamp", {"now"}, DataTypeVector{}, timestamp(), + kResultNullNever, "current_timestamp"), DATE_EXTRACTION_TRUNCATION_FNS(EXTRACT_SAFE_NULL_IF_NULL, extract), DATE_EXTRACTION_TRUNCATION_FNS(TRUNCATE_SAFE_NULL_IF_NULL, date_trunc_), @@ -62,6 +72,10 @@ std::vector GetDateTimeFunctionRegistry() { NEXT_DAY_FNS(next_day), + NativeFunction("date_trunc", {}, DataTypeVector{utf8(), timestamp()}, timestamp(), + kResultNullIfNull, "date_trunc_utf8_timestamp", + NativeFunction::kNeedsContext), + NativeFunction("castDATE", {}, DataTypeVector{utf8()}, date64(), kResultNullIfNull, "castDATE_utf8", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), @@ -85,6 +99,12 @@ std::vector GetDateTimeFunctionRegistry() { NativeFunction::kNeedsContext | NativeFunction::kNeedsFunctionHolder | NativeFunction::kCanReturnErrors), + + NativeFunction("parse_timestamp", {}, DataTypeVector{utf8(), utf8()}, timestamp(), + kResultNullInternal, "gdv_fn_parse_timestamp_utf8_utf8", + NativeFunction::kNeedsContext | + NativeFunction::kNeedsFunctionHolder | + NativeFunction::kCanReturnErrors), NativeFunction("castTIMESTAMP", {}, DataTypeVector{date64()}, timestamp(), kResultNullIfNull, "castTIMESTAMP_date64"), @@ -171,6 +191,18 @@ std::vector GetDateTimeFunctionRegistry() { timestamp(), kResultNullIfNull, "from_utc_timezone_timestamp", NativeFunction::kNeedsContext), + // convert_timezone(timestamp, to_tz) + NativeFunction("convert_timezone", {}, DataTypeVector{timestamp(), utf8()}, + timestamp(), kResultNullIfNull, + "convert_timezone_timestamp_utf8", + NativeFunction::kNeedsContext), + + // convert_timezone(timestamp, from_tz, to_tz) + NativeFunction("convert_timezone", {}, + DataTypeVector{timestamp(), utf8(), utf8()}, timestamp(), + kResultNullIfNull, "convert_timezone_timestamp_utf8_utf8", + NativeFunction::kNeedsContext), + DATE_TYPES(LAST_DAY_SAFE_NULL_IF_NULL, last_day, {}), BASE_NUMERIC_TYPES(TO_TIME_SAFE_NULL_IF_NULL, to_time, {}), BASE_NUMERIC_TYPES(TO_TIMESTAMP_SAFE_NULL_IF_NULL, to_timestamp, {})}; diff --git a/cpp/src/gandiva/function_registry_math_ops.cc b/cpp/src/gandiva/function_registry_math_ops.cc index 232c7c532600..94c0045608ef 100644 --- a/cpp/src/gandiva/function_registry_math_ops.cc +++ b/cpp/src/gandiva/function_registry_math_ops.cc @@ -103,7 +103,16 @@ std::vector GetMathOpsFunctionRegistry() { "gdv_fn_random", NativeFunction::kNeedsFunctionHolder), NativeFunction("random", {"rand"}, DataTypeVector{int32()}, float64(), kResultNullNever, "gdv_fn_random_with_seed", - NativeFunction::kNeedsFunctionHolder)}; + NativeFunction::kNeedsFunctionHolder), + + // Mathematical constant pi + NativeFunction("pi", {}, DataTypeVector{}, float64(), kResultNullNever, "pi"), + + // Random integer functions + NativeFunction("rand_integer", {}, DataTypeVector{int32()}, int32(), + kResultNullNever, "rand_integer_int32"), + NativeFunction("rand_integer", {}, DataTypeVector{int32(), int32()}, int32(), + kResultNullNever, "rand_integer_int32_int32")}; return math_fn_registry_; } diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index 2bc6936d77b3..f27a2d233389 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -99,6 +99,10 @@ std::vector GetStringFunctionRegistry() { "gdv_fn_initcap_utf8", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + NativeFunction("normalize_string", {}, DataTypeVector{utf8(), utf8()}, utf8(), + kResultNullIfNull, "gdv_fn_normalize_string_utf8_utf8", + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + NativeFunction("concat_ws", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(), kResultNullInternal, "concat_ws_utf8_utf8", NativeFunction::kNeedsContext), @@ -431,9 +435,20 @@ std::vector GetStringFunctionRegistry() { kResultNullIfNull, "replace_utf8_utf8_utf8", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + // Convert UTF8 string to binary. Note: binary_string(utf8)->binary supports + // decoding of hex-escaped sequences (e.g. "\\x41" -> 0x41). NativeFunction("binary_string", {}, DataTypeVector{utf8()}, binary(), kResultNullIfNull, "binary_string", NativeFunction::kNeedsContext), + // Pure byte reinterpretation from UTF8 to binary (no decoding). + NativeFunction("string_binary", {}, DataTypeVector{utf8()}, binary(), + kResultNullIfNull, "string_binary", NativeFunction::kNeedsContext), + + // Pure byte reinterpretation from binary to UTF8 (no validation). + NativeFunction("binary_string", {}, DataTypeVector{binary()}, utf8(), + kResultNullIfNull, "binary_string_binary", + NativeFunction::kNeedsContext), + NativeFunction("left", {}, DataTypeVector{utf8(), int32()}, utf8(), kResultNullIfNull, "left_utf8_int32", NativeFunction::kNeedsContext), @@ -494,6 +509,10 @@ std::vector GetStringFunctionRegistry() { kResultNullIfNull, "split_part", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), + NativeFunction("parse_url", {}, DataTypeVector{utf8(), utf8()}, utf8(), + kResultNullInternal, "parse_url_utf8_utf8", + NativeFunction::kNeedsContext), + NativeFunction("aes_encrypt", {}, DataTypeVector{utf8(), utf8()}, utf8(), kResultNullIfNull, "gdv_fn_aes_encrypt", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), @@ -513,6 +532,9 @@ std::vector GetStringFunctionRegistry() { NativeFunction("instr", {}, DataTypeVector{utf8(), utf8()}, int32(), kResultNullIfNull, "instr_utf8"), + NativeFunction("find_in_set", {}, DataTypeVector{utf8(), utf8()}, int32(), + kResultNullIfNull, "find_in_set_utf8_utf8"), + NativeFunction("translate", {}, DataTypeVector{utf8(), utf8(), utf8()}, utf8(), kResultNullIfNull, "translate_utf8_utf8_utf8", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), @@ -541,7 +563,10 @@ std::vector GetStringFunctionRegistry() { kResultNullIfNull, "mask_utf8_utf8", NativeFunction::kNeedsContext), NativeFunction("mask", {}, DataTypeVector{utf8()}, utf8(), kResultNullIfNull, - "mask_utf8", NativeFunction::kNeedsContext)}; + "mask_utf8", NativeFunction::kNeedsContext), + + NativeFunction("uuid", {}, DataTypeVector{}, utf8(), kResultNullNever, "uuid", + NativeFunction::kNeedsContext)}; return string_fn_registry_; } diff --git a/cpp/src/gandiva/function_registry_timestamp_arithmetic.cc b/cpp/src/gandiva/function_registry_timestamp_arithmetic.cc index 188bd60d9eac..368312c715ed 100644 --- a/cpp/src/gandiva/function_registry_timestamp_arithmetic.cc +++ b/cpp/src/gandiva/function_registry_timestamp_arithmetic.cc @@ -50,6 +50,11 @@ namespace gandiva { BINARY_GENERIC_SAFE_NULL_IF_NULL(name, ALIASES, date64, int64, date64), \ BINARY_GENERIC_SAFE_NULL_IF_NULL(name, ALIASES, timestamp, int64, timestamp) + +// Convenience timestamp arithmetic with fixed units. +// Signature matches: (timestamp, int) +#define TIMESTAMP_ADD_FIXED_UNIT_FN(name, ALIASES) BINARY_GENERIC_SAFE_NULL_IF_NULL(name, ALIASES, timestamp, int32, timestamp), BINARY_GENERIC_SAFE_NULL_IF_NULL(name, ALIASES, timestamp, int64, timestamp) + std::vector GetDateTimeArithmeticFunctionRegistry() { static std::vector datetime_fn_registry_ = { BINARY_GENERIC_SAFE_NULL_IF_NULL(months_between, {}, date64, date64, float64), @@ -76,6 +81,13 @@ std::vector GetDateTimeArithmeticFunctionRegistry() { DATE_ADD_FNS(date_add, {}), DATE_ADD_FNS(add, {}), + // add_(timestamp, int) + TIMESTAMP_ADD_FIXED_UNIT_FN(add_days, {}), + TIMESTAMP_ADD_FIXED_UNIT_FN(add_hours, {}), + TIMESTAMP_ADD_FIXED_UNIT_FN(add_minutes, {}), + TIMESTAMP_ADD_FIXED_UNIT_FN(add_seconds, {}), + + NativeFunction("add", {}, DataTypeVector{date64(), int64()}, timestamp(), kResultNullIfNull, "add_date64_int64"), diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index ac967659ae31..59c559f4ff04 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -35,6 +35,7 @@ #include "gandiva/interval_holder.h" #include "gandiva/random_generator_holder.h" #include "gandiva/to_date_holder.h" +#include "gandiva/parse_timestamp_holder.h" extern "C" { @@ -815,6 +816,18 @@ int64_t gdv_fn_to_date_utf8_utf8_int32(int64_t context_ptr, int64_t holder_ptr, return (*holder)(context, data, data_len, in1_validity, out_valid); } +int64_t gdv_fn_parse_timestamp_utf8_utf8(int64_t context_ptr, int64_t holder_ptr, + const char* data, int data_len, + bool in1_validity, const char* pattern, + int pattern_len, bool in2_validity, + bool* out_valid) { + gandiva::ExecutionContext* context = + reinterpret_cast(context_ptr); + gandiva::ParseTimestampHolder* holder = + reinterpret_cast(holder_ptr); + return (*holder)(context, data, data_len, in1_validity, out_valid); +} + int64_t gdv_fn_cast_intervalday_utf8(int64_t context_ptr, int64_t holder_ptr, const char* data, int data_len, bool in1_validity, bool* out_valid) { @@ -889,6 +902,44 @@ gdv_timestamp from_utc_timezone_timestamp(gdv_int64 context, } } +GANDIVA_EXPORT +gdv_timestamp convert_timezone_timestamp_utf8(int64_t context, + gdv_timestamp time_milliseconds, + const char* to_timezone, + int32_t to_length) { + // Alias for converting from UTC to a target timezone. + return from_utc_timezone_timestamp(context, time_milliseconds, to_timezone, to_length); +} + +GANDIVA_EXPORT +gdv_timestamp convert_timezone_timestamp_utf8_utf8( + int64_t context, gdv_timestamp time_milliseconds, const char* from_timezone, + int32_t from_length, const char* to_timezone, int32_t to_length) { + using arrow_vendored::date::locate_zone; + using arrow_vendored::date::sys_time; + using std::chrono::milliseconds; + + const sys_time tp_local{milliseconds{time_milliseconds}}; + + try { + const auto from_tz = locate_zone(std::string(from_timezone, from_length)); + const gdv_timestamp from_offset_ms = + from_tz->get_info(tp_local).offset.count() * 1000; + + const gdv_timestamp utc_time_milliseconds = + time_milliseconds - static_cast(from_offset_ms); + + // Interpret the resulting value as a UTC timestamp and convert to the target timezone. + return from_utc_timezone_timestamp(context, utc_time_milliseconds, to_timezone, + to_length); + } catch (...) { + std::string e_msg = + std::string(from_timezone, from_length) + " is an invalid time zone name."; + gdv_fn_context_set_error_msg(context, e_msg.c_str()); + return 0; + } +} + GANDIVA_EXPORT const char* gdv_mask_show_first_n_utf8_int32(int64_t context, const char* data, int32_t data_len, int32_t n_to_show, @@ -1220,6 +1271,21 @@ arrow::Status ExportedStubFunctions::AddMappings(Engine* engine) const { "gdv_fn_to_date_utf8_utf8_int32", types->i64_type() /*return_type*/, args, reinterpret_cast(gdv_fn_to_date_utf8_utf8_int32)); + // gdv_fn_parse_timestamp_utf8_utf8 + args = {types->i64_type(), // int64_t execution_context + types->i64_type(), // int64_t holder_ptr + types->i8_ptr_type(), // const char* data + types->i32_type(), // int data_len + types->i1_type(), // bool in1_validity + types->i8_ptr_type(), // const char* pattern + types->i32_type(), // int pattern_len + types->i1_type(), // bool in2_validity + types->ptr_type(types->i8_type())}; // bool* out_valid + + engine->AddGlobalMappingForFunc( + "gdv_fn_parse_timestamp_utf8_utf8", types->i64_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_parse_timestamp_utf8_utf8)); + // gdv_fn_cast_intervalday_utf8 args = { types->i64_type(), // context diff --git a/cpp/src/gandiva/gdv_function_stubs.h b/cpp/src/gandiva/gdv_function_stubs.h index 4113f261ad76..71f7983823b1 100644 --- a/cpp/src/gandiva/gdv_function_stubs.h +++ b/cpp/src/gandiva/gdv_function_stubs.h @@ -74,6 +74,11 @@ int64_t gdv_fn_to_date_utf8_utf8_int32(int64_t context, int64_t ptr, const char* bool in2_validity, int32_t suppress_errors, bool in3_validity, bool* out_valid); +int64_t gdv_fn_parse_timestamp_utf8_utf8(int64_t context, int64_t ptr, const char* data, + int data_len, bool in1_validity, + const char* pattern, int pattern_len, + bool in2_validity, bool* out_valid); + GANDIVA_EXPORT void gdv_fn_context_set_error_msg(int64_t context_ptr, const char* err_msg); @@ -350,6 +355,19 @@ gdv_timestamp from_utc_timezone_timestamp(int64_t context, gdv_timestamp time_milliseconds, const char* timezone, int32_t length); +// convert_timezone(timestamp, to_tz) : convert from UTC to the target timezone. +GANDIVA_EXPORT +gdv_timestamp convert_timezone_timestamp_utf8(int64_t context, + gdv_timestamp time_milliseconds, + const char* to_timezone, + int32_t to_length); + +// convert_timezone(timestamp, from_tz, to_tz) : convert between timezones. +GANDIVA_EXPORT +gdv_timestamp convert_timezone_timestamp_utf8_utf8( + int64_t context, gdv_timestamp time_milliseconds, const char* from_timezone, + int32_t from_length, const char* to_timezone, int32_t to_length); + GANDIVA_EXPORT const char* gdv_mask_show_first_n_utf8_int32(int64_t context, const char* data, int32_t data_len, int32_t n_to_show, diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index 134f7dcd27dd..ff6604dad67b 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -747,6 +747,67 @@ TEST(TestGdvFnStubs, TestInitCap) { ctx.Reset(); } +TEST(TestGdvFnStubs, TestNormalizeString) { + gandiva::ExecutionContext ctx; + uint64_t ctx_ptr = reinterpret_cast(&ctx); + gdv_int32 out_len = 0; + + // NFC: compose (e + combining acute) -> é + { + const std::string in = u8"e\u0301"; + const char* out_str = gdv_fn_normalize_string_utf8_utf8( + ctx_ptr, in.data(), static_cast(in.size()), "NFC", 3, &out_len); + + if (ctx.has_error() && ctx.get_error().find("ICU is required") != std::string::npos) { + GTEST_SKIP() << ctx.get_error(); + } + + EXPECT_EQ(std::string(out_str, out_len), u8"\u00E9"); + EXPECT_FALSE(ctx.has_error()); + } + + // NFD: decompose é -> e + combining acute + ctx.Reset(); + { + const std::string in = u8"\u00E9"; + const char* out_str = gdv_fn_normalize_string_utf8_utf8( + ctx_ptr, in.data(), static_cast(in.size()), "NFD", 3, &out_len); + EXPECT_EQ(std::string(out_str, out_len), u8"e\u0301"); + EXPECT_FALSE(ctx.has_error()); + } + + // NFKC: compatibility composition (① -> 1) + ctx.Reset(); + { + const std::string in = u8"\u2460"; + const char* out_str = gdv_fn_normalize_string_utf8_utf8( + ctx_ptr, in.data(), static_cast(in.size()), "NFKC", 4, &out_len); + EXPECT_EQ(std::string(out_str, out_len), "1"); + EXPECT_FALSE(ctx.has_error()); + } + + // NFKD: compatibility decomposition (ff -> ff) + ctx.Reset(); + { + const std::string in = u8"\uFB00"; + const char* out_str = gdv_fn_normalize_string_utf8_utf8( + ctx_ptr, in.data(), static_cast(in.size()), "NFKD", 4, &out_len); + EXPECT_EQ(std::string(out_str, out_len), "ff"); + EXPECT_FALSE(ctx.has_error()); + } + + // Invalid form should set an error + ctx.Reset(); + { + const std::string in = "abc"; + const char* out_str = gdv_fn_normalize_string_utf8_utf8( + ctx_ptr, in.data(), static_cast(in.size()), "BAD", 3, &out_len); + EXPECT_EQ(std::string(out_str, out_len), ""); + EXPECT_TRUE(ctx.has_error()); + EXPECT_THAT(ctx.get_error(), ::testing::HasSubstr("invalid normalization form")); + } +} + TEST(TestGdvFnStubs, TestCastVarbinaryINT) { gandiva::ExecutionContext ctx; @@ -1140,6 +1201,53 @@ TEST(TestGdvFnStubs, TestFromUtcTimezone) { EXPECT_THAT(context.get_error(), "India is an invalid time zone name."); } +TEST(TestGdvFnStubs, TestConvertTimezoneUtcToTz) { + ExecutionContext context; + auto context_ptr = reinterpret_cast(&context); + auto len_ist = static_cast(strlen("Asia/Kolkata")); + + // ts: 1970-01-01 10:00:00 (UTC) + // ts2:1970-01-01 15:30:00 (Asia/Kolkata) + gdv_timestamp ts = 36000000; + gdv_timestamp ts2 = + convert_timezone_timestamp_utf8(context_ptr, ts, "Asia/Kolkata", len_ist); + EXPECT_EQ(ts2, 55800000); + + // Failure case + ts2 = convert_timezone_timestamp_utf8(context_ptr, ts, "India", 5); + EXPECT_THAT(context.get_error(), "India is an invalid time zone name."); +} + +TEST(TestGdvFnStubs, TestConvertTimezoneBetweenZones) { + ExecutionContext context; + auto context_ptr = reinterpret_cast(&context); + auto len_ist = static_cast(strlen("Asia/Kolkata")); + auto len_pst = static_cast(strlen("America/Los_Angeles")); + + // Asia/Kolkata -> America/Los_Angeles + // ts: 2012-02-28 15:30:00 (Asia/Kolkata) + // UTC: 2012-02-28 10:00:00 + // ts2:2012-02-28 02:00:00 (America/Los_Angeles) + gdv_timestamp ts = 1330443000000; + gdv_timestamp ts2 = convert_timezone_timestamp_utf8_utf8( + context_ptr, ts, "Asia/Kolkata", len_ist, "America/Los_Angeles", len_pst); + EXPECT_EQ(ts2, 1330394400000); + + // DST boundary check: America/Los_Angeles -> Asia/Kolkata + // ts: 2018-03-11 01:00:00 (America/Los_Angeles) + // UTC: 2018-03-11 09:00:00 + // ts2:2018-03-11 14:30:00 (Asia/Kolkata) + ts = 1520730000000; + ts2 = convert_timezone_timestamp_utf8_utf8(context_ptr, ts, "America/Los_Angeles", + len_pst, "Asia/Kolkata", len_ist); + EXPECT_EQ(ts2, 1520778600000); + + // Failure case: invalid from_tz + ts2 = convert_timezone_timestamp_utf8_utf8(context_ptr, ts, "America/LA", 10, + "Asia/Kolkata", len_ist); + EXPECT_THAT(context.get_error(), "America/LA is an invalid time zone name."); +} + TEST(TestGdvFnStubs, TestShowFirstN) { gandiva::ExecutionContext ctx; int64_t ctx_ptr = reinterpret_cast(&ctx); diff --git a/cpp/src/gandiva/gdv_string_function_stubs.cc b/cpp/src/gandiva/gdv_string_function_stubs.cc index 17eefbe22e31..ca1900a8a3a1 100644 --- a/cpp/src/gandiva/gdv_string_function_stubs.cc +++ b/cpp/src/gandiva/gdv_string_function_stubs.cc @@ -18,6 +18,18 @@ #include "gandiva/gdv_function_stubs.h" #include + +#ifdef GANDIVA_WITH_ICU +#include +#include +#include +#endif + +#include +#include +#include +#include +#include #include #include #include @@ -539,6 +551,138 @@ const char* gdv_fn_initcap_utf8(int64_t context, const char* data, int32_t data_ *out_len = out_idx; return out; } + +GANDIVA_EXPORT +const char* gdv_fn_normalize_string_utf8_utf8(int64_t context, const char* data, + int32_t data_len, const char* form, + int32_t form_len, int32_t* out_len) { + if (data_len == 0) { + *out_len = 0; + return ""; + } + +#ifndef GANDIVA_WITH_ICU + gdv_fn_context_set_error_msg(context, + "ICU is required for normalize_string but was not " + "available at build time"); + *out_len = 0; + return ""; +#else + std::string_view form_sv(form, static_cast(form_len)); + + // Accept common case variants. + std::string form_upper; + form_upper.reserve(form_sv.size()); + for (char c : form_sv) { + if (c >= 'a' && c <= 'z') { + form_upper.push_back(static_cast(c - 32)); + } else { + form_upper.push_back(c); + } + } + + UErrorCode status = U_ZERO_ERROR; + const UNorm2* normalizer = nullptr; + if (form_upper == "NFC") { + normalizer = unorm2_getNFCInstance(&status); + } else if (form_upper == "NFD") { + normalizer = unorm2_getNFDInstance(&status); + } else if (form_upper == "NFKC") { + normalizer = unorm2_getNFKCInstance(&status); + } else if (form_upper == "NFKD") { + normalizer = unorm2_getNFKDInstance(&status); + } else { + gdv_fn_context_set_error_msg( + context, + "normalize_string(): invalid normalization form (expected NFC, NFD, NFKC, or " + "NFKD)"); + *out_len = 0; + return ""; + } + + if (U_FAILURE(status) || normalizer == nullptr) { + gdv_fn_context_set_error_msg(context, + "normalize_string(): failed to get ICU normalizer"); + *out_len = 0; + return ""; + } + + // Convert UTF-8 -> UTF-16 + status = U_ZERO_ERROR; + int32_t u16_len = 0; + u_strFromUTF8(nullptr, 0, &u16_len, data, data_len, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, "normalize_string(): invalid UTF-8 input"); + *out_len = 0; + return ""; + } + status = U_ZERO_ERROR; + std::vector u16(static_cast(u16_len) + 1); + u_strFromUTF8(u16.data(), u16_len + 1, &u16_len, data, data_len, &status); + if (U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, "normalize_string(): invalid UTF-8 input"); + *out_len = 0; + return ""; + } + + // Normalize UTF-16 + status = U_ZERO_ERROR; + int32_t norm_u16_len = + unorm2_normalize(normalizer, u16.data(), u16_len, nullptr, 0, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, "normalize_string(): ICU normalization failed"); + *out_len = 0; + return ""; + } + + status = U_ZERO_ERROR; + std::vector norm_u16(static_cast(norm_u16_len) + 1); + norm_u16_len = unorm2_normalize(normalizer, u16.data(), u16_len, norm_u16.data(), + norm_u16_len + 1, &status); + if (U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, "normalize_string(): ICU normalization failed"); + *out_len = 0; + return ""; + } + + // Convert UTF-16 -> UTF-8 + status = U_ZERO_ERROR; + int32_t out_utf8_len = 0; + u_strToUTF8(nullptr, 0, &out_utf8_len, norm_u16.data(), norm_u16_len, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, + "normalize_string(): UTF-16 to UTF-8 conversion failed"); + *out_len = 0; + return ""; + } + + if (out_utf8_len == 0) { + *out_len = 0; + return ""; + } + + char* out = + reinterpret_cast(gdv_fn_context_arena_malloc(context, out_utf8_len)); + if (out == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_len = 0; + return ""; + } + + status = U_ZERO_ERROR; + u_strToUTF8(out, out_utf8_len, &out_utf8_len, norm_u16.data(), norm_u16_len, &status); + if (U_FAILURE(status)) { + gdv_fn_context_set_error_msg(context, + "normalize_string(): UTF-16 to UTF-8 conversion failed"); + *out_len = 0; + return ""; + } + + *out_len = out_utf8_len; + return out; +#endif +} + GANDIVA_EXPORT const char* translate_utf8_utf8_utf8(int64_t context, const char* in, int32_t in_len, const char* from, int32_t from_len, const char* to, @@ -755,6 +899,96 @@ const char* translate_utf8_utf8_utf8(int64_t context, const char* in, int32_t in *out_len = result_len; return result; } + +// format_number(number, decimal_places) +GANDIVA_EXPORT +const char* gdv_fn_format_number_float64_int32(int64_t context, gdv_float64 value, + gdv_int32 decimal_places, + gdv_int32* out_len) { + if (out_len == nullptr) { + gdv_fn_context_set_error_msg(context, "Invalid output length pointer"); + return ""; + } + + if (decimal_places < 0) { + gdv_fn_context_set_error_msg(context, "Decimal places cannot be negative"); + *out_len = 0; + return ""; + } + + // Doubles have limited precision; also prevents pathological allocations. + if (decimal_places > 18) { + gdv_fn_context_set_error_msg(context, "Decimal places is too large"); + *out_len = 0; + return ""; + } + + // Match typical formatting conventions for NaN/Infinity. + if (!std::isfinite(value)) { + const char* s = std::isnan(value) ? "NaN" : (value < 0 ? "-Infinity" : "Infinity"); + *out_len = static_cast(strlen(s)); + char* out = reinterpret_cast(gdv_fn_context_arena_malloc(context, *out_len)); + if (out == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_len = 0; + return ""; + } + memcpy(out, s, *out_len); + return out; + } + + std::ostringstream ss; + ss.imbue(std::locale::classic()); + ss.setf(std::ios::fixed); + ss << std::setprecision(decimal_places) << value; + const std::string s = ss.str(); // [-]digits[.digits] + + const bool negative = !s.empty() && s[0] == '-'; + const size_t digits_begin = negative ? 1 : 0; + + const size_t dot_pos = s.find('.'); + std::string int_part = (dot_pos == std::string::npos) + ? s.substr(digits_begin) + : s.substr(digits_begin, dot_pos - digits_begin); + std::string frac_part = (dot_pos == std::string::npos) ? "" : s.substr(dot_pos + 1); + + // Insert commas. + std::string grouped_int; + grouped_int.reserve(int_part.size() + int_part.size() / 3); + const int digits = static_cast(int_part.size()); + for (int i = 0; i < digits; ++i) { + if (i > 0 && ((digits - i) % 3 == 0)) { + grouped_int.push_back(','); + } + grouped_int.push_back(int_part[static_cast(i)]); + } + + std::string out_str; + out_str.reserve(s.size() + s.size() / 3); + if (negative) out_str.push_back('-'); + out_str.append(grouped_int); + + if (decimal_places > 0) { + out_str.push_back('.'); + if (static_cast(frac_part.size()) < decimal_places) { + frac_part.append(static_cast(decimal_places - frac_part.size()), '0'); + } else if (static_cast(frac_part.size()) > decimal_places) { + frac_part.resize(static_cast(decimal_places)); + } + out_str.append(frac_part); + } + + *out_len = static_cast(out_str.size()); + char* out = reinterpret_cast(gdv_fn_context_arena_malloc(context, *out_len)); + if (out == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_len = 0; + return ""; + } + + memcpy(out, out_str.data(), *out_len); + return out; +} } namespace gandiva { @@ -971,6 +1205,20 @@ arrow::Status ExportedStringFunctions::AddMappings(Engine* engine) const { types->i8_ptr_type() /*return_type*/, args, reinterpret_cast(gdv_fn_initcap_utf8)); + // gdv_fn_normalize_string_utf8_utf8 + args = { + types->i64_type(), // context + types->i8_ptr_type(), // data + types->i32_type(), // data_len + types->i8_ptr_type(), // form + types->i32_type(), // form_len + types->i32_ptr_type(), // out_len + }; + + engine->AddGlobalMappingForFunc( + "gdv_fn_normalize_string_utf8_utf8", types->i8_ptr_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_normalize_string_utf8_utf8)); + // translate_utf8_utf8_utf8 args = { types->i64_type(), // context @@ -986,6 +1234,18 @@ arrow::Status ExportedStringFunctions::AddMappings(Engine* engine) const { engine->AddGlobalMappingForFunc("translate_utf8_utf8_utf8", types->i8_ptr_type() /*return_type*/, args, reinterpret_cast(translate_utf8_utf8_utf8)); + + // gdv_fn_format_number_float64_int32 + args = { + types->i64_type(), // context + types->i64_type(), // double value (passed as i64) + types->i32_type(), // decimal_places + types->i32_ptr_type() // out_length + }; + + engine->AddGlobalMappingForFunc("gdv_fn_format_number_float64_int32", + types->i8_ptr_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_format_number_float64_int32)); return arrow::Status::OK(); } } // namespace gandiva diff --git a/cpp/src/gandiva/parse_timestamp_holder.cc b/cpp/src/gandiva/parse_timestamp_holder.cc new file mode 100644 index 000000000000..ea6de7da9e04 --- /dev/null +++ b/cpp/src/gandiva/parse_timestamp_holder.cc @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/parse_timestamp_holder.h" + +#include + +#include "arrow/util/value_parsing.h" +#include "gandiva/date_utils.h" +#include "gandiva/execution_context.h" +#include "gandiva/node.h" + +namespace gandiva { + +Result> ParseTimestampHolder::Make( + const FunctionNode& node) { + if (node.children().size() != 2) { + return Status::Invalid("'parse_timestamp' function requires two parameters"); + } + + auto literal_pattern = dynamic_cast(node.children().at(1).get()); + if (literal_pattern == nullptr) { + return Status::Invalid( + "'parse_timestamp' function requires a literal as the second parameter"); + } + + auto literal_type = literal_pattern->return_type()->id(); + if (literal_type != arrow::Type::STRING && literal_type != arrow::Type::BINARY) { + return Status::Invalid( + "'parse_timestamp' function requires a string literal as the second parameter"); + } + + auto pattern = std::get(literal_pattern->holder()); + return Make(pattern); +} + +Result> ParseTimestampHolder::Make( + const std::string& sql_pattern) { + std::shared_ptr transformed_pattern; + ARROW_RETURN_NOT_OK(DateUtils::ToInternalFormat(sql_pattern, &transformed_pattern)); + return std::shared_ptr( + new ParseTimestampHolder(*transformed_pattern)); +} + +int64_t ParseTimestampHolder::operator()(ExecutionContext* context, const char* data, + int data_len, bool in_valid, + bool* out_valid) { + *out_valid = false; + if (!in_valid) { + return 0; + } + + int64_t millis_since_epoch = 0; + if (!::arrow::internal::ParseTimestampStrptime( + data, data_len, pattern_.c_str(), + /*ignore_time_in_day=*/false, + /*allow_trailing_chars=*/true, ::arrow::TimeUnit::MILLI, + &millis_since_epoch)) { + return_error(context, data, data_len); + return 0; + } + + *out_valid = true; + return millis_since_epoch; +} + +void ParseTimestampHolder::return_error(ExecutionContext* context, const char* data, + int data_len) { + std::string err_msg = + "Error parsing value " + std::string(data, data_len) + " for given format."; + context->set_error_msg(err_msg.c_str()); +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/parse_timestamp_holder.h b/cpp/src/gandiva/parse_timestamp_holder.h new file mode 100644 index 000000000000..ee40d90e4a0e --- /dev/null +++ b/cpp/src/gandiva/parse_timestamp_holder.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "arrow/status.h" + +#include "gandiva/execution_context.h" +#include "gandiva/function_holder.h" +#include "gandiva/node.h" +#include "gandiva/visibility.h" + +namespace gandiva { + +/// Function Holder for SQL 'parse_timestamp' +class GANDIVA_EXPORT ParseTimestampHolder : public FunctionHolder { + public: + ~ParseTimestampHolder() override = default; + + static Result> Make(const FunctionNode& node); + + static Result> Make( + const std::string& sql_pattern); + + /// Parse a timestamp using the configured pattern. + int64_t operator()(ExecutionContext* context, const char* data, int data_len, + bool in_valid, bool* out_valid); + + private: + explicit ParseTimestampHolder(const std::string& pattern) : pattern_(pattern) {} + + void return_error(ExecutionContext* context, const char* data, int data_len); + + std::string pattern_; // internal format string +}; + +} // namespace gandiva diff --git a/cpp/src/gandiva/parse_timestamp_holder_test.cc b/cpp/src/gandiva/parse_timestamp_holder_test.cc new file mode 100644 index 000000000000..5e5fe6bf5787 --- /dev/null +++ b/cpp/src/gandiva/parse_timestamp_holder_test.cc @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "arrow/testing/gtest_util.h" + +#include "gandiva/execution_context.h" +#include "gandiva/parse_timestamp_holder.h" + +#include + +namespace gandiva { + +class TestParseTimestampHolder : public ::testing::Test { + protected: + ExecutionContext execution_context_; +}; + +TEST_F(TestParseTimestampHolder, TestSimpleTimestamp) { + EXPECT_OK_AND_ASSIGN(auto holder, + ParseTimestampHolder::Make("YYYY-MM-DD HH24:MI:SS")); + + auto& parse_timestamp = *holder; + bool out_valid; + + std::string s("1986-12-01 01:01:01"); + int64_t millis_since_epoch = + parse_timestamp(&execution_context_, s.data(), (int)s.length(), true, &out_valid); + EXPECT_TRUE(out_valid); + EXPECT_EQ(millis_since_epoch, 533782861000); + + // allow trailing chars (fractional seconds not present in format) + s = std::string("1986-12-01 01:01:01.11"); + millis_since_epoch = + parse_timestamp(&execution_context_, s.data(), (int)s.length(), true, &out_valid); + EXPECT_TRUE(out_valid); + EXPECT_EQ(millis_since_epoch, 533782861000); +} + +TEST_F(TestParseTimestampHolder, TestQuotedTextInFormat) { + EXPECT_OK_AND_ASSIGN(auto holder, + ParseTimestampHolder::Make("YYYY-MM-DD\"T\"HH24:MI:SS")); + + auto& parse_timestamp = *holder; + bool out_valid; + + std::string s("1986-12-01T01:01:01"); + int64_t millis_since_epoch = + parse_timestamp(&execution_context_, s.data(), (int)s.length(), true, &out_valid); + EXPECT_TRUE(out_valid); + EXPECT_EQ(millis_since_epoch, 533782861000); +} + +TEST_F(TestParseTimestampHolder, TestParseError) { + EXPECT_OK_AND_ASSIGN(auto holder, + ParseTimestampHolder::Make("YYYY-MM-DD HH24:MI:SS")); + + auto& parse_timestamp = *holder; + bool out_valid; + + std::string s("1986-01-40 01:01:01"); + int64_t millis_since_epoch = + parse_timestamp(&execution_context_, s.data(), (int)s.length(), true, &out_valid); + EXPECT_FALSE(out_valid); + EXPECT_EQ(millis_since_epoch, 0); + + std::string expected_error = + "Error parsing value 1986-01-40 01:01:01 for given format"; + EXPECT_TRUE(execution_context_.get_error().find(expected_error) != std::string::npos); + + // not valid should not return error + execution_context_.Reset(); + millis_since_epoch = + parse_timestamp(&execution_context_, "nullptr", 7, false, &out_valid); + EXPECT_EQ(millis_since_epoch, 0); + EXPECT_FALSE(execution_context_.has_error()); +} + +TEST_F(TestParseTimestampHolder, TestMakeError) { + // reject unknown patterns. + ASSERT_RAISES(Invalid, + ParseTimestampHolder::Make("YYYY-MM-DD HH24:MI:SS tzo").status()); +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/arithmetic_ops.cc b/cpp/src/gandiva/precompiled/arithmetic_ops.cc index d800b040241e..41401cbfaa41 100644 --- a/cpp/src/gandiva/precompiled/arithmetic_ops.cc +++ b/cpp/src/gandiva/precompiled/arithmetic_ops.cc @@ -542,4 +542,49 @@ SQRT(float64) #undef NUMERIC_FUNCTION #undef NUMERIC_TYPES +// Bit shift operations +// Left shift: value << bits (logical, zero-fill on right) +// To avoid C++ UB: bits < 0 or bits >= bitwidth returns 0 +#define LSHIFT(TYPE, UTYPE, BITS) \ + FORCE_INLINE \ + gdv_##TYPE lshift_##TYPE##_##TYPE(gdv_##TYPE value, gdv_##TYPE bits) { \ + if (bits < 0 || bits >= BITS) return 0; \ + return static_cast(static_cast(value) << bits); \ + } + +LSHIFT(int32, uint32_t, 32) +LSHIFT(int64, uint64_t, 64) + +#undef LSHIFT + +// Arithmetic right shift: value >> bits (sign-extends) +// bits < 0: no-op (return value) +// bits >= bitwidth: returns 0 for non-negative, -1 for negative +#define RSHIFT(TYPE, BITS) \ + FORCE_INLINE \ + gdv_##TYPE rshift_##TYPE##_##TYPE(gdv_##TYPE value, gdv_##TYPE bits) { \ + if (bits <= 0) return value; \ + if (bits >= BITS) return (value < 0) ? -1 : 0; \ + return value >> bits; \ + } + +RSHIFT(int32, 32) +RSHIFT(int64, 64) + +#undef RSHIFT + +// Logical (unsigned) right shift: value >>> bits (zero-fill on left) +// bits < 0 or bits >= bitwidth returns 0 +#define RSHIFTUNSIGNED(TYPE, UTYPE, BITS) \ + FORCE_INLINE \ + gdv_##TYPE rshiftunsigned_##TYPE##_##TYPE(gdv_##TYPE value, gdv_##TYPE bits) { \ + if (bits < 0 || bits >= BITS) return 0; \ + return static_cast(static_cast(value) >> bits); \ + } + +RSHIFTUNSIGNED(int32, uint32_t, 32) +RSHIFTUNSIGNED(int64, uint64_t, 64) + +#undef RSHIFTUNSIGNED + } // extern "C" diff --git a/cpp/src/gandiva/precompiled/extended_math_ops.cc b/cpp/src/gandiva/precompiled/extended_math_ops.cc index c03db1d5f2b1..bcb7c15228a0 100644 --- a/cpp/src/gandiva/precompiled/extended_math_ops.cc +++ b/cpp/src/gandiva/precompiled/extended_math_ops.cc @@ -25,6 +25,7 @@ extern "C" { #include +#include #include #include #include @@ -466,4 +467,28 @@ BIN_INTEGER(int64) #undef BIN_INTEGER +// Mathematical constant pi +FORCE_INLINE +gdv_float64 pi() { return M_PI; } + +// Random integer in range [min, max] (inclusive) +FORCE_INLINE +gdv_int32 rand_integer_int32_int32(gdv_int32 min_val, gdv_int32 max_val) { + static thread_local std::mt19937 rng(std::random_device{}()); + if (min_val > max_val) { + // Swap if min > max + gdv_int32 tmp = min_val; + min_val = max_val; + max_val = tmp; + } + std::uniform_int_distribution dist(min_val, max_val); + return dist(rng); +} + +// Random integer in range [0, max] (inclusive) +FORCE_INLINE +gdv_int32 rand_integer_int32(gdv_int32 max_val) { + return rand_integer_int32_int32(0, max_val); +} + } // extern "C" diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc index 3849cf7bdf9a..146a08f8495a 100644 --- a/cpp/src/gandiva/precompiled/string_ops.cc +++ b/cpp/src/gandiva/precompiled/string_ops.cc @@ -27,6 +27,7 @@ extern "C" { #include #include #include +#include #include "./types.h" @@ -2288,6 +2289,48 @@ const char* binary_string(gdv_int64 context, const char* text, gdv_int32 text_le return ret; } +FORCE_INLINE +const char* string_binary(gdv_int64 context, const char* text, gdv_int32 text_len, + gdv_int32* out_len) { + if (text_len == 0) { + *out_len = 0; + return ""; + } + + gdv_binary ret = + reinterpret_cast(gdv_fn_context_arena_malloc(context, text_len)); + if (ret == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output binary"); + *out_len = 0; + return ""; + } + + memcpy(ret, text, text_len); + *out_len = text_len; + return ret; +} + +FORCE_INLINE +const char* binary_string_binary(gdv_int64 context, const char* data, gdv_int32 data_len, + gdv_int32* out_len) { + if (data_len == 0) { + *out_len = 0; + return ""; + } + + gdv_utf8 ret = + reinterpret_cast(gdv_fn_context_arena_malloc(context, data_len)); + if (ret == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_len = 0; + return ""; + } + + memcpy(ret, data, data_len); + *out_len = data_len; + return ret; +} + #define CAST_INT_BIGINT_VARBINARY(OUT_TYPE, TYPE_NAME) \ FORCE_INLINE \ OUT_TYPE \ @@ -3013,6 +3056,288 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len, return ret; } +FORCE_INLINE +const char* gdv_substr_copy_out(gdv_int64 context, const char* src, int32_t len, + bool* out_valid, int32_t* out_len) { + if (len <= 0 || src == nullptr) { + *out_valid = true; + *out_len = 0; + return ""; + } + char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, len)); + if (ret == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_valid = false; + *out_len = 0; + return ""; + } + memcpy(ret, src, len); + *out_valid = true; + *out_len = len; + return ret; +} + +FORCE_INLINE +int32_t find_char(const char* data, int32_t data_len, char ch, int32_t start, + int32_t end) { + if (data == nullptr || data_len <= 0) { + return -1; + } + if (start < 0) start = 0; + if (end > data_len) end = data_len; + for (int32_t i = start; i < end; ++i) { + if (data[i] == ch) return i; + } + return -1; +} + +FORCE_INLINE +int32_t find_substr(const char* data, int32_t data_len, const char* needle, + int32_t needle_len, int32_t start, int32_t end) { + if (data == nullptr || needle == nullptr || data_len <= 0 || needle_len <= 0) { + return -1; + } + if (start < 0) start = 0; + if (end > data_len) end = data_len; + if (end - start < needle_len) return -1; + for (int32_t i = start; i <= end - needle_len; ++i) { + if (memcmp(data + i, needle, needle_len) == 0) { + return i; + } + } + return -1; +} + +FORCE_INLINE +const char* parse_url_utf8_utf8(gdv_int64 context, const char* url, int32_t url_len, + bool url_validity, const char* part, int32_t part_len, + bool part_validity, bool* out_valid, + int32_t* out_len) { + if (!url_validity || !part_validity) { + *out_valid = false; + *out_len = 0; + return ""; + } + if (url == nullptr || url_len <= 0 || part == nullptr || part_len <= 0) { + *out_valid = true; + *out_len = 0; + return ""; + } + + auto part_eq = [&](const char* s, int32_t s_len) { + return compare_lower_strings(s, s_len, part, part_len); + }; + + bool want_host = part_eq("host", 4); + bool want_path = part_eq("path", 4); + bool want_query = part_eq("query", 5); + bool want_ref = part_eq("ref", 3); + bool want_protocol = part_eq("protocol", 8); + bool want_file = part_eq("file", 4); + bool want_authority = part_eq("authority", 9); + bool want_userinfo = part_eq("userinfo", 8); + + if (!(want_host || want_path || want_query || want_ref || want_protocol || want_file || + want_authority || want_userinfo)) { + *out_valid = false; + *out_len = 0; + return ""; + } + + int32_t url_end = url_len; + + // Fragment (#ref) + int32_t hash_pos = find_char(url, url_len, '#', 0, url_end); + int32_t frag_start = -1; + int32_t frag_end = -1; + if (hash_pos != -1) { + frag_start = hash_pos + 1; + frag_end = url_end; + url_end = hash_pos; + } + + // Query (?k=v) + int32_t query_pos = find_char(url, url_len, '?', 0, url_end); + int32_t query_start = -1; + int32_t query_end = -1; + if (query_pos != -1) { + query_start = query_pos + 1; + query_end = url_end; + url_end = query_pos; + } + + // Scheme/protocol (scheme: or scheme://) + int32_t proto_start = 0; + int32_t proto_end = -1; + int32_t authority_start = 0; + + const char* scheme_sep = "://"; + int32_t scheme_sep_pos = find_substr(url, url_len, scheme_sep, 3, 0, url_end); + if (scheme_sep_pos != -1) { + proto_end = scheme_sep_pos; + authority_start = scheme_sep_pos + 3; + } else { + int32_t first_delim = url_end; + int32_t slash_pos = find_char(url, url_len, '/', 0, url_end); + if (slash_pos != -1 && slash_pos < first_delim) first_delim = slash_pos; + + int32_t colon_pos = find_char(url, url_len, ':', 0, first_delim); + if (colon_pos != -1) { + proto_end = colon_pos; + } + + if (url_len >= 2 && url[0] == '/' && url[1] == '/') { + authority_start = 2; + } else { + authority_start = 0; + } + } + + // Authority and path + int32_t authority_end = url_end; + int32_t path_start = url_end; + if (authority_start < url_end) { + int32_t slash_pos = find_char(url, url_len, '/', authority_start, url_end); + if (slash_pos != -1) { + authority_end = slash_pos; + path_start = slash_pos; + } + } + + bool has_authority = (scheme_sep_pos != -1) || + (url_len >= 2 && url[0] == '/' && url[1] == '/'); + if (!has_authority) { + authority_start = url_end; + authority_end = url_end; + path_start = 0; + } + + int32_t path_end = url_end; + + const char* protocol_ptr = (proto_end > proto_start) ? (url + proto_start) : nullptr; + int32_t protocol_len = (proto_end > proto_start) ? proto_end - proto_start : 0; + + const char* authority_ptr = + (authority_end > authority_start) ? (url + authority_start) : nullptr; + int32_t authority_len = + (authority_end > authority_start) ? authority_end - authority_start : 0; + + const char* path_ptr = (path_end > path_start) ? (url + path_start) : nullptr; + int32_t path_len = (path_end > path_start) ? path_end - path_start : 0; + + const char* query_ptr = + (query_start != -1 && query_end > query_start) ? (url + query_start) : nullptr; + int32_t query_len = (query_start != -1 && query_end > query_start) + ? query_end - query_start + : 0; + + const char* ref_ptr = + (frag_start != -1 && frag_end > frag_start) ? (url + frag_start) : nullptr; + int32_t ref_len = + (frag_start != -1 && frag_end > frag_start) ? frag_end - frag_start : 0; + + // Parse authority into userinfo and host (no port) + const char* userinfo_ptr = nullptr; + int32_t userinfo_len = 0; + const char* host_ptr = nullptr; + int32_t host_len = 0; + if (authority_len > 0 && authority_ptr != nullptr) { + int32_t at_pos = -1; + for (int32_t i = authority_len - 1; i >= 0; --i) { + if (authority_ptr[i] == '@') { + at_pos = i; + break; + } + } + const char* hostport_ptr = authority_ptr; + int32_t hostport_len = authority_len; + if (at_pos != -1) { + userinfo_ptr = authority_ptr; + userinfo_len = at_pos; + hostport_ptr = authority_ptr + at_pos + 1; + hostport_len = authority_len - at_pos - 1; + } + + if (hostport_len > 0) { + if (hostport_ptr[0] == '[') { + int32_t close = find_char(hostport_ptr, hostport_len, ']', 0, hostport_len); + if (close != -1 && close > 1) { + host_ptr = hostport_ptr + 1; + host_len = close - 1; + } else { + host_ptr = hostport_ptr; + host_len = hostport_len; + } + } else { + int32_t colon = find_char(hostport_ptr, hostport_len, ':', 0, hostport_len); + if (colon == -1) { + host_ptr = hostport_ptr; + host_len = hostport_len; + } else { + host_ptr = hostport_ptr; + host_len = colon; + } + } + } + } + + if (want_protocol) { + return gdv_substr_copy_out(context, protocol_ptr, protocol_len, out_valid, out_len); + } + if (want_authority) { + return gdv_substr_copy_out(context, authority_ptr, authority_len, out_valid, out_len); + } + if (want_userinfo) { + return gdv_substr_copy_out(context, userinfo_ptr, userinfo_len, out_valid, out_len); + } + if (want_host) { + return gdv_substr_copy_out(context, host_ptr, host_len, out_valid, out_len); + } + if (want_path) { + return gdv_substr_copy_out(context, path_ptr, path_len, out_valid, out_len); + } + if (want_query) { + return gdv_substr_copy_out(context, query_ptr, query_len, out_valid, out_len); + } + if (want_ref) { + return gdv_substr_copy_out(context, ref_ptr, ref_len, out_valid, out_len); + } + + // FILE: path + (?query) + if (want_file) { + if (path_len <= 0 && query_len <= 0) { + *out_valid = true; + *out_len = 0; + return ""; + } + int32_t file_len = path_len + (query_len > 0 ? (1 + query_len) : 0); + char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, file_len)); + if (ret == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for output string"); + *out_valid = false; + *out_len = 0; + return ""; + } + int32_t idx = 0; + if (path_len > 0) { + memcpy(ret + idx, path_ptr, path_len); + idx += path_len; + } + if (query_len > 0) { + ret[idx++] = '?'; + memcpy(ret + idx, query_ptr, query_len); + idx += query_len; + } + *out_valid = true; + *out_len = file_len; + return ret; + } + + *out_valid = false; + *out_len = 0; + return ""; +} + FORCE_INLINE int32_t instr_utf8(const char* string, int32_t string_len, const char* substring, int32_t substring_len) { @@ -3034,4 +3359,91 @@ int32_t instr_utf8(const char* string, int32_t string_len, const char* substring } return 0; } + +// Find the position (1-indexed) of a string in a comma-separated list. +// Returns 0 if not found. +// find_in_set('b', 'a,b,c') = 2 +// find_in_set('d', 'a,b,c') = 0 +FORCE_INLINE +int32_t find_in_set_utf8_utf8(const char* needle, int32_t needle_len, + const char* haystack, int32_t haystack_len) { + if (needle_len == 0 && haystack_len == 0) { + return 1; // empty string is at position 1 in empty list + } + + if (haystack_len == 0) { + return 0; // non-empty needle not in empty list + } + + int32_t position = 1; + int32_t start = 0; + + for (int32_t i = 0; i <= haystack_len; i++) { + // Check if we're at a delimiter or end of string + if (i == haystack_len || haystack[i] == ',') { + int32_t item_len = i - start; + + // Compare the current item with the needle + if (item_len == needle_len) { + if (needle_len == 0 || memcmp(haystack + start, needle, needle_len) == 0) { + return position; + } + } + + // Move to next item + start = i + 1; + position++; + } + } + + return 0; // not found +} + +// Generate a random UUID v4 string (RFC 4122) +// Returns a 36-character string like "550e8400-e29b-41d4-a716-446655440000" +FORCE_INLINE +const char* uuid(gdv_int64 context, gdv_int32* out_len) { + static thread_local std::mt19937_64 rng(std::random_device{}()); + + // Generate 128 bits of randomness + uint64_t high = rng(); + uint64_t low = rng(); + + // Set version (4) and variant (10xx) bits per RFC 4122 + high = (high & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; // version 4 + low = (low & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; // variant 10xx + + // Allocate output buffer (36 chars for UUID format) + *out_len = 36; + char* out = reinterpret_cast(gdv_fn_context_arena_malloc(context, *out_len)); + if (out == nullptr) { + gdv_fn_context_set_error_msg(context, "Could not allocate memory for UUID"); + *out_len = 0; + return ""; + } + + // Format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + static const char hex[] = "0123456789abcdef"; + auto write_hex = [&](int pos, uint64_t val, int nibbles) { + for (int i = nibbles - 1; i >= 0; i--) { + out[pos + i] = hex[val & 0xF]; + val >>= 4; + } + }; + + // high: time_low (8) - time_mid (4) - time_hi_and_version (4) + write_hex(0, high >> 32, 8); + out[8] = '-'; + write_hex(9, high >> 16, 4); + out[13] = '-'; + write_hex(14, high, 4); + out[18] = '-'; + + // low: clock_seq_hi_and_reserved + clock_seq_low (4) - node (12) + write_hex(19, low >> 48, 4); + out[23] = '-'; + write_hex(24, low, 12); + + return out; +} } // extern "C" diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc b/cpp/src/gandiva/precompiled/string_ops_test.cc index aaa25db0a9f8..9f001084307d 100644 --- a/cpp/src/gandiva/precompiled/string_ops_test.cc +++ b/cpp/src/gandiva/precompiled/string_ops_test.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include "gandiva/execution_context.h" @@ -1912,6 +1913,31 @@ TEST(TestStringOps, TestBinaryString) { EXPECT_EQ(output, "OM"); } +TEST(TestStringOps, TestStringBinaryAndBinaryString) { + gandiva::ExecutionContext ctx; + uint64_t ctx_ptr = reinterpret_cast(&ctx); + gdv_int32 out_len = 0; + const char* out; + + // string_binary: pure byte reinterpretation (no \\x.. decoding) + out = string_binary(ctx_ptr, "\\x41", 4, &out_len); + EXPECT_EQ(out_len, 4); + EXPECT_EQ(std::string(out, out_len), "\\x41"); + + // works with embedded nulls + std::string with_nulls("A\0B", 3); + out = string_binary(ctx_ptr, with_nulls.data(), static_cast(with_nulls.size()), + &out_len); + EXPECT_EQ(out_len, 3); + EXPECT_EQ(std::string(out, out_len), with_nulls); + + // binary_string(binary): pure byte reinterpretation (no validation) + out = binary_string_binary(ctx_ptr, with_nulls.data(), + static_cast(with_nulls.size()), &out_len); + EXPECT_EQ(out_len, 3); + EXPECT_EQ(std::string(out, out_len), with_nulls); +} + TEST(TestStringOps, TestSplitPart) { gandiva::ExecutionContext ctx; uint64_t ctx_ptr = reinterpret_cast(&ctx); @@ -2704,4 +2730,70 @@ TEST(TestStringOps, TestInstr) { result = instr_utf8(s1.c_str(), s1_len, s2.c_str(), s2_len); EXPECT_EQ(result, 8); } + +TEST(TestStringOps, TestParseUrl) { + gandiva::ExecutionContext ctx; + uint64_t ctx_ptr = reinterpret_cast(&ctx); + bool out_valid = false; + int32_t out_len = 0; + + const char* url = "http://user:pass@example.com:8080/path?query=1#ref"; + const int32_t url_len = static_cast(strlen(url)); + + const char* out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "HOST", 4, true, + &out_valid, &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "example.com"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "PATH", 4, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "/path"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "QUERY", 5, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "query=1"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "REF", 3, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "ref"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "PROTOCOL", 8, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "http"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "AUTHORITY", 9, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "user:pass@example.com:8080"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "USERINFO", 8, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "user:pass"); + + out = parse_url_utf8_utf8(ctx_ptr, url, url_len, true, "FILE", 4, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), "/path?query=1"); + + const char* url2 = "http://example.com/path"; + const int32_t url2_len = static_cast(strlen(url2)); + out = parse_url_utf8_utf8(ctx_ptr, url2, url2_len, true, "QUERY", 5, true, &out_valid, + &out_len); + EXPECT_TRUE(out_valid); + EXPECT_EQ(std::string(out, out_len), ""); + + out = parse_url_utf8_utf8(ctx_ptr, url2, url2_len, true, "BAD", 3, true, &out_valid, + &out_len); + EXPECT_FALSE(out_valid); + + out = parse_url_utf8_utf8(ctx_ptr, url2, url2_len, false, "HOST", 4, true, &out_valid, + &out_len); + EXPECT_FALSE(out_valid); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/time.cc b/cpp/src/gandiva/precompiled/time.cc index 9b09228773b2..cf4531c233f1 100644 --- a/cpp/src/gandiva/precompiled/time.cc +++ b/cpp/src/gandiva/precompiled/time.cc @@ -21,10 +21,13 @@ extern "C" { #define __STDC_FORMAT_MACROS #include +#include #include #include #include +#include + #include "./time_constants.h" #include "./time_fields.h" #include "./types.h" @@ -35,6 +38,29 @@ extern "C" { #define HOURS_IN_DAY 24 +// Return the current timestamp in milliseconds since epoch (UTC). +gdv_timestamp current_timestamp() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +// Alias for current_timestamp(). +gdv_timestamp now() { return current_timestamp(); } + +// Return the current date as milliseconds since epoch at 00:00:00 (UTC). +gdv_date64 current_date() { + gdv_timestamp ts = current_timestamp(); + return static_cast((ts / MILLIS_IN_DAY) * MILLIS_IN_DAY); +} + +// Return the current time as milliseconds since midnight (UTC). +gdv_time32 current_time() { + gdv_timestamp ts = current_timestamp(); + // Note: the cast is safe; MILLIS_IN_DAY fits in int32. + return static_cast(ts % MILLIS_IN_DAY); +} + // Expand inner macro for all date types. #define DATE_TYPES(INNER) \ INNER(date64) \ @@ -356,6 +382,25 @@ gdv_int64 weekOfYear(const EpochTimePoint& tp) { return weekOfCurrentYear(tp); } +FORCE_INLINE +void isoYearWeek(const EpochTimePoint& tp, int* iso_year, int* iso_week) { + const int week = static_cast(weekOfYear(tp)); + int year = 1900 + tp.TmYear(); + + // If we're in late December and week resolves to 1, that week belongs to the next ISO year. + if ((tp.TmMon() == 11) && (tp.TmMday() >= 29) && (week == 1)) { + year += 1; + } + + // If we're in early January and week resolves to 52/53, that week belongs to the previous ISO year. + if ((tp.TmMon() == 0) && (tp.TmMday() <= 3) && (week >= 52)) { + year -= 1; + } + + *iso_year = year; + *iso_week = week; +} + #define EXTRACT_WEEK(TYPE) \ FORCE_INLINE \ gdv_int64 extractWeek##_##TYPE(gdv_##TYPE millis) { \ @@ -365,6 +410,18 @@ gdv_int64 weekOfYear(const EpochTimePoint& tp) { DATE_TYPES(EXTRACT_WEEK) +#define EXTRACT_YEARWEEK(TYPE) \ + FORCE_INLINE \ + gdv_int64 extractYearweek##_##TYPE(gdv_##TYPE millis) { \ + EpochTimePoint tp(millis); \ + int iso_year = 0; \ + int iso_week = 0; \ + isoYearWeek(tp, &iso_year, &iso_week); \ + return static_cast(iso_year) * 100 + iso_week; \ + } + +DATE_TYPES(EXTRACT_YEARWEEK) + #define EXTRACT_DOW(TYPE) \ FORCE_INLINE \ gdv_int64 extractDow##_##TYPE(gdv_##TYPE millis) { \ @@ -508,6 +565,60 @@ EXTRACT_HOUR_TIME(time32) DATE_TRUNC_FUNCTIONS(date64) DATE_TRUNC_FUNCTIONS(timestamp) +FORCE_INLINE +bool equals_ignore_case_ascii(const char* in, gdv_int32 in_len, const char* lit) { + gdv_int32 lit_len = static_cast(strlen(lit)); + if (in_len != lit_len) { + return false; + } + for (gdv_int32 i = 0; i < in_len; ++i) { + unsigned char c = static_cast(in[i]); + if (tolower(c) != lit[i]) { + return false; + } + } + return true; +} + +// date_trunc(precision, timestamp) +// +// precision is one of: year, quarter, month, week, day, hour, minute, second +// +// On invalid precision, sets an error in the execution context. +FORCE_INLINE +gdv_timestamp date_trunc_utf8_timestamp(gdv_int64 context, const char* precision, + gdv_int32 precision_len, gdv_timestamp in) { + if (equals_ignore_case_ascii(precision, precision_len, "year")) { + return date_trunc_Year_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "quarter")) { + return date_trunc_Quarter_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "month")) { + return date_trunc_Month_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "week")) { + return date_trunc_Week_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "day")) { + return date_trunc_Day_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "hour")) { + return date_trunc_Hour_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "minute")) { + return date_trunc_Minute_timestamp(in); + } + if (equals_ignore_case_ascii(precision, precision_len, "second")) { + return date_trunc_Second_timestamp(in); + } + + gdv_fn_context_set_error_msg(context, + "Invalid precision for date_trunc. Expected one of: " + "year, quarter, month, week, day, hour, minute, second"); + return 0; +} + #define LAST_DAY_FUNC(TYPE) \ FORCE_INLINE \ gdv_date64 last_day_from_##TYPE(gdv_date64 millis) { \ diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc index 316e5ecc2602..a12339875f1d 100644 --- a/cpp/src/gandiva/precompiled/time_test.cc +++ b/cpp/src/gandiva/precompiled/time_test.cc @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include "gandiva/execution_context.h" @@ -364,6 +365,38 @@ TEST(TestTime, TimeStampTrunc) { StringToTimestamp("2000-03-06 00:00:00")); } +TEST(TestTime, DateTruncPrecisionTimestamp) { + ExecutionContext context; + int64_t context_ptr = reinterpret_cast(&context); + + gdv_timestamp ts = StringToTimestamp("2026-01-29 15:30:45"); + + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "year", 4, ts), + StringToTimestamp("2026-01-01 00:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "quarter", 7, ts), + StringToTimestamp("2026-01-01 00:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "month", 5, ts), + StringToTimestamp("2026-01-01 00:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "week", 4, ts), + StringToTimestamp("2026-01-26 00:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "day", 3, ts), + StringToTimestamp("2026-01-29 00:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "hour", 4, ts), + StringToTimestamp("2026-01-29 15:00:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "minute", 6, ts), + StringToTimestamp("2026-01-29 15:30:00")); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "second", 6, ts), ts); + + // Case-insensitive + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "MoNtH", 5, ts), + StringToTimestamp("2026-01-01 00:00:00")); + + // Invalid precision should set an error. + context.Reset(); + EXPECT_EQ(date_trunc_utf8_timestamp(context_ptr, "millis", 6, ts), 0); + EXPECT_TRUE(context.has_error()); +} + TEST(TestTime, TimeStampAdd) { EXPECT_EQ( timestampaddSecond_int32_timestamp(30, StringToTimestamp("2000-05-01 10:20:34")), @@ -458,6 +491,20 @@ TEST(TestTime, TimeStampAdd) { EXPECT_EQ(add_date64_int64(StringToTimestamp("2000-02-27 00:00:00"), 4), StringToTimestamp("2000-03-02 00:00:00")); + + // add_ + EXPECT_EQ(add_days_timestamp_int32(StringToTimestamp("2000-05-01 00:00:00"), 7), + StringToTimestamp("2000-05-08 00:00:00")); + EXPECT_EQ(add_days_timestamp_int64(StringToTimestamp("2000-05-01 00:00:00"), -7), + StringToTimestamp("2000-04-24 00:00:00")); + + EXPECT_EQ(add_hours_timestamp_int32(StringToTimestamp("2000-05-01 00:00:00"), 25), + StringToTimestamp("2000-05-02 01:00:00")); + EXPECT_EQ(add_minutes_timestamp_int32(StringToTimestamp("2000-05-01 00:00:00"), -1), + StringToTimestamp("2000-04-30 23:59:00")); + EXPECT_EQ(add_seconds_timestamp_int32(StringToTimestamp("2000-05-01 00:00:00"), 1), + StringToTimestamp("2000-05-01 00:00:01")); + // date_sub EXPECT_EQ(date_sub_timestamp_int32(StringToTimestamp("2000-05-01 00:00:00"), 7), StringToTimestamp("2000-04-24 00:00:00")); @@ -1093,4 +1140,31 @@ TEST(TestTime, TestCastNullableInterval) { context.Reset(); } + + +TEST(TestTime, TestCurrentDateTime) { + using std::chrono::duration_cast; + using std::chrono::milliseconds; + using std::chrono::system_clock; + + auto before = duration_cast(system_clock::now().time_since_epoch()).count(); + gdv_timestamp ts = current_timestamp(); + auto after = duration_cast(system_clock::now().time_since_epoch()).count(); + + EXPECT_GE(ts, before); + EXPECT_LE(ts, after); + + gdv_timestamp ts2 = now(); + EXPECT_GE(ts2, before); + EXPECT_LE(ts2, after); + + gdv_date64 d = current_date(); + EXPECT_EQ(d % MILLIS_IN_DAY, 0); + EXPECT_EQ(d, (ts / MILLIS_IN_DAY) * MILLIS_IN_DAY); + + gdv_time32 t = current_time(); + EXPECT_GE(t, 0); + EXPECT_LT(t, MILLIS_IN_DAY); + EXPECT_EQ(static_cast(t), ts % MILLIS_IN_DAY); +} } // namespace gandiva diff --git a/cpp/src/gandiva/precompiled/timestamp_arithmetic.cc b/cpp/src/gandiva/precompiled/timestamp_arithmetic.cc index 695605b3cc77..114c96cf90e3 100644 --- a/cpp/src/gandiva/precompiled/timestamp_arithmetic.cc +++ b/cpp/src/gandiva/precompiled/timestamp_arithmetic.cc @@ -280,4 +280,22 @@ ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(date64, add, MILLIS_IN_DAY) ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, date_add, MILLIS_IN_DAY) ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, add, MILLIS_IN_DAY) + +// SQL-style convenience helpers (timestamp first, integer second) +// add_days(timestamp, int) +ADD_TIMESTAMP_TO_INT32_FIXED_UNITS(timestamp, add_days, MILLIS_IN_DAY) +ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, add_days, MILLIS_IN_DAY) + +// add_hours(timestamp, int) +ADD_TIMESTAMP_TO_INT32_FIXED_UNITS(timestamp, add_hours, MILLIS_IN_HOUR) +ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, add_hours, MILLIS_IN_HOUR) + +// add_minutes(timestamp, int) +ADD_TIMESTAMP_TO_INT32_FIXED_UNITS(timestamp, add_minutes, MILLIS_IN_MIN) +ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, add_minutes, MILLIS_IN_MIN) + +// add_seconds(timestamp, int) +ADD_TIMESTAMP_TO_INT32_FIXED_UNITS(timestamp, add_seconds, MILLIS_IN_SEC) +ADD_TIMESTAMP_TO_INT64_FIXED_UNITS(timestamp, add_seconds, MILLIS_IN_SEC) + } // extern "C" diff --git a/cpp/src/gandiva/precompiled/types.h b/cpp/src/gandiva/precompiled/types.h index 1277ccff3399..fbabc1a28f89 100644 --- a/cpp/src/gandiva/precompiled/types.h +++ b/cpp/src/gandiva/precompiled/types.h @@ -172,6 +172,32 @@ gdv_int64 date_sub_timestamp_int32(gdv_timestamp, gdv_int32); gdv_int64 subtract_timestamp_int32(gdv_timestamp, gdv_int32); gdv_int64 date_diff_timestamp_int64(gdv_timestamp, gdv_int64); +// date arithmetic helpers +gdv_timestamp add_days_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_days_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_hours_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_hours_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_minutes_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_minutes_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_seconds_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_seconds_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_weeks_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_weeks_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_months_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_months_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_quarters_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_quarters_timestamp_int64(gdv_timestamp, gdv_int64); + +gdv_timestamp add_years_timestamp_int32(gdv_timestamp, gdv_int32); +gdv_timestamp add_years_timestamp_int64(gdv_timestamp, gdv_int64); + + gdv_boolean castBIT_utf8(gdv_int64 context, const char* data, gdv_int32 data_len); bool is_distinct_from_timestamp_timestamp(gdv_int64, bool, gdv_int64, bool); @@ -191,6 +217,10 @@ gdv_int32 datediff_timestamp_timestamp(gdv_timestamp start_millis, gdv_timestamp end_millis); gdv_int64 date_trunc_Week_timestamp(gdv_timestamp); + +gdv_timestamp date_trunc_utf8_timestamp(gdv_int64 context, const char* precision, + gdv_int32 precision_len, gdv_timestamp in); + double months_between_timestamp_timestamp(gdv_uint64, gdv_uint64); gdv_int32 mem_compare(const char* left, gdv_int32 left_len, const char* right, @@ -203,6 +233,14 @@ gdv_float64 mod_float64_float64(gdv_int64 context, gdv_float64 left, gdv_float64 gdv_int64 pmod_int64_int64(int64_t context, gdv_int64 left, gdv_int64 right); +// Bit shift operations +gdv_int32 lshift_int32_int32(gdv_int32 value, gdv_int32 bits); +gdv_int64 lshift_int64_int64(gdv_int64 value, gdv_int64 bits); +gdv_int32 rshift_int32_int32(gdv_int32 value, gdv_int32 bits); +gdv_int64 rshift_int64_int64(gdv_int64 value, gdv_int64 bits); +gdv_int32 rshiftunsigned_int32_int32(gdv_int32 value, gdv_int32 bits); +gdv_int64 rshiftunsigned_int64_int64(gdv_int64 value, gdv_int64 bits); + gdv_int32 positive_int32(gdv_int32 in); gdv_int64 positive_int64(gdv_int64 in); gdv_int32 negative_int32(gdv_int64 context, gdv_int32 in); @@ -281,6 +319,11 @@ gdv_float64 log10_int64(gdv_int64); gdv_float64 log10_float32(gdv_float32); gdv_float64 log10_float64(gdv_float64); +gdv_float64 pi(); + +gdv_int32 rand_integer_int32(gdv_int32 max_val); +gdv_int32 rand_integer_int32_int32(gdv_int32 min_val, gdv_int32 max_val); + gdv_float64 sin_int32(gdv_int32); gdv_float64 sin_int64(gdv_int64); gdv_float64 sin_float32(gdv_float32); @@ -455,6 +498,12 @@ gdv_date64 castDATE_timestamp(gdv_timestamp); gdv_time32 castTIME_utf8(int64_t context, const char* input, int32_t length); gdv_time32 castTIME_timestamp(gdv_timestamp timestamp_in_millis); gdv_time32 castTIME_int32(int32_t int_val); + +// Current date/time functions (UTC). +gdv_date64 current_date(); +gdv_time32 current_time(); +gdv_timestamp current_timestamp(); +gdv_timestamp now(); const char* castVARCHAR_timestamp_int64(int64_t, gdv_timestamp, gdv_int64, gdv_int32*); gdv_date64 last_day_from_timestamp(gdv_date64 millis); @@ -488,6 +537,8 @@ const char* concat_utf8_utf8_utf8_utf8(gdv_int64 context, const char* in1, bool in4_validity, gdv_int32* out_len); const char* space_int32(gdv_int64 ctx, gdv_int32 n, int32_t* out_len); const char* space_int64(gdv_int64 ctx, gdv_int64 n, int32_t* out_len); + +const char* uuid(gdv_int64 context, gdv_int32* out_len); const char* concat_utf8_utf8_utf8_utf8_utf8( gdv_int64 context, const char* in1, gdv_int32 in1_len, bool in1_validity, const char* in2, gdv_int32 in2_len, bool in2_validity, const char* in3, @@ -748,6 +799,24 @@ const char* right_utf8_int32(gdv_int64 context, const char* text, gdv_int32 text const char* binary_string(gdv_int64 context, const char* text, gdv_int32 text_len, gdv_int32* out_len); +// Reinterpret input utf8 bytes as binary (no encoding/decoding). +const char* string_binary(gdv_int64 context, const char* text, gdv_int32 text_len, + gdv_int32* out_len); + +// Reinterpret input binary bytes as utf8 (no encoding/decoding). +const char* binary_string_binary(gdv_int64 context, const char* data, gdv_int32 data_len, + gdv_int32* out_len); + +// Reinterpret input utf8 bytes as binary (no encoding/decoding). +const char* string_binary(gdv_int64 context, const char* text, gdv_int32 text_len, + gdv_int32* out_len); + +// Reinterpret input binary bytes as utf8 (no encoding/decoding). +const char* binary_string_binary(gdv_int64 context, const char* data, gdv_int32 data_len, + gdv_int32* out_len); + +const char* uuid(gdv_int64 context, gdv_int32* out_len); + const char* to_hex_binary(int64_t context, const char* text, int32_t text_len, int32_t* out_len); diff --git a/cpp/src/gandiva/tests/date_time_test.cc b/cpp/src/gandiva/tests/date_time_test.cc index 6208f1ecba9b..7ff133a1b265 100644 --- a/cpp/src/gandiva/tests/date_time_test.cc +++ b/cpp/src/gandiva/tests/date_time_test.cc @@ -828,4 +828,55 @@ TEST_F(DateTimeTestProjector, TestFromUtcTimestamp) { // Validate results EXPECT_ARROW_ARRAY_EQUALS(exp_output, outputs.at(0)); } + +TEST_F(DateTimeTestProjector, TestIsoWeekOfYearAndYearWeek) { + auto field0 = field("ts", timestamp(arrow::TimeUnit::MILLI)); + auto schema = arrow::schema({field0}); + + auto field_week = field("wk", int64()); + auto field_yearweek = field("yw", int64()); + + auto week_expr = TreeExprBuilder::MakeExpression("weekofyear", {field0}, field_week); + auto yearweek_expr = + TreeExprBuilder::MakeExpression("yearweek", {field0}, field_yearweek); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {week_expr, yearweek_expr}, TestConfiguration(), + &projector); + ASSERT_TRUE(status.ok()); + + time_t epoch = Epoch(); + int num_records = 6; + auto validity = {true, true, true, true, true, true}; + + std::vector ts_data = { + // 2015-W53 + MillisSince(epoch, 2015, 12, 31, 0, 0, 0, 0), + MillisSince(epoch, 2016, 1, 1, 0, 0, 0, 0), + MillisSince(epoch, 2016, 1, 3, 0, 0, 0, 0), + // 2016-W01 + MillisSince(epoch, 2016, 1, 4, 0, 0, 0, 0), + // 2015-W01 + MillisSince(epoch, 2014, 12, 29, 0, 0, 0, 0), + // 2020-W53 + MillisSince(epoch, 2021, 1, 1, 0, 0, 0, 0), + }; + + auto ts_array = MakeArrowTypeArray( + arrow::timestamp(arrow::TimeUnit::MILLI), ts_data, validity); + + auto exp_week = MakeArrowArrayInt64({53, 53, 53, 1, 1, 53}, validity); + auto exp_yearweek = MakeArrowArrayInt64({201553, 201553, 201553, 201601, 201501, 202053}, + validity); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {ts_array}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()); + + EXPECT_ARROW_ARRAY_EQUALS(exp_week, outputs.at(0)); + EXPECT_ARROW_ARRAY_EQUALS(exp_yearweek, outputs.at(1)); +} + } // namespace gandiva diff --git a/cpp/src/gandiva/tests/list_test.cc b/cpp/src/gandiva/tests/list_test.cc index abc7b5d7091b..901ade555807 100644 --- a/cpp/src/gandiva/tests/list_test.cc +++ b/cpp/src/gandiva/tests/list_test.cc @@ -344,12 +344,11 @@ TEST_F(TestList, TestListInt32Contains) { // Create a row-batch with some sample data int num_records = 5; ArrayPtr array_a; - _build_list_array( - {1, 5, 19, 42, 57}, - {1, 1, 1, 1, 1}, {true, true, true, true, true}, pool_, &array_a); + _build_list_array({1, 5, 19, 42, 57}, {1, 1, 1, 1, 1}, + {true, true, true, true, true}, pool_, + &array_a); - auto array_b = - MakeArrowArrayInt32({42, 42, 42, 42, 42}); + auto array_b = MakeArrowArrayInt32({42, 42, 42, 42, 42}); // expected output auto exp = MakeArrowArrayBool({false, false, false, true, false}, @@ -376,6 +375,121 @@ TEST_F(TestList, TestListInt32Contains) { EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); } +TEST_F(TestList, TestListArrayLengthAndCardinality) { + // schema for input fields + auto field_a = field("a", list(int32())); + auto schema = arrow::schema({field_a}); + + // output fields + auto res_len = field("res_len", int32()); + auto res_card = field("res_card", int32()); + + int num_records = 5; + ArrayPtr array_a; + // Include a row with a null element, and a null list row. + _build_list_array({1, 2, 3, 5, 6, 9, 2}, + {3, 2, 0, 1, 1}, + {true, true, true, false, true}, pool_, + &array_a, {true, true, false}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a}); + + auto expr_len = + TreeExprBuilder::MakeExpression("array_length", {field_a}, res_len); + auto expr_card = + TreeExprBuilder::MakeExpression("cardinality", {field_a}, res_card); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr_len, expr_card}, TestConfiguration(), + &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + auto exp = MakeArrowArrayInt32({3, 2, 0, 0, 1}, {true, true, true, false, true}); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(1)); +} + +TEST_F(TestList, TestListArrayPositionNullSemantics) { + // schema for input fields + auto field_a = field("a", list(int32())); + auto field_b = field("b", int32()); + auto schema = arrow::schema({field_a, field_b}); + + // output fields + auto res = field("res", int32()); + + int num_records = 5; + ArrayPtr array_a; + // Row 0 has a null element. + _build_list_array({1, 2, 3, 5, 6, 9, 2}, + {3, 2, 0, 1, 1}, + {true, true, true, false, true}, pool_, + &array_a, {true, true, false}); + + // Search for a value not present. + auto array_b = MakeArrowArrayInt32({7, 7, 7, 7, 7}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a, array_b}); + + auto expr = + TreeExprBuilder::MakeExpression("array_position", {field_a, field_b}, res); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + // Row 0: array has a null element and no match -> NULL. + // Row 1: no nulls and no match -> 0. + // Row 2: empty -> 0. + // Row 3: list is NULL -> NULL. + // Row 4: no match -> 0. + auto exp = MakeArrowArrayInt32({0, 0, 0, 0, 0}, {false, true, true, false, true}); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); +} + +TEST_F(TestList, TestListArrayPositionNullSearchValue) { + auto field_a = field("a", list(int32())); + auto field_b = field("b", int32()); + auto schema = arrow::schema({field_a, field_b}); + + auto res = field("res", int32()); + + int num_records = 5; + ArrayPtr array_a; + _build_list_array({1, 2, 3, 5, 6, 9, 2}, + {3, 2, 0, 1, 1}, + {true, true, true, false, true}, pool_, + &array_a, {true, true, false}); + + // Make b NULL on one row. + auto array_b = MakeArrowArrayInt32({2, 2, 2, 2, 2}, {true, false, true, true, true}); + + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a, array_b}); + + auto expr = + TreeExprBuilder::MakeExpression("array_position", {field_a, field_b}, res); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + // Row 1 search value is NULL -> result NULL. + auto exp = MakeArrowArrayInt32({2, 0, 0, 0, 1}, {true, false, true, false, true}); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); +} + TEST_F(TestList, TestListFloat32) { ArrayPtr array; _build_list_array(