diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index 0dbb462893fc..180cb378e635 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -175,7 +175,7 @@ std::vector GetStringFunctionRegistry() { NativeFunction("castVARCHAR", {"varchar"}, DataTypeVector{binary(), int64()}, utf8(), kResultNullIfNull, "castVARCHAR_binary_int64", - NativeFunction::kNeedsContext), + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), NativeFunction("castVARCHAR", {"varchar"}, DataTypeVector{int32(), int64()}, utf8(), kResultNullIfNull, "gdv_fn_castVARCHAR_int32_int64", diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index 229a69624f55..8aee13a03a17 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -29,6 +29,7 @@ #include "arrow/util/base64.h" #include "arrow/util/bit_util.h" #include "arrow/util/double_conversion_internal.h" +#include "arrow/util/utf8.h" #include "arrow/util/value_parsing.h" #include "gandiva/encrypt_utils_ecb.h" @@ -271,6 +272,16 @@ const char* gdv_fn_base64_encode_binary(int64_t context, const char* in, int32_t return ret; } +// Strict UTF-8 validation using Arrow's implementation, which relies on lookup tables and +// so cannot be inlined into the precompiled bitcode. +GANDIVA_EXPORT +bool gdv_fn_is_valid_utf8(const char* data, int32_t data_len) { + // ValidateUTF8 accepts everything until its tables are initialized + static const bool initialized = (arrow::util::InitializeUTF8(), true); + ARROW_UNUSED(initialized); + return arrow::util::ValidateUTF8(reinterpret_cast(data), data_len); +} + GANDIVA_EXPORT const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len, int32_t* out_len) { @@ -1515,6 +1526,16 @@ arrow::Status ExportedStubFunctions::AddMappings(Engine* engine) const { types->i8_ptr_type() /*return_type*/, args, reinterpret_cast(gdv_fn_base64_encode_binary)); + // gdv_fn_is_valid_utf8 + args = { + types->i8_ptr_type(), // data + types->i32_type(), // data_len + }; + + engine->AddGlobalMappingForFunc("gdv_fn_is_valid_utf8", + types->i1_type() /*return_type*/, args, + reinterpret_cast(gdv_fn_is_valid_utf8)); + // gdv_fn_base64_decode_utf8 args = { types->i64_type(), // context diff --git a/cpp/src/gandiva/gdv_function_stubs.h b/cpp/src/gandiva/gdv_function_stubs.h index 89abe3fbf678..d8322df51374 100644 --- a/cpp/src/gandiva/gdv_function_stubs.h +++ b/cpp/src/gandiva/gdv_function_stubs.h @@ -80,6 +80,9 @@ void gdv_fn_context_set_error_msg(int64_t context_ptr, const char* err_msg); GANDIVA_EXPORT uint8_t* gdv_fn_context_arena_malloc(int64_t context_ptr, int32_t data_len); +GANDIVA_EXPORT +bool gdv_fn_is_valid_utf8(const char* data, int32_t data_len); + void gdv_fn_context_arena_reset(int64_t context_ptr); bool in_expr_lookup_int32(int64_t ptr, int32_t value, bool in_validity); diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc index c79a4f355297..86f946706efe 100644 --- a/cpp/src/gandiva/precompiled/string_ops.cc +++ b/cpp/src/gandiva/precompiled/string_ops.cc @@ -691,10 +691,24 @@ const char* castVARCHAR_bool_int64(gdv_int64 context, gdv_boolean value, } CAST_VARCHAR_FROM_VARLEN_TYPE(utf8) -CAST_VARCHAR_FROM_VARLEN_TYPE(binary) #undef CAST_VARCHAR_FROM_VARLEN_TYPE +// Unlike utf8, binary input is not guaranteed to be valid UTF-8, so it is validated +// before being reinterpreted as varchar. +FORCE_INLINE +const char* castVARCHAR_binary_int64(gdv_int64 context, const char* data, + gdv_int32 data_len, int64_t out_len, + int32_t* out_length) { + if (!gdv_fn_is_valid_utf8(data, data_len)) { + gdv_fn_context_set_error_msg( + context, "Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"); + *out_length = 0; + return ""; + } + return castVARCHAR_utf8_int64(context, data, data_len, out_len, out_length); +} + // Add functions for castVARBINARY #define CAST_VARBINARY_FROM_STRING_AND_BINARY(TYPE) \ GANDIVA_EXPORT \ diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc b/cpp/src/gandiva/precompiled/string_ops_test.cc index ff7961ffc86b..ece77dc6bf17 100644 --- a/cpp/src/gandiva/precompiled/string_ops_test.cc +++ b/cpp/src/gandiva/precompiled/string_ops_test.cc @@ -19,6 +19,8 @@ #include #include +#include +#include #include "gandiva/execution_context.h" #include "gandiva/precompiled/types.h" @@ -510,6 +512,57 @@ TEST(TestStringOps, TestCastVarcharToBool) { ctx.Reset(); } +TEST(TestStringOps, TestCastVarcharFromBinaryValidatesUtf8) { + gandiva::ExecutionContext ctx; + uint64_t ctx_ptr = reinterpret_cast(&ctx); + gdv_int32 out_len = 0; + + const std::vector valid_inputs = { + "", + "hello", + "h\xc3\xa9llo \xe2\x82\xac", // é, € + "\xf0\x9f\x98\x80", // U+1F600 + "\xef\xbf\xbf", // U+FFFF + "\xed\x9f\xbf", // U+D7FF, last code point before surrogates + "\xf4\x8f\xbf\xbf", // U+10FFFF + "0123456789abcdef\xc3\xa9 0123456789", // multi-byte after an 8-byte ascii run + }; + for (const auto& input : valid_inputs) { + const char* out_str = castVARCHAR_binary_int64( + ctx_ptr, input.data(), static_cast(input.length()), 0, &out_len); + EXPECT_EQ(std::string(out_str, out_len), input); + EXPECT_FALSE(ctx.has_error()) << ctx.get_error(); + ctx.Reset(); + } + + const std::vector invalid_inputs = { + "\x85\xe9\x65", // UNBASE64('hello world!'): invalid lead byte + "\xc3\x28", // invalid continuation byte + "a\xe2\x82", // truncated 3-byte sequence + "\xc0\xaf", // overlong '/' + "\xe0\x80\xaf", // overlong 3-byte form + "\xf0\x80\x80\xaf", // overlong 4-byte form + "\xed\xa0\x80", // surrogate U+D800 + "\xf4\x90\x80\x80", // above U+10FFFF + "\xf8\x88\x80\x80\x80", // 5-byte form + "\xff", // never valid + "\x80", // lone continuation byte + "\xc1\xbf", // overlong, 0xc1 is never a valid lead byte + "\xf0\x9f\x98\x28", // invalid 4th byte + "\xe2\x28\xa1", // invalid 2nd byte of a 3-byte sequence + "0123456789abcdef\x85", // invalid byte after an 8-byte ascii run + }; + for (const auto& input : invalid_inputs) { + const char* out_str = castVARCHAR_binary_int64( + ctx_ptr, input.data(), static_cast(input.length()), 0, &out_len); + EXPECT_EQ(std::string(out_str, out_len), ""); + EXPECT_THAT(ctx.get_error(), + ::testing::HasSubstr( + "Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8")); + ctx.Reset(); + } +} + TEST(TestStringOps, TestCastVarchar) { gandiva::ExecutionContext ctx; uint64_t ctx_ptr = reinterpret_cast(&ctx); @@ -571,11 +624,15 @@ TEST(TestStringOps, TestCastVarchar) { ::testing::HasSubstr("Output buffer length can't be negative")); ctx.Reset(); + // the whole input is validated, not only the truncated prefix std::string z("aa\xc3"); out_str = castVARCHAR_binary_int64(ctx_ptr, z.data(), static_cast(z.length()), 2, &out_len); - EXPECT_EQ(std::string(out_str, out_len), "aa"); - EXPECT_FALSE(ctx.has_error()); + EXPECT_EQ(std::string(out_str, out_len), ""); + EXPECT_THAT( + ctx.get_error(), + ::testing::HasSubstr("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8")); + ctx.Reset(); out_str = castVARCHAR_binary_int64(ctx_ptr, "1234567812341234", 16, 16, &out_len); EXPECT_EQ(std::string(out_str, out_len), "1234567812341234"); @@ -633,9 +690,9 @@ TEST(TestStringOps, TestCastVarchar) { out_str = castVARCHAR_binary_int64(ctx_ptr, f.data(), static_cast(f.length()), 16, &out_len); EXPECT_EQ(std::string(out_str, out_len), ""); - EXPECT_THAT(ctx.get_error(), - ::testing::HasSubstr( - "unexpected byte \\c3 encountered while decoding utf8 string")); + EXPECT_THAT( + ctx.get_error(), + ::testing::HasSubstr("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8")); ctx.Reset(); // UTF8 TESTS diff --git a/cpp/src/gandiva/tests/utf8_test.cc b/cpp/src/gandiva/tests/utf8_test.cc index 1e408f0e4310..be5d76c10f87 100644 --- a/cpp/src/gandiva/tests/utf8_test.cc +++ b/cpp/src/gandiva/tests/utf8_test.cc @@ -722,6 +722,53 @@ TEST_F(TestUtf8, TestCastVarChar) { EXPECT_ARROW_ARRAY_EQUALS(exp, outputs[0]); } +TEST_F(TestUtf8, TestCastVarCharFromBinary) { + auto field_a = field("a", arrow::binary()); + auto schema = arrow::schema({field_a}); + auto res = field("res", utf8()); + + auto node_a = TreeExprBuilder::MakeField(field_a); + auto node_len = TreeExprBuilder::MakeLiteral(static_cast(3)); + auto cast_varchar = + TreeExprBuilder::MakeFunction("castVARCHAR", {node_a, node_len}, utf8()); + auto expr = TreeExprBuilder::MakeExpression(cast_varchar, res); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // the null row holds invalid UTF-8 bytes, which must not be validated + int num_records = 3; + std::vector offsets = {0, 6, 9, 12}; + std::vector validity = {0x05}; + auto array_a = std::make_shared( + num_records, arrow::Buffer::Wrap(offsets), + arrow::Buffer::FromString("h\xc3\xa9llo" + "\x85\xe9\x65" + "abc"), + arrow::Buffer::Wrap(validity), /*null_count=*/1); + auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a}); + + arrow::ArrayVector outputs; + status = projector->Evaluate(*in_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + + auto exp = MakeArrowArrayUtf8({"h\xc3\xa9l", "", "abc"}, {true, false, true}); + EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0)); + + // UNBASE64('hello world!') produces these bytes + auto array_invalid = MakeArrowArrayBinary({"ok", "\x85\xe9\x65"}, {true, true}); + auto invalid_batch = arrow::RecordBatch::Make(schema, 2, {array_invalid}); + + outputs.clear(); + status = projector->Evaluate(*invalid_batch, pool_, &outputs); + EXPECT_FALSE(status.ok()); + EXPECT_NE( + status.message().find("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"), + std::string::npos) + << status.message(); +} + TEST_F(TestUtf8, TestAscii) { // schema for input fields auto field0 = field("f0", arrow::utf8());