Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/src/gandiva/function_registry_string.cc
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {

NativeFunction("castVARCHAR", {"varchar"}, DataTypeVector{binary(), int64()},
utf8(), kResultNullIfNull, "castVARCHAR_binary_int64",
NativeFunction::kNeedsContext),
NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors),

NativeFunction("castVARCHAR", {"varchar"}, DataTypeVector{int32(), int64()}, utf8(),
kResultNullIfNull, "gdv_fn_castVARCHAR_int32_int64",
Expand Down
21 changes: 21 additions & 0 deletions cpp/src/gandiva/gdv_function_stubs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "arrow/util/base64.h"
#include "arrow/util/bit_util.h"
#include "arrow/util/double_conversion_internal.h"
#include "arrow/util/utf8.h"
#include "arrow/util/value_parsing.h"

#include "gandiva/encrypt_utils_ecb.h"
Expand Down Expand Up @@ -271,6 +272,16 @@ const char* gdv_fn_base64_encode_binary(int64_t context, const char* in, int32_t
return ret;
}

// Strict UTF-8 validation using Arrow's implementation, which relies on lookup tables and
// so cannot be inlined into the precompiled bitcode.
GANDIVA_EXPORT
bool gdv_fn_is_valid_utf8(const char* data, int32_t data_len) {
// ValidateUTF8 accepts everything until its tables are initialized
static const bool initialized = (arrow::util::InitializeUTF8(), true);
ARROW_UNUSED(initialized);
return arrow::util::ValidateUTF8(reinterpret_cast<const uint8_t*>(data), data_len);
}

GANDIVA_EXPORT
const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len,
int32_t* out_len) {
Expand Down Expand Up @@ -1515,6 +1526,16 @@ arrow::Status ExportedStubFunctions::AddMappings(Engine* engine) const {
types->i8_ptr_type() /*return_type*/, args,
reinterpret_cast<void*>(gdv_fn_base64_encode_binary));

// gdv_fn_is_valid_utf8
args = {
types->i8_ptr_type(), // data
types->i32_type(), // data_len
};

engine->AddGlobalMappingForFunc("gdv_fn_is_valid_utf8",
types->i1_type() /*return_type*/, args,
reinterpret_cast<void*>(gdv_fn_is_valid_utf8));

// gdv_fn_base64_decode_utf8
args = {
types->i64_type(), // context
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/gandiva/gdv_function_stubs.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ void gdv_fn_context_set_error_msg(int64_t context_ptr, const char* err_msg);
GANDIVA_EXPORT
uint8_t* gdv_fn_context_arena_malloc(int64_t context_ptr, int32_t data_len);

GANDIVA_EXPORT
bool gdv_fn_is_valid_utf8(const char* data, int32_t data_len);

void gdv_fn_context_arena_reset(int64_t context_ptr);

bool in_expr_lookup_int32(int64_t ptr, int32_t value, bool in_validity);
Expand Down
16 changes: 15 additions & 1 deletion cpp/src/gandiva/precompiled/string_ops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -691,10 +691,24 @@ const char* castVARCHAR_bool_int64(gdv_int64 context, gdv_boolean value,
}

CAST_VARCHAR_FROM_VARLEN_TYPE(utf8)
CAST_VARCHAR_FROM_VARLEN_TYPE(binary)

#undef CAST_VARCHAR_FROM_VARLEN_TYPE

// Unlike utf8, binary input is not guaranteed to be valid UTF-8, so it is validated
// before being reinterpreted as varchar.
FORCE_INLINE
const char* castVARCHAR_binary_int64(gdv_int64 context, const char* data,
gdv_int32 data_len, int64_t out_len,
int32_t* out_length) {
if (!gdv_fn_is_valid_utf8(data, data_len)) {
gdv_fn_context_set_error_msg(
context, "Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8");
*out_length = 0;
return "";
}
return castVARCHAR_utf8_int64(context, data, data_len, out_len, out_length);
}

// Add functions for castVARBINARY
#define CAST_VARBINARY_FROM_STRING_AND_BINARY(TYPE) \
GANDIVA_EXPORT \
Expand Down
67 changes: 62 additions & 5 deletions cpp/src/gandiva/precompiled/string_ops_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#include <gtest/gtest.h>

#include <limits>
#include <string>
#include <vector>

#include "gandiva/execution_context.h"
#include "gandiva/precompiled/types.h"
Expand Down Expand Up @@ -510,6 +512,57 @@ TEST(TestStringOps, TestCastVarcharToBool) {
ctx.Reset();
}

TEST(TestStringOps, TestCastVarcharFromBinaryValidatesUtf8) {
gandiva::ExecutionContext ctx;
uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
gdv_int32 out_len = 0;

const std::vector<std::string> valid_inputs = {
"",
"hello",
"h\xc3\xa9llo \xe2\x82\xac", // é, €
"\xf0\x9f\x98\x80", // U+1F600
"\xef\xbf\xbf", // U+FFFF
"\xed\x9f\xbf", // U+D7FF, last code point before surrogates
"\xf4\x8f\xbf\xbf", // U+10FFFF
"0123456789abcdef\xc3\xa9 0123456789", // multi-byte after an 8-byte ascii run
};
for (const auto& input : valid_inputs) {
const char* out_str = castVARCHAR_binary_int64(
ctx_ptr, input.data(), static_cast<int32_t>(input.length()), 0, &out_len);
EXPECT_EQ(std::string(out_str, out_len), input);
EXPECT_FALSE(ctx.has_error()) << ctx.get_error();
ctx.Reset();
}

const std::vector<std::string> invalid_inputs = {
"\x85\xe9\x65", // UNBASE64('hello world!'): invalid lead byte
"\xc3\x28", // invalid continuation byte
"a\xe2\x82", // truncated 3-byte sequence
"\xc0\xaf", // overlong '/'
"\xe0\x80\xaf", // overlong 3-byte form
"\xf0\x80\x80\xaf", // overlong 4-byte form
"\xed\xa0\x80", // surrogate U+D800
"\xf4\x90\x80\x80", // above U+10FFFF
"\xf8\x88\x80\x80\x80", // 5-byte form
"\xff", // never valid
"\x80", // lone continuation byte
"\xc1\xbf", // overlong, 0xc1 is never a valid lead byte
"\xf0\x9f\x98\x28", // invalid 4th byte
"\xe2\x28\xa1", // invalid 2nd byte of a 3-byte sequence
"0123456789abcdef\x85", // invalid byte after an 8-byte ascii run
};
for (const auto& input : invalid_inputs) {
const char* out_str = castVARCHAR_binary_int64(
ctx_ptr, input.data(), static_cast<int32_t>(input.length()), 0, &out_len);
EXPECT_EQ(std::string(out_str, out_len), "");
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr(
"Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"));
ctx.Reset();
}
}

TEST(TestStringOps, TestCastVarchar) {
gandiva::ExecutionContext ctx;
uint64_t ctx_ptr = reinterpret_cast<gdv_int64>(&ctx);
Expand Down Expand Up @@ -571,11 +624,15 @@ TEST(TestStringOps, TestCastVarchar) {
::testing::HasSubstr("Output buffer length can't be negative"));
ctx.Reset();

// the whole input is validated, not only the truncated prefix
std::string z("aa\xc3");
out_str = castVARCHAR_binary_int64(ctx_ptr, z.data(), static_cast<int>(z.length()), 2,
&out_len);
EXPECT_EQ(std::string(out_str, out_len), "aa");
EXPECT_FALSE(ctx.has_error());
EXPECT_EQ(std::string(out_str, out_len), "");
EXPECT_THAT(
ctx.get_error(),
::testing::HasSubstr("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"));
ctx.Reset();

out_str = castVARCHAR_binary_int64(ctx_ptr, "1234567812341234", 16, 16, &out_len);
EXPECT_EQ(std::string(out_str, out_len), "1234567812341234");
Expand Down Expand Up @@ -633,9 +690,9 @@ TEST(TestStringOps, TestCastVarchar) {
out_str = castVARCHAR_binary_int64(ctx_ptr, f.data(), static_cast<int32_t>(f.length()),
16, &out_len);
EXPECT_EQ(std::string(out_str, out_len), "");
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr(
"unexpected byte \\c3 encountered while decoding utf8 string"));
EXPECT_THAT(
ctx.get_error(),
::testing::HasSubstr("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"));
ctx.Reset();

// UTF8 TESTS
Expand Down
47 changes: 47 additions & 0 deletions cpp/src/gandiva/tests/utf8_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,53 @@ TEST_F(TestUtf8, TestCastVarChar) {
EXPECT_ARROW_ARRAY_EQUALS(exp, outputs[0]);
}

TEST_F(TestUtf8, TestCastVarCharFromBinary) {
auto field_a = field("a", arrow::binary());
auto schema = arrow::schema({field_a});
auto res = field("res", utf8());

auto node_a = TreeExprBuilder::MakeField(field_a);
auto node_len = TreeExprBuilder::MakeLiteral(static_cast<int64_t>(3));
auto cast_varchar =
TreeExprBuilder::MakeFunction("castVARCHAR", {node_a, node_len}, utf8());
auto expr = TreeExprBuilder::MakeExpression(cast_varchar, res);

std::shared_ptr<Projector> projector;
auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();

// the null row holds invalid UTF-8 bytes, which must not be validated
int num_records = 3;
std::vector<int32_t> offsets = {0, 6, 9, 12};
std::vector<uint8_t> validity = {0x05};
auto array_a = std::make_shared<arrow::BinaryArray>(
num_records, arrow::Buffer::Wrap(offsets),
arrow::Buffer::FromString("h\xc3\xa9llo"
"\x85\xe9\x65"
"abc"),
arrow::Buffer::Wrap(validity), /*null_count=*/1);
auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a});

arrow::ArrayVector outputs;
status = projector->Evaluate(*in_batch, pool_, &outputs);
EXPECT_TRUE(status.ok()) << status.message();

auto exp = MakeArrowArrayUtf8({"h\xc3\xa9l", "", "abc"}, {true, false, true});
EXPECT_ARROW_ARRAY_EQUALS(exp, outputs.at(0));

// UNBASE64('hello world!') produces these bytes
auto array_invalid = MakeArrowArrayBinary({"ok", "\x85\xe9\x65"}, {true, true});
auto invalid_batch = arrow::RecordBatch::Make(schema, 2, {array_invalid});

outputs.clear();
status = projector->Evaluate(*invalid_batch, pool_, &outputs);
EXPECT_FALSE(status.ok());
EXPECT_NE(
status.message().find("Cannot cast VARBINARY to VARCHAR: input is not valid UTF-8"),
std::string::npos)
<< status.message();
}

TEST_F(TestUtf8, TestAscii) {
// schema for input fields
auto field0 = field("f0", arrow::utf8());
Expand Down
Loading