diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index 0dbb462893fc..7e3068222541 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -78,7 +78,8 @@ std::vector GetStringFunctionRegistry() { "gdv_fn_base64_encode_binary", NativeFunction::kNeedsContext), NativeFunction("unbase64", {}, DataTypeVector{utf8()}, binary(), kResultNullIfNull, - "gdv_fn_base64_decode_utf8", NativeFunction::kNeedsContext), + "gdv_fn_base64_decode_utf8", + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), NativeFunction("repeat", {}, DataTypeVector{utf8(), int32()}, utf8(), kResultNullIfNull, "repeat_utf8_int32", diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index 229a69624f55..e38a909eedca 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -271,6 +271,28 @@ const char* gdv_fn_base64_encode_binary(int64_t context, const char* in, int32_t return ret; } +namespace { + +// Returns the 6-bit value of a base64 alphabet character (RFC 4648), or -1. +int32_t Base64Value(uint8_t c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; +} + +bool IsBase64Whitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +} // namespace + +// Strict base64 decoding: characters outside the alphabet and malformed tails are errors +// instead of silently truncating the result. Spaces, tabs and line breaks are skipped +// (RFC 2045 allows line breaks, and tools like `base64` end their output with one). +// Padding is optional; when present it must complete the final group of four characters. GANDIVA_EXPORT const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len, int32_t* out_len) { @@ -286,18 +308,77 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i *out_len = 0; return ""; } - // use arrow method to decode base64 string - std::string decoded_str = arrow::util::base64_decode(std::string_view(in, in_len)); - *out_len = static_cast(decoded_str.length()); - // allocate memory for response - char* ret = reinterpret_cast( - gdv_fn_context_arena_malloc(context, static_cast(*out_len))); + + // upper bound of the decoded size; the arena must not be asked for 0 bytes + int32_t capacity = std::max(static_cast(int64_t{in_len} * 3 / 4), 1); + char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, capacity)); if (ret == nullptr) { gdv_fn_context_set_error_msg(context, "Could not allocate memory"); *out_len = 0; return ""; } - memcpy(ret, decoded_str.data(), *out_len); + + int32_t out_pos = 0; + uint32_t accumulator = 0; + int32_t group_size = 0; + int32_t chars = 0; + int32_t padding = 0; + bool valid = true; + char reason[64] = ""; + for (int32_t i = 0; i < in_len; ++i) { + int32_t value = Base64Value(static_cast(in[i])); + if (value >= 0) { + if (padding > 0) { + snprintf(reason, sizeof(reason), "data after padding at position %d", i); + valid = false; + break; + } + accumulator = (accumulator << 6) | static_cast(value); + ++chars; + if (++group_size == 4) { + ret[out_pos++] = static_cast(accumulator >> 16); + ret[out_pos++] = static_cast(accumulator >> 8); + ret[out_pos++] = static_cast(accumulator); + accumulator = 0; + group_size = 0; + } + } else if (in[i] == '=') { + if (++padding > 2) { + snprintf(reason, sizeof(reason), "too many padding characters at position %d", i); + valid = false; + break; + } + } else if (!IsBase64Whitespace(in[i])) { + snprintf(reason, sizeof(reason), "unexpected character 0x%02X at position %d", + static_cast(in[i]), i); + valid = false; + break; + } + } + // padding must complete a group of 4, and a single trailing character carries only 6 + // bits + if (valid && padding > 0 && (chars + padding) % 4 != 0) { + snprintf(reason, sizeof(reason), "padding does not complete the last group of 4"); + valid = false; + } else if (valid && chars % 4 == 1) { + snprintf(reason, sizeof(reason), "incomplete last group"); + valid = false; + } + if (!valid) { + char err_msg[128]; + snprintf(err_msg, sizeof(err_msg), "UNBASE64: input is not valid base64: %s", reason); + gdv_fn_context_set_error_msg(context, err_msg); + *out_len = 0; + return ""; + } + + if (group_size == 2) { + ret[out_pos++] = static_cast(accumulator >> 4); + } else if (group_size == 3) { + ret[out_pos++] = static_cast(accumulator >> 10); + ret[out_pos++] = static_cast(accumulator >> 2); + } + *out_len = out_pos; return ret; } diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index aa253c193987..c1dd6f9ce178 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -22,6 +22,7 @@ #include #include +#include #include "arrow/util/logging.h" #include "gandiva/execution_context.h" @@ -162,6 +163,179 @@ TEST(TestGdvFnStubs, TestBase64Decode) { ctx.Reset(); } +TEST(TestGdvFnStubs, TestBase64DecodeStrict) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + // padding is optional + const std::vector> valid_inputs = { + {"aGVsbG8", "hello"}, + {"aGVsbA==", "hell"}, + {"aGVsbA", "hell"}, + {"ab==", "i"}, + {"abc=", "i\xb7"}, + {"RHLDqW1pbw==", "Dr\xc3\xa9mio"}, + {"+/+/", "\xfb\xff\xbf"}, + // spaces, tabs and line breaks are skipped + {"aGVsbG8=\n", "hello"}, // as printed by the base64 tool + {"aGVsbG8=\r\n", "hello"}, + {" aGVsbG8=", "hello"}, + {"aGVs bG8=", "hello"}, + {"aGVs\nbG8=", "hello"}, // line-wrapped (RFC 2045) + {"aGVs\r\nbG8=\r\n", "hello"}, + {"aGVs\tbG8", "hello"}, + {"aGVsbA= =", "hell"}, // whitespace between padding characters + {" \n", ""}, + // vectors from apache/arrow#49660 + {"Zg==", "f"}, + {"Zm8=", "fo"}, + {"Zm9v", "foo"}, + {"SGVsbG8gd29ybGQ=", "Hello world"}, + {"AAAA", std::string("\x00\x00\x00", 3)}, + {"AP8A", std::string("\x00\xff\x00", 3)}, + }; + for (const auto& [input, expected] : valid_inputs) { + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, input.data(), static_cast(input.length()), &out_len); + EXPECT_EQ(std::string(value, out_len), expected) << input; + EXPECT_FALSE(ctx.has_error()) << input << ": " << ctx.get_error(); + ctx.Reset(); + } + + const std::vector invalid_inputs = { + "hello world!", // used to decode only "hell" and silently drop the rest + "aGVsbG8=aGVs", // data after padding + "aGVsbG8=\naGVs", // data after padding, separated by a line break + "aGVsbG8= x", // any other character after padding + "a\n", // whitespace does not make a single character valid + "hello\nworld!", // not base64 even with the line break skipped + "aGVsbA=", // padding does not complete the group + "aGVsbG8===", // too much padding + "a", // 6 bits cannot form a byte + "aGVsb", // single leftover character + "====", // only padding + "=", // lone padding character + "==", // padding only, no data + "a=", // one data character cannot be completed by padding + "aGVs\vbG8=", // vertical tab is not skipped + "aGVs\fbG8=", // form feed is not skipped + "\v", // not whitespace, so not an empty input + "aGVs-G8_", // url-safe alphabet + "\xe3\x85\x8f \xe3\x85\x91", // non-ascii input + // vectors from apache/arrow#49660 (unpadded "abc" is accepted here, see above) + "ab$=", + "abc\xff", + "aGVs$G8gd29ybGQ=", + "ab=c", + "a===", + "Zm8=Zm8=", + }; + for (const auto& input : invalid_inputs) { + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, input.data(), static_cast(input.length()), &out_len); + EXPECT_EQ(std::string(value, out_len), "") << input; + EXPECT_THAT(ctx.get_error(), + ::testing::HasSubstr("UNBASE64: input is not valid base64")) + << input; + ctx.Reset(); + } +} + +namespace { + +std::string Base64Encode(const std::string& data, bool pad) { + static const char kAlphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + size_t i = 0; + for (; i + 3 <= data.size(); i += 3) { + uint32_t v = static_cast(data[i]) << 16 | + static_cast(data[i + 1]) << 8 | + static_cast(data[i + 2]); + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63], + kAlphabet[v & 63]}; + } + size_t rest = data.size() - i; + if (rest == 1) { + uint32_t v = static_cast(data[i]) << 16; + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63]}; + if (pad) out += "=="; + } else if (rest == 2) { + uint32_t v = static_cast(data[i]) << 16 | static_cast(data[i + 1]) + << 8; + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63]}; + if (pad) out += "="; + } + return out; +} + +} // namespace + +TEST(TestGdvFnStubs, TestBase64DecodeRoundTrip) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + std::mt19937 rng(42); + const std::string whitespace = " \t\n\r"; + + auto check = [&](const std::string& data, bool pad, bool with_whitespace) { + std::string encoded = Base64Encode(data, pad); + if (with_whitespace) { + for (int i = 0; i < 4; ++i) { + encoded.insert(rng() % (encoded.size() + 1), 1, whitespace[rng() % 4]); + } + } + int32_t out_len = 0; + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, encoded.data(), static_cast(encoded.length()), &out_len); + EXPECT_FALSE(ctx.has_error()) << encoded << ": " << ctx.get_error(); + EXPECT_EQ(std::string(value, out_len), data) << encoded; + ctx.Reset(); + }; + + // every byte value, at every position of a group + std::string all_bytes; + for (int b = 0; b < 256; ++b) all_bytes.push_back(static_cast(b)); + for (size_t len = 0; len <= all_bytes.size(); ++len) { + for (bool pad : {true, false}) { + check(all_bytes.substr(0, len), pad, false); + } + } + + // random data of every length up to 300 bytes, with and without whitespace + for (size_t len = 0; len <= 300; ++len) { + for (int round = 0; round < 20; ++round) { + std::string data(len, '\0'); + for (auto& c : data) c = static_cast(rng()); + check(data, rng() % 2 == 0, rng() % 2 == 0); + } + } +} + +TEST(TestGdvFnStubs, TestBase64DecodeErrorReason) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + const std::vector> cases = { + {"hello world!", "unexpected character 0x21 at position 11"}, + {"ab$=", "unexpected character 0x24 at position 2"}, + {"abc\xff", "unexpected character 0xFF at position 3"}, + {"ab=c", "data after padding at position 3"}, + {"a===", "too many padding characters at position 3"}, + {"aGVsbA=", "padding does not complete the last group of 4"}, + {"aGVsb", "incomplete last group"}, + }; + for (const auto& [input, reason] : cases) { + gdv_fn_base64_decode_utf8(ctx_ptr, input.data(), static_cast(input.length()), + &out_len); + EXPECT_THAT(ctx.get_error(), + ::testing::HasSubstr("UNBASE64: input is not valid base64: " + reason)) + << input; + ctx.Reset(); + } +} + TEST(TestGdvFnStubs, TestCastINT) { gandiva::ExecutionContext ctx; diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index fe60ea9c91ef..eb37bc5fe0cc 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -1531,6 +1531,71 @@ TEST_F(TestProjector, TestUnbase64) { EXPECT_ARROW_ARRAY_EQUALS(exp_unbase, outputs.at(0)); } +TEST_F(TestProjector, TestUnbase64NullSlotHoldsInvalidText) { + auto field0 = field("f0", arrow::utf8()); + auto schema = arrow::schema({field0}); + auto expr = TreeExprBuilder::MakeExpression("unbase64", {field0}, + field("base64", arrow::binary())); + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // the NULL row's slot holds "hello world!", which is not valid base64 + std::vector offsets = {0, 7, 19, 19}; + std::vector validity = {0x05}; + auto array = std::make_shared( + 3, arrow::Buffer::Wrap(offsets), arrow::Buffer::FromString("aGVsbG8hello world!"), + arrow::Buffer::Wrap(validity), /*null_count=*/1); + auto batch = arrow::RecordBatch::Make(schema, 3, {array}); + arrow::ArrayVector outputs; + status = projector->Evaluate(*batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}), + outputs.at(0)); +} + +TEST_F(TestProjector, TestUnbase64InvalidInput) { + auto field0 = field("f0", arrow::utf8()); + auto schema = arrow::schema({field0}); + auto field_base = field("base64", arrow::binary()); + auto base_expr = TreeExprBuilder::MakeExpression("unbase64", {field0}, field_base); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {base_expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // the null row is not valid base64, but must not be decoded + auto null_batch = arrow::RecordBatch::Make( + schema, 3, + {MakeArrowArrayUtf8({"aGVsbG8", "hello world!", ""}, {true, false, true})}); + arrow::ArrayVector outputs; + status = projector->Evaluate(*null_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}), + outputs.at(0)); + + // output of the base64 tool: trailing line break and 76-character line wrapping + auto whitespace_batch = arrow::RecordBatch::Make( + schema, 3, + {MakeArrowArrayUtf8({"aGVsbG8=\n", "aGVs\r\nbG8=", " aGVsbG8"}, + {true, true, true})}); + outputs.clear(); + status = projector->Evaluate(*whitespace_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS( + MakeArrowArrayBinary({"hello", "hello", "hello"}, {true, true, true}), + outputs.at(0)); + + auto invalid_batch = arrow::RecordBatch::Make( + schema, 2, {MakeArrowArrayUtf8({"aGVsbG8=", "hello world!"}, {true, true})}); + outputs.clear(); + status = projector->Evaluate(*invalid_batch, pool_, &outputs); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.message().find("UNBASE64: input is not valid base64"), + std::string::npos) + << status.message(); +} + TEST_F(TestProjector, TestLeftString) { // schema for input fields auto field0 = field("f0", arrow::utf8());