From 01e23606178eff3d025336fca8e513bca8d870bb Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Thu, 12 Nov 2026 15:25:17 +0200 Subject: [PATCH 1/6] DX-101651: [C++][Gandiva] unbase64 rejects invalid base64 input unbase64 relied on arrow::util::base64_decode, which stops at the first character outside the alphabet and returns what it decoded so far, so UNBASE64('hello world!') silently produced the 3 bytes 85 E9 A5 instead of failing. Every other malformed input (stray '=', leftover single character, line breaks) was accepted the same way. Decode with a strict RFC 4648 decoder instead and raise "UNBASE64: input is not valid base64". Padding stays optional, as before, but when present it must complete the final group of four characters. The output length is computed up front, and empty output returns "" without calling the arena. Previously a 0-byte allocation could return nullptr, which was reported as "Could not allocate memory" (e.g. for input made only of non-ASCII characters). unbase64 is now flagged kCanReturnErrors so NULL rows, whose slots may hold arbitrary bytes, are not decoded. Co-Authored-By: Claude Sonnet 5.5 --- cpp/src/gandiva/function_registry_string.cc | 21 ++++--- cpp/src/gandiva/gdv_function_stubs.cc | 70 ++++++++++++++++++--- cpp/src/gandiva/gdv_function_stubs_test.cc | 44 +++++++++++++ cpp/src/gandiva/tests/projector_test.cc | 30 +++++++++ 4 files changed, 149 insertions(+), 16 deletions(-) diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index 0dbb462893fc..faa1e5142a5a 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -78,7 +78,8 @@ std::vector GetStringFunctionRegistry() { "gdv_fn_base64_encode_binary", NativeFunction::kNeedsContext), NativeFunction("unbase64", {}, DataTypeVector{utf8()}, binary(), kResultNullIfNull, - "gdv_fn_base64_decode_utf8", NativeFunction::kNeedsContext), + "gdv_fn_base64_decode_utf8", + NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), NativeFunction("repeat", {}, DataTypeVector{utf8(), int32()}, utf8(), kResultNullIfNull, "repeat_utf8_int32", @@ -521,21 +522,23 @@ std::vector GetStringFunctionRegistry() { NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), // Parameters: data, key, mode, iv (e.g. CBC mode) - NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, binary(), - kResultNullIfNull, "gdv_fn_encrypt_dispatcher_4args", + NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, + binary(), kResultNullIfNull, "gdv_fn_encrypt_dispatcher_4args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), - NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, binary(), - kResultNullIfNull, "gdv_fn_decrypt_dispatcher_4args", + NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, + binary(), kResultNullIfNull, "gdv_fn_decrypt_dispatcher_4args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), // Parameters: data, key, mode, iv, fifth_argument (e.g. GCM mode) - NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, binary(), - kResultNullIfNull, "gdv_fn_encrypt_dispatcher_5args", + NativeFunction("encrypt", {}, + DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, + binary(), kResultNullIfNull, "gdv_fn_encrypt_dispatcher_5args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), - NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, binary(), - kResultNullIfNull, "gdv_fn_decrypt_dispatcher_5args", + NativeFunction("decrypt", {}, + DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, + binary(), kResultNullIfNull, "gdv_fn_decrypt_dispatcher_5args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), NativeFunction("mask_first_n", {}, DataTypeVector{utf8(), int32()}, utf8(), diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index 229a69624f55..3f59d771d157 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -271,6 +271,23 @@ const char* gdv_fn_base64_encode_binary(int64_t context, const char* in, int32_t return ret; } +namespace { + +// Returns the 6-bit value of a base64 alphabet character (RFC 4648), or -1. +int32_t Base64Value(uint8_t c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; +} + +} // namespace + +// Strict base64 decoding: characters outside the alphabet and malformed tails are errors +// instead of silently truncating the result. Padding is optional; when present it must +// complete the final group of four characters. GANDIVA_EXPORT const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len, int32_t* out_len) { @@ -282,22 +299,61 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i *out_len = 0; return ""; } - if (in_len == 0) { + + int32_t data_len = in_len; + int32_t padding = 0; + while (data_len > 0 && padding < 2 && in[data_len - 1] == '=') { + --data_len; + ++padding; + } + // padding must complete a group of 4, and a single trailing character carries only 6 + // bits + if ((padding > 0 && (data_len + padding) % 4 != 0) || data_len % 4 == 1) { + gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); + *out_len = 0; + return ""; + } + + int32_t decoded_len = data_len / 4 * 3 + (data_len % 4 == 0 ? 0 : data_len % 4 - 1); + if (decoded_len == 0) { *out_len = 0; return ""; } - // use arrow method to decode base64 string - std::string decoded_str = arrow::util::base64_decode(std::string_view(in, in_len)); - *out_len = static_cast(decoded_str.length()); + // allocate memory for response - char* ret = reinterpret_cast( - gdv_fn_context_arena_malloc(context, static_cast(*out_len))); + char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, decoded_len)); if (ret == nullptr) { gdv_fn_context_set_error_msg(context, "Could not allocate memory"); *out_len = 0; return ""; } - memcpy(ret, decoded_str.data(), *out_len); + + int32_t out_pos = 0; + uint32_t accumulator = 0; + int32_t group_size = 0; + for (int32_t i = 0; i < data_len; ++i) { + int32_t value = Base64Value(static_cast(in[i])); + if (value < 0) { + gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); + *out_len = 0; + return ""; + } + accumulator = (accumulator << 6) | static_cast(value); + if (++group_size == 4) { + ret[out_pos++] = static_cast(accumulator >> 16); + ret[out_pos++] = static_cast(accumulator >> 8); + ret[out_pos++] = static_cast(accumulator); + accumulator = 0; + group_size = 0; + } + } + if (group_size == 2) { + ret[out_pos++] = static_cast(accumulator >> 4); + } else if (group_size == 3) { + ret[out_pos++] = static_cast(accumulator >> 10); + ret[out_pos++] = static_cast(accumulator >> 2); + } + *out_len = out_pos; return ret; } diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index aa253c193987..1327225a7469 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -162,6 +162,50 @@ TEST(TestGdvFnStubs, TestBase64Decode) { ctx.Reset(); } +TEST(TestGdvFnStubs, TestBase64DecodeStrict) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + // padding is optional + const std::vector> valid_inputs = { + {"aGVsbG8", "hello"}, {"aGVsbA==", "hell"}, + {"aGVsbA", "hell"}, {"ab==", "i"}, + {"abc=", "i\xb7"}, {"RHLDqW1pbw==", "Dr\xc3\xa9mio"}, + {"+/+/", "\xfb\xff\xbf"}, + }; + for (const auto& [input, expected] : valid_inputs) { + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, input.data(), static_cast(input.length()), &out_len); + EXPECT_EQ(std::string(value, out_len), expected) << input; + EXPECT_FALSE(ctx.has_error()) << input << ": " << ctx.get_error(); + ctx.Reset(); + } + + const std::vector invalid_inputs = { + "hello world!", // used to decode only "hell" and silently drop the rest + "aGVs bG8=", // space + "aGVs\nbG8=", // line break + "aGVsbG8=aGVs", // data after padding + "aGVsbA=", // padding does not complete the group + "aGVsbG8===", // too much padding + "a", // 6 bits cannot form a byte + "aGVsb", // single leftover character + "====", // only padding + "aGVs-G8_", // url-safe alphabet + "\xe3\x85\x8f \xe3\x85\x91", // non-ascii input + }; + for (const auto& input : invalid_inputs) { + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, input.data(), static_cast(input.length()), &out_len); + EXPECT_EQ(std::string(value, out_len), "") << input; + EXPECT_THAT(ctx.get_error(), + ::testing::HasSubstr("UNBASE64: input is not valid base64")) + << input; + ctx.Reset(); + } +} + TEST(TestGdvFnStubs, TestCastINT) { gandiva::ExecutionContext ctx; diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index fe60ea9c91ef..ccdfd91f9731 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -1531,6 +1531,36 @@ TEST_F(TestProjector, TestUnbase64) { EXPECT_ARROW_ARRAY_EQUALS(exp_unbase, outputs.at(0)); } +TEST_F(TestProjector, TestUnbase64InvalidInput) { + auto field0 = field("f0", arrow::utf8()); + auto schema = arrow::schema({field0}); + auto field_base = field("base64", arrow::binary()); + auto base_expr = TreeExprBuilder::MakeExpression("unbase64", {field0}, field_base); + + std::shared_ptr projector; + auto status = Projector::Make(schema, {base_expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // the null row is not valid base64, but must not be decoded + auto null_batch = arrow::RecordBatch::Make( + schema, 3, + {MakeArrowArrayUtf8({"aGVsbG8", "hello world!", ""}, {true, false, true})}); + arrow::ArrayVector outputs; + status = projector->Evaluate(*null_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}), + outputs.at(0)); + + auto invalid_batch = arrow::RecordBatch::Make( + schema, 2, {MakeArrowArrayUtf8({"aGVsbG8=", "hello world!"}, {true, true})}); + outputs.clear(); + status = projector->Evaluate(*invalid_batch, pool_, &outputs); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.message().find("UNBASE64: input is not valid base64"), + std::string::npos) + << status.message(); +} + TEST_F(TestProjector, TestLeftString) { // schema for input fields auto field0 = field("f0", arrow::utf8()); From 7d0bc69d534af4bd1683d1941fa0a36e0df454e1 Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Thu, 12 Nov 2026 18:20:37 +0200 Subject: [PATCH 2/6] DX-101651: [C++][Gandiva] unbase64 skips spaces, tabs and line breaks The strict decoder rejected whitespace, but output of the base64 tool ends with a line break and RFC 2045 (MIME) allows line breaks anywhere, so inputs such as 'aGVsbG8=\n' that decoded correctly before would have started to fail. Skip ' ', '\t', '\n' and '\r' while decoding. Everything else that is not in the alphabet, data after padding, and malformed padding or length are still errors (whitespace is ignored when checking them). Padding is now handled in the same loop, so the output buffer is sized with an upper bound (3/4 of the input, at least 1 byte) instead of exactly; unrelated to speed, valid-character handling is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- cpp/src/gandiva/gdv_function_stubs.cc | 82 +++++++++++++--------- cpp/src/gandiva/gdv_function_stubs_test.cc | 41 +++++++---- cpp/src/gandiva/tests/projector_test.cc | 12 ++++ 3 files changed, 88 insertions(+), 47 deletions(-) diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index 3f59d771d157..d01c508d5d26 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -283,11 +283,16 @@ int32_t Base64Value(uint8_t c) { return -1; } +bool IsBase64Whitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + } // namespace // Strict base64 decoding: characters outside the alphabet and malformed tails are errors -// instead of silently truncating the result. Padding is optional; when present it must -// complete the final group of four characters. +// instead of silently truncating the result. Spaces, tabs and line breaks are skipped +// (RFC 2045 allows line breaks, and tools like `base64` end their output with one). +// Padding is optional; when present it must complete the final group of four characters. GANDIVA_EXPORT const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len, int32_t* out_len) { @@ -299,29 +304,14 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i *out_len = 0; return ""; } - - int32_t data_len = in_len; - int32_t padding = 0; - while (data_len > 0 && padding < 2 && in[data_len - 1] == '=') { - --data_len; - ++padding; - } - // padding must complete a group of 4, and a single trailing character carries only 6 - // bits - if ((padding > 0 && (data_len + padding) % 4 != 0) || data_len % 4 == 1) { - gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); - *out_len = 0; - return ""; - } - - int32_t decoded_len = data_len / 4 * 3 + (data_len % 4 == 0 ? 0 : data_len % 4 - 1); - if (decoded_len == 0) { + if (in_len == 0) { *out_len = 0; return ""; } - // allocate memory for response - char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, decoded_len)); + // upper bound of the decoded size; the arena must not be asked for 0 bytes + int32_t capacity = std::max(static_cast(int64_t{in_len} * 3 / 4), 1); + char* ret = reinterpret_cast(gdv_fn_context_arena_malloc(context, capacity)); if (ret == nullptr) { gdv_fn_context_set_error_msg(context, "Could not allocate memory"); *out_len = 0; @@ -331,22 +321,46 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i int32_t out_pos = 0; uint32_t accumulator = 0; int32_t group_size = 0; - for (int32_t i = 0; i < data_len; ++i) { + int32_t chars = 0; + int32_t padding = 0; + bool valid = true; + for (int32_t i = 0; i < in_len; ++i) { int32_t value = Base64Value(static_cast(in[i])); - if (value < 0) { - gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); - *out_len = 0; - return ""; - } - accumulator = (accumulator << 6) | static_cast(value); - if (++group_size == 4) { - ret[out_pos++] = static_cast(accumulator >> 16); - ret[out_pos++] = static_cast(accumulator >> 8); - ret[out_pos++] = static_cast(accumulator); - accumulator = 0; - group_size = 0; + if (value >= 0) { + if (padding > 0) { // data after padding + valid = false; + break; + } + accumulator = (accumulator << 6) | static_cast(value); + ++chars; + if (++group_size == 4) { + ret[out_pos++] = static_cast(accumulator >> 16); + ret[out_pos++] = static_cast(accumulator >> 8); + ret[out_pos++] = static_cast(accumulator); + accumulator = 0; + group_size = 0; + } + } else if (in[i] == '=') { + if (++padding > 2) { + valid = false; + break; + } + } else if (!IsBase64Whitespace(in[i])) { + valid = false; + break; } } + // padding must complete a group of 4, and a single trailing character carries only 6 + // bits + if (valid && ((padding > 0 && (chars + padding) % 4 != 0) || chars % 4 == 1)) { + valid = false; + } + if (!valid) { + gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); + *out_len = 0; + return ""; + } + if (group_size == 2) { ret[out_pos++] = static_cast(accumulator >> 4); } else if (group_size == 3) { diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index 1327225a7469..eefa3401d637 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -169,10 +169,23 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { // padding is optional const std::vector> valid_inputs = { - {"aGVsbG8", "hello"}, {"aGVsbA==", "hell"}, - {"aGVsbA", "hell"}, {"ab==", "i"}, - {"abc=", "i\xb7"}, {"RHLDqW1pbw==", "Dr\xc3\xa9mio"}, + {"aGVsbG8", "hello"}, + {"aGVsbA==", "hell"}, + {"aGVsbA", "hell"}, + {"ab==", "i"}, + {"abc=", "i\xb7"}, + {"RHLDqW1pbw==", "Dr\xc3\xa9mio"}, {"+/+/", "\xfb\xff\xbf"}, + // spaces, tabs and line breaks are skipped + {"aGVsbG8=\n", "hello"}, // as printed by the base64 tool + {"aGVsbG8=\r\n", "hello"}, + {" aGVsbG8=", "hello"}, + {"aGVs bG8=", "hello"}, + {"aGVs\nbG8=", "hello"}, // line-wrapped (RFC 2045) + {"aGVs\r\nbG8=\r\n", "hello"}, + {"aGVs\tbG8", "hello"}, + {"aGVsbA= =", "hell"}, // whitespace between padding characters + {" \n", ""}, }; for (const auto& [input, expected] : valid_inputs) { auto value = gdv_fn_base64_decode_utf8( @@ -183,16 +196,18 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { } const std::vector invalid_inputs = { - "hello world!", // used to decode only "hell" and silently drop the rest - "aGVs bG8=", // space - "aGVs\nbG8=", // line break - "aGVsbG8=aGVs", // data after padding - "aGVsbA=", // padding does not complete the group - "aGVsbG8===", // too much padding - "a", // 6 bits cannot form a byte - "aGVsb", // single leftover character - "====", // only padding - "aGVs-G8_", // url-safe alphabet + "hello world!", // used to decode only "hell" and silently drop the rest + "aGVsbG8=aGVs", // data after padding + "aGVsbG8=\naGVs", // data after padding, separated by a line break + "aGVsbG8= x", // any other character after padding + "a\n", // whitespace does not make a single character valid + "hello\nworld!", // not base64 even with the line break skipped + "aGVsbA=", // padding does not complete the group + "aGVsbG8===", // too much padding + "a", // 6 bits cannot form a byte + "aGVsb", // single leftover character + "====", // only padding + "aGVs-G8_", // url-safe alphabet "\xe3\x85\x8f \xe3\x85\x91", // non-ascii input }; for (const auto& input : invalid_inputs) { diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index ccdfd91f9731..09314ce61f6e 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -1551,6 +1551,18 @@ TEST_F(TestProjector, TestUnbase64InvalidInput) { EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}), outputs.at(0)); + // output of the base64 tool: trailing line break and 76-character line wrapping + auto whitespace_batch = arrow::RecordBatch::Make( + schema, 3, + {MakeArrowArrayUtf8({"aGVsbG8=\n", "aGVs\r\nbG8=", " aGVsbG8"}, + {true, true, true})}); + outputs.clear(); + status = projector->Evaluate(*whitespace_batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS( + MakeArrowArrayBinary({"hello", "hello", "hello"}, {true, true, true}), + outputs.at(0)); + auto invalid_batch = arrow::RecordBatch::Make( schema, 2, {MakeArrowArrayUtf8({"aGVsbG8=", "hello world!"}, {true, true})}); outputs.clear(); From 9b7bab2ed786da17460680b60e88b70533d5b229 Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Fri, 13 Nov 2026 12:03:52 +0200 Subject: [PATCH 3/6] DX-101651: [C++][Gandiva] test more invalid unbase64 inputs Cover a lone padding character, padding without data, a single data character followed by padding, and vertical tab / form feed, which are not skipped like space, tab, CR and LF. Co-Authored-By: Claude Sonnet 5.5 --- cpp/src/gandiva/gdv_function_stubs_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index eefa3401d637..7a1360fa781e 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -207,6 +207,12 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { "a", // 6 bits cannot form a byte "aGVsb", // single leftover character "====", // only padding + "=", // lone padding character + "==", // padding only, no data + "a=", // one data character cannot be completed by padding + "aGVs\vbG8=", // vertical tab is not skipped + "aGVs\fbG8=", // form feed is not skipped + "\v", // not whitespace, so not an empty input "aGVs-G8_", // url-safe alphabet "\xe3\x85\x8f \xe3\x85\x91", // non-ascii input }; From c20288d9bbb81b020c017c9ce21417e619644139 Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Wed, 18 Nov 2026 12:28:34 +0200 Subject: [PATCH 4/6] DX-101651: [C++][Gandiva] unbase64 reports why the input is invalid The error now names the rule that failed (unexpected character, data after padding, too many padding characters, incomplete group) and its position. Also test the vectors from apache/arrow#49660, except for the unpadded input that is accepted here. --- cpp/src/gandiva/gdv_function_stubs.cc | 17 ++++++++-- cpp/src/gandiva/gdv_function_stubs_test.cc | 38 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc index d01c508d5d26..e38a909eedca 100644 --- a/cpp/src/gandiva/gdv_function_stubs.cc +++ b/cpp/src/gandiva/gdv_function_stubs.cc @@ -324,10 +324,12 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i int32_t chars = 0; int32_t padding = 0; bool valid = true; + char reason[64] = ""; for (int32_t i = 0; i < in_len; ++i) { int32_t value = Base64Value(static_cast(in[i])); if (value >= 0) { - if (padding > 0) { // data after padding + if (padding > 0) { + snprintf(reason, sizeof(reason), "data after padding at position %d", i); valid = false; break; } @@ -342,21 +344,30 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i } } else if (in[i] == '=') { if (++padding > 2) { + snprintf(reason, sizeof(reason), "too many padding characters at position %d", i); valid = false; break; } } else if (!IsBase64Whitespace(in[i])) { + snprintf(reason, sizeof(reason), "unexpected character 0x%02X at position %d", + static_cast(in[i]), i); valid = false; break; } } // padding must complete a group of 4, and a single trailing character carries only 6 // bits - if (valid && ((padding > 0 && (chars + padding) % 4 != 0) || chars % 4 == 1)) { + if (valid && padding > 0 && (chars + padding) % 4 != 0) { + snprintf(reason, sizeof(reason), "padding does not complete the last group of 4"); + valid = false; + } else if (valid && chars % 4 == 1) { + snprintf(reason, sizeof(reason), "incomplete last group"); valid = false; } if (!valid) { - gdv_fn_context_set_error_msg(context, "UNBASE64: input is not valid base64"); + char err_msg[128]; + snprintf(err_msg, sizeof(err_msg), "UNBASE64: input is not valid base64: %s", reason); + gdv_fn_context_set_error_msg(context, err_msg); *out_len = 0; return ""; } diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index 7a1360fa781e..6231cfe7e72a 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -186,6 +186,13 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { {"aGVs\tbG8", "hello"}, {"aGVsbA= =", "hell"}, // whitespace between padding characters {" \n", ""}, + // vectors from apache/arrow#49660 + {"Zg==", "f"}, + {"Zm8=", "fo"}, + {"Zm9v", "foo"}, + {"SGVsbG8gd29ybGQ=", "Hello world"}, + {"AAAA", std::string("\x00\x00\x00", 3)}, + {"AP8A", std::string("\x00\xff\x00", 3)}, }; for (const auto& [input, expected] : valid_inputs) { auto value = gdv_fn_base64_decode_utf8( @@ -215,6 +222,13 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { "\v", // not whitespace, so not an empty input "aGVs-G8_", // url-safe alphabet "\xe3\x85\x8f \xe3\x85\x91", // non-ascii input + // vectors from apache/arrow#49660 (unpadded "abc" is accepted here, see above) + "ab$=", + "abc\xff", + "aGVs$G8gd29ybGQ=", + "ab=c", + "a===", + "Zm8=Zm8=", }; for (const auto& input : invalid_inputs) { auto value = gdv_fn_base64_decode_utf8( @@ -227,6 +241,30 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { } } +TEST(TestGdvFnStubs, TestBase64DecodeErrorReason) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + + const std::vector> cases = { + {"hello world!", "unexpected character 0x21 at position 11"}, + {"ab$=", "unexpected character 0x24 at position 2"}, + {"abc\xff", "unexpected character 0xFF at position 3"}, + {"ab=c", "data after padding at position 3"}, + {"a===", "too many padding characters at position 3"}, + {"aGVsbA=", "padding does not complete the last group of 4"}, + {"aGVsb", "incomplete last group"}, + }; + for (const auto& [input, reason] : cases) { + gdv_fn_base64_decode_utf8(ctx_ptr, input.data(), static_cast(input.length()), + &out_len); + EXPECT_THAT(ctx.get_error(), + ::testing::HasSubstr("UNBASE64: input is not valid base64: " + reason)) + << input; + ctx.Reset(); + } +} + TEST(TestGdvFnStubs, TestCastINT) { gandiva::ExecutionContext ctx; From 539cb83c57ad346225c3e2d24ed74556371de511 Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Wed, 18 Nov 2026 12:40:56 +0200 Subject: [PATCH 5/6] DX-101651: [C++][Gandiva] unbase64 round-trip test, drop unrelated reformatting Add a test that encodes every byte value and random data of every length up to 300 bytes, with and without padding and with whitespace inserted, and checks that unbase64 returns the original bytes. Restore the encrypt/decrypt registry entries that an earlier commit reflowed. --- cpp/src/gandiva/function_registry_string.cc | 18 +++--- cpp/src/gandiva/gdv_function_stubs_test.cc | 71 +++++++++++++++++++++ 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/cpp/src/gandiva/function_registry_string.cc b/cpp/src/gandiva/function_registry_string.cc index faa1e5142a5a..7e3068222541 100644 --- a/cpp/src/gandiva/function_registry_string.cc +++ b/cpp/src/gandiva/function_registry_string.cc @@ -522,23 +522,21 @@ std::vector GetStringFunctionRegistry() { NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), // Parameters: data, key, mode, iv (e.g. CBC mode) - NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, - binary(), kResultNullIfNull, "gdv_fn_encrypt_dispatcher_4args", + NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, binary(), + kResultNullIfNull, "gdv_fn_encrypt_dispatcher_4args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), - NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, - binary(), kResultNullIfNull, "gdv_fn_decrypt_dispatcher_4args", + NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary()}, binary(), + kResultNullIfNull, "gdv_fn_decrypt_dispatcher_4args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), // Parameters: data, key, mode, iv, fifth_argument (e.g. GCM mode) - NativeFunction("encrypt", {}, - DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, - binary(), kResultNullIfNull, "gdv_fn_encrypt_dispatcher_5args", + NativeFunction("encrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, binary(), + kResultNullIfNull, "gdv_fn_encrypt_dispatcher_5args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), - NativeFunction("decrypt", {}, - DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, - binary(), kResultNullIfNull, "gdv_fn_decrypt_dispatcher_5args", + NativeFunction("decrypt", {}, DataTypeVector{binary(), binary(), utf8(), binary(), binary()}, binary(), + kResultNullIfNull, "gdv_fn_decrypt_dispatcher_5args", NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors), NativeFunction("mask_first_n", {}, DataTypeVector{utf8(), int32()}, utf8(), diff --git a/cpp/src/gandiva/gdv_function_stubs_test.cc b/cpp/src/gandiva/gdv_function_stubs_test.cc index 6231cfe7e72a..c1dd6f9ce178 100644 --- a/cpp/src/gandiva/gdv_function_stubs_test.cc +++ b/cpp/src/gandiva/gdv_function_stubs_test.cc @@ -22,6 +22,7 @@ #include #include +#include #include "arrow/util/logging.h" #include "gandiva/execution_context.h" @@ -241,6 +242,76 @@ TEST(TestGdvFnStubs, TestBase64DecodeStrict) { } } +namespace { + +std::string Base64Encode(const std::string& data, bool pad) { + static const char kAlphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + size_t i = 0; + for (; i + 3 <= data.size(); i += 3) { + uint32_t v = static_cast(data[i]) << 16 | + static_cast(data[i + 1]) << 8 | + static_cast(data[i + 2]); + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63], + kAlphabet[v & 63]}; + } + size_t rest = data.size() - i; + if (rest == 1) { + uint32_t v = static_cast(data[i]) << 16; + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63]}; + if (pad) out += "=="; + } else if (rest == 2) { + uint32_t v = static_cast(data[i]) << 16 | static_cast(data[i + 1]) + << 8; + out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63]}; + if (pad) out += "="; + } + return out; +} + +} // namespace + +TEST(TestGdvFnStubs, TestBase64DecodeRoundTrip) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + std::mt19937 rng(42); + const std::string whitespace = " \t\n\r"; + + auto check = [&](const std::string& data, bool pad, bool with_whitespace) { + std::string encoded = Base64Encode(data, pad); + if (with_whitespace) { + for (int i = 0; i < 4; ++i) { + encoded.insert(rng() % (encoded.size() + 1), 1, whitespace[rng() % 4]); + } + } + int32_t out_len = 0; + auto value = gdv_fn_base64_decode_utf8( + ctx_ptr, encoded.data(), static_cast(encoded.length()), &out_len); + EXPECT_FALSE(ctx.has_error()) << encoded << ": " << ctx.get_error(); + EXPECT_EQ(std::string(value, out_len), data) << encoded; + ctx.Reset(); + }; + + // every byte value, at every position of a group + std::string all_bytes; + for (int b = 0; b < 256; ++b) all_bytes.push_back(static_cast(b)); + for (size_t len = 0; len <= all_bytes.size(); ++len) { + for (bool pad : {true, false}) { + check(all_bytes.substr(0, len), pad, false); + } + } + + // random data of every length up to 300 bytes, with and without whitespace + for (size_t len = 0; len <= 300; ++len) { + for (int round = 0; round < 20; ++round) { + std::string data(len, '\0'); + for (auto& c : data) c = static_cast(rng()); + check(data, rng() % 2 == 0, rng() % 2 == 0); + } + } +} + TEST(TestGdvFnStubs, TestBase64DecodeErrorReason) { gandiva::ExecutionContext ctx; auto ctx_ptr = reinterpret_cast(&ctx); From e513bb05c050739c8546bab09c6f3d08532f04d2 Mon Sep 17 00:00:00 2001 From: vitaliibrodetskyi-prog Date: Wed, 18 Nov 2026 12:55:07 +0200 Subject: [PATCH 6/6] DX-101651: [C++][Gandiva] test unbase64 on a NULL row holding invalid text The existing NULL-row test builds its array with an empty NULL slot, so it passes even without kCanReturnErrors. Put "hello world!" in the NULL slot: the batch only succeeds because the NULL row is not decoded. --- cpp/src/gandiva/tests/projector_test.cc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 09314ce61f6e..eb37bc5fe0cc 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -1531,6 +1531,29 @@ TEST_F(TestProjector, TestUnbase64) { EXPECT_ARROW_ARRAY_EQUALS(exp_unbase, outputs.at(0)); } +TEST_F(TestProjector, TestUnbase64NullSlotHoldsInvalidText) { + auto field0 = field("f0", arrow::utf8()); + auto schema = arrow::schema({field0}); + auto expr = TreeExprBuilder::MakeExpression("unbase64", {field0}, + field("base64", arrow::binary())); + std::shared_ptr projector; + auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); + EXPECT_TRUE(status.ok()) << status.message(); + + // the NULL row's slot holds "hello world!", which is not valid base64 + std::vector offsets = {0, 7, 19, 19}; + std::vector validity = {0x05}; + auto array = std::make_shared( + 3, arrow::Buffer::Wrap(offsets), arrow::Buffer::FromString("aGVsbG8hello world!"), + arrow::Buffer::Wrap(validity), /*null_count=*/1); + auto batch = arrow::RecordBatch::Make(schema, 3, {array}); + arrow::ArrayVector outputs; + status = projector->Evaluate(*batch, pool_, &outputs); + EXPECT_TRUE(status.ok()) << status.message(); + EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}), + outputs.at(0)); +} + TEST_F(TestProjector, TestUnbase64InvalidInput) { auto field0 = field("f0", arrow::utf8()); auto schema = arrow::schema({field0});