Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cpp/src/gandiva/function_registry_string.cc
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ std::vector<NativeFunction> GetStringFunctionRegistry() {
"gdv_fn_base64_encode_binary", NativeFunction::kNeedsContext),

NativeFunction("unbase64", {}, DataTypeVector{utf8()}, binary(), kResultNullIfNull,
"gdv_fn_base64_decode_utf8", NativeFunction::kNeedsContext),
"gdv_fn_base64_decode_utf8",
NativeFunction::kNeedsContext | NativeFunction::kCanReturnErrors),

NativeFunction("repeat", {}, DataTypeVector{utf8(), int32()}, utf8(),
kResultNullIfNull, "repeat_utf8_int32",
Expand Down
95 changes: 88 additions & 7 deletions cpp/src/gandiva/gdv_function_stubs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,28 @@ const char* gdv_fn_base64_encode_binary(int64_t context, const char* in, int32_t
return ret;
}

namespace {

// Returns the 6-bit value of a base64 alphabet character (RFC 4648), or -1.
int32_t Base64Value(uint8_t c) {
if (c >= 'A' && c <= 'Z') return c - 'A';
if (c >= 'a' && c <= 'z') return c - 'a' + 26;
if (c >= '0' && c <= '9') return c - '0' + 52;
if (c == '+') return 62;
if (c == '/') return 63;
return -1;
}

bool IsBase64Whitespace(char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r';
}

} // namespace

// Strict base64 decoding: characters outside the alphabet and malformed tails are errors
// instead of silently truncating the result. Spaces, tabs and line breaks are skipped
// (RFC 2045 allows line breaks, and tools like `base64` end their output with one).
// Padding is optional; when present it must complete the final group of four characters.
GANDIVA_EXPORT
const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t in_len,
int32_t* out_len) {
Expand All @@ -286,18 +308,77 @@ const char* gdv_fn_base64_decode_utf8(int64_t context, const char* in, int32_t i
*out_len = 0;
return "";
}
// use arrow method to decode base64 string
std::string decoded_str = arrow::util::base64_decode(std::string_view(in, in_len));
*out_len = static_cast<int32_t>(decoded_str.length());
// allocate memory for response
char* ret = reinterpret_cast<char*>(
gdv_fn_context_arena_malloc(context, static_cast<int32_t>(*out_len)));

// upper bound of the decoded size; the arena must not be asked for 0 bytes
int32_t capacity = std::max<int32_t>(static_cast<int32_t>(int64_t{in_len} * 3 / 4), 1);
char* ret = reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context, capacity));
if (ret == nullptr) {
gdv_fn_context_set_error_msg(context, "Could not allocate memory");
*out_len = 0;
return "";
}
memcpy(ret, decoded_str.data(), *out_len);

int32_t out_pos = 0;
uint32_t accumulator = 0;
int32_t group_size = 0;
int32_t chars = 0;
int32_t padding = 0;
bool valid = true;
char reason[64] = "";
for (int32_t i = 0; i < in_len; ++i) {
int32_t value = Base64Value(static_cast<uint8_t>(in[i]));
if (value >= 0) {
if (padding > 0) {
snprintf(reason, sizeof(reason), "data after padding at position %d", i);
valid = false;
break;
}
accumulator = (accumulator << 6) | static_cast<uint32_t>(value);
++chars;
if (++group_size == 4) {
ret[out_pos++] = static_cast<char>(accumulator >> 16);
ret[out_pos++] = static_cast<char>(accumulator >> 8);
ret[out_pos++] = static_cast<char>(accumulator);
accumulator = 0;
group_size = 0;
}
} else if (in[i] == '=') {
if (++padding > 2) {
snprintf(reason, sizeof(reason), "too many padding characters at position %d", i);
valid = false;
break;
}
} else if (!IsBase64Whitespace(in[i])) {
snprintf(reason, sizeof(reason), "unexpected character 0x%02X at position %d",
static_cast<uint8_t>(in[i]), i);
valid = false;
break;
}
}
// padding must complete a group of 4, and a single trailing character carries only 6
// bits
if (valid && padding > 0 && (chars + padding) % 4 != 0) {
snprintf(reason, sizeof(reason), "padding does not complete the last group of 4");
valid = false;
} else if (valid && chars % 4 == 1) {
snprintf(reason, sizeof(reason), "incomplete last group");
valid = false;
}
if (!valid) {
char err_msg[128];
snprintf(err_msg, sizeof(err_msg), "UNBASE64: input is not valid base64: %s", reason);
gdv_fn_context_set_error_msg(context, err_msg);
*out_len = 0;
return "";
}

if (group_size == 2) {
ret[out_pos++] = static_cast<char>(accumulator >> 4);
} else if (group_size == 3) {
ret[out_pos++] = static_cast<char>(accumulator >> 10);
ret[out_pos++] = static_cast<char>(accumulator >> 2);
}
*out_len = out_pos;
return ret;
}

Expand Down
174 changes: 174 additions & 0 deletions cpp/src/gandiva/gdv_function_stubs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <gtest/gtest.h>

#include <limits>
#include <random>

#include "arrow/util/logging.h"
#include "gandiva/execution_context.h"
Expand Down Expand Up @@ -162,6 +163,179 @@ TEST(TestGdvFnStubs, TestBase64Decode) {
ctx.Reset();
}

TEST(TestGdvFnStubs, TestBase64DecodeStrict) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = 0;

// padding is optional
const std::vector<std::pair<std::string, std::string>> valid_inputs = {
{"aGVsbG8", "hello"},
{"aGVsbA==", "hell"},
{"aGVsbA", "hell"},
{"ab==", "i"},
{"abc=", "i\xb7"},
{"RHLDqW1pbw==", "Dr\xc3\xa9mio"},
{"+/+/", "\xfb\xff\xbf"},
// spaces, tabs and line breaks are skipped
{"aGVsbG8=\n", "hello"}, // as printed by the base64 tool
{"aGVsbG8=\r\n", "hello"},
{" aGVsbG8=", "hello"},
{"aGVs bG8=", "hello"},
{"aGVs\nbG8=", "hello"}, // line-wrapped (RFC 2045)
{"aGVs\r\nbG8=\r\n", "hello"},
{"aGVs\tbG8", "hello"},
{"aGVsbA= =", "hell"}, // whitespace between padding characters
{" \n", ""},
// vectors from apache/arrow#49660
{"Zg==", "f"},
{"Zm8=", "fo"},
{"Zm9v", "foo"},
{"SGVsbG8gd29ybGQ=", "Hello world"},
{"AAAA", std::string("\x00\x00\x00", 3)},
{"AP8A", std::string("\x00\xff\x00", 3)},
};
for (const auto& [input, expected] : valid_inputs) {
auto value = gdv_fn_base64_decode_utf8(
ctx_ptr, input.data(), static_cast<int32_t>(input.length()), &out_len);
EXPECT_EQ(std::string(value, out_len), expected) << input;
EXPECT_FALSE(ctx.has_error()) << input << ": " << ctx.get_error();
ctx.Reset();
}

const std::vector<std::string> invalid_inputs = {
"hello world!", // used to decode only "hell" and silently drop the rest
"aGVsbG8=aGVs", // data after padding
"aGVsbG8=\naGVs", // data after padding, separated by a line break
"aGVsbG8= x", // any other character after padding
"a\n", // whitespace does not make a single character valid
"hello\nworld!", // not base64 even with the line break skipped
"aGVsbA=", // padding does not complete the group
"aGVsbG8===", // too much padding
"a", // 6 bits cannot form a byte
"aGVsb", // single leftover character
"====", // only padding
"=", // lone padding character
"==", // padding only, no data
"a=", // one data character cannot be completed by padding
"aGVs\vbG8=", // vertical tab is not skipped
"aGVs\fbG8=", // form feed is not skipped
"\v", // not whitespace, so not an empty input
"aGVs-G8_", // url-safe alphabet
"\xe3\x85\x8f \xe3\x85\x91", // non-ascii input
// vectors from apache/arrow#49660 (unpadded "abc" is accepted here, see above)
"ab$=",
"abc\xff",
"aGVs$G8gd29ybGQ=",
"ab=c",
"a===",
"Zm8=Zm8=",
};
for (const auto& input : invalid_inputs) {
auto value = gdv_fn_base64_decode_utf8(
ctx_ptr, input.data(), static_cast<int32_t>(input.length()), &out_len);
EXPECT_EQ(std::string(value, out_len), "") << input;
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr("UNBASE64: input is not valid base64"))
<< input;
ctx.Reset();
}
}

namespace {

std::string Base64Encode(const std::string& data, bool pad) {
static const char kAlphabet[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::string out;
size_t i = 0;
for (; i + 3 <= data.size(); i += 3) {
uint32_t v = static_cast<uint8_t>(data[i]) << 16 |
static_cast<uint8_t>(data[i + 1]) << 8 |
static_cast<uint8_t>(data[i + 2]);
out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63],
kAlphabet[v & 63]};
}
size_t rest = data.size() - i;
if (rest == 1) {
uint32_t v = static_cast<uint8_t>(data[i]) << 16;
out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63]};
if (pad) out += "==";
} else if (rest == 2) {
uint32_t v = static_cast<uint8_t>(data[i]) << 16 | static_cast<uint8_t>(data[i + 1])
<< 8;
out += {kAlphabet[v >> 18], kAlphabet[(v >> 12) & 63], kAlphabet[(v >> 6) & 63]};
if (pad) out += "=";
}
return out;
}

} // namespace

TEST(TestGdvFnStubs, TestBase64DecodeRoundTrip) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
std::mt19937 rng(42);
const std::string whitespace = " \t\n\r";

auto check = [&](const std::string& data, bool pad, bool with_whitespace) {
std::string encoded = Base64Encode(data, pad);
if (with_whitespace) {
for (int i = 0; i < 4; ++i) {
encoded.insert(rng() % (encoded.size() + 1), 1, whitespace[rng() % 4]);
}
}
int32_t out_len = 0;
auto value = gdv_fn_base64_decode_utf8(
ctx_ptr, encoded.data(), static_cast<int32_t>(encoded.length()), &out_len);
EXPECT_FALSE(ctx.has_error()) << encoded << ": " << ctx.get_error();
EXPECT_EQ(std::string(value, out_len), data) << encoded;
ctx.Reset();
};

// every byte value, at every position of a group
std::string all_bytes;
for (int b = 0; b < 256; ++b) all_bytes.push_back(static_cast<char>(b));
for (size_t len = 0; len <= all_bytes.size(); ++len) {
for (bool pad : {true, false}) {
check(all_bytes.substr(0, len), pad, false);
}
}

// random data of every length up to 300 bytes, with and without whitespace
for (size_t len = 0; len <= 300; ++len) {
for (int round = 0; round < 20; ++round) {
std::string data(len, '\0');
for (auto& c : data) c = static_cast<char>(rng());
check(data, rng() % 2 == 0, rng() % 2 == 0);
}
}
}

TEST(TestGdvFnStubs, TestBase64DecodeErrorReason) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = 0;

const std::vector<std::pair<std::string, std::string>> cases = {
{"hello world!", "unexpected character 0x21 at position 11"},
{"ab$=", "unexpected character 0x24 at position 2"},
{"abc\xff", "unexpected character 0xFF at position 3"},
{"ab=c", "data after padding at position 3"},
{"a===", "too many padding characters at position 3"},
{"aGVsbA=", "padding does not complete the last group of 4"},
{"aGVsb", "incomplete last group"},
};
for (const auto& [input, reason] : cases) {
gdv_fn_base64_decode_utf8(ctx_ptr, input.data(), static_cast<int32_t>(input.length()),
&out_len);
EXPECT_THAT(ctx.get_error(),
::testing::HasSubstr("UNBASE64: input is not valid base64: " + reason))
<< input;
ctx.Reset();
}
}

TEST(TestGdvFnStubs, TestCastINT) {
gandiva::ExecutionContext ctx;

Expand Down
65 changes: 65 additions & 0 deletions cpp/src/gandiva/tests/projector_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1531,6 +1531,71 @@ TEST_F(TestProjector, TestUnbase64) {
EXPECT_ARROW_ARRAY_EQUALS(exp_unbase, outputs.at(0));
}

TEST_F(TestProjector, TestUnbase64NullSlotHoldsInvalidText) {
auto field0 = field("f0", arrow::utf8());
auto schema = arrow::schema({field0});
auto expr = TreeExprBuilder::MakeExpression("unbase64", {field0},
field("base64", arrow::binary()));
std::shared_ptr<Projector> projector;
auto status = Projector::Make(schema, {expr}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();

// the NULL row's slot holds "hello world!", which is not valid base64
std::vector<int32_t> offsets = {0, 7, 19, 19};
std::vector<uint8_t> validity = {0x05};
auto array = std::make_shared<arrow::StringArray>(
3, arrow::Buffer::Wrap(offsets), arrow::Buffer::FromString("aGVsbG8hello world!"),
arrow::Buffer::Wrap(validity), /*null_count=*/1);
auto batch = arrow::RecordBatch::Make(schema, 3, {array});
arrow::ArrayVector outputs;
status = projector->Evaluate(*batch, pool_, &outputs);
EXPECT_TRUE(status.ok()) << status.message();
EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}),
outputs.at(0));
}

TEST_F(TestProjector, TestUnbase64InvalidInput) {
auto field0 = field("f0", arrow::utf8());
auto schema = arrow::schema({field0});
auto field_base = field("base64", arrow::binary());
auto base_expr = TreeExprBuilder::MakeExpression("unbase64", {field0}, field_base);

std::shared_ptr<Projector> projector;
auto status = Projector::Make(schema, {base_expr}, TestConfiguration(), &projector);
EXPECT_TRUE(status.ok()) << status.message();

// the null row is not valid base64, but must not be decoded
auto null_batch = arrow::RecordBatch::Make(
schema, 3,
{MakeArrowArrayUtf8({"aGVsbG8", "hello world!", ""}, {true, false, true})});
arrow::ArrayVector outputs;
status = projector->Evaluate(*null_batch, pool_, &outputs);
EXPECT_TRUE(status.ok()) << status.message();
EXPECT_ARROW_ARRAY_EQUALS(MakeArrowArrayBinary({"hello", "", ""}, {true, false, true}),
outputs.at(0));

// output of the base64 tool: trailing line break and 76-character line wrapping
auto whitespace_batch = arrow::RecordBatch::Make(
schema, 3,
{MakeArrowArrayUtf8({"aGVsbG8=\n", "aGVs\r\nbG8=", " aGVsbG8"},
{true, true, true})});
outputs.clear();
status = projector->Evaluate(*whitespace_batch, pool_, &outputs);
EXPECT_TRUE(status.ok()) << status.message();
EXPECT_ARROW_ARRAY_EQUALS(
MakeArrowArrayBinary({"hello", "hello", "hello"}, {true, true, true}),
outputs.at(0));

auto invalid_batch = arrow::RecordBatch::Make(
schema, 2, {MakeArrowArrayUtf8({"aGVsbG8=", "hello world!"}, {true, true})});
outputs.clear();
status = projector->Evaluate(*invalid_batch, pool_, &outputs);
EXPECT_FALSE(status.ok());
EXPECT_NE(status.message().find("UNBASE64: input is not valid base64"),
std::string::npos)
<< status.message();
}

TEST_F(TestProjector, TestLeftString) {
// schema for input fields
auto field0 = field("f0", arrow::utf8());
Expand Down
Loading