Repository navigation
feat: Validate UTF-8 at NANOARROW_VALIDATION_LEVEL_FULL #981
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
rustyconover
wants to merge
3
commits into
apache:main
Choose a base branch
from
rustyconover:feat/validate-utf8-full
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2123,6 +2123,158 @@ static int ArrowAssertInt8In(struct ArrowBufferView view, const int8_t* values, | |
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| // The byte ranges below implement the grammar in RFC 3629, Section 4: | ||
| // https://www.rfc-editor.org/rfc/rfc3629.html#section-4 | ||
| // Returns the length of the longest prefix of data that is valid UTF-8, so | ||
| // data is valid UTF-8 if and only if the result is size_bytes. Overlong encodings, | ||
| // surrogates (U+D800 to U+DFFF), code points above U+10FFFF, stray continuation bytes | ||
| // and truncated sequences are all invalid. | ||
| static int64_t ArrowUtf8ValidPrefix(const uint8_t* data, int64_t size_bytes) { | ||
| int64_t i = 0; | ||
| while (i < size_bytes) { | ||
| const uint8_t lead = data[i]; | ||
|
|
||
| if (lead < 0x80) { | ||
| // After an ASCII byte, skip any further ASCII eight bytes at a time | ||
| i++; | ||
| while (size_bytes - i >= 8) { | ||
| uint64_t word; | ||
| memcpy(&word, data + i, sizeof(word)); | ||
| if ((word & 0x8080808080808080ULL) != 0) { | ||
| break; | ||
| } | ||
| i += 8; | ||
| } | ||
| continue; | ||
| } | ||
|
|
||
| if (lead < 0xE0) { | ||
| // Two bytes (U+0080 to U+07FF); 0xC0 and 0xC1 would be overlong | ||
| if (lead < 0xC2 || size_bytes - i < 2 || (data[i + 1] & 0xC0) != 0x80) { | ||
| return i; | ||
| } | ||
| i += 2; | ||
| continue; | ||
| } | ||
|
|
||
| if (lead < 0xF0) { | ||
| // Three bytes (U+0800 to U+FFFF), excluding overlong encodings (0xE0 followed | ||
| // by less than 0xA0) and surrogates (0xED followed by 0xA0 or more) | ||
| if (size_bytes - i < 3) { | ||
| return i; | ||
| } | ||
| const uint8_t second = data[i + 1]; | ||
| const uint8_t second_min = lead == 0xE0 ? 0xA0 : 0x80; | ||
| const uint8_t second_max = lead == 0xED ? 0x9F : 0xBF; | ||
| if (second < second_min || second > second_max || (data[i + 2] & 0xC0) != 0x80) { | ||
| return i; | ||
| } | ||
| i += 3; | ||
| continue; | ||
| } | ||
|
|
||
| // Four bytes (U+10000 to U+10FFFF), excluding overlong encodings (0xF0 followed by | ||
| // less than 0x90) and code points above U+10FFFF (0xF4 followed by 0x90 or more, or | ||
| // a lead byte above 0xF4) | ||
| if (lead > 0xF4 || size_bytes - i < 4) { | ||
| return i; | ||
| } | ||
| const uint8_t second = data[i + 1]; | ||
| const uint8_t second_min = lead == 0xF0 ? 0x90 : 0x80; | ||
| const uint8_t second_max = lead == 0xF4 ? 0x8F : 0xBF; | ||
| if (second < second_min || second > second_max || (data[i + 2] & 0xC0) != 0x80 || | ||
| (data[i + 3] & 0xC0) != 0x80) { | ||
| return i; | ||
| } | ||
| i += 4; | ||
| } | ||
|
|
||
| return size_bytes; | ||
| } | ||
|
|
||
| static int64_t ArrowArrayViewGetOffsetUnsafe(const struct ArrowArrayView* array_view, | ||
| int64_t i) { | ||
| if (array_view->layout.element_size_bits[1] == 32) { | ||
| return ArrowBufferViewGetInt32Unsafe(&array_view->buffer_views[1], i); | ||
| } else { | ||
| return ArrowBufferViewGetInt64Unsafe(&array_view->buffer_views[1], i); | ||
| } | ||
| } | ||
|
|
||
| static int ArrowArrayViewValidateUtf8Element(const struct ArrowArrayView* array_view, | ||
| int64_t i, const uint8_t* value, | ||
| int64_t size_bytes, | ||
| struct ArrowError* error) { | ||
| if (size_bytes == 0) { | ||
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| int64_t valid_bytes = ArrowUtf8ValidPrefix(value, size_bytes); | ||
| if (valid_bytes != size_bytes) { | ||
| ArrowErrorSet(error, | ||
| "[%" PRId64 | ||
| "] Expected valid UTF-8 %s value but found invalid " | ||
| "sequence at byte %" PRId64, | ||
| i, ArrowTypeString(array_view->storage_type), valid_bytes); | ||
| return EINVAL; | ||
| } | ||
|
|
||
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| // Check that every non-null value of a string or large string array is valid UTF-8. | ||
| // A code point split across two values is invalid. The fast path also checks bytes | ||
| // in null slots, but invalid UTF-8 in a null slot does not fail validation: the | ||
| // fallback checks non-null values individually. | ||
| static int ArrowArrayViewValidateUtf8(const struct ArrowArrayView* array_view, | ||
| struct ArrowError* error) { | ||
| // The offsets buffer of a zero-length array may be empty | ||
| if (array_view->length == 0) { | ||
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| const uint8_t* data = array_view->buffer_views[2].data.as_uint8; | ||
|
|
||
| // The values (including those of null slots) are the contiguous range of bytes | ||
| // between the first and last offsets. If that whole range is valid and every value | ||
| // begins on a code point boundary (i.e., not with a continuation byte), every value is | ||
| // valid, which avoids validating short values one at a time. Otherwise, check the | ||
| // non-null values one at a time to find (and report) the first invalid one. | ||
| { | ||
| int64_t first = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset); | ||
| int64_t last = ArrowArrayViewGetOffsetUnsafe(array_view, | ||
| array_view->offset + array_view->length); | ||
| if (first == last) { | ||
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| int ok = ArrowUtf8ValidPrefix(data + first, last - first) == last - first; | ||
| for (int64_t i = 1; ok && i < array_view->length; i++) { | ||
| int64_t start = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i); | ||
| ok = start == last || (data[start] & 0xC0) != 0x80; | ||
| } | ||
|
|
||
| if (ok) { | ||
| return NANOARROW_OK; | ||
| } | ||
| } | ||
|
|
||
| for (int64_t i = 0; i < array_view->length; i++) { | ||
| if (ArrowArrayViewIsNull(array_view, i)) { | ||
| continue; | ||
| } | ||
|
|
||
| int64_t start = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i); | ||
| int64_t end = ArrowArrayViewGetOffsetUnsafe(array_view, array_view->offset + i + 1); | ||
| if (end > start) { | ||
| NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateUtf8Element( | ||
| array_view, i, data + start, end - start, error)); | ||
| } | ||
| } | ||
|
Comment on lines
+2262
to
+2273
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
|
||
| return NANOARROW_OK; | ||
| } | ||
|
|
||
| static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, | ||
| struct ArrowError* error) { | ||
| for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { | ||
|
|
@@ -2152,6 +2304,15 @@ static int ArrowArrayViewValidateFull(struct ArrowArrayView* array_view, | |
| } | ||
| } | ||
|
|
||
| switch (array_view->storage_type) { | ||
| case NANOARROW_TYPE_STRING: | ||
| case NANOARROW_TYPE_LARGE_STRING: | ||
| NANOARROW_RETURN_NOT_OK(ArrowArrayViewValidateUtf8(array_view, error)); | ||
| break; | ||
| default: | ||
| break; | ||
| } | ||
|
|
||
| if (array_view->storage_type == NANOARROW_TYPE_DENSE_UNION || | ||
| array_view->storage_type == NANOARROW_TYPE_SPARSE_UNION) { | ||
| struct ArrowBufferView sliced_type_ids; | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It may be worth writing two loops here to avoid the branch in ArrowArrayViewGetOffsetUnsafe (maybe the optimizer does this anyway). Views probably don't benefit from this sort of treatment and can fall back to the generic path.