Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ai/skills/review-comet-iceberg-write-pr/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ only the latest.
- [ ] **Nothing nondeterministic reaches the output.** Map iteration order leaks into file names,
manifest order and the row order of unordered reads. The fanout arm sorts its `DataFile`s by
path for this reason ([#5776](https://github.com/apache/datafusion-comet/issues/5776)).
- [ ] **Casts stay strict.** `decorate_batch_with_field_ids` casts with `safe: false`, so a type
- [ ] **Casts stay strict.** `UuidAlignment::convert` casts with `safe: false`, so a type
mismatch fails the task. A safe cast would write NULLs.

## 4. Failure Handling and Cleanup
Expand Down
137 changes: 137 additions & 0 deletions dev/GenerateIcebergUuidOracle.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List;
import java.util.Random;
import java.util.UUID;

/** Run with JDK 17 and 21; the generated fixtures must match byte for byte. */
class GenerateIcebergUuidOracle {
private static final String LICENSE =
"""
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
""";

public static void main(String[] args) throws Exception {
System.err.println(System.getProperty("java.vendor") + " " + Runtime.version());
var lines = new ArrayList<>(LICENSE.lines().toList());
lines.add(
"# Generated by dev/GenerateIcebergUuidOracle.java; UTF-8 hex input, UUID hex or error");
var inputs =
new ArrayList<>(
List.of(
"",
"1-1-1-1-1",
"+1-1-1-1-1",
"100000000-1-1-1-1",
"1-10000-1-1-1",
"1-1-1-1-1",
"١-١-١-١-١",
"A-b-C-d-E",
"𝟙-1-1-1-1",
"-1-1-1-1-1",
"1--1-1-1",
"1-1-1-1-8000000000000000",
"f47ac10b58cc4372a5670e02b2c3d479",
"{f47ac10b-58cc-4372-a567-0e02b2c3d479}",
"f47ac10b-58cc-4372-a567-0e02b2c3d479",
"FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF",
"00000000-0000-0000-0000-000000000000",
" 1-1-1-1-1",
"1-1-1-1-1\n"));
for (int group = 0; group < 5; group++) {
for (String value :
List.of(
"",
"+",
"++1",
"g",
"7",
"+f",
"00000000000000001",
"7fffffffffffffff",
"8000000000000000",
"ffffffffffffffff",
"0000000000000000000001")) {
var groups = new String[] {"1", "1", "1", "1", "1"};
groups[group] = value;
inputs.add(String.join("-", groups));
}
}
// Check Character.digit(char, 16) over the entire BMP, including rejected numeric characters.
var digits = new ArrayList<>(LICENSE.lines().toList());
for (int c = 0; c <= 0xffff; c++) {
int digit = Character.digit((char) c, 16);
if (digit >= 0) {
digits.add(String.format("%04x\t%d", c, digit));
inputs.add((char) c + "-1-1-1-1");
}
}
var random = new Random(6959);
String alphabet = "0123456789abcdefABCDEF+- g\t1١A²Ⅳ𝟙";
// Include valid groups of irregular widths, then mutate separators, signs and characters.
for (int i = 0; i < 1024; i++) {
var groups = new ArrayList<String>();
for (int g = 0; g < 5; g++) {
String hex = Long.toHexString(random.nextLong() & Long.MAX_VALUE);
groups.add(hex.substring(0, 1 + random.nextInt(hex.length())));
}
String value = String.join("-", groups);
inputs.add(value);
int at = random.nextInt(value.length());
// Use code points to avoid creating unpaired surrogates that UTF-8 cannot represent.
int[] chars = alphabet.codePoints().toArray();
String replacement = new String(Character.toChars(chars[random.nextInt(chars.length)]));
inputs.add(value.substring(0, at) + replacement + value.substring(at + 1));
}
for (String input : inputs) {
String result;
try {
result = UUID.fromString(input).toString().replace("-", "");
} catch (IllegalArgumentException e) {
result = "error";
}
lines.add(HexFormat.of().formatHex(input.getBytes(StandardCharsets.UTF_8)) + "\t" + result);
}
Path out = Path.of(args[0]);
Files.createDirectories(out);
Files.write(out.resolve("uuid-java.tsv"), lines, StandardCharsets.UTF_8);
Files.write(out.resolve("uuid-java-digits.tsv"), digits, StandardCharsets.UTF_8);
}
}
Loading