Repository navigation
API, Core: Handle File Type Schema Evolution #18935
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
7ae198a
673bef6
6f2217b
fa6ec9b
9ed30e4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -61,6 +61,10 @@ public class TestReadabilityChecks { | |
| Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY), | ||
| }; | ||
|
|
||
| private static final Schema FILE_SCHEMA = new Schema(optional(1, "photo", Types.FileType.of(1))); | ||
| private static final Schema FILE_AS_STRUCT_SCHEMA = | ||
| new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields()))); | ||
|
|
||
| @Test | ||
| public void testPrimitiveTypes() { | ||
| for (Type.PrimitiveType from : PRIMITIVES) { | ||
|
|
@@ -286,6 +290,27 @@ public void testIncompatibleStructAndPrimitive() { | |
| .contains("struct cannot be read as a string"); | ||
| } | ||
|
|
||
| @Test | ||
| void structCannotBeReadAsAFile() { | ||
| assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_AS_STRUCT_SCHEMA)) | ||
| .singleElement() | ||
| .asString() | ||
| .endsWith("cannot be read as a file"); | ||
| } | ||
|
|
||
| @Test | ||
| void fileCannotBeReadAsAStruct() { | ||
| assertThat(CheckCompatibility.readCompatibilityErrors(FILE_AS_STRUCT_SCHEMA, FILE_SCHEMA)) | ||
| .singleElement() | ||
| .asString() | ||
| .endsWith("file cannot be read as a struct"); | ||
| } | ||
|
|
||
| @Test | ||
| void fileCanBeReadAsAFile() { | ||
| assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_SCHEMA)).isEmpty(); | ||
| } | ||
|
Comment on lines
+293
to
+312
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: this covers three behaviors (struct as file, file as struct, file as file). Can we split it into three tests?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Behaviors are now in their own tests. |
||
|
|
||
| @Test | ||
| public void testMultipleErrors() { | ||
| // required field is optional and cannot be promoted to the read type | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,6 +50,12 @@ | |
| public final class MetricsConfig implements Serializable { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); | ||
| private static final int FILE_URI_ID_OFFSET = 1; | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @dramaticlly the duplicates are here now |
||
| private static final int FILE_SIZE_ID_OFFSET = 3; | ||
| private static final int FILE_CONTENT_TYPE_ID_OFFSET = 4; | ||
| private static final int[] FILE_INFERRED_METRIC_ID_OFFSETS = { | ||
| FILE_URI_ID_OFFSET, FILE_SIZE_ID_OFFSET, FILE_CONTENT_TYPE_ID_OFFSET | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The spec says "Writers should produce statistics for |
||
| }; | ||
|
|
||
| // Disable metrics by default for wide tables to prevent excessive metadata | ||
| private static final MetricsMode DEFAULT_MODE = | ||
|
|
@@ -190,6 +196,17 @@ public Set<Integer> struct(Types.StructType struct, Iterable<Set<Integer>> field | |
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public Set<Integer> file(Types.FileType file, Iterable<Set<Integer>> fieldResults) { | ||
| for (int offset : FILE_INFERRED_METRIC_ID_OFFSETS) { | ||
| if (shouldContinue()) { | ||
| idSet.add(file.baseId() + offset); | ||
| } | ||
| } | ||
|
|
||
| return null; | ||
| } | ||
|
Comment on lines
+200
to
+208
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: is treating a file exactly like a struct the right default for inferred metrics? Every file column now uses six slots of the inferred-column limit, and with several file columns that could push real data columns out of default metrics. It also collects truncated bounds on
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Changed this around pretty dramatically. I didn't have that later in the series. So the new version is to define the exact offsets we consider real fields (uri, size, content) and only those count towards our limit. I put the default for the other file types as "count" unless the table default is "non". So file types now behave a little different
|
||
|
|
||
| @Override | ||
| @SuppressWarnings("ReturnValueIgnored") | ||
| public Set<Integer> field(Types.NestedField field, Supplier<Set<Integer>> fieldResult) { | ||
|
|
@@ -275,6 +292,8 @@ private static MetricsConfig from( | |
|
|
||
| Map<String, MetricsMode> columnModes = Maps.newHashMap(); | ||
|
|
||
| columnModes.putAll(fileColumnModes(schema, defaultMode)); | ||
|
|
||
| columnModes.putAll(defaultColumnConf); | ||
|
|
||
| // Default sort columns to at least truncate (overridden by config) | ||
|
|
@@ -313,6 +332,23 @@ private static Map<String, MetricsMode> defaultColumnModes( | |
| return builder.build(); | ||
| } | ||
|
|
||
| private static Map<String, MetricsMode> fileColumnModes(Schema schema, MetricsMode defaultMode) { | ||
| ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder(); | ||
| if (schema != null) { | ||
| MetricsMode fileMode = | ||
| defaultMode == MetricsModes.None.get() ? defaultMode : MetricsModes.Counts.get(); | ||
| for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { | ||
| if (field.type().isFileType()) { | ||
| for (Types.NestedField nested : field.type().asFileType().fields()) { | ||
| builder.put(schema.findColumnName(nested.fieldId()), fileMode); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
For example, this fails on the current code (size and content_type are counts): @Test
void configuredDefaultAppliesToFileFieldsPastTheInferredLimit() {
Schema schema =
new Schema(
required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2)));
Map<String, String> properties =
Map.of(
TableProperties.DEFAULT_WRITE_METRICS_MODE,
"full",
TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
"1");
MetricsConfig config = MetricsTestUtil.from(properties, schema);
assertThat(config.columnMode("photo.size")).isEqualTo(MetricsModes.Full.get());
assertThat(config.columnMode("photo.content_type")).isEqualTo(MetricsModes.Full.get());
}Should this loop only set offset, checksum, and inline? I tried that locally and the existing tests still pass. |
||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return builder.build(); | ||
| } | ||
|
|
||
| private static Map<String, MetricsMode> sortColumnModes( | ||
| SortOrder order, MetricsMode configuredDefault) { | ||
| ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder(); | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -110,6 +110,11 @@ public List<Integer> map( | |
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public List<Integer> file(Types.FileType file, Iterable<List<Integer>> fieldResults) { | ||
| return struct(file.asStruct(), fieldResults); | ||
|
Comment on lines
+113
to
+115
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Neither this nor the
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That makes sense. I switched this around and added a property of FileType
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Changed this again, it's now a set of fields FILE_INFERRED_METRIC_ID_OFFSETS |
||
| } | ||
|
|
||
| @Override | ||
| public List<Integer> variant(Types.VariantType variant) { | ||
| return null; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: with current logic probably dont even need changes in this class
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah but less magic numbers now :) I think it's worth keeping
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I probably should have done this in the last PR