Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 42 additions & 6 deletions api/src/main/java/org/apache/iceberg/Schema.java
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ public Schema(
this.struct = StructType.of(finalColumns);
this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null;

validateFileBaseIds(struct, null);

// validate IdentifierField
if (identifierFieldIds != null) {
Map<Integer, Integer> idToParent = TypeUtil.indexParents(struct);
Expand All @@ -161,6 +163,27 @@ public Schema(
this.highestFieldId = lazyIdToName().keySet().stream().mapToInt(i -> i).max().orElse(0);
}

private void validateFileBaseIds(Type type, String parentName) {
if (!type.isNestedType()) {
return;
}

for (NestedField field : type.asNestedType().fields()) {
String name = parentName != null ? parentName + "." + field.name() : field.name();
if (field.type().isFileType()) {
int baseId = field.type().asFileType().baseId();
Preconditions.checkArgument(
baseId == field.fieldId(),
"Invalid file column %s: file base ID %s does not match field ID %s",
name,
baseId,
field.fieldId());
} else {
validateFileBaseIds(field.type(), name);
}
}
}

static void validateIdentifierField(
int fieldId, Map<Integer, Types.NestedField> idToField, Map<Integer, Integer> idToParent) {
Types.NestedField field = idToField.get(fieldId);
Expand Down Expand Up @@ -603,20 +626,33 @@ private List<NestedField> reassignIds(List<NestedField> columns, TypeUtil.GetID
if (getID == null) {
return columns;
}

Type res =
TypeUtil.assignIds(
StructType.of(columns),
oldId -> {
int newId = getID.get(oldId);
if (newId != oldId) {
idsToReassigned.put(oldId, newId);
idsToOriginal.put(newId, oldId);
new TypeUtil.GetID() {
@Override
public int get(int oldId) {
return recordIfReassigned(oldId, getID.get(oldId));
}

@Override
public int get(int oldId, Type type) {
return recordIfReassigned(oldId, getID.get(oldId, type));
}
return newId;
});
return res.asStructType().fields();
}

private int recordIfReassigned(int oldId, int newId) {
if (newId != oldId) {
idsToReassigned.put(oldId, newId);
idsToOriginal.put(newId, oldId);
}

return newId;
}

/**
* Check the compatibility of the schema with a format version.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,15 @@ public List<String> variant(Types.VariantType readVariant) {
return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readVariant));
}

@Override
public List<String> file(Types.FileType readFile, Iterable<List<String>> fieldErrorLists) {
if (currentType.isFileType()) {
return NO_ERRORS;
}

return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readFile));
}

@Override
public List<String> primitive(Type.PrimitiveType readPrimitive) {
if (currentType.equals(readPrimitive)) {
Expand Down
14 changes: 8 additions & 6 deletions api/src/main/java/org/apache/iceberg/types/TypeUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -693,22 +693,24 @@ public interface GetID {
/**
* Assigns a new ID for a field of the given type.
*
* <p>Implementations must override this method to assign IDs to file fields.
* <p>Implementations must override this method to assign a different ID to a file field.
*
* @param oldId an existing field ID
* @param type the type of the field
* @return a new field ID
* @throws UnsupportedOperationException if the type is a file and this method is not overridden
* @throws UnsupportedOperationException if the type is a file, this method is not overridden,
* and {@link #get(int)} returns a different ID
*/
default int get(int oldId, Type type) {
if (type.isFileType()) {
int newId = get(oldId);
if (type.isFileType() && newId != oldId) {
throw new UnsupportedOperationException(
String.format(
"Cannot assign IDs to file field %s: this GetID does not implement get(int, Type)",
oldId));
"Cannot reassign file field %s to %s: this GetID does not implement get(int, Type)",
oldId, newId));
}

return get(oldId);
return newId;
}
}

Expand Down
19 changes: 13 additions & 6 deletions api/src/main/java/org/apache/iceberg/types/Types.java
Original file line number Diff line number Diff line change
Expand Up @@ -1158,6 +1158,12 @@ private Map<Integer, NestedField> lazyFieldsById() {
public static final class FileType extends NestedType {
public static final String NAME = "file";
public static final int NUM_NESTED_FIELDS = 6;
private static final int URI_ID_OFFSET = 1;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: with current logic probably dont even need changes in this class

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah but less magic numbers now :) I think it's worth keeping

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I probably should have done this in the last PR

private static final int OFFSET_ID_OFFSET = 2;
private static final int SIZE_ID_OFFSET = 3;
private static final int CONTENT_TYPE_ID_OFFSET = 4;
private static final int CHECKSUM_ID_OFFSET = 5;
private static final int INLINE_ID_OFFSET = 6;

private static final String URI = "uri";
private static final String OFFSET = "offset";
Expand Down Expand Up @@ -1284,12 +1290,13 @@ private List<NestedField> lazyFieldList() {
if (fieldList == null) {
this.fieldList =
ImmutableList.of(
NestedField.optional(baseId + 1, URI, StringType.get()),
NestedField.optional(baseId + 2, OFFSET, LongType.get()),
NestedField.optional(baseId + 3, SIZE, LongType.get()),
NestedField.optional(baseId + 4, CONTENT_TYPE, StringType.get()),
NestedField.optional(baseId + 5, CHECKSUM, StringType.get()),
NestedField.optional(baseId + 6, INLINE, BinaryType.get()));
NestedField.optional(baseId + URI_ID_OFFSET, URI, StringType.get()),
NestedField.optional(baseId + OFFSET_ID_OFFSET, OFFSET, LongType.get()),
NestedField.optional(baseId + SIZE_ID_OFFSET, SIZE, LongType.get()),
NestedField.optional(
baseId + CONTENT_TYPE_ID_OFFSET, CONTENT_TYPE, StringType.get()),
NestedField.optional(baseId + CHECKSUM_ID_OFFSET, CHECKSUM, StringType.get()),
NestedField.optional(baseId + INLINE_ID_OFFSET, INLINE, BinaryType.get()));
}

return fieldList;
Expand Down
47 changes: 47 additions & 0 deletions api/src/test/java/org/apache/iceberg/TestSchema.java
Original file line number Diff line number Diff line change
Expand Up @@ -338,4 +338,51 @@ public void testIndexFieldsNestedSchema() {
assertThat(fields.get(5).name()).isEqualTo("email");
assertThat(((Types.StructType) fields.get(2).type()).fields()).hasSize(3);
}

@Test
void rejectsASchemaWhereAnotherColumnHoldsADerivedId() {
assertThatThrownBy(
() ->
new Schema(
Types.NestedField.required(1, "id", Types.LongType.get()),
Types.NestedField.optional(2, "photo", Types.FileType.of(2)),
Types.NestedField.optional(3, "data", Types.StringType.get())))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageStartingWith("Multiple entries with same key")
.hasMessageContainingAll("3=photo.uri", "3=data");
}

@Test
void rejectsASchemaWhereTwoFileColumnsOverlap() {
assertThatThrownBy(
() ->
new Schema(
Types.NestedField.optional(1, "photo", Types.FileType.of(1)),
Types.NestedField.optional(2, "thumbnail", Types.FileType.of(2))))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageStartingWith("Multiple entries with same key")
.hasMessageContaining("thumbnail");
}

@Test
void rejectsASchemaWithUnderivedNestedIds() {
assertThatThrownBy(
() -> new Schema(Types.NestedField.optional(5, "photo", Types.FileType.of(9))))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid file column photo: file base ID 9 does not match field ID 5");
}

@Test
void rejectsASchemaWithUnderivedNestedIdsInAMap() {
assertThatThrownBy(
() ->
new Schema(
Types.NestedField.optional(
1,
"byName",
Types.MapType.ofOptional(
2, 3, Types.StringType.get(), Types.FileType.of(9)))))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Invalid file column byName.value: file base ID 9 does not match field ID 3");
}
}
16 changes: 15 additions & 1 deletion api/src/test/java/org/apache/iceberg/types/TestFileType.java
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,21 @@ void assignedIdsRejectAnAssignerThatCannotReserve() {
assertThatThrownBy(() -> TypeUtil.assignIds(schema.asStruct(), oldId -> oldId + 10))
.isInstanceOf(UnsupportedOperationException.class)
.hasMessage(
"Cannot assign IDs to file field 1: this GetID does not implement get(int, Type)");
"Cannot reassign file field 1 to 11: this GetID does not implement get(int, Type)");
}

@Test
void assignedIdsKeepAFileWhoseIdIsUnchanged() {
Schema schema =
new Schema(
required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2)));

Type assigned = TypeUtil.assignIds(schema.asStruct(), oldId -> oldId == 1 ? 100 : oldId);

Types.StructType expected =
Types.StructType.of(
required(100, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2)));
assertThat(assigned).isEqualTo(expected);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ public class TestReadabilityChecks {
Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY),
};

private static final Schema FILE_SCHEMA = new Schema(optional(1, "photo", Types.FileType.of(1)));
private static final Schema FILE_AS_STRUCT_SCHEMA =
new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields())));

@Test
public void testPrimitiveTypes() {
for (Type.PrimitiveType from : PRIMITIVES) {
Expand Down Expand Up @@ -286,6 +290,27 @@ public void testIncompatibleStructAndPrimitive() {
.contains("struct cannot be read as a string");
}

@Test
void structCannotBeReadAsAFile() {
assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_AS_STRUCT_SCHEMA))
.singleElement()
.asString()
.endsWith("cannot be read as a file");
}

@Test
void fileCannotBeReadAsAStruct() {
assertThat(CheckCompatibility.readCompatibilityErrors(FILE_AS_STRUCT_SCHEMA, FILE_SCHEMA))
.singleElement()
.asString()
.endsWith("file cannot be read as a struct");
}

@Test
void fileCanBeReadAsAFile() {
assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_SCHEMA)).isEmpty();
}
Comment on lines +293 to +312

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this covers three behaviors (struct as file, file as struct, file as file). Can we split it into three tests? hasSize(1) followed by get(0) also hides the actual error on failure. Prefer .singleElement().asString().endsWith("cannot be read as a file"), and drop the .as() descriptions since they just repeat the assertion.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Behaviors are now in their own tests.


@Test
public void testMultipleErrors() {
// required field is optional and cannot be promoted to the read type
Expand Down
36 changes: 36 additions & 0 deletions core/src/main/java/org/apache/iceberg/MetricsConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@
public final class MetricsConfig implements Serializable {

private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class);
private static final int FILE_URI_ID_OFFSET = 1;

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dramaticlly the duplicates are here now

private static final int FILE_SIZE_ID_OFFSET = 3;
private static final int FILE_CONTENT_TYPE_ID_OFFSET = 4;
private static final int[] FILE_INFERRED_METRIC_ID_OFFSETS = {
FILE_URI_ID_OFFSET, FILE_SIZE_ID_OFFSET, FILE_CONTENT_TYPE_ID_OFFSET

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The spec says "Writers should produce statistics for uri, content_type, and inline fields; other fields may be omitted." Here the inferred fields are uri, size, and content_type, so size gets bounds while inline only gets counts. Should this follow the spec, or should the spec be updated to match?

};

// Disable metrics by default for wide tables to prevent excessive metadata
private static final MetricsMode DEFAULT_MODE =
Expand Down Expand Up @@ -190,6 +196,17 @@ public Set<Integer> struct(Types.StructType struct, Iterable<Set<Integer>> field
return null;
}

@Override
public Set<Integer> file(Types.FileType file, Iterable<Set<Integer>> fieldResults) {
for (int offset : FILE_INFERRED_METRIC_ID_OFFSETS) {
if (shouldContinue()) {
idSet.add(file.baseId() + offset);
}
}

return null;
}
Comment on lines +200 to +208

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: is treating a file exactly like a struct the right default for inferred metrics? Every file column now uses six slots of the inferred-column limit, and with several file columns that could push real data columns out of default metrics. It also collects truncated bounds on inline and checksum, which I doubt help pruning. If that's intended, let's say so. Otherwise, file() could add only uri, size, and content_type by default, possibly in a later part of the series.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed this around pretty dramatically. I didn't have that later in the series.

So the new version is to define the exact offsets we consider real fields (uri, size, content) and only those count towards our limit. I put the default for the other file types as "count" unless the table default is "non".

So file types now behave a little different

  1. Only uri, size and content are included during the "limit" phase of metric collectioN
  2. inline, checksum are default to count if there is still room
  3. Inline, checksum can be made "none" if they set none as the Table default


@Override
@SuppressWarnings("ReturnValueIgnored")
public Set<Integer> field(Types.NestedField field, Supplier<Set<Integer>> fieldResult) {
Expand Down Expand Up @@ -275,6 +292,8 @@ private static MetricsConfig from(

Map<String, MetricsMode> columnModes = Maps.newHashMap();

columnModes.putAll(fileColumnModes(schema, defaultMode));

columnModes.putAll(defaultColumnConf);

// Default sort columns to at least truncate (overridden by config)
Expand Down Expand Up @@ -313,6 +332,23 @@ private static Map<String, MetricsMode> defaultColumnModes(
return builder.build();
}

private static Map<String, MetricsMode> fileColumnModes(Schema schema, MetricsMode defaultMode) {
ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
if (schema != null) {
MetricsMode fileMode =
defaultMode == MetricsModes.None.get() ? defaultMode : MetricsModes.Counts.get();
for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) {
if (field.type().isFileType()) {
for (Types.NestedField nested : field.type().asFileType().fields()) {
builder.put(schema.findColumnName(nested.fieldId()), fileMode);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

write.metadata.metrics.default should apply to all columns, but here the inferred file fields (like size and content_type) are always put in the map as counts. Within write.metadata.metrics.max-inferred-column-defaults (100 by default), defaultColumnConf overrides that later. Past the limit, nothing does, so they stay at counts while a normal column past the limit gets the configured default.

For example, this fails on the current code (size and content_type are counts):

@Test
void configuredDefaultAppliesToFileFieldsPastTheInferredLimit() {
  Schema schema =
      new Schema(
          required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2)));
  Map<String, String> properties =
      Map.of(
          TableProperties.DEFAULT_WRITE_METRICS_MODE,
          "full",
          TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
          "1");

  MetricsConfig config = MetricsTestUtil.from(properties, schema);

  assertThat(config.columnMode("photo.size")).isEqualTo(MetricsModes.Full.get());
  assertThat(config.columnMode("photo.content_type")).isEqualTo(MetricsModes.Full.get());
}

Should this loop only set offset, checksum, and inline? I tried that locally and the existing tests still pass.

}
}
}
}

return builder.build();
}

private static Map<String, MetricsMode> sortColumnModes(
SortOrder order, MetricsMode configuredDefault) {
ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/org/apache/iceberg/SchemaOrder.java
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ public List<Integer> map(
return null;
}

@Override
public List<Integer> file(Types.FileType file, Iterable<List<Integer>> fieldResults) {
return struct(file.asStruct(), fieldResults);
Comment on lines +113 to +115

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Neither this nor the MetricsConfig.limitFieldIds override has a direct test, so nothing pins the order of the nested IDs or which file subfields get default metrics. TestSchemaOrder already has one test per type. Can we add a fileOrder() case there and a limitFieldIds case in TestMetricsConfig? Derive the expected IDs from Types.FileType.of(n).fields() rather than hardcoding them.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That makes sense. I switched this around and added a property of FileType defaultInferredMetricOffsets which gets called here to assign the fields.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed this again, it's now a set of fields

FILE_INFERRED_METRIC_ID_OFFSETS

}

@Override
public List<Integer> variant(Types.VariantType variant) {
return null;
Expand Down
Loading
Loading