diff --git a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto index debacc245d60..781b07f3d5b6 100644 --- a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto +++ b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/external_transforms.proto @@ -109,6 +109,9 @@ message ManagedTransforms { "beam:schematransform:org.apache.beam:delta_lake_read:v1"]; DELTA_LAKE_CDC_READ = 14 [(org.apache.beam.model.pipeline.v1.beam_urn) = "beam:schematransform:org.apache.beam:delta_lake_cdc_read:v1"]; + // Applies a stream or batch of CDC row-level changes to Iceberg tables. + ICEBERG_CDC_WRITE = 15 [(org.apache.beam.model.pipeline.v1.beam_urn) = + "beam:schematransform:org.apache.beam:iceberg_cdc_write:v1"]; } } diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index e2e8a12d01eb..b3228004ed3b 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -46,6 +46,7 @@ dependencies { implementation library.java.vendored_guava_32_1_2_jre implementation project(path: ":sdks:java:core", configuration: "shadow") implementation project(path: ":model:pipeline", configuration: "shadow") + implementation project(path: ":sdks:java:extensions:sorter") implementation library.java.avro implementation library.java.slf4j_api implementation library.java.joda_time diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCdcWriteSchemaTransformProvider.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCdcWriteSchemaTransformProvider.java new file mode 100644 index 000000000000..8e33c8914631 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCdcWriteSchemaTransformProvider.java @@ -0,0 +1,735 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg; + +import static org.apache.beam.sdk.io.iceberg.IcebergCdcWriteSchemaTransformProvider.Configuration; +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.sdk.util.construction.BeamUrns.getUrn; + +import com.google.auto.service.AutoService; +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import org.apache.beam.model.pipeline.v1.ExternalTransforms; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; +import org.apache.beam.sdk.io.iceberg.cdc.sink.WriteCdcRows; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription; +import org.apache.beam.sdk.schemas.transforms.SchemaTransform; +import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider; +import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.util.RowFilter; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionRowTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.catalog.TableIdentifier; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; + +/** + * SchemaTransform implementation for {@link IcebergIO#writeCdcRows}. Applies a stream (or batch) of + * row-level changes ({@code INSERT}/{@code UPDATE_BEFORE}/{@code UPDATE_AFTER}/{@code DELETE}) to + * one or more Iceberg V2+ tables. + * + *

Outputs a {@code snapshots} {@code PCollection} representing the snapshots produced in + * the process (mirroring {@link IcebergWriteSchemaTransformProvider}'s output), and a {@code + * dead_letter} {@code PCollection} of replayable late records (see {@link + * IcebergWriteResult#getDeadLetterRows()}). + * + *

Change kind for cross-language pipelines. A cross-language pipeline (Python, Go, …) + * cannot attach a native Beam {@code ValueKind} to each element, so it must carry the change kind + * in a data column and set {@code change_type_column} (optionally with {@code change_type_map} to + * map source op codes to {@code ValueKind} names). {@code change_type_column} is therefore + * effectively required from those SDKs; without it every record defaults to {@code INSERT}. + * + *

Column flow. The user's {@code keep}/{@code drop}/{@code only} projection is applied as + * a {@code ParDo} upstream of the sink (envelope trimming), and it always preserves the + * control columns (the {@code change_type_column} and the sequence-number column), which the sink + * itself consumes and then strips from the written rows: input → projection (user filter, control + * columns preserved) → sink (consumes + strips control columns). {@link WriteCdcRows} never calls + * {@link DynamicDestinations#getData}/{@link DynamicDestinations#getDataSchema} (destinations are + * routing-only), so a destination-side filter would be inert; the upstream projection also means + * single-table users' filters are honored. For the {@code only} projection the top-level control + * columns are re-appended after the extracted payload row's fields (the Debezium {@code only=after} + * + {@code change_type_column=op} pattern). + * + *

For a templated {@code table} the provider routes via a {@link PortableIcebergDestinations} + * built over the post-projection schema, so template placeholders must reference columns + * that survive the projection (control columns included). Only its routing methods are consulted; + * when a destination table does not exist, the sink auto-creates it from the post-projection input + * schema minus the control columns, honoring only the destination's partition spec, sort order, and + * table properties (none of which this provider configures); the create-config schema built by + * {@code PortableIcebergDestinations#instantiateDestination} is ignored. + */ +@AutoService(SchemaTransformProvider.class) +public class IcebergCdcWriteSchemaTransformProvider + extends TypedSchemaTransformProvider { + + static final String INPUT_TAG = "input"; + static final String SNAPSHOTS_TAG = "snapshots"; + static final String DEAD_LETTER_TAG = "dead_letter"; + + static final Schema OUTPUT_SCHEMA = IcebergWriteSnapshotOutput.OUTPUT_SCHEMA; + + /** The default sequence-number column when {@code sequence_number_column} is unset. */ + private static final String DEFAULT_SEQUENCE_NUMBER_COLUMN = + IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; + + @Override + public String description() { + return "Applies a stream of CDC row-level changes (INSERT/UPDATE_BEFORE/UPDATE_AFTER/DELETE) " + + "to Iceberg V2+ tables via equality deletes; superseded rows are never written.\n" + + "Returns a 'snapshots' PCollection representing the snapshots produced in the process, " + + "with the following schema:\n" + + "{\"table\" (str), \"operation\" (str), \"summary\" (map[str, str]), \"manifestListLocation\" (str)}\n" + + "and a 'dead_letter' PCollection of replayable late records."; + } + + @DefaultSchema(AutoValueSchema.class) + @AutoValue + public abstract static class Configuration { + public static Builder builder() { + return new AutoValue_IcebergCdcWriteSchemaTransformProvider_Configuration.Builder(); + } + + @SchemaFieldDescription( + "A fully-qualified table identifier. You may also provide a template to write to multiple dynamic destinations," + + " for example: `dataset.my_{col1}_{col2.nested}_table`.") + public abstract String getTable(); + + @SchemaFieldDescription("Name of the catalog containing the table.") + public abstract @Nullable String getCatalogName(); + + @SchemaFieldDescription("Properties used to set up the Iceberg catalog.") + public abstract @Nullable Map getCatalogProperties(); + + @SchemaFieldDescription("Properties passed to the Hadoop Configuration.") + public abstract @Nullable Map getConfigProperties(); + + @SchemaFieldDescription( + "A list of field names to keep in the input record. All other fields are dropped before " + + "writing. The change-type and sequence-number control columns are always preserved " + + "and must not be listed. Is mutually exclusive with 'drop' and 'only'.") + public abstract @Nullable List getKeep(); + + @SchemaFieldDescription( + "A list of field names to drop from the input record before writing. The change-type and " + + "sequence-number control columns are stripped automatically and must not be listed. " + + "Is mutually exclusive with 'keep' and 'only'.") + public abstract @Nullable List getDrop(); + + @SchemaFieldDescription( + "The name of a single record field that should be written. The change-type and " + + "sequence-number control columns are carried along automatically and must not be " + + "named. Is mutually exclusive with 'keep' and 'drop'.") + public abstract @Nullable String getOnly(); + + @SchemaFieldDescription( + "Columns defining row identity (the Iceberg equality-delete fields). Defaults to the " + + "destination table's identifier (primary-key) fields.") + public abstract @Nullable List getEqualityColumns(); + + @SchemaFieldDescription( + "The column holding the per-primary-key monotonic sequence number used to order a single " + + "key's changes. Required as a non-nullable INT64 in the input schema. Defaults to " + + "_commit_snapshot_sequence_number. Event timestamps must be non-decreasing with " + + "this column per key; violations are counted by crossWindowSequenceInversions.") + public abstract @Nullable String getSequenceNumberColumn(); + + @SchemaFieldDescription( + "If set, read the change kind from this non-nullable string column (stripped from the " + + "data row and never written to Iceberg) instead of the element's native ValueKind, " + + "for SDKs without ValueKind support.") + public abstract @Nullable String getChangeTypeColumn(); + + @SchemaFieldDescription( + "Optional map from change_type_column value to a ValueKind name " + + "(INSERT|UPDATE_BEFORE|UPDATE_AFTER|DELETE). If omitted, the change_type_column " + + "value must already be a ValueKind name. Represented as Map for " + + "cross-language compatibility.") + public abstract @Nullable Map getChangeTypeMap(); + + @SchemaFieldDescription( + "The number of deterministic primary-key-hash shards (logical write buckets) per " + + "destination. Max write parallelism per destination. Defaults to 16; set it to " + + "about your pipeline's write parallelism. Too low bottlenecks writes (visible as a " + + "growing commit backlog); too high multiplies the sink's file count, which has no " + + "symptom until reads and compaction slow down.") + public abstract @Nullable Integer getNumShards(); + + @SchemaFieldDescription( + "Maximum number of shards a single partition's rows may occupy. Defaults to num_shards" + + " (no cap). Lower values write proportionally fewer files per commit at the cost of" + + " per-partition write parallelism; 1 pins each partition to one writer. Ignored for" + + " unpartitioned tables. Must be between 1 and num_shards.") + public abstract @Nullable Integer getShardsPerPartition(); + + // NOTE: "Mb", not "MB": the config field name is CaseFormat-derived, and "sorterMemoryMB" + // would snake_case to "sorter_memory_m_b". Do not rename to match + // WriteCdcRows#withSorterMemoryMB. + @SchemaFieldDescription( + "The in-memory buffer size (MB) for the pre-write sort; groups larger than this " + + "spill to disk. Must be >= 1. Defaults to 100.") + public abstract @Nullable Integer getSorterMemoryMb(); + + @SchemaFieldDescription( + "If true, only the after-image of each change (INSERT/UPDATE_AFTER) is applied as an " + + "upsert (equality-delete-then-insert on the primary key); UPDATE_BEFORE records are " + + "dropped. Requires every partition source column to be an equality column. Defaults " + + "to false.") + public abstract @Nullable Boolean getUpsert(); + + @SchemaFieldDescription( + "A stable identifier for this sink, used to namespace the idempotency tokens written to " + + "each commit's Iceberg snapshot summary. Set this explicitly (and keep it stable " + + "across relaunches) for cross-relaunch exactly-once commit idempotency. Defaults to " + + "a per-write UUID. BATCH SEMANTICS: in batch all data commits under one global " + + "window, so a stable sink_id makes reruns of the SAME load idempotent, but makes " + + "DIFFERENT loads no-ops (a rerun writes nothing). For periodic batch loads, either " + + "omit sink_id (per-run UUID) or use a per-load value (e.g. suffix the load date).") + public abstract @Nullable String getSinkId(); + + @SchemaFieldDescription( + "The size of each event-time commit window, in seconds. Required for streaming " + + "(unbounded) input; ignored for batch input.") + public abstract @Nullable Integer getTriggeringFrequencySeconds(); + + @SchemaFieldDescription( + "How long (in seconds) a late record may lag behind the watermark before it is dropped " + + "entirely, rather than routed to the dead_letter output. Defaults to 21600 (6 hours). " + + "A larger bound retains more live window state per destination on stateful runners.") + public abstract @Nullable Integer getAllowedLatenessSeconds(); + + @SchemaFieldDescription( + "This option specifies whether and where to output per-record poison rows (null or " + + "missing sequence value on any kind, unknown change type, null equality value, " + + "unresolvable destination) instead of failing the pipeline. Distinct from the " + + "dead_letter output (late-but-valid rows).") + public abstract @Nullable ErrorHandling getErrorHandling(); + + @SchemaFieldDescription( + "Extra key/value properties to add to every commit's Iceberg snapshot summary. Keys " + + "prefixed with 'beam.cdc.' are reserved and rejected.") + public abstract @Nullable Map getSnapshotProperties(); + + @SchemaFieldDescription( + "Streaming only; ignored for batch. If set (> 0), each destination that has committed at " + + "least once emits a periodic empty token-refresh commit while idle, every this many " + + "seconds, so its committed-through snapshot stays recent and is less likely to be " + + "lost to expire_snapshots before the sink resumes. Unset (the default) disables it.") + public abstract @Nullable Integer getTokenHeartbeatSeconds(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setTable(String table); + + public abstract Builder setCatalogName(String catalogName); + + public abstract Builder setCatalogProperties(Map catalogProperties); + + public abstract Builder setConfigProperties(Map confProperties); + + public abstract Builder setKeep(List keep); + + public abstract Builder setDrop(List drop); + + public abstract Builder setOnly(String only); + + public abstract Builder setEqualityColumns(List equalityColumns); + + public abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + public abstract Builder setChangeTypeColumn(String changeTypeColumn); + + public abstract Builder setChangeTypeMap(Map changeTypeMap); + + public abstract Builder setNumShards(Integer numShards); + + public abstract Builder setShardsPerPartition(Integer shardsPerPartition); + + public abstract Builder setSorterMemoryMb(Integer sorterMemoryMb); + + public abstract Builder setUpsert(Boolean upsert); + + public abstract Builder setSinkId(String sinkId); + + public abstract Builder setTriggeringFrequencySeconds(Integer triggeringFrequencySeconds); + + public abstract Builder setAllowedLatenessSeconds(Integer allowedLatenessSeconds); + + public abstract Builder setErrorHandling(ErrorHandling errorHandling); + + public abstract Builder setSnapshotProperties(Map snapshotProperties); + + public abstract Builder setTokenHeartbeatSeconds(Integer tokenHeartbeatSeconds); + + public abstract Configuration build(); + } + + public IcebergCatalogConfig getIcebergCatalog() { + return IcebergCatalogConfig.builder() + .setCatalogName(getCatalogName()) + .setCatalogProperties(getCatalogProperties()) + .setConfigProperties(getConfigProperties()) + .build(); + } + } + + @Override + protected SchemaTransform from(Configuration configuration) { + return new IcebergCdcWriteSchemaTransform(configuration); + } + + @Override + public List inputCollectionNames() { + return Collections.singletonList(INPUT_TAG); + } + + @Override + public List outputCollectionNames() { + return Arrays.asList(SNAPSHOTS_TAG, DEAD_LETTER_TAG); + } + + @Override + public String identifier() { + return getUrn(ExternalTransforms.ManagedTransforms.Urns.ICEBERG_CDC_WRITE); + } + + static class IcebergCdcWriteSchemaTransform extends SchemaTransform { + private final Configuration configuration; + + IcebergCdcWriteSchemaTransform(Configuration configuration) { + this.configuration = configuration; + } + + Row getConfigurationRow() { + return IcebergWriteSnapshotOutput.configurationRow(configuration, Configuration.class); + } + + @Override + public PCollectionRowTuple expand(PCollectionRowTuple input) { + PCollection rows = input.get(INPUT_TAG); + + String table = configuration.getTable(); + @Nullable List drop = configuration.getDrop(); + @Nullable List keep = configuration.getKeep(); + @Nullable String only = configuration.getOnly(); + @Nullable String changeTypeColumn = configuration.getChangeTypeColumn(); + @Nullable String configuredSeq = configuration.getSequenceNumberColumn(); + String seqColumn = configuredSeq != null ? configuredSeq : DEFAULT_SEQUENCE_NUMBER_COLUMN; + boolean projectionConfigured = drop != null || keep != null || only != null; + + validateProjectionConfig(keep, drop, only, changeTypeColumn, seqColumn); + + // Apply the user's projection upstream of the sink (see the class Javadoc's column flow). + if (projectionConfigured) { + UserProjection projection = + UserProjection.of( + rows.getSchema(), + keep, + drop, + only, + controlColumnsPresent(rows.getSchema(), changeTypeColumn, seqColumn)); + rows = + rows.apply("ApplyUserProjection", ParDo.of(new ApplyUserProjectionFn(projection))) + .setRowSchema(projection.outputSchema()); + } + + WriteCdcRows writeTransform = IcebergIO.writeCdcRows(configuration.getIcebergCatalog()); + + if (table.contains("{")) { + // Templated destination: routing/auto-create only (see the class Javadoc), over the + // post-projection schema the sink interpolates against. + try { + writeTransform = + writeTransform.to( + new PortableIcebergDestinations( + table, + FileFormat.PARQUET.toString(), + rows.getSchema(), + /* partitionFields= */ null, + /* sortFields= */ null, + /* tableProperties= */ null, + /* fieldsToDrop= */ null, + /* fieldsToKeep= */ null, + /* onlyField= */ null)); + } catch (IllegalArgumentException e) { + if (projectionConfigured) { + throw new IllegalArgumentException( + "Invalid destination template '" + + table + + "': " + + e.getMessage() + + " Note: the template is resolved against the projected input schema, so a " + + "placeholder may reference a field removed by the configured keep/drop/only " + + "projection; template fields must survive it.", + e); + } + throw e; + } + } else { + writeTransform = writeTransform.to(TableIdentifier.parse(table)); + } + + IcebergWriteResult result = rows.apply(withConfiguredOptions(writeTransform)); + + PCollection snapshots = + result + .getSnapshots() + .apply(MapElements.via(new IcebergWriteSnapshotOutput.SnapshotToRow())) + .setRowSchema(OUTPUT_SCHEMA); + + PCollection deadLetter = checkStateNotNull(result.getDeadLetterRows()); + + PCollectionRowTuple output = + PCollectionRowTuple.of(SNAPSHOTS_TAG, snapshots).and(DEAD_LETTER_TAG, deadLetter); + @Nullable ErrorHandling errorHandling = configuration.getErrorHandling(); + if (ErrorHandling.hasOutput(errorHandling)) { + output = + output.and( + checkStateNotNull(errorHandling).getOutput(), + checkStateNotNull(result.getFailedRows())); + } + return output; + } + + /** + * Rejects a projection that names either control column (the change-type column and the + * sequence-number column) in any of {@code keep}/{@code drop}/{@code only}. Dropping one would + * starve the sink of a column it consumes; naming one in a whitelist is a confusing no-op, + * since a control column is stripped before writing and can never be a table column. + */ + private static void validateProjectionConfig( + @Nullable List keep, + @Nullable List drop, + @Nullable String only, + @Nullable String changeTypeColumn, + String seqColumn) { + Preconditions.checkArgument( + drop == null || !drop.contains(seqColumn), + "drop must not contain sequence_number_column '%s': it is a control column consumed by " + + "the sink to order each key's changes, and stripped from the written rows " + + "automatically.", + seqColumn); + Preconditions.checkArgument( + (keep == null || !keep.contains(seqColumn)) && !seqColumn.equals(only), + "sequence_number_column '%s' must not be named by 'keep' or 'only': it is a control " + + "column stripped before writing, and can never be a table column.", + seqColumn); + if (changeTypeColumn != null) { + Preconditions.checkArgument( + drop == null || !drop.contains(changeTypeColumn), + "drop must not contain change_type_column '%s': it is a control column consumed by " + + "the sink to resolve each record's change kind, and stripped from the written " + + "rows automatically.", + changeTypeColumn); + Preconditions.checkArgument( + (keep == null || !keep.contains(changeTypeColumn)) && !changeTypeColumn.equals(only), + "change_type_column '%s' must not be named by 'keep' or 'only': it is a control " + + "column stripped before writing, and can never be a table column.", + changeTypeColumn); + } + } + + /** Threads every set (non-null) configuration option onto {@code write}. */ + private WriteCdcRows withConfiguredOptions(WriteCdcRows write) { + @Nullable List equalityColumns = configuration.getEqualityColumns(); + if (equalityColumns != null) { + write = write.withEqualityColumns(equalityColumns); + } + + @Nullable String sequenceNumberColumn = configuration.getSequenceNumberColumn(); + if (sequenceNumberColumn != null) { + write = write.withSequenceNumberColumn(sequenceNumberColumn); + } + + @Nullable String changeTypeColumn = configuration.getChangeTypeColumn(); + if (changeTypeColumn != null) { + write = write.withChangeTypeColumn(changeTypeColumn); + } + + @Nullable Map changeTypeMap = configuration.getChangeTypeMap(); + if (changeTypeMap != null) { + write = write.withChangeTypeMap(changeTypeMap); + } + + @Nullable Integer numShards = configuration.getNumShards(); + if (numShards != null) { + write = write.withNumShards(numShards); + } + + @Nullable Integer shardsPerPartition = configuration.getShardsPerPartition(); + if (shardsPerPartition != null) { + write = write.withShardsPerPartition(shardsPerPartition); + } + + @Nullable Integer sorterMemoryMb = configuration.getSorterMemoryMb(); + if (sorterMemoryMb != null) { + write = write.withSorterMemoryMB(sorterMemoryMb); + } + + @Nullable Boolean upsert = configuration.getUpsert(); + if (upsert != null) { + write = write.withUpsert(upsert); + } + + @Nullable String sinkId = configuration.getSinkId(); + if (sinkId != null) { + write = write.withSinkId(sinkId); + } + + @Nullable Integer triggeringFrequencySeconds = configuration.getTriggeringFrequencySeconds(); + if (triggeringFrequencySeconds != null) { + write = write.withTriggeringFrequency(Duration.standardSeconds(triggeringFrequencySeconds)); + } + + @Nullable Integer allowedLatenessSeconds = configuration.getAllowedLatenessSeconds(); + if (allowedLatenessSeconds != null) { + write = write.withAllowedLateness(Duration.standardSeconds(allowedLatenessSeconds)); + } + + if (ErrorHandling.hasOutput(configuration.getErrorHandling())) { + write = write.withErrorHandling(); + } + + @Nullable Map snapshotProperties = configuration.getSnapshotProperties(); + if (snapshotProperties != null) { + write = write.withSnapshotProperties(snapshotProperties); + } + + @Nullable Integer tokenHeartbeatSeconds = configuration.getTokenHeartbeatSeconds(); + if (tokenHeartbeatSeconds != null) { + write = write.withTokenHeartbeat(Duration.standardSeconds(tokenHeartbeatSeconds)); + } + return write; + } + } + + /** + * The control columns actually present in {@code inputSchema}, in {@code [change-type, sequence]} + * order (each is normally present; the sink's own validation fails otherwise). + */ + private static List controlColumnsPresent( + Schema inputSchema, @Nullable String changeTypeColumn, String seqColumn) { + List controls = new ArrayList<>(); + if (changeTypeColumn != null && inputSchema.hasField(changeTypeColumn)) { + controls.add(changeTypeColumn); + } + if (inputSchema.hasField(seqColumn)) { + controls.add(seqColumn); + } + return controls; + } + + /** + * The user's {@code keep}/{@code drop}/{@code only} projection, applied upstream of the sink with + * the control columns preserved: + * + *

+ */ + static class UserProjection implements Serializable { + /** Unused in the {@code only} case, where it serves only to derive the payload schema. */ + private final RowFilter filter; + + private final @Nullable String onlyField; + private final List appendedControlColumns; + private final Schema outputSchema; + + /** The input schema this projection was built for; {@link #positions} are indices into it. */ + private final Schema inputSchema; + + /** + * For each {@link #outputSchema} field, its position in {@link #inputSchema} (the {@code + * keep}/{@code drop} fast path), or {@code null} when the output is not a plain positional + * subset of the input (the {@code only} case, or a field whose type the filter rewrote). + * + *

This exists because {@link RowFilter#filter} is per-record expensive in a way that scales + * with column count: it re-verifies the row's schema against the filter's with a full + * structural {@code assignableTo} walk, then rebuilds the row through a {@code HashMap} keyed + * by field name, resolving a {@code FieldAccessDescriptor} per field. On a wide table that is + * hundreds of name hashes and allocations per record, on the default path for every + * Managed/YAML/Python user who configures {@code keep} or {@code drop}. The positional copy is + * the same shape the sink's own {@code TableSetup.ProjectionPlan} uses immediately downstream + * on these very rows. + * + *

One deliberate difference: {@code RowFilter}'s rebuild also normalizes values it + * copies ({@code ByteBuffer} to {@code byte[]} for {@code BYTES}, any {@code AbstractInstant} + * to {@code Instant} for {@code DATETIME}), which a positional copy does not. That only ever + * mattered for rows carrying non-canonical values, which {@code Row.addValues}/coder-decoded + * rows never do; a row built through the {@code @Internal} {@code attachValues} with such a + * value already fails on the no-projection path, where nothing normalizes it either. + */ + private final int @Nullable [] positions; + + private UserProjection( + RowFilter filter, + @Nullable String onlyField, + List appendedControlColumns, + Schema outputSchema, + Schema inputSchema) { + this.filter = filter; + this.onlyField = onlyField; + this.appendedControlColumns = appendedControlColumns; + this.outputSchema = outputSchema; + this.inputSchema = inputSchema; + this.positions = onlyField == null ? positionsIn(inputSchema, outputSchema) : null; + } + + /** + * For each {@code outputSchema} field, its index in {@code inputSchema}, or {@code null} if any + * output field is not carried through unchanged (same name, identical {@link Schema.Field}), in + * which case the caller must keep using {@link RowFilter}, which knows how to rewrite it. + */ + private static int @Nullable [] positionsIn(Schema inputSchema, Schema outputSchema) { + int[] positions = new int[outputSchema.getFieldCount()]; + for (int i = 0; i < outputSchema.getFieldCount(); i++) { + Schema.Field field = outputSchema.getField(i); + if (!inputSchema.hasField(field.getName())) { + return null; + } + int position = inputSchema.indexOf(field.getName()); + if (!inputSchema.getField(position).equals(field)) { + return null; + } + positions[i] = position; + } + return positions; + } + + static UserProjection of( + Schema inputSchema, + @Nullable List keep, + @Nullable List drop, + @Nullable String only, + List controlColumns) { + // RowFilter also enforces keep/drop/only mutual exclusivity and that every named field + // exists in the input schema. + RowFilter filter = new RowFilter(inputSchema); + if (drop != null) { + filter = filter.drop(drop); + } + if (keep != null) { + LinkedHashSet effectiveKeep = new LinkedHashSet<>(keep); + effectiveKeep.addAll(controlColumns); + filter = filter.keep(new ArrayList<>(effectiveKeep)); + } + if (only == null) { + return new UserProjection( + filter, null, Collections.emptyList(), filter.outputSchema(), inputSchema); + } + filter = filter.only(only); + Schema payloadSchema = filter.outputSchema(); + Schema.Builder outputSchema = Schema.builder().addFields(payloadSchema.getFields()); + for (String control : controlColumns) { + // The control columns are appended to the extracted payload's fields, so a payload field + // of the same name would collide; name the collision rather than letting Schema.Builder + // throw an opaque "Duplicate field" error. + Preconditions.checkArgument( + !payloadSchema.hasField(control), + "The 'only' field '%s' already contains a field named '%s', which collides with the " + + "control column of that name carried alongside it. Rename the nested field, or " + + "configure a different control column name.", + only, + control); + outputSchema.addField(inputSchema.getField(control)); + } + return new UserProjection(filter, only, controlColumns, outputSchema.build(), inputSchema); + } + + Schema outputSchema() { + return outputSchema; + } + + /** + * Whether {@code schema} is the schema {@link #positions} were resolved against. A row that + * fails this (a drifting source schema) takes the {@link RowFilter} path, which re-validates it + * and reports the mismatch itself. + */ + @SuppressWarnings("ReferenceEquality") + private boolean matchesInput(Schema schema) { + return inputSchema == schema || inputSchema.equals(schema); + } + + Row apply(Row row) { + if (onlyField == null) { + int @Nullable [] copyFrom = positions; + if (copyFrom == null || !matchesInput(row.getSchema())) { + return filter.filter(row); + } + List<@Nullable Object> values = new ArrayList<>(copyFrom.length); + for (int position : copyFrom) { + values.add(row.getValue(position)); + } + return Row.withSchema(outputSchema).attachValues(values); + } + @Nullable Row payload = row.getRow(onlyField); + if (payload == null) { + throw new IllegalStateException( + "The 'only' field '" + + onlyField + + "' is null for an input row; a null payload cannot be written. Re-shape such " + + "records upstream (e.g. a Debezium DELETE carries its data in 'before', not " + + "'after')."); + } + List<@Nullable Object> values = new ArrayList<>(outputSchema.getFieldCount()); + for (int i = 0; i < payload.getSchema().getFieldCount(); i++) { + values.add(payload.getValue(i)); + } + for (String control : appendedControlColumns) { + values.add(row.getValue(control)); + } + return Row.withSchema(outputSchema).attachValues(values); + } + } + + /** Applies a {@link UserProjection} to each row, preserving the element's {@link ValueKind}. */ + private static class ApplyUserProjectionFn extends DoFn { + private final UserProjection projection; + + ApplyUserProjectionFn(UserProjection projection) { + this.projection = projection; + } + + @ProcessElement + public void process(@Element Row row, ValueKind kind, OutputReceiver out) { + out.builder(projection.apply(row)).setValueKind(kind).output(); + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java index 080c31ff9126..d2119a241e3e 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergIO.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.io.Read; import org.apache.beam.sdk.io.iceberg.cdc.IncrementalChangelogSource; +import org.apache.beam.sdk.io.iceberg.cdc.sink.WriteCdcRows; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.transforms.PTransform; @@ -393,6 +394,23 @@ public static WriteRows writeRows(IcebergCatalogConfig catalog) { .build(); } + /** + * Returns a {@link WriteCdcRows} transform: the CDC sink, applying inserts/updates/deletes from a + * {@code PCollection} of change records (each carrying a {@link + * org.apache.beam.sdk.values.ValueKind}) to one or more Iceberg V2+ tables via equality deletes; + * superseded rows are never written. + * + * 

{@code
+   * input.apply(IcebergIO.writeCdcRows(catalogConfig)
+   *     .to(tableId)
+   *     .withSequenceNumberColumn("seq")
+   *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+   * }
+ */ + public static WriteCdcRows writeCdcRows(IcebergCatalogConfig catalog) { + return WriteCdcRows.of(catalog); + } + @AutoValue public abstract static class WriteRows extends PTransform, IcebergWriteResult> { diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java index 801464010c69..fc4841dcd3a8 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergSchemaTransformTranslation.java @@ -17,6 +17,7 @@ */ package org.apache.beam.sdk.io.iceberg; +import static org.apache.beam.sdk.io.iceberg.IcebergCdcWriteSchemaTransformProvider.IcebergCdcWriteSchemaTransform; import static org.apache.beam.sdk.io.iceberg.IcebergReadSchemaTransformProvider.IcebergReadSchemaTransform; import static org.apache.beam.sdk.io.iceberg.IcebergWriteSchemaTransformProvider.IcebergWriteSchemaTransform; import static org.apache.beam.sdk.schemas.transforms.SchemaTransformTranslation.SchemaTransformPayloadTranslator; @@ -87,6 +88,19 @@ public Row toConfigRow(IcebergWriteSchemaTransform transform) { } } + static class IcebergCdcWriteSchemaTransformTranslator + extends SchemaTransformPayloadTranslator { + @Override + public SchemaTransformProvider provider() { + return new IcebergCdcWriteSchemaTransformProvider(); + } + + @Override + public Row toConfigRow(IcebergCdcWriteSchemaTransform transform) { + return transform.getConfigurationRow(); + } + } + @AutoService(TransformPayloadTranslatorRegistrar.class) public static class WriteRegistrar implements TransformPayloadTranslatorRegistrar { @Override @@ -97,6 +111,7 @@ public static class WriteRegistrar implements TransformPayloadTranslatorRegistra getTransformPayloadTranslators() { return ImmutableMap., TransformPayloadTranslator>builder() .put(IcebergWriteSchemaTransform.class, new IcebergWriteSchemaTransformTranslator()) + .put(IcebergCdcWriteSchemaTransform.class, new IcebergCdcWriteSchemaTransformTranslator()) .build(); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java index 8e2549b5dadb..8db954428b3f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteResult.java @@ -19,31 +19,110 @@ import java.util.Map; import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PInput; import org.apache.beam.sdk.values.POutput; import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.Row; import org.apache.beam.sdk.values.TupleTag; import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.Nullable; +/** + * The output of an {@code IcebergIO} write: the snapshots each destination table committed, plus + * the two diversion outputs the CDC sink can produce. + * + *

Only {@link #getSnapshots()} is always present. {@link #getDeadLetterRows()} (late-but-valid + * records) and {@link #getFailedRows()} (per-record poison rows) are non-null only for results + * built by {@link #cdc}, that is, by {@code IcebergIO.writeCdcRows}, and {@code getFailedRows()} + * additionally only when error handling was enabled. The append-only sink ({@code + * IcebergIO.writeRows}) leaves both null rather than exposing outputs that can never carry data. + */ public final class IcebergWriteResult implements POutput { private static final TupleTag> SNAPSHOTS_TAG = new TupleTag>() {}; + private static final TupleTag DEAD_LETTER_TAG = new TupleTag() {}; + + private static final TupleTag FAILED_ROWS_TAG = new TupleTag() {}; + private final Pipeline pipeline; private final PCollection> snapshots; + private final @Nullable PCollection deadLetterRows; + + private final @Nullable PCollection failedRows; + + /** + * The committed snapshots, keyed by destination. A window committed in a commit fire that later + * fails may not re-emit its {@link SnapshotInfo} on the retry (table state is unaffected). + */ public PCollection> getSnapshots() { return snapshots; } + /** + * The replayable dead-letter {@link Row}s from the CDC sink: records whose grouped pane fired + * late, i.e. after the watermark had passed their commit window's end. Schema is {@code record + * ROW + change_type STRING + sequence_number INT64 + destination STRING}. To replay, + * unnest {@code record}, map {@code change_type}/{@code sequence_number} as the sink's control + * columns, and route each row by {@code destination}. Replaying is only safe while no newer + * change for those keys has committed; a stale replay's equality delete removes the newer row. + * + * @return the dead-letter {@code PCollection} for results produced by {@code + * IcebergIO.writeCdcRows}/{@link #cdc}, or {@code null} for results produced by the + * append-only sink ({@code IcebergIO.writeRows}), which has no dead-letter output. + */ + public @Nullable PCollection getDeadLetterRows() { + return deadLetterRows; + } + + /** + * The per-record poison rows diverted by the CDC sink when error handling is enabled (see {@code + * WriteCdcRows.withErrorHandling}); schema is {@code failed_row ROW + error_message STRING} (see + * {@link org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling}). Distinct from {@link + * #getDeadLetterRows()} (which carries late-but-valid records). + * + * @return the failed-rows {@code PCollection}, or {@code null} when error handling was not + * enabled (or for the append-only sink). + */ + public @Nullable PCollection getFailedRows() { + return failedRows; + } + IcebergWriteResult(Pipeline pipeline, PCollection> snapshots) { + this(pipeline, snapshots, null, null); + } + + private IcebergWriteResult( + Pipeline pipeline, + PCollection> snapshots, + @Nullable PCollection deadLetterRows, + @Nullable PCollection failedRows) { this.pipeline = pipeline; this.snapshots = snapshots; + this.deadLetterRows = deadLetterRows; + this.failedRows = failedRows; + } + + /** + * Returns an {@link IcebergWriteResult} for the CDC sink, exposing the committed-snapshot {@code + * snapshots}, the replayable {@code deadLetterRows} (see {@link #getDeadLetterRows()}), and the + * optional per-record {@code failedRows} (see {@link #getFailedRows()}, {@code null} when error + * handling is off). + */ + @Internal + public static IcebergWriteResult cdc( + Pipeline pipeline, + PCollection> snapshots, + PCollection deadLetterRows, + @Nullable PCollection failedRows) { + return new IcebergWriteResult(pipeline, snapshots, deadLetterRows, failedRows); } @Override @@ -55,6 +134,12 @@ public Pipeline getPipeline() { public Map, PValue> expand() { ImmutableMap.Builder, PValue> output = ImmutableMap.builder(); output.put(SNAPSHOTS_TAG, snapshots); + if (deadLetterRows != null) { + output.put(DEAD_LETTER_TAG, deadLetterRows); + } + if (failedRows != null) { + output.put(FAILED_ROWS_TAG, failedRows); + } return output.build(); } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSnapshotOutput.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSnapshotOutput.java new file mode 100644 index 000000000000..1e0c19f805bf --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSnapshotOutput.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg; + +import org.apache.beam.sdk.schemas.NoSuchSchemaException; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.SchemaRegistry; +import org.apache.beam.sdk.transforms.SimpleFunction; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.Row; + +/** + * Output-row helpers for the Iceberg CDC write SchemaTransform provider ({@link + * IcebergCdcWriteSchemaTransformProvider}): the {@code snapshots} output schema, the {@code + * SnapshotInfo}-to-Row mapping, and the configuration-row conversion. + * + *

These deliberately mirror {@link IcebergWriteSchemaTransformProvider}'s private equivalents — + * both providers expose an identical {@code snapshots} output shape — but the append provider is + * intentionally left untouched by the CDC change, so it keeps its own copies rather than sharing + * this helper (switching it over would modify a long-stable file for zero behavior change). + */ +final class IcebergWriteSnapshotOutput { + + private IcebergWriteSnapshotOutput() {} + + /** + * The {@code snapshots} output schema: a {@code table} string plus {@link SnapshotInfo}'s fields. + */ + static final Schema OUTPUT_SCHEMA = + Schema.builder() + .addStringField("table") + .addFields(SnapshotInfo.getSchema().getFields()) + .build(); + + /** + * Maps a committed {@code (table, SnapshotInfo)} to a {@link Row} matching {@link + * #OUTPUT_SCHEMA}. + */ + static class SnapshotToRow extends SimpleFunction, Row> { + @Override + public Row apply(KV input) { + return Row.withSchema(OUTPUT_SCHEMA) + .addValue(input.getKey()) + .addValues(input.getValue().toRow().getValues()) + .build(); + } + } + + /** + * Converts a SchemaTransform {@code Configuration} to its config {@link Row}, sorted + * lexicographically and snake_cased to match SchemaTransform config naming conventions. + */ + static Row configurationRow(T configuration, Class configurationClass) { + try { + return SchemaRegistry.createDefault() + .getToRowFunction(configurationClass) + .apply(configuration) + .sorted() + .toSnakeCase(); + } catch (NoSuchSchemaException e) { + throw new RuntimeException(e); + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/PortableIcebergDestinations.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/PortableIcebergDestinations.java index 775020879f67..9185e7eb645f 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/PortableIcebergDestinations.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/PortableIcebergDestinations.java @@ -27,7 +27,7 @@ import org.apache.iceberg.FileFormat; import org.checkerframework.checker.nullness.qual.Nullable; -class PortableIcebergDestinations implements DynamicDestinations { +public class PortableIcebergDestinations implements DynamicDestinations { private final RowFilter rowFilter; private final RowStringInterpolator interpolator; private final String fileFormat; diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java index ceb96d50f8aa..779bc3753c7d 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SerializableDeleteFile.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.schemas.AutoValueSchema; import org.apache.beam.sdk.schemas.annotations.DefaultSchema; import org.apache.beam.sdk.schemas.annotations.SchemaFieldNumber; @@ -37,11 +38,14 @@ import org.apache.iceberg.FileMetadata; import org.apache.iceberg.Metrics; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SingleValueParser; import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StructLike; import org.checkerframework.checker.nullness.qual.Nullable; @DefaultSchema(AutoValueSchema.class) @AutoValue +@Internal public abstract class SerializableDeleteFile { public static SerializableDeleteFile.Builder builder() { return new AutoValue_SerializableDeleteFile.Builder(); @@ -62,7 +66,11 @@ public static SerializableDeleteFile.Builder builder() { @SchemaFieldNumber("4") public abstract long getFileSizeInBytes(); + /** + * @deprecated Use {@link #getJsonPartition()} instead. + */ @SchemaFieldNumber("5") + @Deprecated public abstract String getPartitionPath(); @SchemaFieldNumber("6") @@ -113,6 +121,9 @@ public static SerializableDeleteFile.Builder builder() { @SchemaFieldNumber("21") public abstract @Nullable Long getFileSequenceNumber(); + @SchemaFieldNumber("22") + abstract @Nullable String getJsonPartition(); + @AutoValue.Builder abstract static class Builder { abstract Builder setContentType(FileContent content); @@ -127,6 +138,8 @@ abstract static class Builder { abstract Builder setPartitionPath(String partitionPath); + abstract Builder setJsonPartition(String jsonPartition); + abstract Builder setPartitionSpecId(int partitionSpec); abstract Builder setSortOrderId(@Nullable Integer sortOrderId); @@ -163,7 +176,47 @@ abstract static class Builder { } public static SerializableDeleteFile from( - DeleteFile deleteFile, String partitionPath, boolean includeMetrics) { + DeleteFile deleteFile, Map specs) { + return from(deleteFile, specs, true); + } + + /** + * Creates a {@link SerializableDeleteFile}, resolving the file's {@link PartitionSpec} by its own + * spec id. + * + *

Delete files reached from a scan task may carry a spec id that differs from the spec of the + * data file they apply to, so the lookup has to be per delete file rather than against a single + * "current" spec. + */ + public static SerializableDeleteFile from( + DeleteFile deleteFile, Map specs, boolean includeMetrics) { + return from( + deleteFile, + checkStateNotNull( + specs.get(deleteFile.specId()), + "Could not create a SerializableDeleteFile because DeleteFile is written using a partition spec id '%s' that is not found in the provided specs: %s", + deleteFile.specId(), + specs.keySet()), + includeMetrics); + } + + public static SerializableDeleteFile from(DeleteFile deleteFile, PartitionSpec spec) { + return from(deleteFile, spec, true); + } + + public static SerializableDeleteFile from( + DeleteFile deleteFile, PartitionSpec spec, boolean includeMetrics) { + if (spec.specId() != deleteFile.specId()) { + throw new IllegalArgumentException( + String.format( + "Cannot serialize DeleteFile: its partition spec id %s does not match the provided " + + "spec id %s.", + deleteFile.specId(), spec.specId())); + } + // jsonPartition is the primary (handles evolved specs, special characters). + // partitionPath is the fallback for values that don't round-trip through JSON. + String jsonPartition = SingleValueParser.toJson(spec.partitionType(), deleteFile.partition()); + String partitionPath = spec.partitionToPath(deleteFile.partition()); SerializableDeleteFile.Builder builder = SerializableDeleteFile.builder() @@ -171,6 +224,7 @@ public static SerializableDeleteFile from( .setFileFormat(deleteFile.format().name()) .setFileSizeInBytes(deleteFile.fileSizeInBytes()) .setPartitionPath(partitionPath) + .setJsonPartition(jsonPartition) .setPartitionSpecId(deleteFile.specId()) .setRecordCount(deleteFile.recordCount()) .setColumnSizes(deleteFile.columnSizes()) @@ -228,7 +282,21 @@ public DeleteFile createDeleteFile( .withMetrics(metrics) .withSplitOffsets(getSplitOffsets()) .withEncryptionKeyMetadata(getKeyMetadata()) - .withPartitionPath(getPartitionPath()); + .withReferencedDataFile(getReferencedDataFile()); + + @Nullable String jsonPartition = getJsonPartition(); + if (jsonPartition != null) { + try { + deleteFileBuilder = deleteFileBuilder.withPartition(partition(partitionSpec)); + } catch (RuntimeException e) { + // Some partition values (e.g. NaN / Infinity floating-point) don't round-trip through the + // JSON representation; fall back to the partition-path string + deleteFileBuilder = deleteFileBuilder.withPartitionPath(getPartitionPath()); + } + } else { + // Elements decoded from a pre-jsonPartition release carry only the partition path. + deleteFileBuilder = deleteFileBuilder.withPartitionPath(getPartitionPath()); + } switch (getContentType()) { case POSITION_DELETES: @@ -260,17 +328,22 @@ public DeleteFile createDeleteFile( "Unexpected content type for DeleteFile: " + getContentType()); } - // needed for puffin files + // contentOffset / contentSizeInBytes really are Puffin-only: build() rejects a non-null value + // for either on any other format, and requires both (plus referencedDataFile) on Puffin. if (getFileFormat().equalsIgnoreCase(FileFormat.PUFFIN.name())) { deleteFileBuilder = deleteFileBuilder .withContentOffset(checkStateNotNull(getContentOffset())) - .withContentSizeInBytes(checkStateNotNull(getContentSizeInBytes())) - .withReferencedDataFile(checkStateNotNull(getReferencedDataFile())); + .withContentSizeInBytes(checkStateNotNull(getContentSizeInBytes())); } return deleteFileBuilder.build(); } + private StructLike partition(PartitionSpec spec) { + return (StructLike) + SingleValueParser.fromJson(spec.partitionType(), checkStateNotNull(getJsonPartition())); + } + @Override public final boolean equals(@Nullable Object o) { if (this == o) { @@ -287,6 +360,7 @@ && getRecordCount() == that.getRecordCount() && getFileSizeInBytes() == that.getFileSizeInBytes() && getPartitionPath().equals(that.getPartitionPath()) && getPartitionSpecId() == that.getPartitionSpecId() + && Objects.equals(getJsonPartition(), that.getJsonPartition()) && Objects.equals(getSortOrderId(), that.getSortOrderId()) && Objects.equals(getEqualityFieldIds(), that.getEqualityFieldIds()) && Objects.equals(getKeyMetadata(), that.getKeyMetadata()) @@ -314,6 +388,7 @@ public final int hashCode() { getRecordCount(), getFileSizeInBytes(), getPartitionPath(), + getJsonPartition(), getPartitionSpecId(), getSortOrderId(), getEqualityFieldIds(), diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java index 3410c0a9d7ee..97bcbaa5bec2 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/SerializableChangelogTask.java @@ -17,7 +17,6 @@ */ package org.apache.beam.sdk.io.iceberg.cdc; -import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; @@ -267,13 +266,10 @@ static List getAddedDeleteFiles(ChangelogScanTask task) { private static List toSerializableDeletes( List dfs, Map specs, boolean includeMetrics) { + // Serialize each delete file against its own spec (looked up by its spec id): a delete file may + // carry a different spec id than the data file it applies to. return dfs.stream() - .map( - df -> - SerializableDeleteFile.from( - df, - checkStateNotNull(specs.get(df.specId())).partitionToPath(df.partition()), - includeMetrics)) + .map(df -> SerializableDeleteFile.from(df, specs, includeMetrics)) .collect(Collectors.toList()); } } diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeys.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeys.java new file mode 100644 index 000000000000..12e9622c7b1f --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/AssignCdcKeys.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.coders.ByteArrayCoder; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.transforms.windowing.PaneInfo; +import org.apache.beam.sdk.util.CoderUtils; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.sdk.values.ValueInSingleWindow; +import org.apache.beam.sdk.values.ValueKind; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Instant; + +/** + * Assigns a sort key to input {@link Row}s and groups by destination and shard keys, outputting + * {@code KV, KV>}. + * + *

For each element this: + * + *

    + *
  1. resolves the destination string; + *
  2. resolves and validates the destination table through {@link TableSetup}; + *
  3. resolves the element's {@link ValueKind}; + *
  4. in upsert mode, drops {@code UPDATE_BEFORE} records; + *
  5. reads the sequence number from {@link CdcWriteConfig#getSequenceNumberColumn()}; + *
  6. projects the row to the destination's CDC data schema (stripping control columns); + *
  7. encodes the primary key to bytes, which feed both the shard hash and the sort key; + *
  8. computes the deterministic shard, according to {@code numShards} and {@code + * shardsPerPartition} + *
+ * + *

When {@link CdcWriteConfig#getErrorHandling()} is enabled, a record-level failure (unknown + * change type, missing/null sequence number, null equality value, an unresolvable destination) is + * diverted to the {@link #FAILED} output as an {@link ErrorHandling#errorSchema} row ({@code + * failed_row}, {@code error_message}). When error handling is disabled, the transform fails + * instead. + */ +final class AssignCdcKeys extends PTransform, PCollectionTuple> { + + static final TupleTag, KV>> KEYED = new TupleTag<>() {}; + static final TupleTag FAILED = new TupleTag() {}; + + private final IcebergCatalogConfig catalogConfig; + private final CdcWriteConfig config; + private final DynamicDestinations destinations; + private final String runId; + + AssignCdcKeys( + IcebergCatalogConfig catalogConfig, + CdcWriteConfig config, + DynamicDestinations destinations, + String runId) { + this.catalogConfig = catalogConfig; + this.config = config; + this.destinations = destinations; + this.runId = runId; + } + + @Override + public PCollectionTuple expand(PCollection input) { + Schema inputSchema = input.getSchema(); + Schema errorSchema = ErrorHandling.errorSchema(inputSchema); + Schema cdcDataSchema = config.stripControlColumns(inputSchema); + PCollectionTuple outputs = + input.apply( + "AssignKeys", + ParDo.of( + new AssignFn( + new TableSetup(catalogConfig, config, destinations, runId), + config, + destinations, + errorSchema)) + .withOutputTags(KEYED, TupleTagList.of(FAILED))); + outputs + .get(KEYED) + .setCoder( + KvCoder.of( + KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()), + KvCoder.of(ByteArrayCoder.of(), CdcRecordCoder.of(cdcDataSchema)))); + outputs.get(FAILED).setCoder(RowCoder.of(errorSchema)); + return outputs; + } + + /** Per-record entry point, running the eight steps listed in the main javadoc above. */ + private static final class AssignFn + extends DoFn, KV>> { + + private final TableSetup tableSetup; + private final CdcWriteConfig config; + private final DynamicDestinations destinations; + private final Schema errorSchema; + private final int numShards; + private final int shardsPerPartition; + private final Counter failedRecords = Metrics.counter(AssignCdcKeys.class, "failedRecords"); + private final Counter upsertUpdateBeforeDropped = + Metrics.counter(AssignCdcKeys.class, "upsertUpdateBeforeDropped"); + + /** + * Counts data-projection rebuilds triggered by a drifting source row schema: normally zero; a + * positive value means rows arrive with an unstable schema (correct but slower). + */ + private final Counter schemaDriftRebuilds = + Metrics.counter(AssignCdcKeys.class, "schemaDriftRebuilds"); + + /** The control columns' positions in the current source schema. */ + private @Nullable ControlColumns controls; + + AssignFn( + TableSetup tableSetup, + CdcWriteConfig config, + DynamicDestinations destinations, + Schema errorSchema) { + this.tableSetup = tableSetup; + this.config = config; + this.destinations = destinations; + this.errorSchema = errorSchema; + this.numShards = config.getNumShards(); + this.shardsPerPartition = config.getShardsPerPartition(); + } + + @ProcessElement + public void processElement( + @Element Row element, + ValueKind elementKind, + @Timestamp Instant timestamp, + BoundedWindow window, + PaneInfo pane, + MultiOutputReceiver out) { + try { + Schema schema = element.getSchema(); + String destString = + destinations.getTableStringIdentifier( + ValueInSingleWindow.of(element, timestamp, window, pane)); + TableSetup.Dest dest = tableSetup.get(destString, schema); + + // Resolve the control columns' positions once per source schema. (The local lets the + // nullness checker prove non-nullness, which it cannot for the field.) + ControlColumns cols = controls; + if (cols == null || !cols.matches(schema)) { + cols = ControlColumns.of(schema, config); + controls = cols; + } + + ValueKind kind = resolveKind(element, cols, elementKind); + if (config.getUpsert() && kind == ValueKind.UPDATE_BEFORE) { + upsertUpdateBeforeDropped.inc(); + return; + } + long seq = readSeq(element, cols, kind); + + TableSetup.ProjectionPlan plan = dest.projectionPlan(); + if (!plan.matches(schema)) { + schemaDriftRebuilds.inc(); // project() below rebuilds the plan (and WARNs once). + } + Row data = plan.project(element); + requireNonNullEqualityValues(dest, data); + byte[] pkBytes = encodePk(dest, data); + + out.get(KEYED) + .output( + KV.of( + KV.of(destString, shardFor(dest, data, pkBytes)), + KV.of(CdcSortKey.encode(pkBytes, seq, kind), CdcRecord.of(data, kind, seq)))); + } catch (TableSetup.TableConfigException e) { + throw e; + } catch (RuntimeException e) { + if (!config.getErrorHandling()) { + throw e; + } + failedRecords.inc(); + out.get(FAILED).output(ErrorHandling.errorRecord(errorSchema, element, e)); + } + } + + /** + * Resolves this element's {@link ValueKind}. When configured, uses the {@code + * change_type_column} value (mapped via {@code change_type_map} when configured). Otherwise, + * uses the element's native kind. + */ + private ValueKind resolveKind(Row element, ControlColumns cols, ValueKind elementKind) { + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + if (changeTypeColumn == null) { + return elementKind; + } + if (cols.changeTypeIndex < 0) { + throw new IllegalArgumentException( + "change_type_column '" + + changeTypeColumn + + "' not found in element schema " + + element.getSchema()); + } + @Nullable String raw = element.getString(cols.changeTypeIndex); + if (raw == null) { + throw new IllegalArgumentException( + "change_type_column '" + changeTypeColumn + "' is null for element " + element); + } + @Nullable Map changeTypeMap = config.getChangeTypeMap(); + String name = changeTypeMap != null ? changeTypeMap.getOrDefault(raw, raw) : raw; + try { + return ValueKind.valueOf(name); + } catch (IllegalArgumentException e) { + String mappedClause = name.equals(raw) ? "" : " (mapped to '" + name + "')"; + throw new IllegalArgumentException( + "change_type '" + + raw + + "'" + + mappedClause + + " is not a valid ValueKind name; must be one of " + + Arrays.toString(ValueKind.values()) + + ", or add a change_type_map entry for it.", + e); + } + } + + /** Reads the required non-null sequence number ({@code INT64}) from the full input row. */ + private long readSeq(Row element, ControlColumns cols, ValueKind kind) { + String seqColumn = config.getSequenceNumberColumn(); + Schema schema = element.getSchema(); + @Nullable Long value; + try { + value = cols.seqIndex < 0 ? null : element.getInt64(cols.seqIndex); + } catch (ClassCastException e) { + throw new IllegalArgumentException( + "sequence_number_column '" + + seqColumn + + "' must be INT64 (was: " + + schema.getField(seqColumn).getType() + + ")", + e); + } + if (value == null) { + throw new IllegalArgumentException( + "sequence_number_column '" + + seqColumn + + "' is missing or null for a " + + kind + + " record; every CDC record requires a non-null sequence number."); + } + return value; + } + + /** + * Computes the record's write shard. + * + *

If the table is unpartitioned or if {@code shards_per_partition == num_shards}, the plain + * primary-key shard is returned. + * + *

Otherwise computes the shard using {@link PartitionShardPlan}: each partition owns a block + * of {@code shards_per_partition} consecutive shards. A record's primary key maps to an offset + * within that block. + * + *

Must remain a pure function of the primary key: a key whose same-window records split + * across shards breaks same-commit dedup. + */ + private int shardFor(TableSetup.Dest dest, Row data, byte[] pkBytes) { + @Nullable PartitionShardPlan partitionShardPlan = dest.partitionShardPlan(); + if (partitionShardPlan == null) { + return TableSetup.shardFor(pkBytes, numShards); + } + int offset = Math.floorMod(TableSetup.pkHash(pkBytes), shardsPerPartition); + return partitionShardPlan.shardFor(data, offset, numShards); + } + + /** + * Rejects a projected row with a null equality value: it cannot define row identity, so it + * fails with a clear per-column error rather than an opaque coder failure (or, under + * partition-block sharding, a silently null partition value). + */ + private void requireNonNullEqualityValues(TableSetup.Dest dest, Row data) { + int[] positions = dest.pkFieldPositions(); + for (int i = 0; i < positions.length; i++) { + if (data.getValue(positions[i]) == null) { + throw new IllegalArgumentException( + "null value in equality column '" + + dest.pkSchema().getField(i).getName() + + "'; equality columns must be non-null to define row identity. Row: " + + data); + } + } + } + + /** Extracts the primary key from the projected data row and encodes it to bytes. */ + private byte[] encodePk(TableSetup.Dest dest, Row data) { + int[] pkPositions = dest.pkFieldPositions(); + List<@Nullable Object> pkValues = new ArrayList<>(pkPositions.length); + for (int position : pkPositions) { + pkValues.add(data.getValue(position)); + } + Row pk = Row.withSchema(dest.pkSchema()).attachValues(pkValues); + try { + return CoderUtils.encodeToByteArray(dest.pkCoder(), pk); + } catch (CoderException e) { + throw new RuntimeException("Failed to encode primary key " + pk, e); + } + } + } + + /** The control columns' positions in a source row schema. */ + private static final class ControlColumns { + /** The source schema these positions were resolved against. */ + private final Schema schema; + + /** Position of the sequence-number column, or {@code -1} if the schema has none. */ + private final int seqIndex; + + /** Position of the change-type column, or {@code -1} if unconfigured or absent. */ + private final int changeTypeIndex; + + private ControlColumns(Schema schema, int seqIndex, int changeTypeIndex) { + this.schema = schema; + this.seqIndex = seqIndex; + this.changeTypeIndex = changeTypeIndex; + } + + static ControlColumns of(Schema schema, CdcWriteConfig config) { + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + return new ControlColumns( + schema, + indexOrAbsent(schema, config.getSequenceNumberColumn()), + changeTypeColumn == null ? -1 : indexOrAbsent(schema, changeTypeColumn)); + } + + private static int indexOrAbsent(Schema schema, String name) { + return schema.hasField(name) ? schema.indexOf(name) : -1; + } + + /** Whether these positions were resolved for {@code other}. */ + @SuppressWarnings("ReferenceEquality") + boolean matches(Schema other) { + return schema == other || schema.equals(other); + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecord.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecord.java new file mode 100644 index 000000000000..e3280bfce7f2 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecord.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import java.util.Objects; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * One change record carried through the CDC sink's shuffle. + * + *

{@link ValueKind} is reified because it's not preserved across a {@code GroupByKey}. + */ +final class CdcRecord { + + private final Row data; + private final ValueKind kind; + private final long sequenceNumber; + + private CdcRecord(Row data, ValueKind kind, long sequenceNumber) { + this.data = data; + this.kind = kind; + this.sequenceNumber = sequenceNumber; + } + + public static CdcRecord of(Row data, ValueKind kind, long sequenceNumber) { + return new CdcRecord(data, kind, sequenceNumber); + } + + public Row getData() { + return data; + } + + public ValueKind getKind() { + return kind; + } + + public long getSequenceNumber() { + return sequenceNumber; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CdcRecord)) { + return false; + } + CdcRecord that = (CdcRecord) o; + return sequenceNumber == that.sequenceNumber && kind == that.kind && data.equals(that.data); + } + + @Override + public int hashCode() { + return Objects.hash(data, kind, sequenceNumber); + } + + @Override + public String toString() { + return "CdcRecord{" + + "data=" + + data + + ", kind=" + + kind + + ", sequenceNumber=" + + sequenceNumber + + '}'; + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoder.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoder.java new file mode 100644 index 000000000000..f7988cb8ca20 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoder.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.CoderException; +import org.apache.beam.sdk.coders.CustomCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * {@link CdcRecord} carries a {@link Row} field whose schema is known only at pipeline-construction + * time. We need a custom coder because {@code AutoValueSchema} only infers schemas at the class + * level and cannot infer a dynamic {@link Row} field, so {@code @DefaultSchema} alone cannot + * produce a working coder for {@link CdcRecord}. + */ +final class CdcRecordCoder extends CustomCoder { + + private final RowCoder dataCoder; + private final VarIntCoder kindCoder = VarIntCoder.of(); + private final VarLongCoder seqCoder = VarLongCoder.of(); + + private CdcRecordCoder(Schema dataSchema) { + this.dataCoder = RowCoder.of(dataSchema); + } + + public static CdcRecordCoder of(Schema dataSchema) { + return new CdcRecordCoder(dataSchema); + } + + public Schema getDataSchema() { + return dataCoder.getSchema(); + } + + @Override + public void encode(CdcRecord value, OutputStream outStream) throws IOException { + dataCoder.encode(value.getData(), outStream); + kindCoder.encode(kindToCode(value.getKind()), outStream); + seqCoder.encode(value.getSequenceNumber(), outStream); + } + + @Override + public CdcRecord decode(InputStream inStream) throws IOException { + Row data = dataCoder.decode(inStream); + ValueKind kind = kindFromCode(kindCoder.decode(inStream)); + long seq = seqCoder.decode(inStream); + return CdcRecord.of(data, kind, seq); + } + + private static int kindToCode(ValueKind kind) { + switch (kind) { + case INSERT: + return 0; + case UPDATE_BEFORE: + return 1; + case UPDATE_AFTER: + return 2; + case DELETE: + return 3; + default: + throw new IllegalArgumentException("Unknown ValueKind: " + kind); + } + } + + private static ValueKind kindFromCode(int code) throws CoderException { + switch (code) { + case 0: + return ValueKind.INSERT; + case 1: + return ValueKind.UPDATE_BEFORE; + case 2: + return ValueKind.UPDATE_AFTER; + case 3: + return ValueKind.DELETE; + default: + throw new CoderException("Unknown CdcRecord ValueKind code: " + code); + } + } + + @Override + public void verifyDeterministic() throws NonDeterministicException { + Coder.verifyDeterministic(this, "Data coder must be deterministic", dataCoder); + } + + @Override + public boolean consistentWithEquals() { + // decode() rebuilds the row with this coder's schema object, which may differ from the + // original row's; false is always safe. + return false; + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return getDataSchema().equals(((CdcRecordCoder) o).getDataSchema()); + } + + @Override + public int hashCode() { + return getDataSchema().hashCode(); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java new file mode 100644 index 000000000000..21a297b38240 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.apache.beam.sdk.values.ValueKind; + +/** + * Builds the byte-comparable secondary sort key used by {@code SortValues} (extensions/sorter) to + * order each (destination, shard, window) group: one primary key's records come out contiguous, + * ordered by sequence number then {@link #kindRank(ValueKind)} within the key. + * + *

The key is {@code [pkLen:4][pkBytes][seq ^ Long.MIN_VALUE:8][kindRank:1]}, big-endian. + */ +final class CdcSortKey { + + private CdcSortKey() {} + + /** Ranks change kinds so before-images sort before after-images at an equal {@code seq}. */ + public static byte kindRank(ValueKind kind) { + switch (kind) { + case UPDATE_BEFORE: + return 0; + case DELETE: + return 1; + case UPDATE_AFTER: + return 2; + case INSERT: + return 3; + default: + throw new IllegalArgumentException("Unknown ValueKind: " + kind); + } + } + + /** + * Encodes the deterministic, byte-comparable sort key {@code [pkLen:4][pkBytes][seq ^ + * Long.MIN_VALUE:8][kindRank:1]} for one CDC record. + * + *

SortValues compares unsigned lexicographic byte order. The length prefix is needed to + * accurately compare two primary keys of varying byte-lengths. Flipping the sequence number's + * sign bit makes unsigned byte order match signed numeric order. kindRank breaks equal-seq ties. + */ + public static byte[] encode(byte[] pkBytes, long seq, ValueKind kind) { + return ByteBuffer.allocate(4 + pkBytes.length + 9) + .putInt(pkBytes.length) + .put(pkBytes) + .putLong(seq ^ Long.MIN_VALUE) + .put(kindRank(kind)) + .array(); + } + + /** + * Whether two encoded sort keys carry the same primary key, compared on the raw {@code + * [pkLen:4][pkBytes]} prefix. + */ + public static boolean samePk(byte[] a, byte[] b) { + int aPkEnd = 4 + ByteBuffer.wrap(a).getInt(0); + int bPkEnd = 4 + ByteBuffer.wrap(b).getInt(0); + return Arrays.equals(a, 0, aPkEnd, b, 0, bPkEnd); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java new file mode 100644 index 000000000000..9a23a0277cd7 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcWriteConfig.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.io.iceberg.cdc.IcebergCdcMetadataColumns; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.ValueKind; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Serializable, worker-side configuration for the CDC sink, threaded through every stage of the + * write path: destination/key assignment, commit windowing, delta-file writing, and commit. + * + *

Construct via {@link #builder()}. Every field but {@link #getSinkId()} has a default; {@link + * #getSinkId()} has none here since it is supplied by the public sink API (typically a fresh UUID + * unless the caller pins one explicitly) before {@link Builder#build()} is called. + * + *

{@link #validate()} checks structural invariants only (value bounds, mutually exclusive + * options, reserved names). It is pure and side-effect-free: no I/O, no catalog or table access. + */ +@AutoValue +abstract class CdcWriteConfig implements Serializable { + + /** + * Default value for {@link #getSequenceNumberColumn()}. Pinned to {@link + * IcebergCdcMetadataColumns#COMMIT_SNAPSHOT_SEQUENCE_NUMBER}, the column name the Iceberg CDC + * read source populates, so a source-to-sink pipeline works with no explicit wiring. + */ + static final String DEFAULT_SEQUENCE_NUMBER_COLUMN = + IcebergCdcMetadataColumns.COMMIT_SNAPSHOT_SEQUENCE_NUMBER; + + /** + * Default value for {@link #getNumShards()}: a starting point of roughly one shard per worker, to + * be raised to about the pipeline's write parallelism. + * + *

Sharding is the sink's write-parallelism knob, but unlike Flink's equivalent (writer + * parallelism, which the operator has already sized to their cluster), it is a fixed number that + * most users never revisit. It also multiplies straight into file count: every shard that touches + * a partition writes a file per commit window, so {@code num_shards x touched partitions x + * windows per day} files. At 64 shards and a 1-minute triggering frequency that is ~92k files a + * day for a single unpartitioned table, whatever the actual data rate. On a partitioned + * table {@link #getShardsPerPartition()} caps the per-partition factor without lowering this + * number, trading per-partition write parallelism for proportionally fewer files. + * + *

The two failure modes are not symmetric, which is what decides the default. Too few shards + * shows up immediately and legibly, as a write bottleneck with a growing commit backlog, and is + * fixed by raising this number. Too many shards shows up as nothing at all until reads and + * compaction get slow, months later, by which time the small files are already written. So the + * default errs low and the guidance (see {@code package-info}) tells operators to raise it toward + * their worker parallelism. + */ + static final int DEFAULT_NUM_SHARDS = 16; + + /** Default value for {@link #getSorterMemoryMB()}. */ + static final int DEFAULT_SORTER_MEMORY_MB = 100; + + /** + * Columns that define a row's identity (the Iceberg equality-delete fields). + * + * @return the configured equality columns, or {@code null} to use the destination table's + * identifier fields (the default). + */ + abstract @Nullable List getEqualityColumns(); + + /** + * The column holding the per-primary-key monotonic sequence number used to order a single key's + * changes. Defaults to {@value #DEFAULT_SEQUENCE_NUMBER_COLUMN}. + */ + abstract String getSequenceNumberColumn(); + + /** + * If set, the change kind is read from this string column instead of the element's native {@link + * ValueKind}. The column is stripped from the data row and never written to Iceberg. + * + * @return the change-type column name, or {@code null} to use the element's native {@link + * ValueKind}. + */ + abstract @Nullable String getChangeTypeColumn(); + + /** + * Optional mapping from {@link #getChangeTypeColumn()} values to {@link ValueKind} names (e.g. + * Debezium {@code {"c": "INSERT", "u": "UPDATE_AFTER", "d": "DELETE"}}). If {@code null}, {@link + * #getChangeTypeColumn()} values must already be {@link ValueKind} names. + * + *

Only the map's values are constrained (each must name a {@link ValueKind} constant); the + * keys are arbitrary source-system codes. Requires {@link #getChangeTypeColumn()} to also be set. + * See {@link #validate()}. + */ + abstract @Nullable Map getChangeTypeMap(); + + /** + * The number of deterministic primary-key-hash shards (logical write buckets) per destination: + * the sink's write-parallelism knob. Must be {@code >= 1}. Defaults to {@value + * #DEFAULT_NUM_SHARDS}; set it to about your pipeline's write parallelism, and see {@link + * #DEFAULT_NUM_SHARDS} for why the default errs low. + */ + abstract int getNumShards(); + + /** + * The maximum number of shards a single partition's rows may occupy on a partitioned + * destination, always resolved to a concrete value at construction ({@code num_shards}, no cap, + * when the user left it unset, so downstream code never sees null). A {@code (destination, + * window)} then writes about {@code min(shards_per_partition, distinct keys)} files per touched + * partition per file kind, and per-partition write parallelism is capped at this value: {@code 1} + * pins each partition to a single writer, {@code num_shards} is plain primary-key sharding. + * Ignored (no partition plan is built) for an unpartitioned destination, which always shards by + * primary key. See {@link WriteCdcRows#withShardsPerPartition} for the trade-off. + */ + abstract int getShardsPerPartition(); + + /** + * The in-memory buffer size (MB) for the sorter that orders each shard's records by primary key, + * then sequence number, then change kind, before writing. Must be {@code >= 1}. Defaults to + * {@value #DEFAULT_SORTER_MEMORY_MB}. + */ + abstract int getSorterMemoryMB(); + + /** + * If {@code true}, {@code UPDATE_BEFORE} records are dropped and {@code INSERT}/{@code + * UPDATE_AFTER} are applied as upserts (equality-delete-then-insert on the primary key). Defaults + * to {@code false}. + */ + abstract boolean getUpsert(); + + /** + * If set, a destination that has committed at least once emits a periodic empty token-refresh + * commit while idle, keeping this sink's committed-through token snapshot recent. Disabled + * ({@code null}) by default. + */ + abstract @Nullable Long getTokenHeartbeatMillis(); + + /** + * A stable identifier for this sink, used to namespace the idempotency tokens written to each + * commit's Iceberg snapshot summary. + */ + abstract String getSinkId(); + + /** + * Extra user properties to add to every commit's Iceberg snapshot summary. Keys prefixed with + * {@code beam.cdc.} are reserved for the sink's own idempotency/diagnostic tokens; see {@link + * #validate()}. + */ + abstract @Nullable Map getSnapshotProperties(); + + /** + * If {@code true}, a poison record (unknown change type, missing/null sequence number, null + * equality value, an unresolvable destination) is diverted to the sink's failed-rows output + * instead of failing the pipeline. Defaults to {@code false} (fail-fast). + */ + abstract boolean getErrorHandling(); + + static Builder builder() { + return new AutoValue_CdcWriteConfig.Builder() + .setSequenceNumberColumn(DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setNumShards(DEFAULT_NUM_SHARDS) + .setShardsPerPartition(DEFAULT_NUM_SHARDS) + .setSorterMemoryMB(DEFAULT_SORTER_MEMORY_MB) + .setUpsert(false) + .setErrorHandling(false); + } + + void validate() { + checkArgument(getNumShards() >= 1, "num_shards must be >= 1, got %s", getNumShards()); + checkArgument( + getShardsPerPartition() >= 1 && getShardsPerPartition() <= getNumShards(), + "shards_per_partition must be between 1 and num_shards (%s); got %s", + getNumShards(), + getShardsPerPartition()); + checkArgument( + getSorterMemoryMB() >= 1, "sorter_memory_mb must be >= 1, got %s", getSorterMemoryMB()); + + @Nullable List equalityColumns = getEqualityColumns(); + checkArgument( + equalityColumns == null || !equalityColumns.isEmpty(), + "equality_columns must be non-empty or unset (leave unset to use the table's identifier " + + "fields)."); + + checkArgument( + !getSequenceNumberColumn().equals(getChangeTypeColumn()), + "sequence_number_column and change_type_column must be distinct, both are '%s'.", + getSequenceNumberColumn()); + + @Nullable Map changeTypeMap = getChangeTypeMap(); + checkArgument( + changeTypeMap == null || getChangeTypeColumn() != null, + "change_type_map requires change_type_column to also be set (it defines the source " + + "values mapped for that column)."); + if (changeTypeMap != null) { + for (String value : changeTypeMap.values()) { + checkArgument( + isValueKindName(value), + "change_type_map value '%s' is not a valid ValueKind name; must be one of %s.", + value, + Arrays.toString(ValueKind.values())); + } + } + + @Nullable Long heartbeatMillis = getTokenHeartbeatMillis(); + checkArgument( + heartbeatMillis == null || heartbeatMillis > 0, + "token heartbeat (withTokenHeartbeat / token_heartbeat_seconds) must be > 0 when set, " + + "got %s ms", + heartbeatMillis); + + @Nullable Map snapshotProperties = getSnapshotProperties(); + if (snapshotProperties != null) { + for (String key : snapshotProperties.keySet()) { + checkArgument( + !key.startsWith("beam.cdc."), + "snapshot_properties key '%s' uses the reserved 'beam.cdc.' prefix; choose a " + + "different key.", + key); + } + } + } + + /** + * Returns {@code source} minus the control columns ({@link #getSequenceNumberColumn()} always, + * {@link #getChangeTypeColumn()} when configured). This is the single strip rule shared by the + * coder derivation, table-schema validation, and table auto-creation, so the three always agree. + */ + Schema stripControlColumns(Schema source) { + String sequenceNumberColumn = getSequenceNumberColumn(); + @Nullable String changeTypeColumn = getChangeTypeColumn(); + Schema.Builder builder = Schema.builder(); + for (Schema.Field field : source.getFields()) { + String name = field.getName(); + if (name.equals(sequenceNumberColumn) || name.equals(changeTypeColumn)) { + continue; + } + builder.addField(field); + } + return builder.build(); + } + + private static boolean isValueKindName(String value) { + for (ValueKind kind : ValueKind.values()) { + if (kind.name().equals(value)) { + return true; + } + } + return false; + } + + @AutoValue.Builder + abstract static class Builder { + + abstract Builder setEqualityColumns(@Nullable List equalityColumns); + + abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + abstract Builder setChangeTypeColumn(@Nullable String changeTypeColumn); + + abstract Builder setChangeTypeMap(@Nullable Map changeTypeMap); + + abstract Builder setNumShards(int numShards); + + abstract Builder setShardsPerPartition(int shardsPerPartition); + + abstract Builder setSorterMemoryMB(int sorterMemoryMB); + + abstract Builder setUpsert(boolean upsert); + + abstract Builder setTokenHeartbeatMillis(@Nullable Long tokenHeartbeatMillis); + + abstract Builder setSinkId(String sinkId); + + abstract Builder setSnapshotProperties(@Nullable Map snapshotProperties); + + abstract Builder setErrorHandling(boolean errorHandling); + + abstract CdcWriteConfig build(); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitDeltas.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitDeltas.java new file mode 100644 index 000000000000..c0cda220080a --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitDeltas.java @@ -0,0 +1,1011 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects.firstNonNull; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.LongSupplier; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarIntCoder; +import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.SerializableDataFile; +import org.apache.beam.sdk.io.iceberg.SerializableDeleteFile; +import org.apache.beam.sdk.io.iceberg.SnapshotInfo; +import org.apache.beam.sdk.io.iceberg.TableCache; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Distribution; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.NoSuchSchemaException; +import org.apache.beam.sdk.schemas.SchemaRegistry; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldNumber; +import org.apache.beam.sdk.state.BagState; +import org.apache.beam.sdk.state.StateSpec; +import org.apache.beam.sdk.state.StateSpecs; +import org.apache.beam.sdk.state.TimeDomain; +import org.apache.beam.sdk.state.Timer; +import org.apache.beam.sdk.state.TimerSpec; +import org.apache.beam.sdk.state.TimerSpecs; +import org.apache.beam.sdk.state.ValueState; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.WithKeys; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.transforms.windowing.GlobalWindow; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists; +import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotUpdate; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.util.ThreadPools; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The CDC sink's commit stage: commits each {@code (destination, window)}'s merged writer outputs + * (represented as {@link ShardDeltaFiles}) as a single Iceberg snapshot, in ascending window-end + * order. Re-keys by destination, gathers all shards per {@code (dest, window)}, captures the window + * end, then re-windows into the global window for the stateful {@link OrderedCommitFn}. + * + *

Each commit writes the window's end millis to the snapshot summary as an idempotency token, + * keyed by the sink's unique {@code sinkId}. The committer recovers it by scanning snapshot + * ancestry: once on first touch of a destination, and again on every commit fire. Any window whose + * end is at or below the recovered token has already been committed, so it is skipped. + */ +class CommitDeltas + extends PTransform, PCollection>> { + + private static final Logger LOG = LoggerFactory.getLogger(CommitDeltas.class); + + // test-only attributes + @VisibleForTesting static volatile @Nullable Runnable preCommitHookForTest = null; + @VisibleForTesting static volatile @Nullable BiConsumer> onFireForTest = null; + + private final IcebergCatalogConfig catalogConfig; + private final String sinkId; + private final Map snapshotProperties; + private final long heartbeatMillis; + + /** The expansion's runId. */ + private final String runId; + + private Clock clock = System::currentTimeMillis; + + @VisibleForTesting + CommitDeltas(IcebergCatalogConfig catalogConfig, String sinkId) { + this(catalogConfig, sinkId, null, null); + } + + @VisibleForTesting + CommitDeltas( + IcebergCatalogConfig catalogConfig, + String sinkId, + @Nullable Map snapshotProperties, + @Nullable Long tokenHeartbeatMillis) { + this( + catalogConfig, + sinkId, + snapshotProperties, + tokenHeartbeatMillis, + UUID.randomUUID().toString()); + } + + CommitDeltas( + IcebergCatalogConfig catalogConfig, + String sinkId, + @Nullable Map snapshotProperties, + @Nullable Long tokenHeartbeatMillis, + String runId) { + this.catalogConfig = catalogConfig; + this.sinkId = sinkId; + this.snapshotProperties = + snapshotProperties == null ? Collections.emptyMap() : snapshotProperties; + this.heartbeatMillis = tokenHeartbeatMillis == null ? 0L : tokenHeartbeatMillis; + this.runId = runId; + } + + /** Overrides the committer's clock to test skew deterministically. */ + @VisibleForTesting + CommitDeltas withClockForTest(Clock clock) { + this.clock = clock; + return this; + } + + @Override + public PCollection> expand(PCollection input) { + boolean streaming = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + return input + .apply("KeyByDestination", WithKeys.of(ShardDeltaFiles::getTableIdentifierString)) + .setCoder(KvCoder.of(StringUtf8Coder.of(), WriteDeltas.shardDeltaFilesCoder())) + // One element per (dest, window): every shard's output for the pair. + .apply("GatherShardsPerWindow", GroupByKey.create()) + .apply("CaptureWindowEnd", ParDo.of(new CaptureWindowEndFn())) + .setCoder(KvCoder.of(StringUtf8Coder.of(), windowedCommitCoder())) + .apply("ToGlobalWindow", Window.into(new GlobalWindows())) + .apply( + "OrderedCommit", + ParDo.of( + new OrderedCommitFn( + catalogConfig, + sinkId, + runId, + snapshotProperties, + heartbeatMillis, + clock, + streaming))) + .setCoder(KvCoder.of(StringUtf8Coder.of(), snapshotInfoCoder())); + } + + private static Coder snapshotInfoCoder() { + try { + return SchemaRegistry.createDefault().getSchemaCoder(SnapshotInfo.class); + } catch (NoSuchSchemaException e) { + throw new RuntimeException("Could not build a coder for SnapshotInfo.", e); + } + } + + static Coder windowedCommitCoder() { + try { + return SchemaRegistry.createDefault().getSchemaCoder(WindowedCommit.class); + } catch (NoSuchSchemaException e) { + throw new RuntimeException("Could not build a coder for WindowedCommit.", e); + } + } + + /** Max number of file paths listed in a skip-path WARN before truncating. */ + private static final int SKIP_PATHS_LOGGED = 5; + + /** Every file path a window carries: data files first then delete files. */ + @VisibleForTesting + static List filePaths(WindowedCommit wc) { + List paths = new ArrayList<>(); + List deletePaths = new ArrayList<>(); + for (ShardDeltaFiles shard : wc.getFiles()) { + for (SerializableDataFile dataFile : shard.getDataFiles()) { + paths.add(dataFile.getPath()); + } + for (SerializableDeleteFile deleteFile : shard.getDeleteFiles()) { + deletePaths.add(deleteFile.getLocation()); + } + } + paths.addAll(deletePaths); + return paths; + } + + /** Renders up to {@link #SKIP_PATHS_LOGGED} of a skipped window's file paths, plus a count. */ + @VisibleForTesting + static String describeSkippedFiles(WindowedCommit wc) { + return describePaths(filePaths(wc)); + } + + private static String describePaths(List paths) { + StringBuilder sb = new StringBuilder(); + int shown = Math.min(SKIP_PATHS_LOGGED, paths.size()); + for (int i = 0; i < shown; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(paths.get(i)); + } + if (paths.size() > SKIP_PATHS_LOGGED) { + sb.append(" (… ").append(paths.size() - SKIP_PATHS_LOGGED).append(" more)"); + } + return sb.toString(); + } + + /** + * One {@code (destination, window)}'s merged writer outputs, tagged with the window's end. + * + *

The window end is a deterministic {@code FixedWindows} boundary and doubles as the + * restart-safe idempotency token. A window whose end is at or below the recovered + * committed-through token is skipped. + */ + @AutoValue + @DefaultSchema(AutoValueSchema.class) + abstract static class WindowedCommit { + @SchemaFieldNumber("0") + public abstract long getWindowEndMs(); + + @SchemaFieldNumber("1") + public abstract List getFiles(); + + public static WindowedCommit of(long windowEndMs, List files) { + return new AutoValue_CommitDeltas_WindowedCommit(windowEndMs, files); + } + } + + /** Folds a {@code (dest, window)}'s shard outputs into one {@link WindowedCommit}. */ + static class CaptureWindowEndFn + extends DoFn>, KV> { + @ProcessElement + public void process( + @Element KV> element, + BoundedWindow window, + OutputReceiver> out) { + long windowEndMs = window.maxTimestamp().getMillis(); + List files = Lists.newArrayList(element.getValue()); + out.outputWithTimestamp( + KV.of(element.getKey(), WindowedCommit.of(windowEndMs, files)), window.maxTimestamp()); + } + } + + /** + * A serializable millisecond clock; injectable via {@link #withClockForTest} so heartbeat tests + * can skew "now" deterministically. + */ + @FunctionalInterface + interface Clock extends LongSupplier, Serializable {} + + /** The committer's metrics, all namespaced under {@link CommitDeltas}. */ + static final class CommitterMetrics implements Serializable { + + final Counter snapshotsCreated = Metrics.counter(CommitDeltas.class, "snapshotsCreated"); + final Counter committedDataFiles = Metrics.counter(CommitDeltas.class, "committedDataFiles"); + final Counter committedDeleteFiles = + Metrics.counter(CommitDeltas.class, "committedDeleteFiles"); + final Counter committedRecords = Metrics.counter(CommitDeltas.class, "committedRecords"); + final Counter committedEqualityDeleteRecords = + Metrics.counter(CommitDeltas.class, "committedEqualityDeleteRecords"); + final Counter committedBytes = Metrics.counter(CommitDeltas.class, "committedBytes"); + final Distribution commitDurationMs = + Metrics.distribution(CommitDeltas.class, "commitDurationMs"); + final Counter commitFailures = Metrics.counter(CommitDeltas.class, "commitFailures"); + + final Counter alreadyCommittedWindowsSkipped = + Metrics.counter(CommitDeltas.class, "alreadyCommittedWindowsSkipped"); + final Counter orphanFiles = Metrics.counter(CommitDeltas.class, "orphanFiles"); + final Counter tokenParseFailures = Metrics.counter(CommitDeltas.class, "tokenParseFailures"); + final Counter suspectedTokenExpiry = + Metrics.counter(CommitDeltas.class, "suspectedTokenExpiry"); + final Counter crossWindowSequenceInversions = + Metrics.counter(CommitDeltas.class, "crossWindowSequenceInversions"); + final Counter specMismatchedWindows = + Metrics.counter(CommitDeltas.class, "specMismatchedWindows"); + + final Counter heartbeatCommits = Metrics.counter(CommitDeltas.class, "heartbeatCommits"); + } + + /** + * The ordered, idempotent committer. Keyed by destination; keeps the last-committed window-end + * and a bag of pending windows, plus one event-time timer armed at the earliest pending end. On + * fire, it commits every pending window at or below the input watermark, in ascending order, each + * as its own single-snapshot commit. A configured {@code tokenHeartbeatMillis} adds an idle + * token-refresh timer ({@link #onHeartbeat}). + */ + static class OrderedCommitFn extends DoFn, KV> { + + private final CommitterMetrics metrics = new CommitterMetrics(); + + private final IcebergCatalogConfig catalogConfig; + private final String sinkId; + private final String runId; + private final Map snapshotProperties; + private final CommitToken token; + + /** Idle token-refresh heartbeat interval in millis; {@code 0} = disabled. */ + private final long heartbeatMillis; + + private final Clock clock; + private final boolean streaming; + + @StateId("lastCommittedEndMs") + private final StateSpec> lastCommittedEndMsSpec = + StateSpecs.value(VarLongCoder.of()); + + /** The max source sequence number committed so far. */ + @StateId("lastCommittedMaxSeq") + private final StateSpec> lastCommittedMaxSeqSpec = + StateSpecs.value(VarLongCoder.of()); + + @StateId("pending") + private final StateSpec> pendingSpec; + + /** + * The earliest uncommitted window-end in {@link #pendingSpec} (what the commit timer must be + * armed at). + */ + @StateId("earliestPending") + private final StateSpec> earliestPendingSpec = + StateSpecs.value(VarLongCoder.of()); + + /** The partition-spec id pinned for this destination under {@link #pinnedRunIdSpec}'s runId. */ + @StateId("pinnedSpecId") + private final StateSpec> pinnedSpecIdSpec = + StateSpecs.value(VarIntCoder.of()); + + /** The runId the spec pin was taken under; a different live runId re-pins. */ + @StateId("pinnedRunId") + private final StateSpec> pinnedRunIdSpec = + StateSpecs.value(StringUtf8Coder.of()); + + @TimerId("commit") + private final TimerSpec commitTimerSpec = TimerSpecs.timer(TimeDomain.EVENT_TIME); + + /** Processing-time timer that fires the idle token-refresh heartbeat (when configured). */ + @TimerId("heartbeat") + private final TimerSpec heartbeatTimerSpec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME); + + OrderedCommitFn( + IcebergCatalogConfig catalogConfig, + String sinkId, + String runId, + Map snapshotProperties, + long heartbeatMillis, + Clock clock, + boolean streaming) { + this.catalogConfig = catalogConfig; + this.sinkId = sinkId; + this.runId = runId; + this.snapshotProperties = snapshotProperties; + this.heartbeatMillis = heartbeatMillis; + this.clock = clock; + this.streaming = streaming; + this.token = + new CommitToken(sinkId, runId, metrics.tokenParseFailures, metrics.suspectedTokenExpiry); + this.pendingSpec = StateSpecs.bag(windowedCommitCoder()); + } + + @RequiresStableInput + @ProcessElement + public void process( + @Element KV element, + @StateId("lastCommittedEndMs") ValueState lastCommittedEndMs, + @StateId("lastCommittedMaxSeq") ValueState lastMaxSeq, + @StateId("pending") BagState pending, + @StateId("earliestPending") ValueState earliestPending, + @TimerId("commit") Timer commitTimer, + @TimerId("heartbeat") Timer heartbeatTimer) { + String dest = element.getKey(); + WindowedCommit wc = element.getValue(); + + long lastCommittedMs = recoverOrReadCommitted(dest, lastCommittedEndMs, lastMaxSeq); + // Sets the idle heartbeat timer on every element; it fires after a full interval of idleness. + setHeartbeat(heartbeatTimer); + if (wc.getWindowEndMs() <= lastCommittedMs) { + // Already committed (retry/duplicate, or a rerun under a stable sink_id). + skipAlreadyCommitted(dest, wc, lastCommittedMs); + return; + } + + long earliest = + Math.min(firstNonNull(earliestPending.read(), Long.MAX_VALUE), wc.getWindowEndMs()); + pending.add(wc); + setCommitTimer(earliest, earliestPending, commitTimer); + } + + /** Returns the last committed window-end for {@code dest}. */ + private long recoverOrReadCommitted( + String dest, ValueState lastCommittedEndMs, ValueState lastMaxSeq) { + @Nullable Long stored = lastCommittedEndMs.read(); + if (stored != null) { + return stored; + } + CommitToken.Recovered recovered = token.recoverFromTable(catalogConfig, dest); + // Throw before writing to state + checkRecoveredTokenNotBatchEnd(dest, recovered.committedThroughMs); + lastCommittedEndMs.write(recovered.committedThroughMs); + lastMaxSeq.write(recovered.maxCommittedSeq); + return recovered.committedThroughMs; + } + + /** + * Fails a streaming destination whose recovered token is the batch token (the global-window end + * every bounded load commits under). Every real-time window's end falls below it, so the run + * would silently skip every window forever. + */ + private void checkRecoveredTokenNotBatchEnd(String dest, long recoveredMs) { + if (!streaming || recoveredMs != GlobalWindow.INSTANCE.maxTimestamp().getMillis()) { + return; + } + throw new IllegalStateException( + "CDC sink '" + + sinkId + + "' recovered a committed-through token for table '" + + dest + + "' equal to the global-window end (" + + recoveredMs + + " ms): this sink_id was last used by a batch (bounded) load, whose single " + + "global-window commit claims every event-time window. A streaming run reusing it " + + "would skip every window forever. Use a different sink_id for the streaming " + + "continuation."); + } + + @RequiresStableInput + @OnTimer("commit") + public void onCommit( + OnTimerContext c, + @Key String dest, + @StateId("lastCommittedEndMs") ValueState lastCommittedEndMs, + @StateId("lastCommittedMaxSeq") ValueState lastMaxSeq, + @StateId("pending") BagState pending, + @StateId("earliestPending") ValueState earliestPending, + @StateId("pinnedSpecId") ValueState pinnedSpecId, + @StateId("pinnedRunId") ValueState pinnedRunId, + @TimerId("commit") Timer timer, + OutputReceiver> out) { + // The "commit" timer is always armed at the earliest pending window's end. + long earliestPendingWindowEnd = c.timestamp().getMillis(); + // Current input watermark tells us that all past pending windows are safe to commit. + long inputWatermark = timer.getCurrentRelativeTime().getMillis(); + long fireWatermarkMs = Math.max(earliestPendingWindowEnd, inputWatermark); + + List all = Lists.newArrayList(pending.read()); + all.sort(Comparator.comparingLong(WindowedCommit::getWindowEndMs)); + + long committedMaxSeq = firstNonNull(lastMaxSeq.read(), Long.MIN_VALUE); + // One table load + ancestry scan per fire + Table table = TableCache.getRefreshed(catalogConfig, dest); + + long tableTokenMs = token.recoverFrom(table, dest).committedThroughMs; + checkRecoveredTokenNotBatchEnd(dest, tableTokenMs); + + long lastCommittedMs = + Math.max(firstNonNull(lastCommittedEndMs.read(), Long.MIN_VALUE), tableTokenMs); + + // Ascending triage: + // 1. already-committed ends skip loudly + // 2. ends past the fire watermark stay pending + // 3. the rest commit now (in this fire) + // Duplicate window-ends are parked and skipped only after its window's commit lands. + List committable = new ArrayList<>(); + List duplicates = new ArrayList<>(); + List remaining = new ArrayList<>(); + long selectedThrough = lastCommittedMs; + for (WindowedCommit wc : all) { + if (wc.getWindowEndMs() <= lastCommittedMs) { + skipAlreadyCommitted(dest, wc, lastCommittedMs); + } else if (wc.getWindowEndMs() > fireWatermarkMs) { + // not yet safe to commit. there could be earlier windows that haven't reached the pending + // bag yet + remaining.add(wc); + } else if (wc.getWindowEndMs() <= selectedThrough) { + duplicates.add(wc); // same end as a window this fire is about to commit + } else { + committable.add(wc); + selectedThrough = wc.getWindowEndMs(); + } + } + + // Commit each committable window as its own snapshot + List committedEnds = new ArrayList<>(committable.size()); + // The spec this run pinned, or null when the pin is from an earlier run. A pipeline update + // keeps committer state but regenerates the runId. We drop the pin if it's stale (if the + // construction-time runId doesn't match the preserved state's pinnedRunId). Doing this lets + // the new run re-pin onto the current live spec. + @Nullable Integer currentSpecId = + runId.equals(pinnedRunId.read()) ? pinnedSpecId.read() : null; + for (WindowedCommit wc : committable) { + CommitSummary summary; + try { + summary = commitOneWindow(table, wc, currentSpecId); + } catch (RuntimeException e) { + // Later windows must not commit past a failed one. + // Count and rethrow before any state write, so the retry re-fires with the pending bag. + metrics.commitFailures.inc(); + throw commitFailure(dest, wc, all, lastCommittedMs, e); + } + + currentSpecId = + rePinAndWarnOnMismatch(dest, wc, summary, currentSpecId, pinnedSpecId, pinnedRunId); + + // Pair the window with its own snapshot by token + SnapshotInfo info = identifyCommitted(table, dest, wc, summary); + lastCommittedMs = wc.getWindowEndMs(); + out.output(KV.of(dest, info)); + committedMaxSeq = detectSequenceInversion(dest, wc, committedMaxSeq); + committedEnds.add(wc.getWindowEndMs()); + for (WindowedCommit dup : duplicates) { + if (dup.getWindowEndMs() == wc.getWindowEndMs()) { + // Release this duplicate now that its window's commit succeeded. + skipSameFireDuplicate(dest, dup, wc); + } + } + } + lastCommittedEndMs.write(lastCommittedMs); + lastMaxSeq.write(committedMaxSeq); + pending.clear(); + long earliest = Long.MAX_VALUE; + for (WindowedCommit wc : remaining) { + pending.add(wc); + earliest = Math.min(earliest, wc.getWindowEndMs()); + } + // Re-arm the timer at the next-earliest pending end to account for remaining windows + setCommitTimer(earliest, earliestPending, timer); + BiConsumer> onFire = onFireForTest; + if (onFire != null) { + onFire.accept(fireWatermarkMs, committedEnds); + } + } + + /** + * Commits one window's merged files as a single Iceberg snapshot and returns its volume + * summary. + */ + private CommitSummary commitOneWindow(Table table, WindowedCommit wc, @Nullable Integer pin) { + WindowFiles files = reconstructFiles(table, wc); + CommitSummary summary = CommitSummary.of(files); + // The spec id to stamp this commit with. + // If the run has no pinned spec yet, rePinAndWarnOnMismatch() pins this same value after + // the commit. + @Nullable Integer stampSpecId = pin != null ? pin : summary.firstSpecId; + long commitStart = clock.getAsLong(); + applyCommit(table, wc.getWindowEndMs(), files, stampSpecId); + metrics.commitDurationMs.update(clock.getAsLong() - commitStart); + return summary; + } + + /** + * Establishes the run's spec pin on the first window committed under this runId. Returns the + * current spec id pin. + * + *

Future calls will warn and count {@code specMismatchedWindows} when a committed window + * mixes spec ids against the pin and carries any equality deletes. This can happen when a user + * evolves a table's spec mid-run (bad practice). + * + *

This case is problematic because equality deletes apply only to data files of its own + * {@code (spec id, partition)}, so a mixed-spec window means deletes may be intended for rows + * written under another spec. In such a case, those rows will incorrectly remain live. The + * window is already committed either way (reconstruction is per-file-spec). Deletes written + * under an unpartitioned spec are global and do reach all rows. + */ + private @Nullable Integer rePinAndWarnOnMismatch( + String dest, + WindowedCommit wc, + CommitSummary summary, + @Nullable Integer currentSpecId, + ValueState pinnedSpecId, + ValueState pinnedRunId) { + if (currentSpecId == null) { + currentSpecId = summary.firstSpecId; + if (currentSpecId == null) { + return null; // a window with no files cannot seed a pin + } + pinnedSpecId.write(currentSpecId); + pinnedRunId.write(runId); + } + if (!summary.hasEqualityDeletes) { + return currentSpecId; + } + for (int specId : summary.specIds) { + if (specId != currentSpecId) { + metrics.specMismatchedWindows.inc(); + LOG.warn( + "CDC sink '{}' committed window-end {} ms for table '{}' with equality deletes in " + + "a window mixing partition specs (pinned spec id {}, saw spec id {}): the " + + "partition spec evolved mid-run. An equality delete applies only to data " + + "files of its own (spec id, partition), so deletes may not reach rows " + + "written under the other spec. Run rewrite_data_files so that data is correctly " + + "repartitioned, then drain and restart the pipeline to converge on one spec.", + sinkId, + wc.getWindowEndMs(), + dest, + currentSpecId, + specId); + break; + } + } + return currentSpecId; + } + + /** + * Pairs a just-committed window with the snapshot carrying its token, records its metrics, and + * returns its {@link SnapshotInfo}. We match by token because there could be concurrent foreign + * writers that overwrite {@code currentSnapshot()}. + */ + private SnapshotInfo identifyCommitted( + Table table, String dest, WindowedCommit wc, CommitSummary summary) { + Snapshot snapshot = + token.findRecentlyCommittedTokenSnapshot(table, dest, wc.getWindowEndMs()); + recordCommitMetrics(dest, wc.getWindowEndMs(), summary, snapshot); + return SnapshotInfo.fromSnapshot(snapshot); + } + + /** + * Reconstructs live {@link DataFile}/{@link DeleteFile} objects from one window's serialized + * shard outputs. Each file is rebuilt against its own recorded partition-spec id (and + * sort-order id) so a bundle written before a spec evolution still reconstructs under the spec + * it was written with. + */ + private static WindowFiles reconstructFiles(Table table, WindowedCommit wc) { + Map specs = table.specs(); + Map sortOrders = sortOrdersForReconstruction(table); + WindowFiles files = new WindowFiles(); + for (ShardDeltaFiles shard : wc.getFiles()) { + for (SerializableDataFile dataFile : shard.getDataFiles()) { + files.dataFiles.add(dataFile.createDataFile(specs)); + } + for (SerializableDeleteFile deleteFile : shard.getDeleteFiles()) { + files.deleteFiles.add(deleteFile.createDeleteFile(specs, sortOrders)); + } + files.maxSequenceNumber = Math.max(files.maxSequenceNumber, shard.getMaxSequenceNumber()); + } + return files; + } + + /** + * {@code table.sortOrders()}, guaranteed to contain the unsorted order (id {@code 0}). A table + * created with a sort order stores only that order, without the unsorted id {@code 0}. + */ + private static Map sortOrdersForReconstruction(Table table) { + Map sortOrders = table.sortOrders(); + int unsortedId = SortOrder.unsorted().orderId(); + if (sortOrders.containsKey(unsortedId)) { + return sortOrders; + } + Map withUnsorted = new HashMap<>(sortOrders); + withUnsorted.put(unsortedId, SortOrder.unsorted()); + return withUnsorted; + } + + /** + * Builds and commits one window's Iceberg operation: {@link AppendFiles} when the window has no + * delete files, else {@link RowDelta}. Windows commit in strict ascending order, so this + * window's equality deletes apply to all lower-sequence-number data. + */ + private void applyCommit( + Table table, long windowEndMs, WindowFiles files, @Nullable Integer stampSpecId) { + Runnable hook = preCommitHookForTest; + if (hook != null) { + hook.run(); + } + SnapshotUpdate op; + if (files.deleteFiles.isEmpty()) { + AppendFiles append = table.newAppend(); // append fast path: the window has no deletes + files.dataFiles.forEach(append::appendFile); + op = append; + } else { + RowDelta rowDelta = table.newRowDelta(); + files.dataFiles.forEach(rowDelta::addRows); + files.deleteFiles.forEach(rowDelta::addDeletes); + op = rowDelta; + } + // User snapshot-summary properties first + snapshotProperties.forEach(op::set); + token.writeTo(op, windowEndMs, files.maxSequenceNumber, stampSpecId); + // Parallelize manifest scanning on Iceberg's process-global worker pool. + op.scanManifestsWith(ThreadPools.getWorkerPool()); + op.commit(); + } + + /** Records a successful commit's volume/latency/liveness metrics, from data already in hand. */ + private void recordCommitMetrics( + String dest, long windowEndMs, CommitSummary summary, Snapshot snapshot) { + metrics.committedDataFiles.inc(summary.dataFileCount); + metrics.committedDeleteFiles.inc(summary.deleteFileCount); + metrics.committedRecords.inc(summary.dataRecords); + metrics.committedEqualityDeleteRecords.inc(summary.equalityDeleteRecords); + metrics.committedBytes.inc(summary.bytes); + metrics.snapshotsCreated.inc(); + LOG.info( + "CDC sink '{}' committed window-end {} ms for table '{}' as snapshot {}.", + sinkId, + windowEndMs, + dest, + snapshot.snapshotId()); + } + + /** + * Wraps a window's commit failure with operator-triage context; {@code committed} is already + * advanced past this fire's earlier commits, so the pending numbers describe the retry. + */ + private RuntimeException commitFailure( + String dest, + WindowedCommit failing, + List all, + long committed, + RuntimeException cause) { + long earliestPending = earliestUncommitted(all, committed); + int pendingCount = countUncommitted(all, committed); + String message = + "CDC sink '" + + sinkId + + "' failed to commit table '" + + dest + + "' at window-end " + + failing.getWindowEndMs() + + " ms (earliest pending window-end " + + earliestPending + + " ms, " + + pendingCount + + " pending window(s)). The destination is halted until this commit succeeds; " + + "see commitFailures. The pending bag is " + + "untouched, so the retry re-fires with the same windows and the same files. " + + "Windows committed earlier in this same fire stay committed with their tokens " + + "durable in the table, so the retry recovers those tokens and skips them as " + + "already-committed (counted by alreadyCommittedWindowsSkipped)."; + return new RuntimeException(message, cause); + } + + /** + * Skips a window if the committed-through token already covers it. This only proves a window + * with this end has already committed. It does not prove that the same content was committed. + * For that reason, we log every file name and count it as a potential orphan. + */ + private void skipAlreadyCommitted(String dest, WindowedCommit wc, long committed) { + metrics.alreadyCommittedWindowsSkipped.inc(); + metrics.orphanFiles.inc(filePaths(wc).size()); + + LOG.warn( + "CDC sink '{}' skipping window-end {} ms for table '{}': already committed " + + "(committed-through token = {} ms). The files below were written by this attempt. " + + "If a different attempt committed this window, they are unreferenced and hold rows " + + "the table never received. Files: {}", + sinkId, + wc.getWindowEndMs(), + dest, + committed, + describeSkippedFiles(wc)); + } + + /** + * The idempotent skip of a duplicate window: this very fire already published a window with the + * identical end, so the duplicate's rows are in the table by construction. + * + *

Compares file paths to count and name orphans: a redelivered duplicate carries the + * identical files (live table data, nothing orphaned), a second pane of the same window carries + * distinct files (genuine orphans). + */ + private void skipSameFireDuplicate(String dest, WindowedCommit dup, WindowedCommit committed) { + metrics.alreadyCommittedWindowsSkipped.inc(); + Set committedPaths = new HashSet<>(filePaths(committed)); + List orphaned = new ArrayList<>(); + for (String path : filePaths(dup)) { + if (!committedPaths.contains(path)) { + orphaned.add(path); + } + } + metrics.orphanFiles.inc(orphaned.size()); + if (orphaned.isEmpty()) { + LOG.info( + "CDC sink '{}' skipping window-end {} ms for table '{}': this commit fire already " + + "published a window with the same end. Every " + + "file this entry names was published by that commit, so it is a pure " + + "redelivery and nothing is orphaned.", + sinkId, + committed.getWindowEndMs(), + dest); + } else { + LOG.warn( + "CDC sink '{}' skipping window-end {} ms for table '{}': this commit fire already " + + "published a window with the same end. {} of this " + + "entry's files are not among the ones that commit published. The sink cannot prove whether " + + "they are redundant copies or hold rows the table never received. Files: {}", + sinkId, + committed.getWindowEndMs(), + dest, + orphaned.size(), + describePaths(orphaned)); + } + } + + /** + * Flags a committed window whose min source sequence is below an earlier window's committed + * max, a possible ordering violation. Returns the updated running max. + */ + private long detectSequenceInversion(String dest, WindowedCommit wc, long prevMax) { + long windowMinSeq = Long.MAX_VALUE; + long windowMaxSeq = Long.MIN_VALUE; + for (ShardDeltaFiles shard : wc.getFiles()) { + windowMinSeq = Math.min(windowMinSeq, shard.getMinSequenceNumber()); + windowMaxSeq = Math.max(windowMaxSeq, shard.getMaxSequenceNumber()); + } + if (windowMinSeq < prevMax) { + metrics.crossWindowSequenceInversions.inc(); + LOG.warn( + "CDC sink '{}' committed window-end {} ms for table '{}' with min sequence {} below " + + "the previously committed max sequence {}: possible ordering violation (source " + + "event-time not monotonic with the sequence number); final table state for keys " + + "spanning these windows may be incorrect; this check has benign false positives " + + "when the windows touch disjoint keys.", + sinkId, + wc.getWindowEndMs(), + dest, + windowMinSeq, + prevMax); + } + return Math.max(prevMax, windowMaxSeq); + } + + /** Records the earliest uncommitted pending window and resets the commit timer at it. */ + private void setCommitTimer( + long earliest, ValueState earliestPending, Timer commitTimer) { + earliestPending.write(earliest); + if (earliest != Long.MAX_VALUE) { + commitTimer.set(new Instant(earliest)); + } + } + + /** Sets the processing-time heartbeat timer {@code heartbeatMillis} from now. */ + private void setHeartbeat(Timer heartbeatTimer) { + if (heartbeatMillis > 0) { + heartbeatTimer.offset(Duration.millis(heartbeatMillis)).setRelative(); + } + } + + /** + * Idle token-refresh heartbeat: when a destination has committed, has nothing pending, and its + * newest token snapshot has aged past the interval, emit an empty append re-writing the current + * token. The purpose is to always maintain a recent token-bearing snapshot that is young enough + * to survive {@code expire_snapshots}. + */ + @OnTimer("heartbeat") + public void onHeartbeat( + @Key String dest, + @StateId("lastCommittedEndMs") ValueState lastCommittedEndMs, + @StateId("lastCommittedMaxSeq") ValueState lastMaxSeq, + @StateId("pending") BagState pending, + @StateId("pinnedSpecId") ValueState pinnedSpecId, + @StateId("pinnedRunId") ValueState pinnedRunId, + @TimerId("heartbeat") Timer heartbeatTimer) { + try { + Long lastCommitted = lastCommittedEndMs.read(); + if (lastCommitted == null || lastCommitted == Long.MIN_VALUE) { + return; // never committed for this destination; no token to refresh + } + if (!pending.isEmpty().read()) { + return; // a real commit is imminent; the token will refresh naturally + } + + Table table = TableCache.getRefreshed(catalogConfig, dest); + if (!token.shouldHeartbeat(table, heartbeatMillis, clock.getAsLong())) { + return; + } + AppendFiles append = table.newAppend(); // empty append: a token refresh, no new files + snapshotProperties.forEach(append::set); + @Nullable Integer pin = runId.equals(pinnedRunId.read()) ? pinnedSpecId.read() : null; + long maxCommittedSeq = firstNonNull(lastMaxSeq.read(), Long.MIN_VALUE); + token.writeHeartbeatTo(append, lastCommitted, maxCommittedSeq, pin); + append.commit(); + + metrics.heartbeatCommits.inc(); + LOG.info( + "CDC sink '{}' emitted an idle token-refresh (heartbeat) commit for table '{}' at " + + "committed-through {} ms.", + sinkId, + dest, + lastCommitted); + } finally { + setHeartbeat(heartbeatTimer); // keep firing through idle + } + } + + /** The earliest window-end strictly greater than {@code committed} (else {@code MIN}). */ + private static long earliestUncommitted(List windows, long committed) { + long min = Long.MAX_VALUE; + for (WindowedCommit wc : windows) { + if (wc.getWindowEndMs() > committed) { + min = Math.min(min, wc.getWindowEndMs()); + } + } + return min == Long.MAX_VALUE ? Long.MIN_VALUE : min; + } + + /** Count of windows whose end is strictly greater than {@code committed}. */ + private static int countUncommitted(List windows, long committed) { + int count = 0; + for (WindowedCommit wc : windows) { + if (wc.getWindowEndMs() > committed) { + count++; + } + } + return count; + } + + /** One window's reconstructed files plus the max source sequence they cover. */ + private static final class WindowFiles { + final List dataFiles = new ArrayList<>(); + final List deleteFiles = new ArrayList<>(); + long maxSequenceNumber = Long.MIN_VALUE; + } + + private static final class CommitSummary { + final long dataFileCount; + final long deleteFileCount; + final long dataRecords; + final long equalityDeleteRecords; + final long bytes; + final @Nullable Integer firstSpecId; + final Set specIds; + final boolean hasEqualityDeletes; + + private CommitSummary( + long dataFileCount, + long deleteFileCount, + long dataRecords, + long equalityDeleteRecords, + long bytes, + @Nullable Integer firstSpecId, + Set specIds, + boolean hasEqualityDeletes) { + this.dataFileCount = dataFileCount; + this.deleteFileCount = deleteFileCount; + this.dataRecords = dataRecords; + this.equalityDeleteRecords = equalityDeleteRecords; + this.bytes = bytes; + this.firstSpecId = firstSpecId; + this.specIds = specIds; + this.hasEqualityDeletes = hasEqualityDeletes; + } + + static CommitSummary of(WindowFiles files) { + long dataRecords = 0; + long bytes = 0; + @Nullable Integer firstSpecId = null; + Set specIds = new HashSet<>(); + for (DataFile f : files.dataFiles) { + dataRecords += f.recordCount(); + bytes += f.fileSizeInBytes(); + firstSpecId = firstSpecId == null ? f.specId() : firstSpecId; + specIds.add(f.specId()); + } + long equalityDeleteRecords = 0; + boolean hasEqualityDeletes = false; + for (DeleteFile f : files.deleteFiles) { + bytes += f.fileSizeInBytes(); + firstSpecId = firstSpecId == null ? f.specId() : firstSpecId; + specIds.add(f.specId()); + if (f.content() == FileContent.EQUALITY_DELETES) { + equalityDeleteRecords += f.recordCount(); + hasEqualityDeletes = true; + } + } + return new CommitSummary( + files.dataFiles.size(), + files.deleteFiles.size(), + dataRecords, + equalityDeleteRecords, + bytes, + firstSpecId, + specIds, + hasEqualityDeletes); + } + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitToken.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitToken.java new file mode 100644 index 000000000000..793b7639da13 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitToken.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.io.Serializable; +import java.util.Map; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.TableCache; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotUpdate; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.util.SnapshotUtil; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The CDC sink's idempotency-token contract: the sink-id-namespaced snapshot-summary keys that make + * commits idempotent, plus every token-keyed ancestry walk the committer performs. + */ +final class CommitToken implements Serializable { + + private static final Logger LOG = LoggerFactory.getLogger(CommitToken.class); + + /** Marks a snapshot as committed by the CDC sink instance named by its value. */ + static final String SINK_ID_KEY = "beam.cdc.sink-id"; + + /** Prefix of the committed-through window-end token key ({@code + sinkId}). */ + static final String COMMITTED_THROUGH_MS_PREFIX = "beam.cdc.committed-through-ms."; + + /** Prefix of the max-committed source-sequence key ({@code + sinkId}). */ + static final String MAX_COMMITTED_SEQ_PREFIX = "beam.cdc.max-committed-seq."; + + /** Prefix of the run-spec stamp key ({@code + sinkId}); value {@code :}. */ + static final String RUN_SPEC_PREFIX = "beam.cdc.run-spec."; + + private final String sinkId; + private final String runId; + private final Counter tokenParseFailures; + private final Counter suspectedTokenExpiry; + + /** + * @param runId the run runId stamped into the run-spec key + * @param tokenParseFailures counts unparseable token/max-seq summary values met during recovery + * @param suspectedTokenExpiry counts recoveries where the sink-id marker survives but no token + * does (the token-bearing snapshots were likely expired away) + */ + CommitToken( + String sinkId, String runId, Counter tokenParseFailures, Counter suspectedTokenExpiry) { + this.sinkId = sinkId; + this.runId = runId; + this.tokenParseFailures = tokenParseFailures; + this.suspectedTokenExpiry = suspectedTokenExpiry; + } + + /** Writes the three token keys and the {@code pinnedSpecId} onto a pending snapshot operation. */ + void writeTo( + SnapshotUpdate op, + long committedThroughMs, + long maxCommittedSeq, + @Nullable Integer pinnedSpecId) { + op.set(COMMITTED_THROUGH_MS_PREFIX + sinkId, Long.toString(committedThroughMs)); + op.set(MAX_COMMITTED_SEQ_PREFIX + sinkId, Long.toString(maxCommittedSeq)); + if (pinnedSpecId != null) { + op.set(RUN_SPEC_PREFIX + sinkId, runId + ":" + pinnedSpecId); + } + op.set(SINK_ID_KEY, sinkId); + } + + /** + * Writes the token keys for an idle token-refresh (heartbeat) commit. Unlike {@link #writeTo}, + * the max-committed-seq key is omitted when unknown ({@code MIN}, meaning recovery found a token + * whose snapshot carried no parseable max-seq). + */ + void writeHeartbeatTo( + SnapshotUpdate op, + long committedThroughMs, + long maxCommittedSeq, + @Nullable Integer pinnedSpecId) { + op.set(COMMITTED_THROUGH_MS_PREFIX + sinkId, Long.toString(committedThroughMs)); + if (maxCommittedSeq != Long.MIN_VALUE) { + op.set(MAX_COMMITTED_SEQ_PREFIX + sinkId, Long.toString(maxCommittedSeq)); + } + if (pinnedSpecId != null) { + op.set(RUN_SPEC_PREFIX + sinkId, runId + ":" + pinnedSpecId); + } + op.set(SINK_ID_KEY, sinkId); + } + + /** + * Returns the spec id stamped for {@code sinkId} under {@code runId}, read from the most recent + * stamp-bearing snapshot on {@code table}'s current branch. {@code null} when that stamp is + * absent, unparseable, or another run's (the caller falls back to the current spec). + */ + static @Nullable Integer readRunSpec(Table table, String sinkId, String runId) { + Snapshot current = table.currentSnapshot(); + if (current == null) { + return null; + } + String key = RUN_SPEC_PREFIX + sinkId; + String wantedPrefix = runId + ":"; + for (Snapshot s : SnapshotUtil.ancestorsOf(current.snapshotId(), table::snapshot)) { + Map summary = s.summary(); + if (summary == null) { + continue; + } + String value = summary.get(key); + if (value == null) { + continue; + } + // Only the newest stamp counts; a foreign runId or garbage value reads as no stamp. + if (!value.startsWith(wantedPrefix)) { + return null; + } + try { + return Integer.parseInt(value.substring(wantedPrefix.length())); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + /** + * The state recovered from a table's ancestry: the committed-through-ms window token and the + * max-committed sequence from the same snapshot. The sequence seeds the cross-window inversion + * detector when committer state is empty, which is a relaunch or the first time a destination is + * seen. {@link #FRESH_START} means neither was found. + */ + static final class Recovered { + static final Recovered FRESH_START = new Recovered(Long.MIN_VALUE, Long.MIN_VALUE); + + final long committedThroughMs; + final long maxCommittedSeq; + + private Recovered(long committedThroughMs, long maxCommittedSeq) { + this.committedThroughMs = committedThroughMs; + this.maxCommittedSeq = maxCommittedSeq; + } + } + + /** + * Loads {@code dest} (forcing a refresh) and recovers this sink's token from its ancestry. The + * table may not exist yet, so a missing table is tolerated as a fresh start. + */ + Recovered recoverFromTable(IcebergCatalogConfig catalogConfig, String dest) { + Table table; + try { + table = TableCache.getRefreshed(catalogConfig, dest); + } catch (RuntimeException e) { + if (hasCause(e, NoSuchTableException.class)) { + return Recovered.FRESH_START; + } + throw e; + } + return recoverFrom(table, dest); + } + + /** + * Recovers this sink's committed-through-ms token (and corresponding max-committed sequence) by + * scanning a table's snapshot ancestry and returning the first {@code + * beam.cdc.committed-through-ms.} found, else {@link Long#MIN_VALUE}. + */ + Recovered recoverFrom(Table table, String dest) { + Snapshot current = table.currentSnapshot(); + if (current == null) { + return Recovered.FRESH_START; + } + String tokenKey = COMMITTED_THROUGH_MS_PREFIX + sinkId; + String maxSeqKey = MAX_COMMITTED_SEQ_PREFIX + sinkId; + boolean sawSinkMarker = false; + for (Snapshot s : SnapshotUtil.ancestorsOf(current.snapshotId(), table::snapshot)) { + Map summary = s.summary(); + if (summary == null) { + continue; + } + if (sinkId.equals(summary.get(SINK_ID_KEY))) { + sawSinkMarker = true; + } + String tokenValue = summary.get(tokenKey); + if (tokenValue == null) { + continue; + } + long committedThroughMs; + try { + committedThroughMs = Long.parseLong(tokenValue); + } catch (NumberFormatException e) { + // An older intact token is better than crash-looping + tokenParseFailures.inc(); + LOG.error( + "CDC sink '{}' found an unparseable committed-through token '{}' in snapshot {} " + + "of table '{}'; ignoring it and scanning older ancestors.", + sinkId, + tokenValue, + s.snapshotId(), + dest); + continue; + } + // Both values come from this snapshot: the pair must describe one commit. + return new Recovered(committedThroughMs, parseMaxSeq(summary.get(maxSeqKey), s, dest)); + } + if (sawSinkMarker) { + // This sink has committed to the table before, yet no token survived the ancestry scan. + // Rare but can happen if expire_snapshots removes the token-bearing snapshots. + suspectedTokenExpiry.inc(); + LOG.warn( + "CDC sink '{}' found its sink-id marker in table '{}' ancestry but no " + + "committed-through token; the token-bearing snapshot(s) may have been expired. " + + "Falling back to MIN, which may replay retained windows.", + sinkId, + dest); + } + return Recovered.FRESH_START; + } + + /** {@link Long#MIN_VALUE} when the max-committed-seq is absent or unparseable. */ + private long parseMaxSeq(@Nullable String value, Snapshot s, String dest) { + if (value == null) { + return Long.MIN_VALUE; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + tokenParseFailures.inc(); + LOG.error( + "CDC sink '{}' found an unparseable max-committed-seq '{}' in snapshot {} of " + + "table '{}'; ignoring it.", + sinkId, + value, + s.snapshotId(), + dest); + return Long.MIN_VALUE; + } + } + + /** + * Whether an idle destination should emit an empty token-refresh (heartbeat) commit: {@code true} + * iff the most recent table snapshot bearing this sink's committed-through token is older than + * {@code intervalMillis} relative to {@code nowMs}. + */ + boolean shouldHeartbeat(Table table, long intervalMillis, long nowMs) { + @Nullable Snapshot current = table.currentSnapshot(); + if (current == null) { + return false; + } + String tokenKey = COMMITTED_THROUGH_MS_PREFIX + sinkId; + for (Snapshot s : SnapshotUtil.ancestorsOf(current.snapshotId(), table::snapshot)) { + Map summary = s.summary(); + if (summary != null && summary.get(tokenKey) != null) { + return s.timestampMillis() < nowMs - intervalMillis; + } + } + return false; + } + + /** + * Finds and returns the snapshot corresponding to a just-committed window by looking for the + * specified {@code windowEndMs}. Expects that the caller has just committed the window, so throws + * if no such snapshot exists. + */ + Snapshot findRecentlyCommittedTokenSnapshot(Table table, String dest, long windowEndMs) { + table.refresh(); + Snapshot current = + checkStateNotNull( + table.currentSnapshot(), + "table '%s' has no current snapshot right after a commit", + dest); + String tokenKey = COMMITTED_THROUGH_MS_PREFIX + sinkId; + String wanted = Long.toString(windowEndMs); + for (Snapshot s : SnapshotUtil.ancestorsOf(current.snapshotId(), table::snapshot)) { + Map summary = s.summary(); + if (summary != null && wanted.equals(summary.get(tokenKey))) { + return s; + } + } + throw new IllegalStateException( + "CDC sink '" + + sinkId + + "' committed window-end " + + windowEndMs + + " ms to table '" + + dest + + "' but found no snapshot carrying its committed-through token in the refreshed " + + "ancestry."); + } + + private static boolean hasCause(Throwable t, Class type) { + for (Throwable cause = t; cause != null; cause = cause.getCause()) { + if (type.isInstance(cause)) { + return true; + } + } + return false; + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitWindows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitWindows.java new file mode 100644 index 000000000000..89ee9379fc6c --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CommitWindows.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.util.Map; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.extensions.sorter.BufferedExternalSorter; +import org.apache.beam.sdk.extensions.sorter.SortValues; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.GroupByKey; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.AfterPane; +import org.apache.beam.sdk.transforms.windowing.AfterWatermark; +import org.apache.beam.sdk.transforms.windowing.DefaultTrigger; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.GlobalWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollection.IsBounded; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PInput; +import org.apache.beam.sdk.values.POutput; +import org.apache.beam.sdk.values.PValue; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; +import org.apache.beam.sdk.values.TupleTagList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; + +/** + * Windows sharded, sort-keyed records into event-time commit windows, grouped by the {@code + * KV} key. Each {@code (destination, shard, window)} becomes one commit unit + * group and the event-time watermark is the commit barrier. Each group is sorted by the byte sort + * key applied from {@link AssignCdcKeys}: one primary key's records contiguous, in {@code (seq, + * kind)} order within the key. + * + *

Late panes are routed to the DLQ (via {@link SplitLateData}) before any ordering happens. This + * is necessary because the downstream commit step skips panes if their window token is present in + * an already committed snapshot. If late panes are let through, their contents will never get to + * the table. + * + *

Windowing by input mode: + * + *

    + *
  • Bounded: a single {@link GlobalWindows} window. One commit per destination, no late + * data possible. + *
  • Unbounded: event-time {@link FixedWindows} of {@code triggeringFrequency}, firing on + * the watermark with late firings per element, discarding fired panes. + *
+ */ +final class CommitWindows + extends PTransform< + PCollection, KV>>, CommitWindows.Result> { + + private static final TupleTag, Iterable>>> + ON_TIME_TAG = new TupleTag<>("onTime"); + private static final TupleTag DEAD_LETTER_TAG = new TupleTag<>("deadLetter"); + + private final CdcWriteConfig config; + private final @Nullable Duration triggeringFrequency; + private final Duration allowedLateness; + + CommitWindows( + CdcWriteConfig config, @Nullable Duration triggeringFrequency, Duration allowedLateness) { + this.config = config; + this.triggeringFrequency = triggeringFrequency; + this.allowedLateness = allowedLateness; + } + + @Override + public Result expand(PCollection, KV>> input) { + Schema deadLetterSchema = SplitLateData.deadLetterSchema(dataSchemaOf(input.getCoder())); + + PCollection, KV>> windowed = applyCommitWindow(input); + + // Exactly one group per (destination, shard, window) + PCollection, Iterable>>> grouped = + windowed.apply("GroupByShardKey", GroupByKey.create()); + + // Late-data split before sorting anything + PCollectionTuple split = + grouped.apply( + "SplitLateData", + ParDo.of(new SplitLateData(deadLetterSchema, ON_TIME_TAG, DEAD_LETTER_TAG)) + .withOutputTags(ON_TIME_TAG, TupleTagList.of(DEAD_LETTER_TAG))); + PCollection, Iterable>>> onTimeUnsorted = + split.get(ON_TIME_TAG).setCoder(grouped.getCoder()); + PCollection deadLetter = + split.get(DEAD_LETTER_TAG).setCoder(RowCoder.of(deadLetterSchema)); + + // Sort each surviving group's records by the byte sort key. The secondary key is byte[] + + // ByteArrayCoder, so SortValues compares the raw CdcSortKey bytes (no coder framing): + // each primary key's records come out contiguous, in (seq, kind) order within the key. + PCollection, Iterable>>> sorted = + onTimeUnsorted.apply( + "SortBySeqKind", + SortValues.create( + BufferedExternalSorter.options().withMemoryMB(config.getSorterMemoryMB()))); + + return new Result(input.getPipeline(), sorted, deadLetter, deadLetterSchema); + } + + /** Applies the commit-window assignment for the input's boundedness; see the class Javadoc. */ + private PCollection, KV>> applyCommitWindow( + PCollection, KV>> input) { + if (input.isBounded() == IsBounded.BOUNDED) { + return input.apply( + "GlobalWindows", + Window., KV>>into(new GlobalWindows()) + .triggering(DefaultTrigger.of()) + .discardingFiredPanes()); + } + return input.apply( + "EventTimeWindows", + Window., KV>>into( + FixedWindows.of( + checkStateNotNull( + triggeringFrequency, + "triggeringFrequency is required for unbounded input"))) + .triggering( + AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(1))) + .withAllowedLateness(allowedLateness) + .discardingFiredPanes()); + } + + /** Extracts the CDC data schema carried by the input's nested {@link CdcRecordCoder}. */ + private static Schema dataSchemaOf(Coder inputCoder) { + checkArgument( + inputCoder instanceof KvCoder, + "expected a KvCoder input element coder, got %s", + inputCoder); + Coder valueCoder = ((KvCoder) inputCoder).getValueCoder(); + checkArgument( + valueCoder instanceof KvCoder, "expected a KvCoder input value coder, got %s", valueCoder); + Coder recordCoder = ((KvCoder) valueCoder).getValueCoder(); + checkArgument( + recordCoder instanceof CdcRecordCoder, + "expected a CdcRecordCoder input record coder, got %s", + recordCoder); + return ((CdcRecordCoder) recordCoder).getDataSchema(); + } + + /** + * The output of {@link CommitWindows}: the surviving sorted groups ready for the delta writer, + * and the replayable dead-letter {@link Row}s from late panes. + */ + public static final class Result implements POutput { + + private final Pipeline pipeline; + private final PCollection, Iterable>>> + sortedGroups; + private final PCollection deadLetterRows; + private final Schema deadLetterSchema; + + private Result( + Pipeline pipeline, + PCollection, Iterable>>> sortedGroups, + PCollection deadLetterRows, + Schema deadLetterSchema) { + this.pipeline = pipeline; + this.sortedGroups = sortedGroups; + this.deadLetterRows = deadLetterRows; + this.deadLetterSchema = deadLetterSchema; + } + + /** The surviving sorted groups: one per {@code (destination, shard, window)}. */ + public PCollection, Iterable>>> getSortedGroups() { + return sortedGroups; + } + + /** Replayable dead-letter rows from late panes; {@link SplitLateData} describes the shape. */ + public PCollection getDeadLetterRows() { + return deadLetterRows; + } + + /** The schema of {@link #getDeadLetterRows()}. */ + public Schema getDeadLetterSchema() { + return deadLetterSchema; + } + + @Override + public Pipeline getPipeline() { + return pipeline; + } + + @Override + public Map, PValue> expand() { + return ImmutableMap., PValue>builder() + .put(ON_TIME_TAG, sortedGroups) + .put(DEAD_LETTER_TAG, deadLetterRows) + .build(); + } + + @Override + public void finishSpecifyingOutput( + String transformName, PInput input, PTransform transform) { + // no-op + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/PartitionShardPlan.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/PartitionShardPlan.java new file mode 100644 index 000000000000..73fbd334ab6c --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/PartitionShardPlan.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.Row; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionKey; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.data.InternalRecordWrapper; +import org.apache.iceberg.types.JavaHash; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Derives a record's write shard from its Iceberg partition tuple. Each partition owns a block of + * {@code shards_per_partition} consecutive shards, and the caller's {@code offset} (derived from + * the primary-key hash) selects one of the shards. Used when {@code shards_per_partition < + * num_shards}. + * + *

Correctness rests on one property: this plan exists only under a {@code shards_per_partition} + * cap below {@code num_shards}, where {@link TableSetup#validatePartitioning} still requires + * partition source columns to be equality columns, so the shard is a pure function of the primary + * key and one key's records never split across shards. + */ +final class PartitionShardPlan { + + /** Beam schema of the partition source columns, in the projected Iceberg schema's order. */ + private final Schema sourceSchema; + + /** For each {@link #sourceSchema} field, its position in the CDC data schema. */ + private final int[] sourcePositions; + + /** Iceberg schema of the partition source columns. */ + private final org.apache.iceberg.Schema sourceIcebergSchema; + + /** Adapts a converted record to the internal representation the transforms expect. */ + private final InternalRecordWrapper wrapper; + + /** The spec's bound transforms over a reused partition tuple. */ + private final PartitionKey partitionKey; + + /** Type-aware, JVM-stable hash of the partition tuple. */ + private final JavaHash partitionHash; + + private PartitionShardPlan( + Schema sourceSchema, + int[] sourcePositions, + org.apache.iceberg.Schema sourceIcebergSchema, + InternalRecordWrapper wrapper, + PartitionKey partitionKey, + JavaHash partitionHash) { + this.sourceSchema = sourceSchema; + this.sourcePositions = sourcePositions; + this.sourceIcebergSchema = sourceIcebergSchema; + this.wrapper = wrapper; + this.partitionKey = partitionKey; + this.partitionHash = partitionHash; + } + + /** Builds the plan for a partitioned spec. Converts only the partition source columns. */ + static PartitionShardPlan of( + PartitionSpec spec, org.apache.iceberg.Schema tableSchema, Schema cdcDataSchema) { + // Find distinct source ids since one column can feed several partition fields + Set sourceIds = new LinkedHashSet<>(); + for (PartitionField field : spec.fields()) { + sourceIds.add(field.sourceId()); + } + org.apache.iceberg.Schema sourceIcebergSchema = TypeUtil.select(tableSchema, sourceIds); + + List sourceColumns = sourceIcebergSchema.columns(); + Schema.Builder sourceBeamSchemaBuilder = Schema.builder(); + int[] sourcePositions = new int[sourceColumns.size()]; + // convert to a Beam schema using input data schema fields + for (int i = 0; i < sourceColumns.size(); i++) { + String name = sourceColumns.get(i).name(); + sourceBeamSchemaBuilder.addField(cdcDataSchema.getField(name)); + sourcePositions[i] = cdcDataSchema.indexOf(name); + } + Schema sourceBeamSchema = sourceBeamSchemaBuilder.build(); + + return new PartitionShardPlan( + sourceBeamSchema, + sourcePositions, + sourceIcebergSchema, + new InternalRecordWrapper(sourceIcebergSchema.asStruct()), + new PartitionKey(spec, sourceIcebergSchema), + JavaHash.forType(spec.partitionType())); + } + + /** + * Computes the shard for {@code data}: the partition tuple's hash picks the block base, and + * {@code offset} (in {@code [0, shardsPerPartition)}) selects the shard within the block. + */ + int shardFor(Row data, int offset, int numShards) { + List<@Nullable Object> values = new ArrayList<>(sourcePositions.length); + for (int position : sourcePositions) { + values.add(data.getValue(position)); + } + Row sourceRow = Row.withSchema(sourceSchema).attachValues(values); + partitionKey.partition( + wrapper.wrap(IcebergUtils.beamRowToIcebergRecord(sourceIcebergSchema, sourceRow))); + int base = TableSetup.shardForHash(partitionHash.hash(partitionKey), numShards); + return Math.floorMod(base + offset, numShards); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/RecordDeltaTaskWriter.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/RecordDeltaTaskWriter.java new file mode 100644 index 000000000000..0fa13df54c09 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/RecordDeltaTaskWriter.java @@ -0,0 +1,462 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.Ints; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionKey; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.data.GenericFileWriterFactory; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.InternalRecordWrapper; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.FileWriterFactory; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.io.RollingDataWriter; +import org.apache.iceberg.io.RollingEqualityDeleteWriter; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.Tasks; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * Writes one sorted {@code (destination, shard, window)} group, collapsing each primary key's + * changes into at most one equality delete and one data row. + * + *

The group arrives sorted by {@link CdcSortKey}, so one key's records are contiguous and in + * (sequence, kind) order. The writer holds a block (the sequence of records for the current key) + * and flushes it when the key changes. + * + *

The block's last record is the key's final state; its first record tells us whether + * anything preceded this window. + * + *

    + *
  • Opens with INSERT: the key was born this window, so no earlier commit holds it and + * no delete is written, even if the key dies again before the window ends. + *
  • Opens with anything else: an earlier commit may hold the key, so a delete is written + * once an UPDATE_BEFORE or DELETE appears. A block of only UPDATE_AFTERs writes none. + *
  • Upsert mode: always creates a delete. + *
  • Ends with INSERT or UPDATE_AFTER: the final row is written. Otherwise, the key is + * gone and nothing is written. + *
+ * + *

Partition routing

+ * + *

The data row routes by the block's last record (the partition the key now lives in). + * The equality delete routes by the block's first record (the partition the committed row + * still lives in). Those differ whenever an update moved the row. {@code kindRank} ranks + * UPDATE_BEFORE and DELETE ahead of the after-images at an equal sequence, so the block opens with + * a before-image whenever the window's first change carries one, a guarantee that holds within one + * commit window only. Upsert has no before-images, but it requires partition source columns to be + * equality columns, so there every record of a block routes alike. + * + *

Hence the input contract for a table partitioned on non-key columns: every update must carry + * its UPDATE_BEFORE. A block opening with an after-image can only route its delete to the partition + * the row moved to, leaving the committed row unreachable in the one it moved from. + * + *

This never writes position deletes, and no deletion vectors on V3. Those exist to retract a + * row that was already flushed when a later change superseded it. Collapsing means the superseded + * row is never written at all. + */ +abstract class RecordDeltaTaskWriter { + + private final PartitionSpec spec; + private final FileWriterFactory writerFactory; + private final OutputFileFactory fileFactory; + private final FileIO io; + private final long targetFileSize; + private final Schema deleteSchema; + + /** Column position in the table schema of each {@link #deleteSchema} field. */ + private final int[] pkPos; + + private final boolean upsert; + + private final List partitionWriters = new ArrayList<>(); + + /** The previous record's sort key, for the unsorted-input tripwire in {@link #write}. */ + private byte @Nullable [] lastSortKey; + + /** The current block: sort key, opening and latest records/kinds, and delete-trigger flag. */ + private byte @Nullable [] blockKey; + + private @Nullable Record latestRecord; + private @Nullable ValueKind latestKind; + private @Nullable Record firstRecord; + private @Nullable ValueKind firstKind; + private boolean sawUbOrDelete; + + RecordDeltaTaskWriter( + PartitionSpec spec, + FileWriterFactory writerFactory, + OutputFileFactory fileFactory, + FileIO io, + long targetFileSize, + Schema schema, + Schema deleteSchema, + boolean upsert) { + this.spec = spec; + this.writerFactory = writerFactory; + this.fileFactory = fileFactory; + this.io = io; + this.targetFileSize = targetFileSize; + this.deleteSchema = deleteSchema; + List pkFields = deleteSchema.columns(); + this.pkPos = new int[pkFields.size()]; + List allFields = schema.columns(); + for (int i = 0; i < pkFields.size(); i++) { + int fieldId = pkFields.get(i).fieldId(); + int pos = -1; + for (int j = 0; j < allFields.size(); j++) { + if (allFields.get(j).fieldId() == fieldId) { + pos = j; + break; + } + } + if (pos < 0) { + throw new IllegalStateException( + "Equality field " + + pkFields.get(i).name() + + " is not a top-level column of schema: " + + schema); + } + this.pkPos[i] = pos; + } + this.upsert = upsert; + } + + /** Routes a record to the {@link PartitionDeltaWriter} responsible for its partition. */ + abstract PartitionDeltaWriter route(Record row); + + /** + * Buffers {@code row} into the current block, flushing the previous block first when {@code + * sortKey} starts a new primary key. + */ + public void write(byte[] sortKey, Record row, ValueKind kind) { + // The collapse is only correct over sorted input, so a regressing key must not be accepted. + if (lastSortKey != null && Arrays.compareUnsigned(sortKey, lastSortKey) < 0) { + throw new IllegalStateException( + "RecordDeltaTaskWriter received unsorted input: a record's sort key sorts below its " + + "predecessor's within the group."); + } + lastSortKey = sortKey.clone(); + if (blockKey != null && !CdcSortKey.samePk(blockKey, sortKey)) { + // we're encountering a new PK. flush the current one + flushBlock(); + } + if (blockKey == null) { + blockKey = sortKey.clone(); + firstRecord = row; + firstKind = kind; + } + if (kind == ValueKind.UPDATE_BEFORE || kind == ValueKind.DELETE) { + sawUbOrDelete = true; + } + latestRecord = row; + latestKind = kind; + } + + /** Flushes the current block per the class javadoc's rule and resets the block state. */ + private void flushBlock() { + Record row = checkStateNotNull(latestRecord); + boolean deleteExistingRow; + if (upsert) { + deleteExistingRow = true; // any key may replace a row from an earlier commit + } else if (firstKind == ValueKind.INSERT) { + deleteExistingRow = false; // key born this window: no earlier commit holds it + } else { + // delete if we see a UPDATE_BEFORE/DELETE + deleteExistingRow = sawUbOrDelete; + } + boolean writeRow = latestKind == ValueKind.INSERT || latestKind == ValueKind.UPDATE_AFTER; + + // The delete routes (and projects its key) by the block's first record: kindRank sorts + // UPDATE_BEFORE/DELETE ahead of after-images at an equal sequence, so the block opens with a + // before-image whenever the window's first change carries one. + // Upsert drops before-images, but it also requires partition sources to be equality columns, + // so there every record of the block routes alike. + // The write routes by the latest record, the key's final state: the block is sorted by + // sequence, with kindRank putting the after-image last at an equal sequence. + if (deleteExistingRow) { + Record first = checkStateNotNull(firstRecord); + route(first).delete(projectKey(first)); + } + if (writeRow) { + route(row).write(row); + } + blockKey = null; + latestRecord = null; + latestKind = null; + firstRecord = null; + firstKind = null; + sawUbOrDelete = false; + } + + /** Flushes the last block, closes every file, and returns the completed files. */ + public WriteResult complete() throws IOException { + if (blockKey != null) { + flushBlock(); + } + close(); + WriteResult.Builder result = WriteResult.builder(); + for (PartitionDeltaWriter writer : partitionWriters) { + result.addDataFiles(writer.dataFiles()); + result.addDeleteFiles(writer.deleteFiles()); + } + return result.build(); + } + + /** Closes every file and deletes it: a failed group must leave nothing behind. */ + public void abort() throws IOException { + close(); + List locations = new ArrayList<>(); + for (PartitionDeltaWriter writer : partitionWriters) { + for (DataFile file : writer.dataFiles()) { + locations.add(file.location()); + } + for (DeleteFile file : writer.deleteFiles()) { + locations.add(file.location()); + } + } + Tasks.foreach(locations).throwFailureWhenFinished().noRetry().run(io::deleteFile); + } + + private void close() throws IOException { + Tasks.foreach(partitionWriters) + .throwFailureWhenFinished() + .noRetry() + .run(PartitionDeltaWriter::close, IOException.class); + } + + /** Projects a full record onto a PK-only {@link Record} matching {@link #deleteSchema}. */ + private Record projectKey(Record row) { + GenericRecord key = GenericRecord.create(deleteSchema); + for (int i = 0; i < pkPos.length; i++) { + key.set(i, row.get(pkPos[i], Object.class)); + } + return key; + } + + PartitionDeltaWriter newPartitionWriter(@Nullable PartitionKey partition) { + PartitionDeltaWriter writer = new PartitionDeltaWriter(partition); + partitionWriters.add(writer); + return writer; + } + + @SuppressWarnings("argument") + private RollingDataWriter newDataWriter(@Nullable PartitionKey partition) { + return new RollingDataWriter<>(writerFactory, fileFactory, io, targetFileSize, spec, partition); + } + + @SuppressWarnings("argument") + private RollingEqualityDeleteWriter newDeleteWriter(@Nullable PartitionKey partition) { + return new RollingEqualityDeleteWriter<>( + writerFactory, fileFactory, io, targetFileSize, spec, partition); + } + + /** One partition's rolling data and equality-delete writers, each opened on first use. */ + protected class PartitionDeltaWriter { + private final @Nullable PartitionKey partition; + private @Nullable RollingDataWriter dataWriter; + private @Nullable RollingEqualityDeleteWriter deleteWriter; + + PartitionDeltaWriter(@Nullable PartitionKey partition) { + this.partition = partition; + } + + void write(Record row) { + @Nullable RollingDataWriter writer = dataWriter; + if (writer == null) { + writer = newDataWriter(partition); + dataWriter = writer; + } + writer.write(row); + } + + void delete(Record key) { + @Nullable RollingEqualityDeleteWriter writer = deleteWriter; + if (writer == null) { + writer = newDeleteWriter(partition); + deleteWriter = writer; + } + writer.write(key); + } + + void close() throws IOException { + try { + if (dataWriter != null) { + dataWriter.close(); + } + } finally { + if (deleteWriter != null) { + deleteWriter.close(); + } + } + } + + List dataFiles() { + return dataWriter == null ? ImmutableList.of() : dataWriter.result().dataFiles(); + } + + List deleteFiles() { + return deleteWriter == null ? ImmutableList.of() : deleteWriter.result().deleteFiles(); + } + } + + /** Record writer for an unpartitioned table. */ + static class UnpartitionedRecordDeltaWriter extends RecordDeltaTaskWriter { + private final PartitionDeltaWriter writer; + + @SuppressWarnings("method.invocation") + UnpartitionedRecordDeltaWriter( + PartitionSpec spec, + FileWriterFactory writerFactory, + OutputFileFactory fileFactory, + FileIO io, + long targetFileSize, + Schema schema, + Schema deleteSchema, + boolean upsert) { + super(spec, writerFactory, fileFactory, io, targetFileSize, schema, deleteSchema, upsert); + this.writer = newPartitionWriter(null); + } + + @Override + PartitionDeltaWriter route(Record row) { + return writer; + } + } + + /** + * Partitioned table: a fanout delta writer per partition key, created lazily on first touch and + * held open, because the group is sorted by PK and partitions interleave. + */ + static class PartitionedRecordDeltaWriter extends RecordDeltaTaskWriter { + private final PartitionKey partitionKey; + private final InternalRecordWrapper wrapper; + private final Map writers = Maps.newHashMap(); + + PartitionedRecordDeltaWriter( + PartitionSpec spec, + FileWriterFactory writerFactory, + OutputFileFactory fileFactory, + FileIO io, + long targetFileSize, + Schema schema, + Schema deleteSchema, + boolean upsert) { + super(spec, writerFactory, fileFactory, io, targetFileSize, schema, deleteSchema, upsert); + this.partitionKey = new PartitionKey(spec, schema); + this.wrapper = new InternalRecordWrapper(schema.asStruct()); + } + + @Override + PartitionDeltaWriter route(Record row) { + partitionKey.partition(wrapper.wrap(row)); + + @Nullable PartitionDeltaWriter writer = writers.get(partitionKey); + if (writer == null) { + // The shared partitionKey is mutated on every route() call; copy before keying the map. + PartitionKey copiedKey = partitionKey.copy(); + writer = newPartitionWriter(copiedKey); + writers.put(copiedKey, writer); + } + + return writer; + } + } + + /** Builds a {@link RecordDeltaTaskWriter} writing under a specified {@code spec}. */ + static RecordDeltaTaskWriter create( + Table table, + PartitionSpec spec, + Set equalityFieldIds, + boolean upsert, + long targetFileSizeBytes, + OutputFileFactory fileFactory, + FileFormat dataFormat, + FileFormat deleteFormat) { + Schema deleteSchema = TypeUtil.select(table.schema(), Sets.newHashSet(equalityFieldIds)); + FileWriterFactory writerFactory = + new GenericFileWriterFactory.Builder(table) + .dataSchema(table.schema()) + .dataFileFormat(dataFormat) + .deleteFileFormat(deleteFormat) + .equalityFieldIds(Ints.toArray(equalityFieldIds)) + .equalityDeleteRowSchema(deleteSchema) + .build(); + + if (spec.isUnpartitioned()) { + return new UnpartitionedRecordDeltaWriter( + spec, + writerFactory, + fileFactory, + table.io(), + targetFileSizeBytes, + table.schema(), + deleteSchema, + upsert); + } else { + return new PartitionedRecordDeltaWriter( + spec, + writerFactory, + fileFactory, + table.io(), + targetFileSizeBytes, + table.schema(), + deleteSchema, + upsert); + } + } + + /** The table's default data file format ({@code write.format.default}, Parquet fallback). */ + static FileFormat dataFileFormat(Table table) { + return FileFormat.fromString( + PropertyUtil.propertyAsString( + table.properties(), + TableProperties.DEFAULT_FILE_FORMAT, + TableProperties.DEFAULT_FILE_FORMAT_DEFAULT)); + } + + /** The equality-delete file format: {@code write.delete.format.default}, else the data format. */ + static FileFormat deleteFileFormat(Table table, FileFormat dataFormat) { + return FileFormat.fromString( + PropertyUtil.propertyAsString( + table.properties(), TableProperties.DELETE_DEFAULT_FILE_FORMAT, dataFormat.name())); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/ShardDeltaFiles.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/ShardDeltaFiles.java new file mode 100644 index 000000000000..27fce7fed956 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/ShardDeltaFiles.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import com.google.auto.value.AutoValue; +import java.util.List; +import org.apache.beam.sdk.io.iceberg.SerializableDataFile; +import org.apache.beam.sdk.io.iceberg.SerializableDeleteFile; +import org.apache.beam.sdk.schemas.AutoValueSchema; +import org.apache.beam.sdk.schemas.annotations.DefaultSchema; +import org.apache.beam.sdk.schemas.annotations.SchemaFieldNumber; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; + +/** + * Holds the serialized metadata for {@link DataFile}s and {@link DeleteFile}s of one {@code + * (destination, shard, window)} group, plus the source sequence range the bundle covers. + */ +@AutoValue +@DefaultSchema(AutoValueSchema.class) +public abstract class ShardDeltaFiles { + + @SchemaFieldNumber("0") + public abstract String getTableIdentifierString(); + + /** Reconstructed into live {@link DataFile}s at commit time. */ + @SchemaFieldNumber("1") + public abstract List getDataFiles(); + + /** Reconstructed into live {@link DeleteFile}s at commit time. */ + @SchemaFieldNumber("2") + public abstract List getDeleteFiles(); + + /** The minimum source sequence number covered by this bundle. */ + @SchemaFieldNumber("3") + public abstract long getMinSequenceNumber(); + + /** The maximum source sequence number covered by this bundle. */ + @SchemaFieldNumber("4") + public abstract long getMaxSequenceNumber(); + + public static Builder builder() { + return new AutoValue_ShardDeltaFiles.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setTableIdentifierString(String tableIdString); + + public abstract Builder setDataFiles(List dataFiles); + + public abstract Builder setDeleteFiles(List deleteFiles); + + public abstract Builder setMinSequenceNumber(long minSequenceNumber); + + public abstract Builder setMaxSequenceNumber(long maxSequenceNumber); + + public abstract ShardDeltaFiles build(); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/SplitLateData.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/SplitLateData.java new file mode 100644 index 000000000000..bf60ffae5654 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/SplitLateData.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.windowing.PaneInfo; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.TupleTag; + +/** + * Splits late data in {@link CommitWindows}: every {@linkplain PaneInfo.Timing#LATE late} pane is + * diverted to a DLQ side output; on-time and early panes pass through unchanged. + * + *

Pane history is per {@code (destination, shard, window)} while the committer's skip is per + * {@code (destination, window)}: a late pane on a shard with no on-time data cannot prove its + * destination-window uncommitted, and a record let through into a committed window would reach + * neither the table nor the dead-letter output. + * + *

Each dead letter nests the untouched data row under {@value #DL_RECORD} beside {@value + * #DL_CHANGE_TYPE}, {@value #DL_SEQ}, and {@value #DL_DEST}; to replay, unnest {@value #DL_RECORD} + * and map {@value #DL_CHANGE_TYPE}/{@value #DL_SEQ} as the sink's control columns. Replaying is + * only safe while no newer change for those keys has committed; a stale replay's equality delete + * removes the newer row. + */ +final class SplitLateData + extends DoFn< + KV, Iterable>>, + KV, Iterable>>> { + + static final String DL_RECORD = "record"; + static final String DL_CHANGE_TYPE = "change_type"; + static final String DL_SEQ = "sequence_number"; + static final String DL_DEST = "destination"; + + private final Counter deadLetterRecords = + Metrics.counter(SplitLateData.class, "deadLetterRecords"); + + private final Schema deadLetterSchema; + private final TupleTag, Iterable>>> onTimeTag; + private final TupleTag deadLetterTag; + + SplitLateData( + Schema deadLetterSchema, + TupleTag, Iterable>>> onTimeTag, + TupleTag deadLetterTag) { + this.deadLetterSchema = deadLetterSchema; + this.onTimeTag = onTimeTag; + this.deadLetterTag = deadLetterTag; + } + + static Schema deadLetterSchema(Schema cdcDataSchema) { + return Schema.builder() + .addRowField(DL_RECORD, cdcDataSchema) + .addStringField(DL_CHANGE_TYPE) + .addInt64Field(DL_SEQ) + .addStringField(DL_DEST) + .build(); + } + + @ProcessElement + public void process( + @Element KV, Iterable>> group, + PaneInfo pane, + MultiOutputReceiver out) { + if (pane.getTiming() == PaneInfo.Timing.LATE) { + String dest = group.getKey().getKey(); + for (KV kv : group.getValue()) { + CdcRecord record = kv.getValue(); + Row deadLetter = + Row.withSchema(deadLetterSchema) + .addValue(record.getData()) + .addValue(record.getKind().name()) + .addValue(record.getSequenceNumber()) + .addValue(dest) + .build(); + out.get(deadLetterTag).output(deadLetter); + deadLetterRecords.inc(); + } + } else { + out.get(onTimeTag).output(group); + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/TableSetup.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/TableSetup.java new file mode 100644 index 000000000000..87c36e694dac --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/TableSetup.java @@ -0,0 +1,845 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.RowCoder; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergDestination; +import org.apache.beam.sdk.io.iceberg.IcebergTableCreateConfig; +import org.apache.beam.sdk.io.iceberg.IcebergUtils; +import org.apache.beam.sdk.io.iceberg.TableCache; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.hash.Hashing; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.types.Types; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Per-destination table resolution for the CDC sink: loads (or auto-creates) the destination {@link + * Table}, validates that it can accept CDC writes, and precomputes the per-destination artifacts + * the write path needs. All catalog I/O and table-level validation lives here; failures are thrown + * as {@link TableConfigException}. + * + *

One instance lives inside each worker {@code DoFn}, single-owner and not thread-safe. Results + * are memoized per destination string; the memo is what pins a destination's resolution, including + * its {@link Dest#spec()}, for the worker's lifetime. + */ +final class TableSetup implements Serializable { + + private static final Logger LOG = LoggerFactory.getLogger(TableSetup.class); + + private final IcebergCatalogConfig catalogConfig; + private final CdcWriteConfig config; + private final DynamicDestinations destinations; + + /** The run runId; a run-spec stamp carrying it names the spec {@link #resolve} pins to. */ + private final String runId; + + /** + * Matches {@link TableCache}'s bound so this memo (whose {@link Dest}s strongly reference their + * {@link Table}s) can never pin more table metadata than that cache would hold. Exceeding it + * costs a re-resolve, not a failure. + */ + private static final int MAX_MEMOIZED_DESTS = 1000; + + /** Per-destination memo, lazily initialized (never serialized). */ + private transient @Nullable Map dests; + + TableSetup( + IcebergCatalogConfig catalogConfig, + CdcWriteConfig config, + DynamicDestinations destinations, + String runId) { + this.catalogConfig = catalogConfig; + this.config = config; + this.destinations = destinations; + this.runId = runId; + } + + /** + * Returns the resolved, validated {@link Dest} for {@code destString}, memoized per destination + * string. Under block sharding a memoized {@link Dest} is re-checked against the table's live + * partition spec before it is handed back ({@link #requireResolvedPartitionSpec}). + * + * @throws TableConfigException for any table-level problem (including catalog failures) + */ + Dest get(String destString, Schema sourceRowSchema) { + Map memo = dests; + if (memo == null) { + // Access-ordered LRU: this class is single-threaded by contract, so a LinkedHashMap is the + // whole mechanism needed to keep the memo bounded. + memo = + new LinkedHashMap(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_MEMOIZED_DESTS; + } + }; + dests = memo; + } + @Nullable Dest existing = memo.get(destString); + if (existing != null) { + requireResolvedPartitionSpec(destString, existing); + return existing; + } + Dest dest; + try { + dest = resolve(destString, sourceRowSchema); + } catch (TableConfigException e) { + throw e; + } catch (RuntimeException e) { + throw new TableConfigException("Failed resolving destination table '" + destString + "'.", e); + } + memo.put(destString, dest); + return dest; + } + + /** Type-stable murmur3 hash of encoded primary-key bytes. Stable across workers and JVMs. */ + static int pkHash(byte[] pkBytes) { + return Hashing.murmur3_32_fixed().hashBytes(pkBytes).asInt(); + } + + /** + * The deterministic shard for {@code pkBytes}: {@code floorMod(murmur3_32(pkBytes), numShards)}. + */ + static int shardFor(byte[] pkBytes, int numShards) { + return Math.floorMod(pkHash(pkBytes), numShards); + } + + /** + * The deterministic shard for an already-computed value hash, used by {@link PartitionShardPlan} + * for the partition tuple's block base: {@code floorMod(murmur3_32(hash), numShards)}. + */ + static int shardForHash(int hash, int numShards) { + return Math.floorMod(Hashing.murmur3_32_fixed().hashInt(hash).asInt(), numShards); + } + + private Dest resolve(String destString, Schema sourceRowSchema) { + TableIdentifier identifier = IcebergUtils.parseTableIdentifier(destString); + Table table = + TableCache.getAndRefreshIfStale( + catalogConfig, + identifier, + () -> loadOrCreateTable(identifier, destString, sourceRowSchema)); + + int formatVersion = TableUtil.formatVersion(table); + if (formatVersion < 2) { + throw new TableConfigException( + "CDC sink requires an Iceberg format-version >= 2 table, but '" + + destString + + "' is format-version " + + formatVersion + + ". Use IcebergIO.writeRows (the append sink) for V1 tables."); + } + + org.apache.iceberg.Schema tableSchema = table.schema(); + Schema cdcDataSchema = cdcDataSchema(destString, sourceRowSchema, tableSchema); + Set equalityFieldIds = equalityFieldIds(destString, tableSchema); + requireNonNullableEqualityFields(destString, tableSchema, equalityFieldIds); + Schema pkSchema = pkSchema(tableSchema, cdcDataSchema, equalityFieldIds); + + RowCoder pkCoder = RowCoder.of(pkSchema); + try { + pkCoder.verifyDeterministic(); + } catch (Coder.NonDeterministicException e) { + throw new TableConfigException( + "Primary-key coder for table '" + + destString + + "' (schema " + + pkSchema + + ") is not deterministic; Iceberg identifier fields should be primitive and " + + "required.", + e); + } + + PartitionSpec spec = runSpec(table); + validatePartitioning(destString, spec, tableSchema, equalityFieldIds); + + int[] pkFieldPositions = new int[pkSchema.getFieldCount()]; + for (int i = 0; i < pkSchema.getFieldCount(); i++) { + pkFieldPositions[i] = cdcDataSchema.indexOf(pkSchema.getField(i).getName()); + } + ProjectionPlan projectionPlan = ProjectionPlan.of(cdcDataSchema, sourceRowSchema, destString); + + return new Dest( + table, + spec, + ImmutableSet.copyOf(equalityFieldIds), + pkSchema, + pkCoder, + pkFieldPositions, + cdcDataSchema, + projectionPlan, + partitionShardPlan(destString, spec, tableSchema, cdcDataSchema)); + } + + /** + * The spec to resolve and validate against. Resolves to the most recently stamped spec in this + * run. Otherwise, falls back to the live {@code table.spec()}. + */ + private PartitionSpec runSpec(Table table) { + @Nullable Integer stamped = CommitToken.readRunSpec(table, config.getSinkId(), runId); + if (stamped != null) { + @Nullable PartitionSpec pinned = table.specs().get(stamped); + if (pinned != null) { + return pinned; + } + } + return table.spec(); + } + + /** + * The destination's {@link PartitionShardPlan} when {@code shards_per_partition} is below {@code + * num_shards} AND the table is partitioned, else {@code null} (plain primary-key sharding). A cap + * on an unpartitioned destination is a no-op with a WARN, not a rejection: rejecting would fail a + * whole dynamic-destinations pipeline over one table, and the fallback is what the operator wants + * anyway. + */ + private @Nullable PartitionShardPlan partitionShardPlan( + String destString, + PartitionSpec spec, + org.apache.iceberg.Schema tableSchema, + Schema cdcDataSchema) { + if (config.getShardsPerPartition() >= config.getNumShards()) { + return null; + } + if (spec.isUnpartitioned()) { + LOG.warn( + "shards_per_partition ({}) is below num_shards ({}) but destination '{}' is " + + "unpartitioned, so there is no partition to bound; ignoring the cap and sharding " + + "by primary key across num_shards shards. The option only helps partitioned " + + "tables.", + config.getShardsPerPartition(), + config.getNumShards(), + destString); + return null; + } + return PartitionShardPlan.of(spec, tableSchema, cdcDataSchema); + } + + /** + * Block-sharding-only drift check, run on every memo hit. Under {@code shards_per_partition} the + * assigners derive each record's shard from the partition tuple under the resolved spec, so + * workers resolving different specs would split one primary key's window across shards: silent + * same-commit duplicates nothing downstream detects. The default path needs no check: the write + * path builds writers from the pinned {@link Dest#spec()}, never the live {@code table.spec()}. + * Freshness is best-effort: the compared spec is the process-cached table's, refreshed only when + * something in the process refreshes it. + */ + private static void requireResolvedPartitionSpec(String destString, Dest dest) { + if (dest.partitionShardPlan() == null) { + return; + } + int currentSpecId = dest.table().spec().specId(); + if (currentSpecId != dest.spec().specId()) { + throw new TableConfigException( + "Table '" + + destString + + "' changed its partition spec while the CDC sink was running with a " + + "shards_per_partition cap (spec id " + + dest.spec().specId() + + " when the sink resolved the table, spec id " + + currentSpecId + + " now). Partition-block sharding derives each record's shard from the partition " + + "tuple under the resolved spec, so workers resolving different specs would split " + + "one primary key's window across shards and silently duplicate rows within a " + + "commit. " + + "Drain the pipeline before evolving the partition spec, and restart it " + + "afterwards."); + } + } + + /** + * Loads the table, auto-creating it (namespace first) if it does not exist. Auto-creation + * requires configured equality columns (a brand-new table has no identifier fields to infer + * from); the created schema is the source row schema minus the control columns, with the equality + * columns as identifier fields, honoring the destination's {@link IcebergTableCreateConfig} plus + * a format-version 2 default. + */ + private Table loadOrCreateTable( + TableIdentifier identifier, String destString, Schema sourceRowSchema) { + Catalog catalog = catalogConfig.catalog(); + try { + return catalog.loadTable(identifier); + } catch (NoSuchTableException e) { + // Missing table: fall through to auto-create (parity with the append sink). + } + + @Nullable List equalityColumns = config.getEqualityColumns(); + if (equalityColumns == null || equalityColumns.isEmpty()) { + throw new TableConfigException( + "Table '" + + destString + + "' does not exist and no equality_columns are configured, so its identifier " + + "(primary-key) fields cannot be determined for auto-creation. Configure " + + "equality_columns, or pre-create the table with identifier fields."); + } + + Schema createSchema = config.stripControlColumns(sourceRowSchema); + org.apache.iceberg.Schema base = IcebergUtils.beamSchemaToIcebergSchema(createSchema); + Set identifierFieldIds = new LinkedHashSet<>(); + for (String column : equalityColumns) { + requireTopLevelEqualityColumn(column); + Types.NestedField field = base.findField(column); + if (field == null) { + throw new TableConfigException( + "Cannot auto-create table '" + + destString + + "': equality column '" + + column + + "' is not present in the input data schema " + + createSchema + + "."); + } + // Iceberg refuses an OPTIONAL identifier field with a cryptic error; detect the nullable + // Beam field here so the message names the input field the user controls. + if (createSchema.getField(column).getType().getNullable()) { + throw new TableConfigException( + "Cannot auto-create table '" + + destString + + "': equality column '" + + column + + "' must be non-nullable in the input schema (a nullable column cannot be an " + + "Iceberg identifier field). Make the input field non-nullable, or pre-create " + + "the table with required identifier fields."); + } + identifierFieldIds.add(field.fieldId()); + } + org.apache.iceberg.Schema schemaWithIds = + new org.apache.iceberg.Schema(base.columns(), identifierFieldIds); + + IcebergDestination destination = destinations.instantiateDestination(destString); + @Nullable IcebergTableCreateConfig createConfig = destination.getTableCreateConfig(); + PartitionSpec partitionSpec = + createConfig != null ? createConfig.getPartitionSpec() : PartitionSpec.unpartitioned(); + SortOrder sortOrder = createConfig != null ? createConfig.getSortOrder() : SortOrder.unsorted(); + Map properties = new HashMap<>(); + if (createConfig != null) { + @Nullable Map createProperties = createConfig.getTableProperties(); + if (createProperties != null) { + properties.putAll(createProperties); + } + } + properties.putIfAbsent(TableProperties.FORMAT_VERSION, "2"); + + Namespace namespace = identifier.namespace(); + if (!namespace.isEmpty() && catalog instanceof SupportsNamespaces) { + SupportsNamespaces supportsNamespaces = (SupportsNamespaces) catalog; + if (!supportsNamespaces.namespaceExists(namespace)) { + try { + supportsNamespaces.createNamespace(namespace); + LOG.info("Created new namespace '{}'.", namespace); + } catch (AlreadyExistsException ignored) { + // Race: another worker created the namespace first. + } + } + } + + try { + Table table = + catalog + .buildTable(identifier, schemaWithIds) + .withPartitionSpec(partitionSpec) + .withSortOrder(sortOrder) + .withProperties(properties) + .create(); + LOG.info( + "CDC sink auto-created table '{}' with schema {}, partition spec {}, sort order {}, " + + "properties {}.", + identifier, + schemaWithIds, + partitionSpec, + sortOrder, + properties); + return table; + } catch (AlreadyExistsException ignored) { + // Race: another worker created the table first. + return catalog.loadTable(identifier); + } + } + + /** + * Returns the table schema as a Beam {@link Schema}, validating that the source row schema + * (control columns stripped) matches the table's top-level column names exactly AND in the same + * order. Order matters: the written rows and the shuffle coder are built positionally, so a + * column reorder would silently write values into the wrong columns. + */ + private Schema cdcDataSchema( + String destString, Schema sourceRowSchema, org.apache.iceberg.Schema tableSchema) { + Schema canonical = IcebergUtils.icebergSchemaToBeamSchema(tableSchema); + + String sequenceNumberColumn = config.getSequenceNumberColumn(); + rejectControlColumnCollision( + destString, canonical, sequenceNumberColumn, "sequence_number_column"); + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + if (changeTypeColumn != null) { + rejectControlColumnCollision(destString, canonical, changeTypeColumn, "change_type_column"); + } + + Schema stripped = config.stripControlColumns(sourceRowSchema); + List remaining = stripped.getFieldNames(); + List canonicalNames = canonical.getFieldNames(); + if (!remaining.equals(canonicalNames)) { + throw new TableConfigException(schemaMismatchMessage(destString, canonicalNames, remaining)); + } + requireMatchingColumnTypes(destString, canonical, stripped); + return canonical; + } + + /** + * Column-by-column type and nullability check behind the name check: rows are encoded against the + * table-derived schema, so a mismatched type would only fail later as an opaque coder error. A + * non-null input column on an optional table column is fine; the reverse is not. + */ + private static void requireMatchingColumnTypes( + String destString, Schema canonical, Schema stripped) { + for (int i = 0; i < canonical.getFieldCount(); i++) { + String name = canonical.getField(i).getName(); + Schema.FieldType tableType = canonical.getField(i).getType(); + Schema.FieldType inputType = stripped.getField(i).getType(); + if (!tableType.withNullable(false).equals(inputType.withNullable(false))) { + throw new TableConfigException( + "CDC data schema mismatch for table '" + + destString + + "': column '" + + name + + "' is " + + inputType + + " in the input but " + + tableType + + " in the table. Align the input schema with the table."); + } + if (inputType.getNullable() && !tableType.getNullable()) { + throw new TableConfigException( + "CDC data schema mismatch for table '" + + destString + + "': column '" + + name + + "' is nullable in the input but required in the table. Align the input schema " + + "with the table."); + } + } + } + + /** The mismatch message: unexpected/missing columns (or the order difference) plus hints. */ + private String schemaMismatchMessage( + String destString, List canonicalNames, List remaining) { + Set unexpected = new LinkedHashSet<>(remaining); + unexpected.removeAll(canonicalNames); + Set missing = new LinkedHashSet<>(canonicalNames); + missing.removeAll(remaining); + StringBuilder msg = + new StringBuilder("CDC data schema mismatch for table '").append(destString).append("':"); + if (!unexpected.isEmpty()) { + msg.append(" unexpected columns (in the input, not in the table): ") + .append(unexpected) + .append(";"); + } + if (!missing.isEmpty()) { + msg.append(" missing columns (in the table, not supplied by the input): ") + .append(missing) + .append(";"); + } + if (unexpected.isEmpty() && missing.isEmpty()) { + msg.append( + " the input's data columns match the table's columns but in a different order;" + + " column order must match the table (rows are projected and encoded" + + " positionally);"); + } + msg.append(" the sequence-number column ('") + .append(config.getSequenceNumberColumn()) + .append("') and the change-type column are stripped before writing (configured via ") + .append("sequence_number_column / change_type_column). Table columns: ") + .append(canonicalNames) + .append("; input columns after stripping: ") + .append(remaining) + .append("."); + return msg.toString(); + } + + /** + * Equality columns must be top-level: Iceberg resolves dotted paths to nested fields, which are + * out of scope as identifier columns (and a same-named leaf could silently misbind). + */ + private static void requireTopLevelEqualityColumn(String name) { + if (name.contains(".")) { + throw new TableConfigException( + "equality_columns must be top-level columns; got '" + + name + + "' (nested fields are not supported)."); + } + } + + /** Rejects a table column whose name collides with the named control-column option. */ + private static void rejectControlColumnCollision( + String destString, Schema canonical, String controlColumn, String optionName) { + if (canonical.hasField(controlColumn)) { + throw new TableConfigException( + "Table '" + + destString + + "' has a column '" + + controlColumn + + "' that collides with " + + optionName + + "; the sink strips that column from written rows. Rename the table column, choose " + + "a different " + + optionName + + ", or duplicate the value into a differently-named data column upstream."); + } + } + + /** + * The Iceberg field ids that define a row's identity: the configured equality columns (resolved + * by name) when set, else the table's identifier fields. + */ + private Set equalityFieldIds(String destString, org.apache.iceberg.Schema tableSchema) { + @Nullable List override = config.getEqualityColumns(); + if (override != null) { + if (override.isEmpty()) { + // An empty override is a misconfiguration, not a request for the identifier fields. + throw new TableConfigException( + "equality_columns must be non-empty or unset (leave unset to use the identifier " + + "fields of table '" + + destString + + "')."); + } + ImmutableSet.Builder ids = ImmutableSet.builder(); + for (String name : override) { + requireTopLevelEqualityColumn(name); + Types.NestedField field = tableSchema.findField(name); + if (field == null) { + throw new TableConfigException( + "Configured equality column '" + + name + + "' does not exist in table '" + + destString + + "'. Table columns: " + + columnNames(tableSchema) + + "."); + } + ids.add(field.fieldId()); + } + return ids.build(); + } + Set identifierFieldIds = tableSchema.identifierFieldIds(); + if (identifierFieldIds.isEmpty()) { + throw new TableConfigException( + "Table '" + + destString + + "' has no identifier (primary-key) fields and no equality_columns are " + + "configured. Configure equality_columns, or add identifier fields to the table."); + } + return identifierFieldIds; + } + + /** Equality columns must be required (non-null): a nullable column cannot define row identity. */ + private static void requireNonNullableEqualityFields( + String destString, org.apache.iceberg.Schema tableSchema, Set equalityFieldIds) { + for (int fieldId : equalityFieldIds) { + Types.NestedField field = checkStateNotNull(tableSchema.findField(fieldId)); + if (!field.isRequired()) { + throw new TableConfigException( + "Equality column '" + + field.name() + + "' (field id " + + fieldId + + ") of table '" + + destString + + "' must be required (non-null); a nullable column cannot define row identity."); + } + } + } + + /** + * The Beam schema of the equality columns, in ascending Iceberg field-id order (a stable, + * table-derived order independent of how the identifier fields or overrides were declared). + */ + private static Schema pkSchema( + org.apache.iceberg.Schema tableSchema, Schema cdcDataSchema, Set equalityFieldIds) { + Schema.Builder builder = Schema.builder(); + for (int fieldId : new TreeSet<>(equalityFieldIds)) { + Types.NestedField field = checkStateNotNull(tableSchema.findField(fieldId)); + builder.addField(cdcDataSchema.getField(field.name())); + } + return builder.build(); + } + + /** + * Tables may be partitioned on any columns; two options additionally require every partition + * source field to be an equality field, because they need a row's partition to be a pure function + * of its primary key: {@code upsert} (before-images are dropped, so a moved row's equality delete + * could only ever route to its new partition) and a {@code shards_per_partition} cap (the shard + * is derived from the partition tuple). + */ + private void validatePartitioning( + String destString, + PartitionSpec spec, + org.apache.iceberg.Schema tableSchema, + Set equalityFieldIds) { + if (spec.isUnpartitioned()) { + return; + } + String requirement; + if (config.getUpsert()) { + requirement = + "upsert drops before-images, so a row that moved partitions could never be deleted " + + "from its old partition"; + } else if (config.getShardsPerPartition() < config.getNumShards()) { + requirement = + "shards_per_partition (" + + config.getShardsPerPartition() + + ") is below num_shards (" + + config.getNumShards() + + ") and derives each record's shard from its partition tuple, which must therefore " + + "be a pure function of the primary key"; + } else { + return; + } + List nonKeySources = new ArrayList<>(); + for (PartitionField field : spec.fields()) { + if (!equalityFieldIds.contains(field.sourceId())) { + nonKeySources.add("'" + tableSchema.findColumnName(field.sourceId()) + "'"); + } + } + if (!nonKeySources.isEmpty()) { + throw new TableConfigException( + "Table '" + + destString + + "' has partition source columns " + + nonKeySources + + " that are not equality columns, but " + + requirement + + ". Partition only on equality columns, or drop the option."); + } + } + + private static List columnNames(org.apache.iceberg.Schema tableSchema) { + List names = new ArrayList<>(tableSchema.columns().size()); + for (Types.NestedField field : tableSchema.columns()) { + names.add(field.name()); + } + return names; + } + + /** + * Precomputed per-destination state. All fields are fixed at resolution time; the {@link + * #projectionPlan()} additionally self-heals when a row's source schema drifts. + */ + static final class Dest { + private final Table table; + private final PartitionSpec spec; + private final Set equalityFieldIds; + private final Schema pkSchema; + private final RowCoder pkCoder; + private final int[] pkFieldPositions; + private final Schema cdcDataSchema; + private final ProjectionPlan projectionPlan; + private final @Nullable PartitionShardPlan partitionShardPlan; + + private Dest( + Table table, + PartitionSpec spec, + Set equalityFieldIds, + Schema pkSchema, + RowCoder pkCoder, + int[] pkFieldPositions, + Schema cdcDataSchema, + ProjectionPlan projectionPlan, + @Nullable PartitionShardPlan partitionShardPlan) { + this.table = table; + this.spec = spec; + this.equalityFieldIds = equalityFieldIds; + this.pkSchema = pkSchema; + this.pkCoder = pkCoder; + this.pkFieldPositions = pkFieldPositions; + this.cdcDataSchema = cdcDataSchema; + this.projectionPlan = projectionPlan; + this.partitionShardPlan = partitionShardPlan; + } + + /** + * The destination table, loaded or auto-created. This is the process-shared {@link TableCache} + * instance, which may be refreshed in place, so its live metadata can be newer than the + * memoized schemas held here. + */ + Table table() { + return table; + } + + /** + * The partition spec this destination was resolved and validated against: the worker's pin. The + * live {@link #table()} can be refreshed onto a newer spec; the write path must build writers + * from this pinned spec. + */ + PartitionSpec spec() { + return spec; + } + + /** The Iceberg field ids of the equality (primary-key) columns. */ + Set equalityFieldIds() { + return equalityFieldIds; + } + + /** The Beam schema of the equality columns, in ascending Iceberg field-id order. */ + Schema pkSchema() { + return pkSchema; + } + + /** A deterministic coder for {@link #pkSchema()} rows. */ + RowCoder pkCoder() { + return pkCoder; + } + + /** + * Position of each {@link #pkSchema()} field within {@link #cdcDataSchema()}; do not mutate. + */ + int[] pkFieldPositions() { + return pkFieldPositions; + } + + /** The written-row schema: the table's schema in Beam form (control columns stripped). */ + Schema cdcDataSchema() { + return cdcDataSchema; + } + + /** The positional source-to-{@link #cdcDataSchema()} projection. */ + ProjectionPlan projectionPlan() { + return projectionPlan; + } + + /** The partition-block sharding plan, or {@code null} to shard by plain primary-key hash. */ + @Nullable PartitionShardPlan partitionShardPlan() { + return partitionShardPlan; + } + } + + /** + * Positional projection from a source row schema to the CDC data schema, dropping the control + * columns. One mapping is cached; {@link #project} rebuilds it by field name when a row's schema + * drifts from the build-time source schema, and {@link #matches} lets callers observe the drift. + */ + static final class ProjectionPlan { + private final Schema targetSchema; + + private final String destString; + + /** The source schema the current {@link #positions} were computed from. */ + private Schema sourceSchema; + + /** For each {@link #targetSchema} field, its index in {@link #sourceSchema}. */ + private int[] positions; + + private boolean driftWarned; + + private ProjectionPlan( + Schema targetSchema, Schema sourceSchema, int[] positions, String destString) { + this.targetSchema = targetSchema; + this.sourceSchema = sourceSchema; + this.positions = positions; + this.destString = destString; + } + + static ProjectionPlan of(Schema targetSchema, Schema sourceSchema, String destString) { + return new ProjectionPlan( + targetSchema, sourceSchema, positionsFor(targetSchema, sourceSchema), destString); + } + + /** Whether the current mapping was built for {@code schema}. */ + @SuppressWarnings("ReferenceEquality") + boolean matches(Schema schema) { + return sourceSchema == schema || sourceSchema.equals(schema); + } + + /** Projects {@code source} to the target schema positionally. */ + Row project(Row source) { + Schema schema = source.getSchema(); + if (!matches(schema)) { + if (!driftWarned) { + driftWarned = true; + LOG.warn( + "CDC sink destination '{}' received a row whose schema drifted from the cached " + + "projection plan; positions were re-resolved by name. This is correct but " + + "slower. Supply rows with a stable schema to avoid it.", + destString); + } + positions = positionsFor(targetSchema, schema); + sourceSchema = schema; + } + List<@Nullable Object> values = new ArrayList<>(positions.length); + for (int position : positions) { + values.add(source.getValue(position)); + } + return Row.withSchema(targetSchema).attachValues(values); + } + + /** + * For each {@code targetSchema} field, its index in {@code sourceSchema}; {@code indexOf} makes + * a drifted-away column fail loudly by name. + */ + private static int[] positionsFor(Schema targetSchema, Schema sourceSchema) { + int[] positions = new int[targetSchema.getFieldCount()]; + for (int i = 0; i < targetSchema.getFieldCount(); i++) { + positions[i] = sourceSchema.indexOf(targetSchema.getField(i).getName()); + } + return positions; + } + } + + /** + * A table-level (as opposed to record-level) configuration failure: the destination table (or the + * sink configuration as applied to it) cannot accept CDC writes at all. Callers rethrow this + * fail-fast, bypassing any per-record poison-record handling. + */ + static final class TableConfigException extends RuntimeException { + TableConfigException(String message) { + super(message); + } + + TableConfigException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java new file mode 100644 index 000000000000..d47b747b9753 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java @@ -0,0 +1,523 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull; + +import com.google.auto.value.AutoValue; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.IcebergWriteResult; +import org.apache.beam.sdk.io.iceberg.SnapshotInfo; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.Row; +import org.apache.beam.sdk.values.ValueKind; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Predicates; +import org.apache.iceberg.catalog.TableIdentifier; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.joda.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The top-level CDC sink transform: applies a collection of change records to one or more Iceberg + * V2+ tables: + * + * 

{@code
+ * changeRecords.apply(IcebergIO.writeCdcRows(catalogConfig)
+ *     .to(tableId)
+ *     .withSequenceNumberColumn("seq")
+ *     .withTriggeringFrequency(Duration.standardMinutes(1)));
+ * }
+ * + *

Input contract

+ * + *

Each input {@link Row} is one change record. Its change kind is its native {@link ValueKind}, + * but can be overridden with a string column value using {@link #withChangeTypeColumn}, optionally + * translated via {@link #withChangeTypeMap}. Each row must also carry a per-key monotonic + * (long) sequence number specified by {@link #withSequenceNumberColumn}, which orders a single + * primary key's changes; the column is required in the input schema as a non-nullable {@code + * INT64}; set defaults upstream. These control columns are stripped from the input rows before + * writing to the table. + * + *

Ordering requirement (not validated): for a given primary key, the input element's + * event-time must be non-decreasing with its (sequence number, kind rank): a higher-sequence change + * never carries an earlier event time, and equal-sequence records (an update's before and after + * images) carry equal event times, so neither half lands in an earlier commit window. A violation + * can corrupt final table state (a lower-sequence equality delete deleting a higher-sequence row + * committed in a later snapshot). + * + *

Semantics

+ * + *

The sink commits one new snapshot to the destination table per commit window, in ascending + * window order, with an idempotency token written to each snapshot's summary. A retried or + * restarted commit finds the token and skips already-committed windows, making the commit + * effectively-once on runners that honor {@code @RequiresStableInput}. Records whose grouped pane + * fires late (watermark is past their commit window's end) are diverted to the DLQ output, + * accessible with {@link IcebergWriteResult#getDeadLetterRows()}. + * + *

Each dead-lettered record nests the data row under {@code record}, beside {@code change_type}, + * {@code sequence_number}, and {@code destination}. + * + *

Sink id

+ * + *

The sink creates a fresh unique id by default. All commits in a single pipeline run share the + * same id. Commits get stamped with the sink id and a window-end millis token. Set {@link + * #withSinkId} explicitly (and keep it stable) for cross-relaunch idempotency. Don't reuse sink ids + * for batch runs though because all commits fall under a single global window, so a second load + * with the same sink id will recognize the same window-end millis token and skip the commit. A + * batch load's sink id must likewise not be carried into a streaming continuation. + */ +@Internal +@AutoValue +public abstract class WriteCdcRows extends PTransform, IcebergWriteResult> { + + private static final Logger LOG = LoggerFactory.getLogger(WriteCdcRows.class); + + public static final Duration DEFAULT_ALLOWED_LATENESS = Duration.standardHours(6); + + abstract IcebergCatalogConfig getCatalogConfig(); + + abstract @Nullable TableIdentifier getTableIdentifier(); + + abstract @Nullable DynamicDestinations getDynamicDestinations(); + + abstract @Nullable List getEqualityColumns(); + + abstract String getSequenceNumberColumn(); + + abstract @Nullable String getChangeTypeColumn(); + + abstract @Nullable Map getChangeTypeMap(); + + abstract int getNumShards(); + + abstract @Nullable Integer getShardsPerPartition(); + + abstract int getSorterMemoryMB(); + + abstract boolean getUpsert(); + + abstract @Nullable Long getTokenHeartbeatMillis(); + + abstract boolean getErrorHandlingEnabled(); + + abstract @Nullable Map getSnapshotProperties(); + + /** The sink id namespacing the {@code beam.cdc.} snapshot-summary tokens. */ + abstract String getSinkId(); + + abstract @Nullable Duration getTriggeringFrequency(); + + abstract @Nullable Duration getAllowedLateness(); + + abstract Builder toBuilder(); + + @AutoValue.Builder + abstract static class Builder { + abstract Builder setCatalogConfig(IcebergCatalogConfig catalogConfig); + + abstract Builder setTableIdentifier(TableIdentifier tableIdentifier); + + abstract Builder setDynamicDestinations(DynamicDestinations destinations); + + abstract Builder setEqualityColumns(List equalityColumns); + + abstract Builder setSequenceNumberColumn(String sequenceNumberColumn); + + abstract Builder setChangeTypeColumn(String changeTypeColumn); + + abstract Builder setChangeTypeMap(Map changeTypeMap); + + abstract Builder setNumShards(int numShards); + + abstract Builder setShardsPerPartition(Integer shardsPerPartition); + + abstract Builder setSorterMemoryMB(int sorterMemoryMB); + + abstract Builder setUpsert(boolean upsert); + + abstract Builder setTokenHeartbeatMillis(@Nullable Long tokenHeartbeatMillis); + + abstract Builder setErrorHandlingEnabled(boolean errorHandlingEnabled); + + abstract Builder setSnapshotProperties(Map snapshotProperties); + + abstract Builder setSinkId(String sinkId); + + abstract Builder setTriggeringFrequency(Duration triggeringFrequency); + + abstract Builder setAllowedLateness(Duration allowedLateness); + + abstract WriteCdcRows build(); + } + + public static WriteCdcRows of(IcebergCatalogConfig catalogConfig) { + return new AutoValue_WriteCdcRows.Builder() + .setCatalogConfig(catalogConfig) + .setSequenceNumberColumn(CdcWriteConfig.DEFAULT_SEQUENCE_NUMBER_COLUMN) + .setNumShards(CdcWriteConfig.DEFAULT_NUM_SHARDS) + .setSorterMemoryMB(CdcWriteConfig.DEFAULT_SORTER_MEMORY_MB) + .setUpsert(false) + .setErrorHandlingEnabled(false) + .setSinkId(UUID.randomUUID().toString()) + .build(); + } + + /** Writes to a single table. Mutually exclusive with {@link #to(DynamicDestinations)}. */ + public WriteCdcRows to(TableIdentifier tableIdentifier) { + return toBuilder().setTableIdentifier(tableIdentifier).build(); + } + + /** + * Writes to multiple tables. Mutually exclusive with {@link #to(TableIdentifier)}. + * + *

The sink projects the raw input row (minus the control columns) against each destination + * table's schema. + */ + public WriteCdcRows to(DynamicDestinations destinations) { + return toBuilder().setDynamicDestinations(destinations).build(); + } + + /** + * Columns that define a row's identity (the Iceberg equality-delete fields). Defaults to the + * destination table's identifier (primary-key) fields. Tables may be partitioned on non-key + * columns; partition source columns must be equality columns only under {@link #withUpsert} or a + * {@link #withShardsPerPartition} cap. + */ + public WriteCdcRows withEqualityColumns(List columns) { + return toBuilder().setEqualityColumns(columns).build(); + } + + /** + * The column holding the per-primary-key monotonic sequence number used to order a single key's + * changes. Must be declared as a non-nullable {@code INT64} in the input schema. The column is + * stripped from the written rows. Defaults to {@value + * CdcWriteConfig#DEFAULT_SEQUENCE_NUMBER_COLUMN}. + */ + public WriteCdcRows withSequenceNumberColumn(String column) { + return toBuilder().setSequenceNumberColumn(column).build(); + } + + /** + * When set, reads the change kind from this column instead of the element's native {@link + * ValueKind}. Must be declared as a non-nullable {@code STRING} in the input schema. The column + * is stripped from the written rows. + */ + public WriteCdcRows withChangeTypeColumn(String column) { + return toBuilder().setChangeTypeColumn(column).build(); + } + + /** + * Mapping from {@link #withChangeTypeColumn} values to {@link ValueKind} names (e.g. for + * Debezium: {@code {"c": "INSERT", "u": "UPDATE_AFTER", "d": "DELETE"}}). Requires {@link + * #withChangeTypeColumn} to also be set. + */ + public WriteCdcRows withChangeTypeMap(Map changeTypeMap) { + return toBuilder().setChangeTypeMap(changeTypeMap).build(); + } + + /** + * The number of deterministic primary-key-hash shards per destination. Controls the sink's + * write-parallelism knob. Defaults to {@value CdcWriteConfig#DEFAULT_NUM_SHARDS}. + * + *

Too low may cause a write bottleneck with a growing commit backlog, too high increases the + * sink's output file count ({@code num_shards x touched partitions} files per commit window). + * + *

On a partitioned table, a commit window writes up to this many files per touched partition. + * {@link #withShardsPerPartition} lowers that per-partition count while leaving this number + * unchanged, so the file count drops without lowering the ceiling on total write parallelism. + * Each individual partition is then written by at most that many shards. + */ + public WriteCdcRows withNumShards(int numShards) { + return toBuilder().setNumShards(numShards).build(); + } + + /** + * Caps how many shards one partition may occupy (default: {@code num_shards}, i.e. all shards may + * write to all partitions). A lower cap reduces write parallelism per partition but also + * concentrates the writes in fewer files. A cap of {@code 1} pins each partition to a single + * writer. Ignored for unpartitioned tables. A cap below {@code num_shards} requires every + * partition source column to be an equality column, making the partition (and with it the shard) + * a pure function of the primary key. + */ + public WriteCdcRows withShardsPerPartition(int shardsPerPartition) { + return toBuilder().setShardsPerPartition(shardsPerPartition).build(); + } + + /** + * The in-memory buffer size (MB) for the sorter that orders each shard's records by primary key, + * then sequence number, then change kind, before writing; groups larger than this spill to disk. + * Must be {@code >= 1}. Defaults to {@value CdcWriteConfig#DEFAULT_SORTER_MEMORY_MB}. + */ + public WriteCdcRows withSorterMemoryMB(int sorterMemoryMB) { + Preconditions.checkArgument( + sorterMemoryMB >= 1, "sorter_memory_mb must be >= 1, got %s", sorterMemoryMB); + return toBuilder().setSorterMemoryMB(sorterMemoryMB).build(); + } + + /** + * If {@code true}, only the after-image of each change ({@code INSERT}/{@code UPDATE_AFTER}) is + * required; {@code UPDATE_BEFORE} records are dropped and {@code INSERT}/{@code UPDATE_AFTER} are + * applied as upserts (equality-delete-then-insert on the primary key). Defaults to {@code false}. + * Requires every partition source column to be an equality column: with before-images dropped, a + * row that moved partitions could never be deleted from its old one. + */ + public WriteCdcRows withUpsert(boolean upsert) { + return toBuilder().setUpsert(upsert).build(); + } + + /** + * Enables a periodic empty token-refresh (heartbeat) commit for each destination with a + * committed-through token, whether committed in this run or recovered from the table: while idle, + * the committer re-writes the token into a fresh snapshot every {@code interval}, so the + * token-bearing snapshot stays recent and is less likely to be lost to {@code expire_snapshots} + * before the sink resumes. Disabled by default, and ignored for bounded (batch) input. + * + *

With heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming + * processing-time timer can keep a drain from completing. + */ + public WriteCdcRows withTokenHeartbeat(Duration interval) { + Preconditions.checkArgument( + interval != null && interval.getMillis() > 0, "token heartbeat interval must be positive"); + return toBuilder().setTokenHeartbeatMillis(interval.getMillis()).build(); + } + + /** + * Enables per-record error handling. Poison records (unknown change type, missing/null sequence + * number, null equality value, an unresolvable destination) are diverted to {@link + * IcebergWriteResult#getFailedRows()} (schema {@code failed_row ROW + error_message STRING}). + * Defaults to off where the DoFn throws an error instead. Diversion covers poison detectable in + * the assignment stage; a failure later in the pipeline fails and retries its bundle as usual. + */ + public WriteCdcRows withErrorHandling() { + return toBuilder().setErrorHandlingEnabled(true).build(); + } + + /** + * Extra user properties to add to every commit's Iceberg snapshot summary. Keys prefixed with + * {@code beam.cdc.} are reserved for the sink's own idempotency/diagnostic tokens and are + * rejected at construction. + */ + public WriteCdcRows withSnapshotProperties(Map snapshotProperties) { + return toBuilder().setSnapshotProperties(snapshotProperties).build(); + } + + /** + * A stable identifier for this sink, used to namespace the idempotency tokens written to each + * commit's Iceberg snapshot summary ({@code beam.cdc.sink-id}, {@code + * beam.cdc.committed-through-ms.}). Mostly useful when re-using the same {@code sinkId} + * across subsequent streaming relaunches to maintain exactly-once commit idempotency from the + * same source. If unset, a fresh UUID is used per launch. + * + *

Also useful in batch to make runs idempotent for the same {@code sinkId}. A stable {@code + * sinkId} should only be used once though. In batch, the idempotency token is always + * global-window end, so re-using it will lead to no-op for subsequent loads (they will be skipped + * at commit time). Do not carry a batch job's {@code sinkId} into a streaming job, + * as it would skip every commit window forever. + */ + public WriteCdcRows withSinkId(String sinkId) { + return toBuilder().setSinkId(sinkId).build(); + } + + /** The size of each event-time commit window (streaming only). */ + public WriteCdcRows withTriggeringFrequency(Duration triggeringFrequency) { + return toBuilder().setTriggeringFrequency(triggeringFrequency).build(); + } + + /** + * How far behind the watermark an element's event-time may be before it is dropped entirely, + * otherwise routed to {@link IcebergWriteResult#getDeadLetterRows()} as a late firing. Defaults + * to {@link #DEFAULT_ALLOWED_LATENESS} (6 hours) if unset. + */ + public WriteCdcRows withAllowedLateness(Duration allowedLateness) { + return toBuilder().setAllowedLateness(allowedLateness).build(); + } + + private CdcWriteConfig cdcWriteConfig() { + @Nullable Integer shardsPerPartition = getShardsPerPartition(); + return CdcWriteConfig.builder() + .setEqualityColumns(getEqualityColumns()) + .setSequenceNumberColumn(getSequenceNumberColumn()) + .setChangeTypeColumn(getChangeTypeColumn()) + .setChangeTypeMap(getChangeTypeMap()) + .setNumShards(getNumShards()) + .setShardsPerPartition(shardsPerPartition == null ? getNumShards() : shardsPerPartition) + .setSorterMemoryMB(getSorterMemoryMB()) + .setUpsert(getUpsert()) + .setTokenHeartbeatMillis(getTokenHeartbeatMillis()) + .setSinkId(getSinkId()) + .setSnapshotProperties(getSnapshotProperties()) + .setErrorHandling(getErrorHandlingEnabled()) + .build(); + } + + /** Construction-time validation. */ + private void validate(PCollection input, CdcWriteConfig config) { + Preconditions.checkArgument( + 1 + == Stream.of(getTableIdentifier(), getDynamicDestinations()) + .filter(Predicates.notNull()) + .count(), + "Must set exactly one of to(TableIdentifier) or to(DynamicDestinations)."); + + // Structural config invariants (value bounds, mutually exclusive options, reserved names). + config.validate(); + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration triggeringFrequency = getTriggeringFrequency(); + if (unbounded) { + if (triggeringFrequency == null) { + throw new IllegalArgumentException( + "Streaming CDC writes require withTriggeringFrequency(...)"); + } + Preconditions.checkArgument( + triggeringFrequency.isLongerThan(Duration.ZERO), + "triggering frequency must be positive for streaming CDC writes, got %s", + triggeringFrequency); + } + + @Nullable String changeTypeColumn = config.getChangeTypeColumn(); + if (changeTypeColumn != null) { + if (!input.getSchema().hasField(changeTypeColumn)) { + throw new IllegalArgumentException( + "CDC input schema is missing the change-type column '" + + changeTypeColumn + + "' named by change_type_column. Check the spelling, or drop change_type_column " + + "to use native change kind. Input schema: " + + input.getSchema()); + } + Schema.FieldType changeTypeType = input.getSchema().getField(changeTypeColumn).getType(); + if (changeTypeType.getTypeName() != Schema.TypeName.STRING) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be STRING, but the input schema declares it " + + changeTypeType + + "."); + } + if (changeTypeType.getNullable()) { + throw new IllegalArgumentException( + "CDC change-type column '" + + changeTypeColumn + + "' must be non-nullable, but the input schema declares it nullable."); + } + } + + String seqCol = config.getSequenceNumberColumn(); + if (!input.getSchema().hasField(seqCol)) { + throw new IllegalArgumentException( + "CDC input schema is missing the sequence-number column '" + + seqCol + + "', required to order each primary key's changes. Input schema: " + + input.getSchema()); + } + Schema.FieldType seqType = input.getSchema().getField(seqCol).getType(); + if (seqType.getTypeName() != Schema.TypeName.INT64) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be INT64, but the input schema declares it " + + seqType + + "."); + } + if (seqType.getNullable()) { + throw new IllegalArgumentException( + "CDC sequence-number column '" + + seqCol + + "' must be non-nullable, but the input schema declares it nullable."); + } + + // The idle token-refresh heartbeat is a streaming-only concept. + if (!unbounded && config.getTokenHeartbeatMillis() != null) { + LOG.warn( + "Token heartbeat is ignored for bounded input. Batch loads will commit once and exit."); + } + } + + @Override + public IcebergWriteResult expand(PCollection input) { + CdcWriteConfig config = cdcWriteConfig(); + validate(input, config); + + @Nullable DynamicDestinations destinations = getDynamicDestinations(); + if (destinations == null) { + destinations = + DynamicDestinations.singleTable( + checkStateNotNull(getTableIdentifier()), input.getSchema()); + } + + boolean unbounded = input.isBounded() == PCollection.IsBounded.UNBOUNDED; + @Nullable Duration allowedLateness = getAllowedLateness(); + + String filePrefix = UUID.randomUUID().toString(); + + // Stage 1: destination + shard/sort-key assignment, and handle error rows + PCollectionTuple assigned = + input.apply( + "AssignCdcKeys", + new AssignCdcKeys(getCatalogConfig(), config, destinations, filePrefix)); + PCollection, KV>> keyed = + assigned.get(AssignCdcKeys.KEYED); + // getFailedRows() is null when error handling is disabled + @Nullable PCollection failedRows = + getErrorHandlingEnabled() ? assigned.get(AssignCdcKeys.FAILED) : null; + + // Stage 2: create commit windows, group by (destination, shard) and split late rows to the + // replayable dead-letter, and sort groups by (pk, seq, kind) + CommitWindows.Result windowed = + keyed.apply( + "CommitWindows", + new CommitWindows( + config, + getTriggeringFrequency(), + allowedLateness != null ? allowedLateness : DEFAULT_ALLOWED_LATENESS)); + + // Stage 3: write delta files and emit their serialized metadata, one element per shard. + PCollection written = + windowed + .getSortedGroups() + .apply( + "WriteDeltas", + new WriteDeltas(getCatalogConfig(), config, destinations, filePrefix)); + + // Commit stage: ordered, idempotent commit + PCollection> snapshots = + written.apply( + "CommitDeltas", + new CommitDeltas( + getCatalogConfig(), + getSinkId(), + config.getSnapshotProperties(), + unbounded ? config.getTokenHeartbeatMillis() : null, + filePrefix)); + + return IcebergWriteResult.cdc( + input.getPipeline(), snapshots, windowed.getDeadLetterRows(), failedRows); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteDeltas.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteDeltas.java new file mode 100644 index 000000000000..efbab9d3403e --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/WriteDeltas.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; + +import static org.apache.beam.sdk.io.iceberg.IcebergUtils.beamRowToIcebergRecord; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.IterableCoder; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.io.iceberg.DynamicDestinations; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.SerializableDataFile; +import org.apache.beam.sdk.io.iceberg.SerializableDeleteFile; +import org.apache.beam.sdk.schemas.NoSuchSchemaException; +import org.apache.beam.sdk.schemas.Schema; +import org.apache.beam.sdk.schemas.SchemaRegistry; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.BoundedWindow; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.io.WriteResult; +import org.apache.iceberg.util.PropertyUtil; + +/** + * Stage 3 of the CDC sink: consumes each complete sorted {@code (destination, shard, window)} group + * from {@link CommitWindows}, collapses each primary key to its final state through a {@link + * RecordDeltaTaskWriter}, and emits one {@link ShardDeltaFiles} with the serialized file metadata. + */ +final class WriteDeltas + extends PTransform< + PCollection, Iterable>>>, + PCollection> { + + private final IcebergCatalogConfig catalogConfig; + private final CdcWriteConfig config; + private final DynamicDestinations destinations; + private final String filePrefix; + + WriteDeltas( + IcebergCatalogConfig catalogConfig, + CdcWriteConfig config, + DynamicDestinations destinations, + String filePrefix) { + this.catalogConfig = catalogConfig; + this.config = config; + this.destinations = destinations; + this.filePrefix = filePrefix; + } + + @Override + public PCollection expand( + PCollection, Iterable>>> input) { + Schema dataSchema = dataSchemaOf(input.getCoder()); + return input + .apply( + "WriteDeltas", + ParDo.of( + new WriteDeltasFn( + new TableSetup(catalogConfig, config, destinations, filePrefix), + config, + filePrefix, + dataSchema))) + .setCoder(shardDeltaFilesCoder()); + } + + static Coder shardDeltaFilesCoder() { + try { + return SchemaRegistry.createDefault().getSchemaCoder(ShardDeltaFiles.class); + } catch (NoSuchSchemaException e) { + throw new RuntimeException("Could not build a coder for ShardDeltaFiles.", e); + } + } + + /** Extracts the CDC data schema carried by the input's nested {@link CdcRecordCoder}. */ + private static Schema dataSchemaOf(Coder inputCoder) { + checkArgument( + inputCoder instanceof KvCoder, + "expected a KvCoder input element coder, got %s", + inputCoder); + Coder groupCoder = ((KvCoder) inputCoder).getValueCoder(); + checkArgument( + groupCoder instanceof IterableCoder, + "expected an IterableCoder input value coder, got %s", + groupCoder); + Coder elementCoder = ((IterableCoder) groupCoder).getElemCoder(); + checkArgument( + elementCoder instanceof KvCoder, + "expected a KvCoder group element coder, got %s", + elementCoder); + Coder recordCoder = ((KvCoder) elementCoder).getValueCoder(); + checkArgument( + recordCoder instanceof CdcRecordCoder, + "expected a CdcRecordCoder input record coder, got %s", + recordCoder); + return ((CdcRecordCoder) recordCoder).getDataSchema(); + } + + /** + * Writes one {@link RecordDeltaTaskWriter} per input group and emits the serialized {@link + * ShardDeltaFiles}. On any failure {@link RecordDeltaTaskWriter#abort()} deletes the group's + * written files. + */ + @VisibleForTesting + static final class WriteDeltasFn + extends DoFn, Iterable>>, ShardDeltaFiles> { + + private final TableSetup tableSetup; + private final CdcWriteConfig config; + private final String filePrefix; + private final Schema dataSchema; + + WriteDeltasFn( + TableSetup tableSetup, CdcWriteConfig config, String filePrefix, Schema dataSchema) { + this.tableSetup = tableSetup; + this.config = config; + this.filePrefix = filePrefix; + this.dataSchema = dataSchema; + } + + @ProcessElement + public void process( + @Element KV, Iterable>> group, + BoundedWindow window, + OutputReceiver out) + throws IOException { + String destString = group.getKey().getKey(); + int shardId = group.getKey().getValue(); + TableSetup.Dest dest = tableSetup.get(destString, dataSchema); + Table table = dest.table(); + PartitionSpec spec = dest.spec(); + + FileFormat dataFormat = RecordDeltaTaskWriter.dataFileFormat(table); + FileFormat deleteFormat = RecordDeltaTaskWriter.deleteFileFormat(table, dataFormat); + long targetFileSize = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.WRITE_TARGET_FILE_SIZE_BYTES, + TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT); + + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(table, shardId, /* taskId= */ 0) + .format(dataFormat) + .defaultSpec(spec) + .operationId( + filePrefix + "-" + window.maxTimestamp().getMillis() + "-" + UUID.randomUUID()) + .build(); + RecordDeltaTaskWriter writer = + RecordDeltaTaskWriter.create( + table, + spec, + dest.equalityFieldIds(), + config.getUpsert(), + targetFileSize, + fileFactory, + dataFormat, + deleteFormat); + + org.apache.iceberg.Schema tableSchema = table.schema(); + long minSeq = Long.MAX_VALUE; + long maxSeq = Long.MIN_VALUE; + try { + // Sorted upstream by the pk-prefixed sort key: the collapse writer relies on each key's + // records arriving contiguous, in (seq, kind) order, and reads the key's pk prefix to + // find block boundaries. + for (KV keyed : group.getValue()) { + CdcRecord record = keyed.getValue(); + writer.write( + keyed.getKey(), + beamRowToIcebergRecord(tableSchema, record.getData()), + record.getKind()); + minSeq = Math.min(minSeq, record.getSequenceNumber()); + maxSeq = Math.max(maxSeq, record.getSequenceNumber()); + } + + WriteResult result = writer.complete(); + if (result.dataFiles().length == 0 && result.deleteFiles().length == 0) { + return; // Empty group: emit nothing. + } + out.output(serialize(destString, table, result, minSeq, maxSeq)); + } catch (Throwable t) { + // Delete this group's written files; preserve the root cause if abort also fails. + try { + writer.abort(); + } catch (Exception abortEx) { + t.addSuppressed(abortEx); + } + throw t; + } + } + } + + /** Serializes {@code result}'s files and returns the {@link ShardDeltaFiles} carrying them. */ + static ShardDeltaFiles serialize( + String destString, Table table, WriteResult result, long minSeq, long maxSeq) { + Map specs = table.specs(); + List dataFiles = new ArrayList<>(result.dataFiles().length); + for (DataFile dataFile : result.dataFiles()) { + dataFiles.add(SerializableDataFile.from(dataFile, specs)); + } + List deleteFiles = new ArrayList<>(result.deleteFiles().length); + for (DeleteFile deleteFile : result.deleteFiles()) { + deleteFiles.add(SerializableDeleteFile.from(deleteFile, specs)); + } + return ShardDeltaFiles.builder() + .setTableIdentifierString(destString) + .setDataFiles(dataFiles) + .setDeleteFiles(deleteFiles) + .setMinSequenceNumber(minSeq) + .setMaxSequenceNumber(maxSeq) + .build(); + } +} diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java new file mode 100644 index 000000000000..5f835f1cc9f0 --- /dev/null +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Iceberg CDC sink: applies a stream or batch of inserts, updates and deletes to Iceberg V2+ + * tables, exposed as {@code IcebergIO.writeCdcRows(catalogConfig)}. For the transform's API and + * semantics see {@code WriteCdcRows}. + * + *

This is a merge-on-read sink. Within one commit window the writer collapses each primary key's + * changes to their final state, so a superseded intermediate row is never written at all; a change + * that reaches back into an already-committed snapshot is written as a PK-only equality + * delete. No position deletes or deletion vectors are ever written. + * + *

Monitoring

+ * + *

All metrics below are queryable through {@code PipelineResult.metrics()}; the namespace is the + * owning class's fully-qualified name, so filter on {@code + * org.apache.beam.sdk.io.iceberg.cdc.sink.*}. + * + *

Committer volume and latency

+ * + *

The normal throughput picture. None of these indicate a problem by themselves; watch their + * shape over time. + * + *

    + *
  • snapshotsCreated: CDC snapshots committed. In streaming this should track one per + * commit window per destination. A flat line while data is flowing means commits have + * stalled. + *
  • committedDataFiles / committedDeleteFiles: files published. Divide by {@code + * snapshotsCreated} for files per commit. A large ratio points to the small-file problem + * described below. + *
  • committedRecords: rows committed. + *
  • committedEqualityDeleteRecords: equality-delete rows committed. + *
  • committedBytes: total bytes of committed files. + *
  • commitDurationMs (distribution): wall-clock time spent waiting on the Iceberg commit + * operation. Its tail is your catalog's health: a growing maximum usually means catalog + * contention, not a Beam problem. + *
  • commitFailures: fires whenever the commit path throws. Iceberg's own optimistic + * retry runs underneath, so a genuine commit collision is counted only once that is + * exhausted. The failure is rethrown, so the bundle fails and the runner retries it; + * occasional increments under heavy concurrent writing are survivable. A sustained rate means + * the table has too many concurrent writers, and the sink will not make progress past the + * failing window (commits are strictly ordered). + *
  • heartbeatCommits: empty token-refresh commits emitted while a destination is idle + * (only if {@code withTokenHeartbeat} is configured). Each one is a real snapshot. With + * heartbeat enabled prefer cancel-and-resubmit over drain: the self-re-arming processing-time + * timer can keep a drain from completing. + *
+ * + *

Safety tripwires

+ * + *

These should be zero. A nonzero value is not necessarily an outage, but each one means + * something specific happened that you want to know about. + * + *

    + *
  • alreadyCommittedWindowsSkipped: a commit window was skipped because the table + * already carried a committed-through token at or past its end. Expected and benign after a + * restart or a retried bundle (we skip to ensure idempotency). Alarming in two cases: (1) in + * batch, where the whole load commits under one window, a nonzero value on a fresh + * load means a stable {@code sink_id} matched a previous load and this run wrote + * nothing at all. Use a unique {@code sink_id} per load, or omit it. Never carry a batch + * load's {@code sink_id} into a subsequent streaming run (the batch token is the + * global-window end, above every streaming window; the streaming committer will refuse a + * recovered batch token); and (2) in streaming, a persistently nonzero rate means data keeps + * arriving for windows that already closed, which is a source-lateness problem. The token + * proves that a window with that end committed, not that the skipped rows are the rows that + * committed. A logged warning names up to 5 of the window's files plus a count: your + * handle on the skipped rows until {@code remove_orphan_files} reclaims them. + *
  • orphanFiles: files (data and delete) carried by a skipped window: potential + * orphans (a pure redelivery's files are the committed live ones). Wasted storage until + * {@code remove_orphan_files} runs. + *
  • tokenParseFailures: an unparseable {@code beam.cdc.*} value was found while scanning + * snapshot ancestry. The committed-through scan continues to older ancestors rather than + * crash-looping; a bad max-seq or run-spec stamp just reads as absent. Nonzero means + * something wrote a malformed value into a snapshot summary; investigate before trusting the + * recovered position. + *
  • suspectedTokenExpiry: the sink's own {@code beam.cdc.sink-id} marker was found in + * the ancestry but no committed-through token was. Strongly implies that {@code + * expire_snapshots} removed the token-bearing snapshots while the pipeline was down. Recovery + * then falls back to the beginning and may re-apply retained windows. If you see this, either + * lengthen snapshot retention or enable {@code withTokenHeartbeat} so the token stays young. + *
  • crossWindowSequenceInversions: a committed window's minimum source sequence number + * was below an earlier window's committed maximum. This is the detector for a violated + * ordering contract (see below) and the only tripwire here that can mean silently wrong + * table contents. It has benign false positives when the two windows touch entirely + * disjoint primary keys, so treat it as "go and check the source's ordering", not as proof of + * corruption. + *
  • specMismatchedWindows: signals the partition spec was evolved mid-run. Incremented + * when a committed window carries equality deletes and uses partition-spec ids different from + * the run's pinned spec. Those deletes may not reach rows written under the other spec. + *
+ * + *

Input health

+ * + *

These describe what is arriving, not what the sink did with it. + * + *

    + *
  • deadLetterRecords: records whose grouped pane fired late (the watermark had already + * passed their commit window's end), diverted to the replayable dead-letter output ({@code + * IcebergWriteResult.getDeadLetterRows()}) instead of being applied out of order. + * Every late pane is diverted, including a window's first ({@code SplitLateData}'s + * javadoc explains why). Nonzero means your source is lagging past the commit window. The + * records are not lost, but nothing consumes them unless you wire that output somewhere. + * Records later than {@code withAllowedLateness} are a different case: see below. + *
  • failedRecords: poison records diverted to {@code IcebergWriteResult.getFailedRows()} + * when {@code withErrorHandling()} is on: unknown change type, missing sequence number, null + * equality value, unresolvable destination. Without error handling these fail the pipeline + * instead. Like the dead letters above, the records are not lost, but you need to wire the + * output somewhere to avoid dropping them. + *
  • schemaDriftRebuilds: the per-record projection had to be rebuilt because the + * incoming row schema changed. Normally zero. A steady rate is correct but slow: your source + * is emitting rows with an unstable schema. + *
  • upsertUpdateBeforeDropped: {@code UPDATE_BEFORE} records discarded because the sink + * is in upsert mode, which needs only after-images. Tells you how much of your input was + * redundant. + *
+ * + *

Records more than {@code withAllowedLateness} behind the watermark are dropped by the runner + * at the {@code GroupByShardKey} step. Raising {@code withAllowedLateness} is the only way to + * capture them (at the cost of more live window state per destination). + * + *

Files and maintenance

+ * + *

File count per commit is governed by {@code num_shards} × touched partitions, and + * total file count by that times the number of commit windows. A table with many partitions and a + * short {@code triggering_frequency_seconds} produces a lot of small files very quickly, regardless + * of the data rate. {@code num_shards} (default=16) is the sink's write-parallelism knob. Too low + * may lead to a write bottleneck and commit backlog. Too high is fine for the sink but may lead to + * slower downstream reads due to small files. + * + *

There are three levers that you can use: + * + *

    + *
  • {@code triggering_frequency_seconds}: increasing it leads to fewer, larger commits, + * at the cost of end-to-end latency. + *
  • {@code num_shards}: trades throughput for file count. Lowering it reduces file + * count, but also the write-parallelism ceiling. + *
  • {@code shards_per_partition} (default={@code num_shards}, i.e. no cap): on a + * partitioned table, cap how many shards one partition's rows may occupy. A touched + * partition then writes about {@code min(shards_per_partition, distinct keys)} files per file + * kind per commit, with its write parallelism capped to match; {@code 1} is the pure + * partition-affine endpoint (one writer, about one data file, per partition). + *
+ * + *

The trade-off a lowered {@code shards_per_partition} makes, stated plainly: effective + * write parallelism per destination becomes at most the cap times the number of distinct partitions + * receiving data in a window: at {@code 1}, one writer per partition. A cap of {@code 1} is right + * for a table with many partitions and wrong for a table with three, which it would hold at + * three concurrent writers however many workers the pipeline has; such a table wants an + * intermediate cap, sized so cap × touched partitions still covers the pipeline's write + * parallelism. The cap is moot for an unpartitioned table, where it is simply ignored (with a WARN) + * rather than funnelling everything through a single shard. At {@code 1} this is the same + * limitation Flink's {@code hash} write-distribution mode carries; the dial exists because the + * alternative (leaving {@code num_shards} as the only lever) forces the same trade with none of the + * benefit. It is safe with {@code upsert}: both options require every partition source column to be + * an equality column (see the partitioning section below), which makes a row's partition a pure + * function of its primary key. + * + *

Snapshot expiry bounds the sink's own cost, not just read performance

+ * + *

This sink creates one snapshot per commit window per table. The following sink costs + * grow as snapshot count grows: + * + *

    + *
  • Commit latency: Every commit fire loads the destination's metadata once, then walks its + * ancestry to find the latest committed-through token. Each commit within that fire publishes + * a snapshot, which requires refreshing and re-parsing the metadata. Both scale with the + * number of snapshots in that file. + *
  • Worker heap: Each cached table pins a {@code TableMetadata} holding every retained + * snapshot, per worker, multiplied by the number of dynamic destinations that worker touches. + *
+ * + *

Orphan files

+ * + *

The writer stage writes data files and delete files and passes their metadata to the committer + * stage in the pipeline. Every orphan is an ordinary data or delete file that no snapshot ended up + * referencing. Orphan files can be produced in three ways: + * + *

    + *
  • A retried or failed writer bundle. Each attempt names its files with a fresh UUID, + * so whichever attempt's output element loses the retry race leaves its files unreferenced. + *
  • A window skipped because the committed-through token already covers its end. This + * happens when a relaunch reuses a stable {@code sink_id}. The incoming element is targeting + * a window that has already been committed with different files. If the incoming element's + * files contain different content than the committed ones, they may hold rows the table never + * received. Those files are left in place so you can recover them. Counted by {@code + * orphanFiles}; the skip WARN names up to 5 of them plus a count. + *
  • A failed commit attempt. Iceberg normally deletes the manifests it wrote when a + * commit fails, but it cannot when the outcome is unknown (i.e. {@code + * CommitStateUnknownException}). The commit may in fact have succeeded, and deleting those + * manifests would corrupt a live snapshot. Only the manifests are left orphaned. The window's + * own data and delete files are untouched, because the pending bag survives the failure and + * the retry commits those same files. + *
+ * + *

All of these are reclaimed by Iceberg's {@code remove_orphan_files}. The sink writes + * everything through the table's own write path. Give the procedure an age threshold much longer + * than your longest in-flight bundle to avoid deleting files a live pipeline is about to reference. + * + *

Partition-spec evolution mid-run

+ * + *

Iceberg matches an equality delete to its data files by {@code (spec id, partition)}, so if a + * table's spec changes during streaming writes, the sink will choose to keep writing with the old + * spec. Otherwise, it could leave new deletes that never reach old-spec rows, which is data + * corruption. If a window happens to still mix specs while carrying equality deletes, it commits + * anyway, WARNs, and increments the {@code specMismatchedWindows} counter. + * + *

If you update a table's spec, run {@code rewrite_data_files} first so old-spec rows are + * rewritten under the new spec. A relaunched pipeline or in-place update will adopt the new spec. + * + *

Partitioning

+ * + *

Tables may be partitioned on any columns, key or not, with any transforms. Two options still + * require every partition source column to be an equality column: {@code upsert} (before-images are + * dropped, so a row that moved partitions could never be deleted from its old one) and a {@code + * shards_per_partition} cap below {@code num_shards} (the shard is derived from the partition + * tuple, which must therefore be a pure function of the primary key). + * + *

Non-key partitioning sharpens the input contract: every update must carry its {@code + * UPDATE_BEFORE}, because a moved row whose before-image never arrives leaves a permanent duplicate + * in the old partition that no later delete reaches; and {@code UPDATE_BEFORE}/{@code DELETE} rows + * must carry the row's actual old values in the partition source columns, because a nulled non-key + * column routes the equality delete to the null partition. + * + *

The ordering contract

+ * + *

The sink orders each primary key's changes by the sequence-number column, and commits one + * snapshot per commit window in ascending window order. That is only sound if the two orderings + * agree at the source: for a given key, an element's event time must be non-decreasing with its + * (sequence number, kind rank), so equal-sequence records (an update's before and after images) + * carry equal event times. If a higher-sequence change lands in an earlier commit window + * than a lower-sequence one, a stale equality delete can be committed after the row it should not + * have touched, and the table's final contents are wrong. + * + *

This is a source contract. Violations are detected at commit time and visible with the {@code + * crossWindowSequenceInversions} counter and warning logs, but the commit still proceeds. The fix + * is in how event times are assigned upstream. Replaying dead letters is bound by the same + * contract: see the replay caveat at {@code IcebergWriteResult#getDeadLetterRows}. + */ +package org.apache.beam.sdk.io.iceberg.cdc.sink; diff --git a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java index 27c647478e17..2af593fdbd6d 100644 --- a/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java +++ b/sdks/java/managed/src/main/java/org/apache/beam/sdk/managed/Managed.java @@ -119,6 +119,7 @@ DELTA_LAKE_CDC, getUrn(ExternalTransforms.ManagedTransforms.Urns.DELTA_LAKE_CDC_ public static final Map WRITE_TRANSFORMS = ImmutableMap.builder() .put(ICEBERG, getUrn(ExternalTransforms.ManagedTransforms.Urns.ICEBERG_WRITE)) + .put(ICEBERG_CDC, getUrn(ExternalTransforms.ManagedTransforms.Urns.ICEBERG_CDC_WRITE)) .put(KAFKA, getUrn(ExternalTransforms.ManagedTransforms.Urns.KAFKA_WRITE)) .put(BIGQUERY, getUrn(ExternalTransforms.ManagedTransforms.Urns.BIGQUERY_WRITE)) .put(POSTGRES, getUrn(ExternalTransforms.ManagedTransforms.Urns.POSTGRES_WRITE)) @@ -164,6 +165,8 @@ public static ManagedTransform read(String source) { *

    *
  • {@link Managed#ICEBERG} : Write to Apache Iceberg tables IcebergIO + *
  • {@link Managed#ICEBERG_CDC} : CDC Write to Apache Iceberg tables using IcebergIO *
  • {@link Managed#KAFKA} : Write to Apache Kafka topics KafkaIO *
  • {@link Managed#BIGQUERY} : Write to GCP BigQuery tables