[IcebergIO] CDC record and sort key

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecord.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg.cdc.sink;
+
+import com.google.common.base.MoreObjects;
+import java.util.Objects;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.ValueKind;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+/**
+ * One change record carried through the CDC sink's shuffle.
+ *
+ * <p>{@link ValueKind} is reified because it's not preserved across a {@code GroupByKey}.
+ */
+final class CdcRecord {
+
+  private final Row data;
+  private final ValueKind kind;
+  private final long sequenceNumber;
+
+  private CdcRecord(Row data, ValueKind kind, long sequenceNumber) {
+    this.data = data;
+    this.kind = kind;
+    this.sequenceNumber = sequenceNumber;
+  }
+
+  public static CdcRecord of(Row data, ValueKind kind, long sequenceNumber) {
+    return new CdcRecord(data, kind, sequenceNumber);
+  }
+
+  public Row getData() {
+    return data;
+  }
+
+  public ValueKind getKind() {
+    return kind;
+  }
+
+  public long getSequenceNumber() {
+    return sequenceNumber;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof CdcRecord)) {
+      return false;
+    }
+    CdcRecord that = (CdcRecord) o;
+    return sequenceNumber == that.sequenceNumber && kind == that.kind && data.equals(that.data);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(data, kind, sequenceNumber);
+  }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(CdcRecord.class)
+        .add("data", data)
+        .add("kind", kind)
+        .add("sequenceNumber", sequenceNumber)
+        .toString();
+  }
+}

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcRecordCoder.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg.cdc.sink;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.RowCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.ValueKind;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+/**
+ * {@link CdcRecord} carries a {@link Row} field whose schema is known only at pipeline-construction
+ * time. We need a custom coder because {@code AutoValueSchema} only infers schemas at the class
+ * level and cannot infer a dynamic {@link Row} field, so {@code @DefaultSchema} alone cannot
+ * produce a working coder for {@link CdcRecord}.
+ */
+final class CdcRecordCoder extends CustomCoder<CdcRecord> {
+
+  private final RowCoder dataCoder;
+  private final VarIntCoder kindCoder = VarIntCoder.of();
+  private final VarLongCoder seqCoder = VarLongCoder.of();
+
+  private CdcRecordCoder(Schema dataSchema) {
+    this.dataCoder = RowCoder.of(dataSchema);
+  }
+
+  public static CdcRecordCoder of(Schema dataSchema) {
+    return new CdcRecordCoder(dataSchema);
+  }
+
+  public Schema getDataSchema() {
+    return dataCoder.getSchema();
+  }
+
+  @Override
+  public void encode(CdcRecord value, OutputStream outStream) throws IOException {
+    dataCoder.encode(value.getData(), outStream);
+    kindCoder.encode(kindToCode(value.getKind()), outStream);
+    seqCoder.encode(value.getSequenceNumber(), outStream);
+  }
+
+  @Override
+  public CdcRecord decode(InputStream inStream) throws IOException {
+    Row data = dataCoder.decode(inStream);
+    ValueKind kind = kindFromCode(kindCoder.decode(inStream));
+    long seq = seqCoder.decode(inStream);
+    return CdcRecord.of(data, kind, seq);
+  }
+
+  private static int kindToCode(ValueKind kind) {
+    switch (kind) {
+      case INSERT:
+        return 0;
+      case UPDATE_BEFORE:
+        return 1;
+      case UPDATE_AFTER:
+        return 2;
+      case DELETE:
+        return 3;
+      default:
+        throw new IllegalArgumentException("Unknown ValueKind: " + kind);
+    }
+  }
+
+  private static ValueKind kindFromCode(int code) throws CoderException {
+    switch (code) {
+      case 0:
+        return ValueKind.INSERT;
+      case 1:
+        return ValueKind.UPDATE_BEFORE;
+      case 2:
+        return ValueKind.UPDATE_AFTER;
+      case 3:
+        return ValueKind.DELETE;
+      default:
+        throw new CoderException("Unknown CdcRecord ValueKind code: " + code);
+    }
+  }
+
+  @Override
+  public void verifyDeterministic() throws NonDeterministicException {
+    Coder.verifyDeterministic(this, "Data coder must be deterministic", dataCoder);
+  }
+
+  @Override
+  public boolean consistentWithEquals() {
+    // decode() rebuilds the row with this coder's schema object, which may differ from the
+    // original row's; false is always safe.
+    return false;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    return getDataSchema().equals(((CdcRecordCoder) o).getDataSchema());
+  }
+
+  @Override
+  public int hashCode() {
+    return getDataSchema().hashCode();
+  }
+}

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/CdcSortKey.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.iceberg.cdc.sink;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.apache.beam.sdk.values.ValueKind;
+
+/**
+ * Builds the byte-comparable secondary sort key used by {@code SortValues} (extensions/sorter) to
+ * order each (destination, shard, window) group: one primary key's records come out contiguous,
+ * ordered by sequence number then {@link #kindRank(ValueKind)} within the key.
+ *
+ * <p>The key is {@code [pkLen:4][pkBytes][seq ^ Long.MIN_VALUE:8][kindRank:1]}, big-endian.
+ */
+final class CdcSortKey {
+
+  private CdcSortKey() {}
+
+  /** Ranks change kinds so before-images sort before after-images at an equal {@code seq}. */
+  public static byte kindRank(ValueKind kind) {
+    switch (kind) {
+      case UPDATE_BEFORE:
+        return 0;
+      case DELETE:
+        return 1;
+      case UPDATE_AFTER:
+        return 2;
+      case INSERT:
+        return 3;
+      default:
+        throw new IllegalArgumentException("Unknown ValueKind: " + kind);
+    }
+  }
+
+  /**
+   * Encodes the deterministic, byte-comparable sort key {@code [pkLen:4][pkBytes][seq ^
+   * Long.MIN_VALUE:8][kindRank:1]} for one CDC record.
+   *
+   * <p>SortValues compares unsigned lexicographic byte order. The length prefix is needed to
+   * accurately compare two primary keys of varying byte-lengths. Flipping the sequence number's
+   * sign bit makes unsigned byte order match signed numeric order. kindRank breaks equal-seq ties.
+   */
+  public static byte[] encode(byte[] pkBytes, long seq, ValueKind kind) {
+    return ByteBuffer.allocate(4 + pkBytes.length + 9)
+        .putInt(pkBytes.length)
+        .put(pkBytes)
+        .putLong(seq ^ Long.MIN_VALUE)
+        .put(kindRank(kind))
+        .array();
+  }
+
+  /**
+   * Whether two encoded sort keys carry the same primary key, compared on the raw {@code
+   * [pkLen:4][pkBytes]} prefix.
+   */
+  public static boolean samePk(byte[] a, byte[] b) {
+    int aPkEnd = 4 + ByteBuffer.wrap(a).getInt(0);
+    int bPkEnd = 4 + ByteBuffer.wrap(b).getInt(0);
+    return Arrays.equals(a, 0, aPkEnd, b, 0, bPkEnd);
+  }
+}

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/cdc/sink/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A change data capture (CDC) write sink for Apache Iceberg: applies inserts, updates, and deletes
+ * to a table, identifying rows by equality columns.
+ */
+package org.apache.beam.sdk.io.iceberg.cdc.sink;