From 4923f3d7aa706582a16558af19bca50d7a40b65a Mon Sep 17 00:00:00 2001 From: mikepapadim Date: Fri, 13 Nov 2026 16:47:23 +0000 Subject: [PATCH 1/2] [lake] Let the lake tiering factory declare record batch write support The tiering job wrote ARROW log tables batch by batch only when the table's lake format was Paimon, by comparing DataLakeFormat.PAIMON in TieringSplitReader. A lake writer of another format that implements SupportsRecordBatchWrite was never given batches. LakeTieringFactory#supportsRecordBatchWrite(TableInfo), false by default, now declares it, the same way supportsPartitionMarkDone() does for mark-done. PaimonLakeTieringFactory returns true, so Paimon tiering is unchanged; other lakes keep the row path until they opt in. fluss-flink-common gets unshaded Arrow as a test dependency, so the record batch path can be tested there. --- .../fluss/lake/writer/LakeTieringFactory.java | 16 +++ fluss-flink/fluss-flink-common/pom.xml | 16 +++ .../tiering/source/TieringSplitReader.java | 6 +- .../source/TieringSplitReaderTest.java | 123 ++++++++++++++++++ .../tiering/PaimonLakeTieringFactory.java | 6 + 5 files changed, 163 insertions(+), 4 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java b/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java index 2d153aef16e..b3c4fb15df6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java @@ -22,6 +22,7 @@ import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.committer.PartitionMarkDoneCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.fluss.metadata.TableInfo; import java.io.IOException; import java.io.Serializable; @@ -50,6 +51,21 @@ public interface LakeTieringFactory extends Serializa LakeWriter createLakeWriter(WriterInitContext writerInitContext) throws IOException; + /** + * Returns whether the lake writers of this factory accept Arrow record batches for the given + * table. If true, every {@link LakeWriter} created for the table must implement {@link + * SupportsRecordBatchWrite}, and the tiering job writes append-only tables in the ARROW log + * format batch by batch instead of row by row. + * + *

This check must not access lake resources. + * + * @param tableInfo the Fluss table to tier + * @return true if the lake writers accept Arrow record batches for the table + */ + default boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return false; + } + /** * Returns the serializer for write results. * diff --git a/fluss-flink/fluss-flink-common/pom.xml b/fluss-flink/fluss-flink-common/pom.xml index adfa268afc7..fcdf9afb1fe 100644 --- a/fluss-flink/fluss-flink-common/pom.xml +++ b/fluss-flink/fluss-flink-common/pom.xml @@ -200,6 +200,22 @@ test + + + org.apache.arrow + arrow-vector + ${arrow.version} + test + + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + test + + diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java index 7a2a56ad902..0a4fef9c61d 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java @@ -40,7 +40,6 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; -import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.TableBucket; @@ -430,7 +429,7 @@ private void mayCreateLogScanner() { /** * Determines whether the current table should use the Arrow record batch path for tiering. The * batch path is used when the table is an ARROW format append-only (log) table and the lake - * writer supports batch writing. + * tiering factory supports record batch writes for it. */ private boolean useRecordBatchPath() { if (currentTableUseRecordBatchPath != null) { @@ -442,8 +441,7 @@ private boolean useRecordBatchPath() { unshadedArrowAvailable && !tableInfo.hasPrimaryKey() && tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW - && tableInfo.getTableConfig().getDataLakeFormat().orElse(null) - == DataLakeFormat.PAIMON; + && lakeTieringFactory.supportsRecordBatchWrite(tableInfo); return currentTableUseRecordBatchPath; } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java index cf7d30d4a7a..cdc58415d63 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java @@ -33,11 +33,15 @@ import org.apache.fluss.flink.tiering.source.metrics.TieringMetrics; import org.apache.fluss.flink.tiering.source.split.FlinkTieringSplit; import org.apache.fluss.flink.utils.FlinkTestBase; +import org.apache.fluss.lake.batch.ArrowRecordBatch; +import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.InternalRow; @@ -539,6 +543,72 @@ void testTieringFirstRowMergeEngineFinishes() throws Exception { } } + @Test + void testArrowLogTableUsesRecordBatchWriteWhenFactorySupportsIt() throws Exception { + RecordBatchCountingLakeTieringFactory lakeTieringFactory = + new RecordBatchCountingLakeTieringFactory(true); + TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_batch_write"); + + tierLogTableAndVerifyRows(tablePath, lakeTieringFactory); + + assertThat(lakeTieringFactory.writers).isNotEmpty(); + assertThat(lakeTieringFactory.writers) + .allSatisfy( + writer -> { + assertThat(writer.writtenRows).isZero(); + assertThat(writer.writtenBatchRows).isPositive(); + }); + } + + @Test + void testArrowLogTableWritesRowsWhenFactoryDoesNotSupportRecordBatchWrite() throws Exception { + RecordBatchCountingLakeTieringFactory lakeTieringFactory = + new RecordBatchCountingLakeTieringFactory(false); + TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_row_write"); + + tierLogTableAndVerifyRows(tablePath, lakeTieringFactory); + + assertThat(lakeTieringFactory.writers).isNotEmpty(); + assertThat(lakeTieringFactory.writers) + .allSatisfy( + writer -> { + assertThat(writer.writtenRows).isPositive(); + assertThat(writer.writtenBatchRows).isZero(); + }); + } + + private void tierLogTableAndVerifyRows( + TablePath tablePath, TestingLakeTieringFactory lakeTieringFactory) throws Exception { + long tableId = createTable(tablePath, DEFAULT_LOG_TABLE_DESCRIPTOR); + Map> rows = putRows(tableId, tablePath, 10); + try (Connection connection = + ConnectionFactory.createConnection( + FLUSS_CLUSTER_EXTENSION.getClientConfig()); + TieringSplitReader tieringSplitReader = + createTieringReader(connection, lakeTieringFactory)) { + List splits = new ArrayList<>(); + Map expectedRowCount = new HashMap<>(); + for (Map.Entry> entry : rows.entrySet()) { + TableBucket tableBucket = entry.getKey(); + int rowCount = entry.getValue().size(); + splits.add( + createLogSplit( + tablePath, + tableId, + tableBucket.getBucket(), + EARLIEST_OFFSET, + rowCount)); + expectedRowCount.put(tableBucket, rowCount); + } + tieringSplitReader.handleSplitsChanges(new SplitsAddition<>(splits)); + verifyTieringRows( + tieringSplitReader, + tableId, + expectedRowCount, + splits.stream().map(FlinkTieringSplit::splitId).collect(Collectors.toSet())); + } + } + private TieringSplitReader createTieringReader(Connection connection) { final TieringMetrics tieringMetrics = new TieringMetrics( @@ -693,4 +763,57 @@ public TestingWriteResult complete() throws IOException { @Override public void close() throws IOException {} } + + /** + * A {@link TestingLakeTieringFactory} whose writers accept both rows and Arrow record batches, + * and that declares record batch support as configured. + */ + private static class RecordBatchCountingLakeTieringFactory extends TestingLakeTieringFactory { + + private final boolean supportsRecordBatchWrite; + private final List writers = new ArrayList<>(); + + private RecordBatchCountingLakeTieringFactory(boolean supportsRecordBatchWrite) { + this.supportsRecordBatchWrite = supportsRecordBatchWrite; + } + + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return supportsRecordBatchWrite; + } + + @Override + public LakeWriter createLakeWriter(WriterInitContext writerInitContext) + throws IOException { + RecordBatchCountingLakeWriter writer = new RecordBatchCountingLakeWriter(); + writers.add(writer); + return writer; + } + } + + private static class RecordBatchCountingLakeWriter + implements LakeWriter, SupportsRecordBatchWrite { + + private int writtenRows; + private int writtenBatchRows; + + @Override + public void write(LogRecord record) throws IOException { + writtenRows++; + } + + @Override + public void write(RecordBatch recordBatch) throws IOException { + writtenBatchRows += + ((ArrowRecordBatch) recordBatch).getArrowBatchData().getRecordCount(); + } + + @Override + public TestingWriteResult complete() throws IOException { + return new TestingWriteResult(writtenRows + writtenBatchRows); + } + + @Override + public void close() throws IOException {} + } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java index da607a4de35..4b3eda5dc15 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java @@ -24,6 +24,7 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.WriterInitContext; +import org.apache.fluss.metadata.TableInfo; import java.io.IOException; @@ -45,6 +46,11 @@ public LakeWriter createLakeWriter(WriterInitContext writerIn return new PaimonLakeWriter(paimonCatalogProvider, writerInitContext); } + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return true; + } + @Override public SimpleVersionedSerializer getWriteResultSerializer() { return new PaimonWriteResultSerializer(); From 3036ca5c0a593c705ae958db467b43da4d03a413 Mon Sep 17 00:00:00 2001 From: mikepapadim Date: Tue, 17 Nov 2026 15:20:12 +0000 Subject: [PATCH 2/2] [flink] Test that a non-Paimon lake format can opt in to batch writes Extract the record batch path decision of TieringSplitReader into a static method, and check it on an Iceberg TableInfo with a factory that declares batch support for Iceberg only. The test cluster creates only Paimon tables, so the existing positive test could not tell the capability check from a hard-coded Paimon check; this one fails with the latter. --- .../tiering/source/TieringSplitReader.java | 20 +++++-- .../source/TieringSplitReaderTest.java | 56 +++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java index 0a4fef9c61d..42ddb124ed4 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java @@ -438,13 +438,25 @@ private boolean useRecordBatchPath() { TableInfo tableInfo = checkNotNull(currentTable).getTableInfo(); currentTableUseRecordBatchPath = - unshadedArrowAvailable - && !tableInfo.hasPrimaryKey() - && tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW - && lakeTieringFactory.supportsRecordBatchWrite(tableInfo); + useRecordBatchPath(tableInfo, lakeTieringFactory, unshadedArrowAvailable); return currentTableUseRecordBatchPath; } + /** + * Whether a table is tiered batch by batch: an ARROW log table whose lake tiering factory + * declares record batch write support for it, whatever the table's lake format. + */ + @VisibleForTesting + static boolean useRecordBatchPath( + TableInfo tableInfo, + LakeTieringFactory lakeTieringFactory, + boolean unshadedArrowAvailable) { + return unshadedArrowAvailable + && !tableInfo.hasPrimaryKey() + && tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW + && lakeTieringFactory.supportsRecordBatchWrite(tableInfo); + } + /** * Generic template method for processing tiering log records. Encapsulates the shared workflow * of bucket traversal, stopping offset checks, LakeWriter management, offset/timestamp diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java index cdc58415d63..c08b65fa223 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java @@ -38,6 +38,7 @@ import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; @@ -577,6 +578,48 @@ void testArrowLogTableWritesRowsWhenFactoryDoesNotSupportRecordBatchWrite() thro }); } + @Test + void testNonPaimonLakeFormatCanOptInToRecordBatchWrite() { + // The test cluster only creates Paimon tables, so the decision is checked on a TableInfo + // of an Iceberg table, with a factory that declares batch support for Iceberg only. + TableInfo icebergLogTable = tableInfo(DataLakeFormat.ICEBERG, DEFAULT_LOG_TABLE_DESCRIPTOR); + TableInfo paimonLogTable = tableInfo(DataLakeFormat.PAIMON, DEFAULT_LOG_TABLE_DESCRIPTOR); + TestingLakeTieringFactory icebergBatchFactory = new IcebergBatchLakeTieringFactory(); + + assertThat( + TieringSplitReader.useRecordBatchPath( + icebergLogTable, icebergBatchFactory, true)) + .isTrue(); + assertThat(TieringSplitReader.useRecordBatchPath(paimonLogTable, icebergBatchFactory, true)) + .isFalse(); + // The capability is still gated on Arrow being on the class path and on log tables. + assertThat( + TieringSplitReader.useRecordBatchPath( + icebergLogTable, icebergBatchFactory, false)) + .isFalse(); + assertThat( + TieringSplitReader.useRecordBatchPath( + tableInfo(DataLakeFormat.ICEBERG, DEFAULT_PK_TABLE_DESCRIPTOR), + icebergBatchFactory, + true)) + .isFalse(); + } + + private static TableInfo tableInfo(DataLakeFormat format, TableDescriptor descriptor) { + Map properties = new HashMap<>(descriptor.getProperties()); + properties.put(ConfigOptions.TABLE_DATALAKE_FORMAT.key(), format.toString()); + TableDescriptor withFormat = descriptor.withProperties(properties); + long now = System.currentTimeMillis(); + return TableInfo.of( + TablePath.of("fluss", "tiering_" + format + "_table"), + 1L, + 1, + withFormat, + "/tmp/remote-data", + now, + now); + } + private void tierLogTableAndVerifyRows( TablePath tablePath, TestingLakeTieringFactory lakeTieringFactory) throws Exception { long tableId = createTable(tablePath, DEFAULT_LOG_TABLE_DESCRIPTOR); @@ -791,6 +834,19 @@ public LakeWriter createLakeWriter(WriterInitContext writerI } } + /** A {@link TestingLakeTieringFactory} that declares record batch support for Iceberg only. */ + private static class IcebergBatchLakeTieringFactory extends TestingLakeTieringFactory { + + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return tableInfo + .getTableConfig() + .getDataLakeFormat() + .map(DataLakeFormat.ICEBERG::equals) + .orElse(false); + } + } + private static class RecordBatchCountingLakeWriter implements LakeWriter, SupportsRecordBatchWrite {