diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java b/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java index 2d153aef16e..b3c4fb15df6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/writer/LakeTieringFactory.java @@ -22,6 +22,7 @@ import org.apache.fluss.lake.committer.LakeCommitter; import org.apache.fluss.lake.committer.PartitionMarkDoneCommitter; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; +import org.apache.fluss.metadata.TableInfo; import java.io.IOException; import java.io.Serializable; @@ -50,6 +51,21 @@ public interface LakeTieringFactory extends Serializa LakeWriter createLakeWriter(WriterInitContext writerInitContext) throws IOException; + /** + * Returns whether the lake writers of this factory accept Arrow record batches for the given + * table. If true, every {@link LakeWriter} created for the table must implement {@link + * SupportsRecordBatchWrite}, and the tiering job writes append-only tables in the ARROW log + * format batch by batch instead of row by row. + * + *

This check must not access lake resources. + * + * @param tableInfo the Fluss table to tier + * @return true if the lake writers accept Arrow record batches for the table + */ + default boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return false; + } + /** * Returns the serializer for write results. * diff --git a/fluss-flink/fluss-flink-common/pom.xml b/fluss-flink/fluss-flink-common/pom.xml index adfa268afc7..fcdf9afb1fe 100644 --- a/fluss-flink/fluss-flink-common/pom.xml +++ b/fluss-flink/fluss-flink-common/pom.xml @@ -200,6 +200,22 @@ test + + + org.apache.arrow + arrow-vector + ${arrow.version} + test + + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + test + + diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java index 7a2a56ad902..42ddb124ed4 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/tiering/source/TieringSplitReader.java @@ -40,7 +40,6 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; -import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PartitionInfo; import org.apache.fluss.metadata.TableBucket; @@ -430,7 +429,7 @@ private void mayCreateLogScanner() { /** * Determines whether the current table should use the Arrow record batch path for tiering. The * batch path is used when the table is an ARROW format append-only (log) table and the lake - * writer supports batch writing. + * tiering factory supports record batch writes for it. */ private boolean useRecordBatchPath() { if (currentTableUseRecordBatchPath != null) { @@ -439,14 +438,25 @@ private boolean useRecordBatchPath() { TableInfo tableInfo = checkNotNull(currentTable).getTableInfo(); currentTableUseRecordBatchPath = - unshadedArrowAvailable - && !tableInfo.hasPrimaryKey() - && tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW - && tableInfo.getTableConfig().getDataLakeFormat().orElse(null) - == DataLakeFormat.PAIMON; + useRecordBatchPath(tableInfo, lakeTieringFactory, unshadedArrowAvailable); return currentTableUseRecordBatchPath; } + /** + * Whether a table is tiered batch by batch: an ARROW log table whose lake tiering factory + * declares record batch write support for it, whatever the table's lake format. + */ + @VisibleForTesting + static boolean useRecordBatchPath( + TableInfo tableInfo, + LakeTieringFactory lakeTieringFactory, + boolean unshadedArrowAvailable) { + return unshadedArrowAvailable + && !tableInfo.hasPrimaryKey() + && tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW + && lakeTieringFactory.supportsRecordBatchWrite(tableInfo); + } + /** * Generic template method for processing tiering log records. Encapsulates the shared workflow * of bucket traversal, stopping offset checks, LakeWriter management, offset/timestamp diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java index cf7d30d4a7a..c08b65fa223 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/source/TieringSplitReaderTest.java @@ -33,11 +33,16 @@ import org.apache.fluss.flink.tiering.source.metrics.TieringMetrics; import org.apache.fluss.flink.tiering.source.split.FlinkTieringSplit; import org.apache.fluss.flink.utils.FlinkTestBase; +import org.apache.fluss.lake.batch.ArrowRecordBatch; +import org.apache.fluss.lake.batch.RecordBatch; import org.apache.fluss.lake.writer.LakeWriter; +import org.apache.fluss.lake.writer.SupportsRecordBatchWrite; import org.apache.fluss.lake.writer.WriterInitContext; +import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.LogRecord; import org.apache.fluss.row.InternalRow; @@ -539,6 +544,114 @@ void testTieringFirstRowMergeEngineFinishes() throws Exception { } } + @Test + void testArrowLogTableUsesRecordBatchWriteWhenFactorySupportsIt() throws Exception { + RecordBatchCountingLakeTieringFactory lakeTieringFactory = + new RecordBatchCountingLakeTieringFactory(true); + TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_batch_write"); + + tierLogTableAndVerifyRows(tablePath, lakeTieringFactory); + + assertThat(lakeTieringFactory.writers).isNotEmpty(); + assertThat(lakeTieringFactory.writers) + .allSatisfy( + writer -> { + assertThat(writer.writtenRows).isZero(); + assertThat(writer.writtenBatchRows).isPositive(); + }); + } + + @Test + void testArrowLogTableWritesRowsWhenFactoryDoesNotSupportRecordBatchWrite() throws Exception { + RecordBatchCountingLakeTieringFactory lakeTieringFactory = + new RecordBatchCountingLakeTieringFactory(false); + TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_row_write"); + + tierLogTableAndVerifyRows(tablePath, lakeTieringFactory); + + assertThat(lakeTieringFactory.writers).isNotEmpty(); + assertThat(lakeTieringFactory.writers) + .allSatisfy( + writer -> { + assertThat(writer.writtenRows).isPositive(); + assertThat(writer.writtenBatchRows).isZero(); + }); + } + + @Test + void testNonPaimonLakeFormatCanOptInToRecordBatchWrite() { + // The test cluster only creates Paimon tables, so the decision is checked on a TableInfo + // of an Iceberg table, with a factory that declares batch support for Iceberg only. + TableInfo icebergLogTable = tableInfo(DataLakeFormat.ICEBERG, DEFAULT_LOG_TABLE_DESCRIPTOR); + TableInfo paimonLogTable = tableInfo(DataLakeFormat.PAIMON, DEFAULT_LOG_TABLE_DESCRIPTOR); + TestingLakeTieringFactory icebergBatchFactory = new IcebergBatchLakeTieringFactory(); + + assertThat( + TieringSplitReader.useRecordBatchPath( + icebergLogTable, icebergBatchFactory, true)) + .isTrue(); + assertThat(TieringSplitReader.useRecordBatchPath(paimonLogTable, icebergBatchFactory, true)) + .isFalse(); + // The capability is still gated on Arrow being on the class path and on log tables. + assertThat( + TieringSplitReader.useRecordBatchPath( + icebergLogTable, icebergBatchFactory, false)) + .isFalse(); + assertThat( + TieringSplitReader.useRecordBatchPath( + tableInfo(DataLakeFormat.ICEBERG, DEFAULT_PK_TABLE_DESCRIPTOR), + icebergBatchFactory, + true)) + .isFalse(); + } + + private static TableInfo tableInfo(DataLakeFormat format, TableDescriptor descriptor) { + Map properties = new HashMap<>(descriptor.getProperties()); + properties.put(ConfigOptions.TABLE_DATALAKE_FORMAT.key(), format.toString()); + TableDescriptor withFormat = descriptor.withProperties(properties); + long now = System.currentTimeMillis(); + return TableInfo.of( + TablePath.of("fluss", "tiering_" + format + "_table"), + 1L, + 1, + withFormat, + "/tmp/remote-data", + now, + now); + } + + private void tierLogTableAndVerifyRows( + TablePath tablePath, TestingLakeTieringFactory lakeTieringFactory) throws Exception { + long tableId = createTable(tablePath, DEFAULT_LOG_TABLE_DESCRIPTOR); + Map> rows = putRows(tableId, tablePath, 10); + try (Connection connection = + ConnectionFactory.createConnection( + FLUSS_CLUSTER_EXTENSION.getClientConfig()); + TieringSplitReader tieringSplitReader = + createTieringReader(connection, lakeTieringFactory)) { + List splits = new ArrayList<>(); + Map expectedRowCount = new HashMap<>(); + for (Map.Entry> entry : rows.entrySet()) { + TableBucket tableBucket = entry.getKey(); + int rowCount = entry.getValue().size(); + splits.add( + createLogSplit( + tablePath, + tableId, + tableBucket.getBucket(), + EARLIEST_OFFSET, + rowCount)); + expectedRowCount.put(tableBucket, rowCount); + } + tieringSplitReader.handleSplitsChanges(new SplitsAddition<>(splits)); + verifyTieringRows( + tieringSplitReader, + tableId, + expectedRowCount, + splits.stream().map(FlinkTieringSplit::splitId).collect(Collectors.toSet())); + } + } + private TieringSplitReader createTieringReader(Connection connection) { final TieringMetrics tieringMetrics = new TieringMetrics( @@ -693,4 +806,70 @@ public TestingWriteResult complete() throws IOException { @Override public void close() throws IOException {} } + + /** + * A {@link TestingLakeTieringFactory} whose writers accept both rows and Arrow record batches, + * and that declares record batch support as configured. + */ + private static class RecordBatchCountingLakeTieringFactory extends TestingLakeTieringFactory { + + private final boolean supportsRecordBatchWrite; + private final List writers = new ArrayList<>(); + + private RecordBatchCountingLakeTieringFactory(boolean supportsRecordBatchWrite) { + this.supportsRecordBatchWrite = supportsRecordBatchWrite; + } + + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return supportsRecordBatchWrite; + } + + @Override + public LakeWriter createLakeWriter(WriterInitContext writerInitContext) + throws IOException { + RecordBatchCountingLakeWriter writer = new RecordBatchCountingLakeWriter(); + writers.add(writer); + return writer; + } + } + + /** A {@link TestingLakeTieringFactory} that declares record batch support for Iceberg only. */ + private static class IcebergBatchLakeTieringFactory extends TestingLakeTieringFactory { + + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return tableInfo + .getTableConfig() + .getDataLakeFormat() + .map(DataLakeFormat.ICEBERG::equals) + .orElse(false); + } + } + + private static class RecordBatchCountingLakeWriter + implements LakeWriter, SupportsRecordBatchWrite { + + private int writtenRows; + private int writtenBatchRows; + + @Override + public void write(LogRecord record) throws IOException { + writtenRows++; + } + + @Override + public void write(RecordBatch recordBatch) throws IOException { + writtenBatchRows += + ((ArrowRecordBatch) recordBatch).getArrowBatchData().getRecordCount(); + } + + @Override + public TestingWriteResult complete() throws IOException { + return new TestingWriteResult(writtenRows + writtenBatchRows); + } + + @Override + public void close() throws IOException {} + } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java index da607a4de35..4b3eda5dc15 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/tiering/PaimonLakeTieringFactory.java @@ -24,6 +24,7 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.lake.writer.LakeWriter; import org.apache.fluss.lake.writer.WriterInitContext; +import org.apache.fluss.metadata.TableInfo; import java.io.IOException; @@ -45,6 +46,11 @@ public LakeWriter createLakeWriter(WriterInitContext writerIn return new PaimonLakeWriter(paimonCatalogProvider, writerInitContext); } + @Override + public boolean supportsRecordBatchWrite(TableInfo tableInfo) { + return true; + } + @Override public SimpleVersionedSerializer getWriteResultSerializer() { return new PaimonWriteResultSerializer();