Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.fluss.lake.committer.LakeCommitter;
import org.apache.fluss.lake.committer.PartitionMarkDoneCommitter;
import org.apache.fluss.lake.serializer.SimpleVersionedSerializer;
import org.apache.fluss.metadata.TableInfo;

import java.io.IOException;
import java.io.Serializable;
Expand Down Expand Up @@ -50,6 +51,21 @@ public interface LakeTieringFactory<WriteResult, CommittableT> extends Serializa
LakeWriter<WriteResult> createLakeWriter(WriterInitContext writerInitContext)
throws IOException;

/**
* Returns whether the lake writers of this factory accept Arrow record batches for the given
* table. If true, every {@link LakeWriter} created for the table must implement {@link
* SupportsRecordBatchWrite}, and the tiering job writes append-only tables in the ARROW log
* format batch by batch instead of row by row.
*
* <p>This check must not access lake resources.
*
* @param tableInfo the Fluss table to tier
* @return true if the lake writers accept Arrow record batches for the table
*/
default boolean supportsRecordBatchWrite(TableInfo tableInfo) {
return false;
}

/**
* Returns the serializer for write results.
*
Expand Down
16 changes: 16 additions & 0 deletions fluss-flink/fluss-flink-common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,22 @@
<scope>test</scope>
</dependency>

<!-- Unshaded Arrow, which the tiering job needs to take the Arrow record batch path.
At runtime a lake plugin supplies it (e.g. paimon-arrow). -->
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
<version>${arrow.version}</version>
<scope>test</scope>
</dependency>

</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.apache.fluss.lake.writer.LakeTieringFactory;
import org.apache.fluss.lake.writer.LakeWriter;
import org.apache.fluss.lake.writer.SupportsRecordBatchWrite;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PartitionInfo;
import org.apache.fluss.metadata.TableBucket;
Expand Down Expand Up @@ -430,7 +429,7 @@ private void mayCreateLogScanner() {
/**
* Determines whether the current table should use the Arrow record batch path for tiering. The
* batch path is used when the table is an ARROW format append-only (log) table and the lake
* writer supports batch writing.
* tiering factory supports record batch writes for it.
*/
private boolean useRecordBatchPath() {
if (currentTableUseRecordBatchPath != null) {
Expand All @@ -439,14 +438,25 @@ private boolean useRecordBatchPath() {
TableInfo tableInfo = checkNotNull(currentTable).getTableInfo();

currentTableUseRecordBatchPath =
unshadedArrowAvailable
&& !tableInfo.hasPrimaryKey()
&& tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW
&& tableInfo.getTableConfig().getDataLakeFormat().orElse(null)
== DataLakeFormat.PAIMON;
useRecordBatchPath(tableInfo, lakeTieringFactory, unshadedArrowAvailable);
return currentTableUseRecordBatchPath;
}

/**
* Whether a table is tiered batch by batch: an ARROW log table whose lake tiering factory
* declares record batch write support for it, whatever the table's lake format.
*/
@VisibleForTesting
static boolean useRecordBatchPath(
TableInfo tableInfo,
LakeTieringFactory<?, ?> lakeTieringFactory,
boolean unshadedArrowAvailable) {
return unshadedArrowAvailable
&& !tableInfo.hasPrimaryKey()
&& tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW
&& lakeTieringFactory.supportsRecordBatchWrite(tableInfo);
}

/**
* Generic template method for processing tiering log records. Encapsulates the shared workflow
* of bucket traversal, stopping offset checks, LakeWriter management, offset/timestamp
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,16 @@
import org.apache.fluss.flink.tiering.source.metrics.TieringMetrics;
import org.apache.fluss.flink.tiering.source.split.FlinkTieringSplit;
import org.apache.fluss.flink.utils.FlinkTestBase;
import org.apache.fluss.lake.batch.ArrowRecordBatch;
import org.apache.fluss.lake.batch.RecordBatch;
import org.apache.fluss.lake.writer.LakeWriter;
import org.apache.fluss.lake.writer.SupportsRecordBatchWrite;
import org.apache.fluss.lake.writer.WriterInitContext;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.row.InternalRow;
Expand Down Expand Up @@ -539,6 +544,114 @@ void testTieringFirstRowMergeEngineFinishes() throws Exception {
}
}

@Test
void testArrowLogTableUsesRecordBatchWriteWhenFactorySupportsIt() throws Exception {
RecordBatchCountingLakeTieringFactory lakeTieringFactory =
new RecordBatchCountingLakeTieringFactory(true);
TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_batch_write");

tierLogTableAndVerifyRows(tablePath, lakeTieringFactory);

assertThat(lakeTieringFactory.writers).isNotEmpty();
assertThat(lakeTieringFactory.writers)
.allSatisfy(
writer -> {
assertThat(writer.writtenRows).isZero();
assertThat(writer.writtenBatchRows).isPositive();
});
}

@Test
void testArrowLogTableWritesRowsWhenFactoryDoesNotSupportRecordBatchWrite() throws Exception {
RecordBatchCountingLakeTieringFactory lakeTieringFactory =
new RecordBatchCountingLakeTieringFactory(false);
TablePath tablePath = TablePath.of("fluss", "tiering_arrow_log_table_row_write");

tierLogTableAndVerifyRows(tablePath, lakeTieringFactory);

assertThat(lakeTieringFactory.writers).isNotEmpty();
assertThat(lakeTieringFactory.writers)
.allSatisfy(
writer -> {
assertThat(writer.writtenRows).isPositive();
assertThat(writer.writtenBatchRows).isZero();
});
}

@Test
void testNonPaimonLakeFormatCanOptInToRecordBatchWrite() {
// The test cluster only creates Paimon tables, so the decision is checked on a TableInfo
// of an Iceberg table, with a factory that declares batch support for Iceberg only.
TableInfo icebergLogTable = tableInfo(DataLakeFormat.ICEBERG, DEFAULT_LOG_TABLE_DESCRIPTOR);
TableInfo paimonLogTable = tableInfo(DataLakeFormat.PAIMON, DEFAULT_LOG_TABLE_DESCRIPTOR);
TestingLakeTieringFactory icebergBatchFactory = new IcebergBatchLakeTieringFactory();

assertThat(
TieringSplitReader.useRecordBatchPath(
icebergLogTable, icebergBatchFactory, true))
.isTrue();
assertThat(TieringSplitReader.useRecordBatchPath(paimonLogTable, icebergBatchFactory, true))
.isFalse();
// The capability is still gated on Arrow being on the class path and on log tables.
assertThat(
TieringSplitReader.useRecordBatchPath(
icebergLogTable, icebergBatchFactory, false))
.isFalse();
assertThat(
TieringSplitReader.useRecordBatchPath(
tableInfo(DataLakeFormat.ICEBERG, DEFAULT_PK_TABLE_DESCRIPTOR),
icebergBatchFactory,
true))
.isFalse();
}

private static TableInfo tableInfo(DataLakeFormat format, TableDescriptor descriptor) {
Map<String, String> properties = new HashMap<>(descriptor.getProperties());
properties.put(ConfigOptions.TABLE_DATALAKE_FORMAT.key(), format.toString());
TableDescriptor withFormat = descriptor.withProperties(properties);
long now = System.currentTimeMillis();
return TableInfo.of(
TablePath.of("fluss", "tiering_" + format + "_table"),
1L,
1,
withFormat,
"/tmp/remote-data",
now,
now);
}

private void tierLogTableAndVerifyRows(
TablePath tablePath, TestingLakeTieringFactory lakeTieringFactory) throws Exception {
long tableId = createTable(tablePath, DEFAULT_LOG_TABLE_DESCRIPTOR);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we make the positive test use a non-Paimon lake format? DEFAULT_LOG_TABLE_DESCRIPTOR is configured for Paimon, so the previous hard-coded format check would also take the batch path here. The negative test covers the fallback, but we still need a test showing that a non-Paimon factory can opt in to batch writes.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good point. The test cluster only accepts tables whose lake format matches the cluster's (Paimon here), so I couldn't create an Iceberg table end to end. Instead, 3036ca5 extracts the decision into TieringSplitReader#useRecordBatchPath(TableInfo, LakeTieringFactory, boolean) and adds testNonPaimonLakeFormatCanOptInToRecordBatchWrite. It uses an Iceberg TableInfo and a factory that enables batch writes for Iceberg only, and checks that the Iceberg log table takes the batch path while a Paimon table does not, and that the Arrow and primary-key conditions still apply. With the old hard-coded Paimon check in place, this test fails.

Map<TableBucket, List<InternalRow>> rows = putRows(tableId, tablePath, 10);
try (Connection connection =
ConnectionFactory.createConnection(
FLUSS_CLUSTER_EXTENSION.getClientConfig());
TieringSplitReader<TestingWriteResult> tieringSplitReader =
createTieringReader(connection, lakeTieringFactory)) {
List<FlinkTieringSplit> splits = new ArrayList<>();
Map<TableBucket, Integer> expectedRowCount = new HashMap<>();
for (Map.Entry<TableBucket, List<InternalRow>> entry : rows.entrySet()) {
TableBucket tableBucket = entry.getKey();
int rowCount = entry.getValue().size();
splits.add(
createLogSplit(
tablePath,
tableId,
tableBucket.getBucket(),
EARLIEST_OFFSET,
rowCount));
expectedRowCount.put(tableBucket, rowCount);
}
tieringSplitReader.handleSplitsChanges(new SplitsAddition<>(splits));
verifyTieringRows(
tieringSplitReader,
tableId,
expectedRowCount,
splits.stream().map(FlinkTieringSplit::splitId).collect(Collectors.toSet()));
}
}

private TieringSplitReader<TestingWriteResult> createTieringReader(Connection connection) {
final TieringMetrics tieringMetrics =
new TieringMetrics(
Expand Down Expand Up @@ -693,4 +806,70 @@ public TestingWriteResult complete() throws IOException {
@Override
public void close() throws IOException {}
}

/**
* A {@link TestingLakeTieringFactory} whose writers accept both rows and Arrow record batches,
* and that declares record batch support as configured.
*/
private static class RecordBatchCountingLakeTieringFactory extends TestingLakeTieringFactory {

private final boolean supportsRecordBatchWrite;
private final List<RecordBatchCountingLakeWriter> writers = new ArrayList<>();

private RecordBatchCountingLakeTieringFactory(boolean supportsRecordBatchWrite) {
this.supportsRecordBatchWrite = supportsRecordBatchWrite;
}

@Override
public boolean supportsRecordBatchWrite(TableInfo tableInfo) {
return supportsRecordBatchWrite;
}

@Override
public LakeWriter<TestingWriteResult> createLakeWriter(WriterInitContext writerInitContext)
throws IOException {
RecordBatchCountingLakeWriter writer = new RecordBatchCountingLakeWriter();
writers.add(writer);
return writer;
}
}

/** A {@link TestingLakeTieringFactory} that declares record batch support for Iceberg only. */
private static class IcebergBatchLakeTieringFactory extends TestingLakeTieringFactory {

@Override
public boolean supportsRecordBatchWrite(TableInfo tableInfo) {
return tableInfo
.getTableConfig()
.getDataLakeFormat()
.map(DataLakeFormat.ICEBERG::equals)
.orElse(false);
}
}

private static class RecordBatchCountingLakeWriter
implements LakeWriter<TestingWriteResult>, SupportsRecordBatchWrite {

private int writtenRows;
private int writtenBatchRows;

@Override
public void write(LogRecord record) throws IOException {
writtenRows++;
}

@Override
public void write(RecordBatch recordBatch) throws IOException {
writtenBatchRows +=
((ArrowRecordBatch) recordBatch).getArrowBatchData().getRecordCount();
}

@Override
public TestingWriteResult complete() throws IOException {
return new TestingWriteResult(writtenRows + writtenBatchRows);
}

@Override
public void close() throws IOException {}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.fluss.lake.writer.LakeTieringFactory;
import org.apache.fluss.lake.writer.LakeWriter;
import org.apache.fluss.lake.writer.WriterInitContext;
import org.apache.fluss.metadata.TableInfo;

import java.io.IOException;

Expand All @@ -45,6 +46,11 @@ public LakeWriter<PaimonWriteResult> createLakeWriter(WriterInitContext writerIn
return new PaimonLakeWriter(paimonCatalogProvider, writerInitContext);
}

@Override
public boolean supportsRecordBatchWrite(TableInfo tableInfo) {
return true;
}

@Override
public SimpleVersionedSerializer<PaimonWriteResult> getWriteResultSerializer() {
return new PaimonWriteResultSerializer();
Expand Down