Skip to content

Commit 268a98f

Browse files
committed
feat: support scan planning and pushdown for Java table providers
Add an opt-in ScanTableProvider contract with immutable projection, expression-proto filters and limit requests. Plan a lazy TableScan before execution while retaining existing full-schema provider behavior. Forward DataFusion filter capabilities and preserve residual-filter/limit semantics without source-specific dependencies or DataFusion-core changes. Cover the Java/JNI/Arrow path with selective-source integration tests.
1 parent 57f0947 commit 268a98f

12 files changed

Lines changed: 854 additions & 57 deletions

File tree

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.Collections;
23+
import java.util.List;
24+
import java.util.Optional;
25+
import java.util.OptionalLong;
26+
27+
import org.apache.arrow.memory.BufferAllocator;
28+
import org.apache.arrow.vector.ipc.ArrowReader;
29+
import org.apache.datafusion.protobuf.LogicalExprNode;
30+
31+
/**
32+
* Optional scan planning and source pushdown for a Java-implemented {@link TableProvider}.
33+
*
34+
* <p>The Rust adapter forwards DataFusion's scan request without interpreting source predicates.
35+
* Implementations choose their own access path and return a lazy {@link TableScan}. Its readers
36+
* must produce the requested projected schema, including a zero-column schema when requested.
37+
*
38+
* <p>Planning and capability callbacks may run more than once and must not consume source rows.
39+
* Sources that cannot execute a requested access path may reject it during planning rather than
40+
* opening an unwanted unbounded scan.
41+
*
42+
* <p>Callbacks may run on native worker threads. Shared provider state and reader factories must be
43+
* thread-safe; each execution owns an independent reader.
44+
*/
45+
public interface ScanTableProvider extends TableProvider {
46+
/** DataFusion's per-filter pushdown contract. */
47+
enum FilterPushdown {
48+
/** The source does not apply this filter; DataFusion keeps it above the scan. */
49+
UNSUPPORTED,
50+
/** The source prunes candidates; DataFusion also applies the filter to the returned rows. */
51+
INEXACT,
52+
/** The source completely applies the filter with DataFusion-compatible semantics. */
53+
EXACT
54+
}
55+
56+
/**
57+
* Return one capability per input filter, in the same order. The default declines every filter.
58+
* An inexact filter must never discard a matching row.
59+
*/
60+
default List<FilterPushdown> supportsFiltersPushdown(List<LogicalExprNode> filters) {
61+
return Collections.nCopies(filters.size(), FilterPushdown.UNSUPPORTED);
62+
}
63+
64+
/** Plan a source scan without reading rows. Only supported/inexact filters are included. */
65+
TableScan planScan(TableScanRequest request);
66+
67+
/** Compatibility entry point for callers requesting a full-schema unfiltered scan. */
68+
@Override
69+
default ArrowReader scan(BufferAllocator allocator) {
70+
return planScan(new TableScanRequest(Optional.empty(), List.of(), OptionalLong.empty()))
71+
.execute(allocator);
72+
}
73+
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -559,10 +559,14 @@ public void registerUdf(ScalarUdf udf) {
559559
* reference {@code name} call back into {@code provider} to fetch batches.
560560
*
561561
* <p>{@link TableProvider#schema()} is called once here, on the calling thread, and cached on the
562-
* native side. {@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called once
563-
* per query that touches the table, on a Tokio worker thread; it must return a fresh, independent
564-
* {@link org.apache.arrow.vector.ipc.ArrowReader} on every call, with its buffers allocated from
565-
* the {@link org.apache.arrow.memory.BufferAllocator} the framework supplies.
562+
* native side. For base providers, {@link
563+
* TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called once per physical scan,
564+
* on a Tokio worker thread; it must return a fresh, independent {@link
565+
* org.apache.arrow.vector.ipc.ArrowReader} on every call, with its buffers allocated from the
566+
* {@link org.apache.arrow.memory.BufferAllocator} the framework supplies.
567+
*
568+
* <p>Implement {@link ScanTableProvider} to plan a lazy scan with projection, supported filters
569+
* and a limit supplied by DataFusion. Planning does not open its reader.
566570
*
567571
* <p>This is the Java counterpart to DataFusion's Rust {@code SessionContext::register_table}.
568572
*

‎core/src/main/java/org/apache/datafusion/TableProvider.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,8 @@
2626
/**
2727
* A Java-implemented table that can be registered with a {@link SessionContext} via {@link
2828
* SessionContext#registerTable(String, TableProvider)}. Mirrors the role of DataFusion's Rust
29-
* {@code TableProvider} trait, but at present only exposes the methods needed for a full table
30-
* scan; future versions may add filter/projection pushdown and multi-partition support as default
31-
* methods so existing implementations keep working.
29+
* {@code TableProvider} trait. This base interface provides a full-schema scan; implement {@link
30+
* ScanTableProvider} to opt into scan planning and projection/filter/limit pushdown.
3231
*
3332
* <p>{@link SimpleTableProvider} is a ready-made implementation for the common case of "I have a
3433
* schema and a function that returns an {@link ArrowReader}".
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.function.Function;
23+
24+
import org.apache.arrow.memory.BufferAllocator;
25+
import org.apache.arrow.vector.ipc.ArrowReader;
26+
27+
/**
28+
* A bounded, single-partition leaf scan planned by a {@link ScanTableProvider}.
29+
*
30+
* <p>Constructing the scan must not read source rows. Each execution opens a fresh independent
31+
* reader whose schema is the requested projection of the registered table schema. Reader buffers
32+
* must use the supplied allocator or a child of it. DataFusion owns and closes the reader.
33+
*
34+
* <p>This is a Java-backed scan, not an API for arbitrary child-consuming physical operators.
35+
*/
36+
public final class TableScan {
37+
private final Function<BufferAllocator, ArrowReader> readerFactory;
38+
39+
public TableScan(Function<BufferAllocator, ArrowReader> readerFactory) {
40+
if (readerFactory == null) {
41+
throw new IllegalArgumentException("readerFactory must be non-null");
42+
}
43+
this.readerFactory = readerFactory;
44+
}
45+
46+
/** Open one independent stream. Called by the framework during physical execution. */
47+
public ArrowReader execute(BufferAllocator allocator) {
48+
return readerFactory.apply(allocator);
49+
}
50+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.List;
23+
import java.util.Optional;
24+
import java.util.OptionalLong;
25+
26+
import org.apache.datafusion.protobuf.LogicalExprNode;
27+
28+
/**
29+
* A physical scan request using DataFusion's projection, expression and limit semantics.
30+
*
31+
* <p>Projection indices refer to the registered table schema. An absent projection means all
32+
* columns; a present empty list means no columns (for example, {@code COUNT(*)}), not no rows.
33+
* Filters use the same expression protobufs as DataFusion logical plans, not SQL strings.
34+
*
35+
* <p>The optional limit is the value supplied by DataFusion's planner. Sources may return more
36+
* rows; they must not return fewer than requested when that many matching rows are available.
37+
* DataFusion retains responsibility for deciding whether a limit can safely cross a residual
38+
* filter. Limits outside the non-negative Java {@code long} range are not forwarded; DataFusion
39+
* still applies the query's limit.
40+
*/
41+
public record TableScanRequest(
42+
Optional<List<Integer>> projection, List<LogicalExprNode> filters, OptionalLong limit) {
43+
public TableScanRequest {
44+
projection = projection.map(List::copyOf);
45+
filters = List.copyOf(filters);
46+
}
47+
}

‎core/src/main/java/org/apache/datafusion/internal/JniBridge.java‎

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
import java.util.ArrayList;
2323
import java.util.List;
24+
import java.util.Optional;
25+
import java.util.OptionalLong;
2426

2527
import org.apache.arrow.c.ArrowArray;
2628
import org.apache.arrow.c.ArrowArrayStream;
@@ -33,14 +35,21 @@
3335
import org.apache.datafusion.ColumnarValue;
3436
import org.apache.datafusion.ScalarFunction;
3537
import org.apache.datafusion.ScalarFunctionArgs;
38+
import org.apache.datafusion.ScanTableProvider;
3639
import org.apache.datafusion.TableProvider;
40+
import org.apache.datafusion.TableScan;
41+
import org.apache.datafusion.TableScanRequest;
42+
import org.apache.datafusion.protobuf.LogicalExprNode;
43+
import org.apache.datafusion.protobuf.TableScanOptionsProto;
3744

38-
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
45+
import com.google.protobuf.InvalidProtocolBufferException;
46+
47+
/** Internal trampolines for native UDF and table-provider callbacks. Not part of the public API. */
3948
public final class JniBridge {
4049
/**
41-
* Shared allocator for UDF inputs/outputs. Created once at class-load time; never closed.
42-
* Outstanding allocations are released by the FFI structs' release callbacks when the native side
43-
* drops them after {@code from_ffi}.
50+
* Shared allocator for UDF inputs/outputs and table scans. Created once at class-load time; never
51+
* closed. Outstanding allocations are released by the FFI structs' release callbacks when the
52+
* native side drops them after {@code from_ffi}.
4453
*/
4554
private static final RootAllocator ALLOCATOR = new RootAllocator();
4655

@@ -143,20 +152,74 @@ public static byte invokeScalarUdf(
143152
}
144153
}
145154

155+
/** Plan one physical scan without opening its reader. Called from native code. */
156+
public static TableScan planTableScan(
157+
TableProvider provider, byte[] optionsBytes, byte[][] filterProtos)
158+
throws InvalidProtocolBufferException {
159+
if (!(provider instanceof ScanTableProvider planned)) {
160+
return new TableScan(provider::scan);
161+
}
162+
TableScanOptionsProto options = TableScanOptionsProto.parseFrom(optionsBytes);
163+
Optional<List<Integer>> projection =
164+
options.hasProjection()
165+
? Optional.of(options.getProjection().getIndicesList())
166+
: Optional.empty();
167+
TableScan scan =
168+
planned.planScan(
169+
new TableScanRequest(
170+
projection,
171+
parseFilters(filterProtos),
172+
options.hasLimit() ? OptionalLong.of(options.getLimit()) : OptionalLong.empty()));
173+
if (scan == null) {
174+
throw new IllegalStateException("ScanTableProvider.planScan returned null");
175+
}
176+
return scan;
177+
}
178+
179+
/** Report per-expression capabilities using the native adapter's closed numeric contract. */
180+
public static int[] supportsTableFilterPushdown(TableProvider provider, byte[][] filterProtos)
181+
throws InvalidProtocolBufferException {
182+
List<ScanTableProvider.FilterPushdown> capabilities =
183+
((ScanTableProvider) provider).supportsFiltersPushdown(parseFilters(filterProtos));
184+
if (capabilities == null || capabilities.size() != filterProtos.length) {
185+
throw new IllegalArgumentException(
186+
"supportsFiltersPushdown must return one capability per filter");
187+
}
188+
int[] result = new int[capabilities.size()];
189+
for (int i = 0; i < result.length; i++) {
190+
result[i] =
191+
switch (capabilities.get(i)) {
192+
case UNSUPPORTED -> 1;
193+
case INEXACT -> 2;
194+
case EXACT -> 3;
195+
};
196+
}
197+
return result;
198+
}
199+
200+
private static List<LogicalExprNode> parseFilters(byte[][] filterProtos)
201+
throws InvalidProtocolBufferException {
202+
List<LogicalExprNode> filters = new ArrayList<>(filterProtos.length);
203+
for (byte[] filter : filterProtos) {
204+
filters.add(LogicalExprNode.parseFrom(filter));
205+
}
206+
return filters;
207+
}
208+
146209
/**
147210
* Open a fresh batch stream from a Java {@link TableProvider} and export it through the supplied
148211
* Arrow C Data Interface address. Called from native code; not for application use.
149212
*
150-
* <p>{@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called with {@link
213+
* <p>{@link TableScan#execute(org.apache.arrow.memory.BufferAllocator)} is called with {@link
151214
* #ALLOCATOR} so that the reader's buffers share the same allocator root required by {@link
152215
* Data#exportArrayStream}.
153216
*
154217
* <p>On success, ownership of the returned reader transfers to the FFI stream's release callback,
155218
* so the native side closing the stream also closes the reader. On any failure during export, the
156219
* reader is closed here before the exception propagates.
157220
*/
158-
public static void invokeTableScan(TableProvider provider, long ffiStreamAddr) {
159-
ArrowReader reader = provider.scan(ALLOCATOR);
221+
public static void invokeTableScan(TableScan scan, long ffiStreamAddr) {
222+
ArrowReader reader = scan.execute(ALLOCATOR);
160223
if (reader == null) {
161224
throw new IllegalStateException("TableProvider.scan returned null");
162225
}

0 commit comments

Comments
 (0)