Skip to content

Commit 1df2bd2

Browse files
committed
refactor(datasource)!: rename DataSource API to TableProvider
Address PR #65 review: align Java-side naming with DataFusion's Rust TableProvider trait and free up the DataSource name for the separate datafusion-datasource concept in the future. Add SimpleTableProvider as a convenience wrapper for the (schema, scan-fn) case. - DataSource -> TableProvider (Java interface) - SessionContext.registerDataSource -> registerTable - JniBridge.invokeDataSourceScan -> invokeTableScan - Native JavaDataSource struct + module renamed to JavaTableProvider / table_provider.rs; JNI entry point + signature updated accordingly - New SimpleTableProvider class wraps a Schema and a Function<BufferAllocator, ArrowReader> for the common no-pushdown case - Test, example, and user-guide docs updated to match
1 parent 82c740a commit 1df2bd2

10 files changed

Lines changed: 219 additions & 100 deletions

File tree

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -393,36 +393,38 @@ public void registerUdf(ScalarUdf udf) {
393393
}
394394

395395
/**
396-
* Register a Java-implemented data source as a table. SQL queries that reference {@code name}
397-
* call back into {@code source} to fetch batches.
396+
* Register a Java-implemented {@link TableProvider} under {@code name}. SQL queries that
397+
* reference {@code name} call back into {@code provider} to fetch batches.
398398
*
399-
* <p>{@link DataSource#schema()} is called once here, on the calling thread, and cached on the
400-
* native side. {@link DataSource#scan(org.apache.arrow.memory.BufferAllocator)} is called once
399+
* <p>{@link TableProvider#schema()} is called once here, on the calling thread, and cached on the
400+
* native side. {@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called once
401401
* per query that touches the table, on a Tokio worker thread; it must return a fresh, independent
402402
* {@link org.apache.arrow.vector.ipc.ArrowReader} on every call, with its buffers allocated from
403403
* the {@link org.apache.arrow.memory.BufferAllocator} the framework supplies.
404404
*
405-
* @throws IllegalArgumentException if {@code name} or {@code source} is {@code null}.
406-
* @throws IllegalStateException if {@code source.schema()} returns {@code null}, or this context
407-
* is closed.
405+
* <p>This is the Java counterpart to DataFusion's Rust {@code SessionContext::register_table}.
406+
*
407+
* @throws IllegalArgumentException if {@code name} or {@code provider} is {@code null}.
408+
* @throws IllegalStateException if {@code provider.schema()} returns {@code null}, or this
409+
* context is closed.
408410
* @throws RuntimeException if native registration fails.
409411
*/
410-
public void registerDataSource(String name, DataSource source) {
412+
public void registerTable(String name, TableProvider provider) {
411413
if (nativeHandle == 0) {
412414
throw new IllegalStateException("SessionContext is closed");
413415
}
414416
if (name == null) {
415-
throw new IllegalArgumentException("registerDataSource name must be non-null");
417+
throw new IllegalArgumentException("registerTable name must be non-null");
416418
}
417-
if (source == null) {
418-
throw new IllegalArgumentException("registerDataSource source must be non-null");
419+
if (provider == null) {
420+
throw new IllegalArgumentException("registerTable provider must be non-null");
419421
}
420-
Schema schema = source.schema();
422+
Schema schema = provider.schema();
421423
if (schema == null) {
422-
throw new IllegalStateException("DataSource.schema returned null");
424+
throw new IllegalStateException("TableProvider.schema returned null");
423425
}
424426
byte[] schemaIpc = serializeSchemaIpc(schema);
425-
registerDataSourceNative(nativeHandle, name, schemaIpc, source);
427+
registerTableNative(nativeHandle, name, schemaIpc, provider);
426428
}
427429

428430
private static byte[] serializeSchemaIpc(Schema schema) {
@@ -487,6 +489,6 @@ private static native long readJsonWithOptions(
487489
private static native void registerScalarUdf(
488490
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarFunction impl);
489491

490-
private static native void registerDataSourceNative(
491-
long handle, String name, byte[] schemaIpcBytes, DataSource source);
492+
private static native void registerTableNative(
493+
long handle, String name, byte[] schemaIpcBytes, TableProvider provider);
492494
}
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.function.Function;
23+
24+
import org.apache.arrow.memory.BufferAllocator;
25+
import org.apache.arrow.vector.ipc.ArrowReader;
26+
import org.apache.arrow.vector.types.pojo.Schema;
27+
28+
/**
29+
* A {@link TableProvider} that pairs a fixed {@link Schema} with a function that opens a fresh
30+
* {@link ArrowReader} for each scan. Provided as a convenience for the common case where there is
31+
* no projection / filter pushdown to implement.
32+
*
33+
* <p>Each call to {@link #scan(BufferAllocator)} invokes the supplied function and returns whatever
34+
* {@link ArrowReader} it produces, so the function MUST return a fresh, independent reader on every
35+
* invocation (see the contract on {@link TableProvider#scan(BufferAllocator)}).
36+
*
37+
* <p>As {@link TableProvider} grows additional methods in the future, this class will provide
38+
* defaults so existing callers keep working without changes.
39+
*/
40+
public final class SimpleTableProvider implements TableProvider {
41+
42+
private final Schema schema;
43+
private final Function<BufferAllocator, ArrowReader> scanFn;
44+
45+
/**
46+
* @param schema the table schema; returned as-is from {@link #schema()}
47+
* @param scanFn called on every {@link #scan(BufferAllocator)} with the framework-supplied
48+
* allocator; must return a fresh, independent {@link ArrowReader} each time
49+
*/
50+
public SimpleTableProvider(Schema schema, Function<BufferAllocator, ArrowReader> scanFn) {
51+
if (schema == null) {
52+
throw new IllegalArgumentException("schema must be non-null");
53+
}
54+
if (scanFn == null) {
55+
throw new IllegalArgumentException("scanFn must be non-null");
56+
}
57+
this.schema = schema;
58+
this.scanFn = scanFn;
59+
}
60+
61+
@Override
62+
public Schema schema() {
63+
return schema;
64+
}
65+
66+
@Override
67+
public ArrowReader scan(BufferAllocator allocator) {
68+
return scanFn.apply(allocator);
69+
}
70+
}

core/src/main/java/org/apache/datafusion/DataSource.java renamed to core/src/main/java/org/apache/datafusion/TableProvider.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,14 @@
2424
import org.apache.arrow.vector.types.pojo.Schema;
2525

2626
/**
27-
* A Java-implemented table that can be registered with a {@link SessionContext}.
27+
* A Java-implemented table that can be registered with a {@link SessionContext} via {@link
28+
* SessionContext#registerTable(String, TableProvider)}. Mirrors the role of DataFusion's Rust
29+
* {@code TableProvider} trait, but at present only exposes the methods needed for a full table
30+
* scan; future versions may add filter/projection pushdown and multi-partition support as default
31+
* methods so existing implementations keep working.
32+
*
33+
* <p>{@link SimpleTableProvider} is a ready-made implementation for the common case of "I have a
34+
* schema and a function that returns an {@link ArrowReader}".
2835
*
2936
* <p>Each call to {@link #scan(BufferAllocator)} must return a fresh, independent {@link
3037
* ArrowReader} so that queries which touch the table more than once (self-joins, {@code UNION ALL},
@@ -35,7 +42,7 @@
3542
* produced by every {@code ArrowReader} returned from {@link #scan(BufferAllocator)} must conform
3643
* to it; a mismatch fails the query.
3744
*/
38-
public interface DataSource {
45+
public interface TableProvider {
3946
/** The fixed schema of this table. Called once, at registration time. */
4047
Schema schema();
4148

‎core/src/main/java/org/apache/datafusion/internal/JniBridge.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
import org.apache.arrow.vector.VectorSchemaRoot;
3232
import org.apache.arrow.vector.ipc.ArrowReader;
3333
import org.apache.datafusion.ColumnarValue;
34-
import org.apache.datafusion.DataSource;
3534
import org.apache.datafusion.ScalarFunction;
3635
import org.apache.datafusion.ScalarFunctionArgs;
36+
import org.apache.datafusion.TableProvider;
3737

3838
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
3939
public final class JniBridge {
@@ -144,21 +144,21 @@ public static byte invokeScalarUdf(
144144
}
145145

146146
/**
147-
* Open a fresh batch stream from a Java {@link DataSource} and export it through the supplied
147+
* Open a fresh batch stream from a Java {@link TableProvider} and export it through the supplied
148148
* Arrow C Data Interface address. Called from native code; not for application use.
149149
*
150-
* <p>{@link DataSource#scan(org.apache.arrow.memory.BufferAllocator)} is called with {@link
150+
* <p>{@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called with {@link
151151
* #ALLOCATOR} so that the reader's buffers share the same allocator root required by {@link
152152
* Data#exportArrayStream}.
153153
*
154154
* <p>On success, ownership of the returned reader transfers to the FFI stream's release callback,
155155
* so the native side closing the stream also closes the reader. On any failure during export, the
156156
* reader is closed here before the exception propagates.
157157
*/
158-
public static void invokeDataSourceScan(DataSource source, long ffiStreamAddr) {
159-
ArrowReader reader = source.scan(ALLOCATOR);
158+
public static void invokeTableScan(TableProvider provider, long ffiStreamAddr) {
159+
ArrowReader reader = provider.scan(ALLOCATOR);
160160
if (reader == null) {
161-
throw new IllegalStateException("DataSource.scan returned null");
161+
throw new IllegalStateException("TableProvider.scan returned null");
162162
}
163163
ArrowArrayStream stream = ArrowArrayStream.wrap(ffiStreamAddr);
164164
try {

0 commit comments

Comments
 (0)