Skip to content

Commit 4bd2ad1

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/dataframe-cancellation
2 parents 05f0d3d + 89d5496 commit 4bd2ad1

22 files changed

Lines changed: 1865 additions & 46 deletions

File tree

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
382382
writeCsvWithOptions(nativeHandle, path, options.toBytes());
383383
}
384384

385+
/**
386+
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
387+
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
388+
* receiver remains usable and must still be closed independently.
389+
*
390+
* @throws RuntimeException if the write fails.
391+
*/
392+
public void writeJson(String path) {
393+
writeJson(path, new JsonWriteOptions());
394+
}
395+
396+
/**
397+
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
398+
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
399+
*
400+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
401+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
402+
* etc.).
403+
*/
404+
public void writeJson(String path, JsonWriteOptions options) {
405+
if (nativeHandle == 0) {
406+
throw new IllegalStateException("DataFrame is closed or already collected");
407+
}
408+
if (path == null) {
409+
throw new IllegalArgumentException("writeJson path must be non-null");
410+
}
411+
if (options == null) {
412+
throw new IllegalArgumentException("writeJson options must be non-null");
413+
}
414+
writeJsonWithOptions(nativeHandle, path, options.toBytes());
415+
}
416+
385417
@Override
386418
public void close() {
387419
if (nativeHandle != 0) {
@@ -427,4 +459,6 @@ private static native void writeParquetWithOptions(
427459
boolean singleFileOutputValue);
428460

429461
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
462+
463+
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
430464
}
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Configuration knobs for writing JSON, passed to {@link DataFrame#writeJson(String,
27+
* JsonWriteOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
30+
* JsonOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
31+
* null} or empty (meaning the DataFusion default is used).
32+
*
33+
* <p>Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
34+
* {@code writeJson} is the literal output filename. When left unset (the default) and there are no
35+
* partition columns, the path is treated as a directory that DataFusion populates with one or more
36+
* part-files.
37+
*
38+
* <p>The output is always newline-delimited JSON (NDJSON). DataFusion's JSON writer does not emit
39+
* the bracketed array form, so there is no toggle for it here.
40+
*
41+
* <p>Compression reuses {@link FileCompressionType} -- the same codec set ({@code UNCOMPRESSED},
42+
* {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}) the read side and the CSV writer accept.
43+
*/
44+
public final class JsonWriteOptions {
45+
46+
private Boolean singleFileOutput;
47+
private final List<String> partitionCols = new ArrayList<>();
48+
private FileCompressionType fileCompressionType;
49+
50+
/**
51+
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
52+
* and no partition columns are configured, the path is treated as a directory and DataFusion
53+
* writes one or more part-files.
54+
*/
55+
public JsonWriteOptions singleFileOutput(boolean v) {
56+
this.singleFileOutput = v;
57+
return this;
58+
}
59+
60+
/**
61+
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
62+
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
63+
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
64+
*/
65+
public JsonWriteOptions partitionCols(String... cols) {
66+
this.partitionCols.clear();
67+
for (String c : cols) {
68+
this.partitionCols.add(c);
69+
}
70+
return this;
71+
}
72+
73+
/** Output compression codec. Defaults to uncompressed. */
74+
public JsonWriteOptions fileCompressionType(FileCompressionType t) {
75+
this.fileCompressionType = t;
76+
return this;
77+
}
78+
79+
byte[] toBytes() {
80+
org.apache.datafusion.protobuf.JsonWriteOptionsProto.Builder b =
81+
org.apache.datafusion.protobuf.JsonWriteOptionsProto.newBuilder();
82+
if (singleFileOutput != null) {
83+
b.setSingleFileOutput(singleFileOutput);
84+
}
85+
b.addAllPartitionCols(partitionCols);
86+
if (fileCompressionType != null) {
87+
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
88+
}
89+
return b.build().toByteArray();
90+
}
91+
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,41 @@ public void registerUdf(ScalarUdf udf) {
474474
registerScalarUdf(nativeHandle, name, signatureBytes, volatility.code(), impl);
475475
}
476476

477+
/**
478+
* Register a Java-implemented {@link TableProvider} under {@code name}. SQL queries that
479+
* reference {@code name} call back into {@code provider} to fetch batches.
480+
*
481+
* <p>{@link TableProvider#schema()} is called once here, on the calling thread, and cached on the
482+
* native side. {@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called once
483+
* per query that touches the table, on a Tokio worker thread; it must return a fresh, independent
484+
* {@link org.apache.arrow.vector.ipc.ArrowReader} on every call, with its buffers allocated from
485+
* the {@link org.apache.arrow.memory.BufferAllocator} the framework supplies.
486+
*
487+
* <p>This is the Java counterpart to DataFusion's Rust {@code SessionContext::register_table}.
488+
*
489+
* @throws IllegalArgumentException if {@code name} or {@code provider} is {@code null}.
490+
* @throws IllegalStateException if {@code provider.schema()} returns {@code null}, or this
491+
* context is closed.
492+
* @throws RuntimeException if native registration fails.
493+
*/
494+
public void registerTable(String name, TableProvider provider) {
495+
if (nativeHandle == 0) {
496+
throw new IllegalStateException("SessionContext is closed");
497+
}
498+
if (name == null) {
499+
throw new IllegalArgumentException("registerTable name must be non-null");
500+
}
501+
if (provider == null) {
502+
throw new IllegalArgumentException("registerTable provider must be non-null");
503+
}
504+
Schema schema = provider.schema();
505+
if (schema == null) {
506+
throw new IllegalStateException("TableProvider.schema returned null");
507+
}
508+
byte[] schemaIpc = serializeSchemaIpc(schema);
509+
registerTableNative(nativeHandle, name, schemaIpc, provider);
510+
}
511+
477512
private static byte[] serializeSchemaIpc(Schema schema) {
478513
ByteArrayOutputStream baos = new ByteArrayOutputStream();
479514
try (BufferAllocator allocator = new RootAllocator();
@@ -543,4 +578,7 @@ private static native void registerScalarUdf(
543578
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarFunction impl);
544579

545580
private static native long createCancellationToken();
581+
582+
private static native void registerTableNative(
583+
long handle, String name, byte[] schemaIpcBytes, TableProvider provider);
546584
}
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.function.Function;
23+
24+
import org.apache.arrow.memory.BufferAllocator;
25+
import org.apache.arrow.vector.ipc.ArrowReader;
26+
import org.apache.arrow.vector.types.pojo.Schema;
27+
28+
/**
29+
* A {@link TableProvider} that pairs a fixed {@link Schema} with a function that opens a fresh
30+
* {@link ArrowReader} for each scan. Provided as a convenience for the common case where there is
31+
* no projection / filter pushdown to implement.
32+
*
33+
* <p>Each call to {@link #scan(BufferAllocator)} invokes the supplied function and returns whatever
34+
* {@link ArrowReader} it produces, so the function MUST return a fresh, independent reader on every
35+
* invocation (see the contract on {@link TableProvider#scan(BufferAllocator)}).
36+
*
37+
* <p>As {@link TableProvider} grows additional methods in the future, this class will provide
38+
* defaults so existing callers keep working without changes.
39+
*/
40+
public final class SimpleTableProvider implements TableProvider {
41+
42+
private final Schema schema;
43+
private final Function<BufferAllocator, ArrowReader> scanFn;
44+
45+
/**
46+
* @param schema the table schema; returned as-is from {@link #schema()}
47+
* @param scanFn called on every {@link #scan(BufferAllocator)} with the framework-supplied
48+
* allocator; must return a fresh, independent {@link ArrowReader} each time
49+
*/
50+
public SimpleTableProvider(Schema schema, Function<BufferAllocator, ArrowReader> scanFn) {
51+
if (schema == null) {
52+
throw new IllegalArgumentException("schema must be non-null");
53+
}
54+
if (scanFn == null) {
55+
throw new IllegalArgumentException("scanFn must be non-null");
56+
}
57+
this.schema = schema;
58+
this.scanFn = scanFn;
59+
}
60+
61+
@Override
62+
public Schema schema() {
63+
return schema;
64+
}
65+
66+
@Override
67+
public ArrowReader scan(BufferAllocator allocator) {
68+
return scanFn.apply(allocator);
69+
}
70+
}
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.memory.BufferAllocator;
23+
import org.apache.arrow.vector.ipc.ArrowReader;
24+
import org.apache.arrow.vector.types.pojo.Schema;
25+
26+
/**
27+
* A Java-implemented table that can be registered with a {@link SessionContext} via {@link
28+
* SessionContext#registerTable(String, TableProvider)}. Mirrors the role of DataFusion's Rust
29+
* {@code TableProvider} trait, but at present only exposes the methods needed for a full table
30+
* scan; future versions may add filter/projection pushdown and multi-partition support as default
31+
* methods so existing implementations keep working.
32+
*
33+
* <p>{@link SimpleTableProvider} is a ready-made implementation for the common case of "I have a
34+
* schema and a function that returns an {@link ArrowReader}".
35+
*
36+
* <p>Each call to {@link #scan(BufferAllocator)} must return a fresh, independent {@link
37+
* ArrowReader} so that queries which touch the table more than once (self-joins, {@code UNION ALL},
38+
* repeated reads) work correctly. The returned reader is closed by the framework when the stream
39+
* ends.
40+
*
41+
* <p>The schema returned by {@link #schema()} is captured once at registration time. Every batch
42+
* produced by every {@code ArrowReader} returned from {@link #scan(BufferAllocator)} must conform
43+
* to it; a mismatch fails the query.
44+
*/
45+
public interface TableProvider {
46+
/** The fixed schema of this table. Called once, at registration time. */
47+
Schema schema();
48+
49+
/**
50+
* Open a fresh batch stream for this table. Called once per physical scan of the table — a single
51+
* query may invoke this more than once (self-joins, {@code UNION ALL} over the same table, etc.).
52+
*
53+
* <p>Each invocation MUST return an independent {@link ArrowReader}. The reader's schema MUST
54+
* equal {@link #schema()}. The reader's buffers MUST be allocated from {@code allocator} (or from
55+
* a child of it) — the framework needs the reader's allocator hierarchy to share a root with the
56+
* one it passes here. The allocator contract mirrors the one on {@link ScalarFunction#evaluate}.
57+
*/
58+
ArrowReader scan(BufferAllocator allocator);
59+
}

‎core/src/main/java/org/apache/datafusion/internal/JniBridge.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,17 @@
2323
import java.util.List;
2424

2525
import org.apache.arrow.c.ArrowArray;
26+
import org.apache.arrow.c.ArrowArrayStream;
2627
import org.apache.arrow.c.ArrowSchema;
2728
import org.apache.arrow.c.Data;
2829
import org.apache.arrow.memory.RootAllocator;
2930
import org.apache.arrow.vector.FieldVector;
3031
import org.apache.arrow.vector.VectorSchemaRoot;
32+
import org.apache.arrow.vector.ipc.ArrowReader;
3133
import org.apache.datafusion.ColumnarValue;
3234
import org.apache.datafusion.ScalarFunction;
3335
import org.apache.datafusion.ScalarFunctionArgs;
36+
import org.apache.datafusion.TableProvider;
3437

3538
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
3639
public final class JniBridge {
@@ -139,4 +142,34 @@ public static byte invokeScalarUdf(
139142
return resultKind;
140143
}
141144
}
145+
146+
/**
147+
* Open a fresh batch stream from a Java {@link TableProvider} and export it through the supplied
148+
* Arrow C Data Interface address. Called from native code; not for application use.
149+
*
150+
* <p>{@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called with {@link
151+
* #ALLOCATOR} so that the reader's buffers share the same allocator root required by {@link
152+
* Data#exportArrayStream}.
153+
*
154+
* <p>On success, ownership of the returned reader transfers to the FFI stream's release callback,
155+
* so the native side closing the stream also closes the reader. On any failure during export, the
156+
* reader is closed here before the exception propagates.
157+
*/
158+
public static void invokeTableScan(TableProvider provider, long ffiStreamAddr) {
159+
ArrowReader reader = provider.scan(ALLOCATOR);
160+
if (reader == null) {
161+
throw new IllegalStateException("TableProvider.scan returned null");
162+
}
163+
ArrowArrayStream stream = ArrowArrayStream.wrap(ffiStreamAddr);
164+
try {
165+
Data.exportArrayStream(ALLOCATOR, reader, stream);
166+
} catch (Throwable t) {
167+
try {
168+
reader.close();
169+
} catch (Exception ignored) {
170+
// best-effort cleanup; original error wins
171+
}
172+
throw t;
173+
}
174+
}
142175
}

0 commit comments

Comments
 (0)