Skip to content

Commit d94d946

Browse files
committed
feat(write): add DataFrame.writeParquet with native binding
1 parent aefdd9f commit d94d946

3 files changed

Lines changed: 148 additions & 0 deletions

File tree

‎native/src/lib.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ use datafusion::arrow::datatypes::{Schema, SchemaRef};
2525
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
2626
use datafusion::arrow::ipc::reader::StreamReader;
2727
use datafusion::arrow::record_batch::RecordBatchIterator;
28+
use datafusion::config::TableParquetOptions;
2829
use datafusion::dataframe::DataFrame;
30+
use datafusion::dataframe::DataFrameWriteOptions;
2931
use datafusion::error::DataFusionError;
3032
use datafusion::prelude::{ParquetReadOptions, SessionContext};
3133
use jni::objects::{JByteArray, JClass, JObjectArray, JString};
@@ -195,6 +197,42 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_filterRows<'local>(
195197
})
196198
}
197199

200+
#[no_mangle]
201+
pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeParquetWithOptions<'local>(
202+
mut env: JNIEnv<'local>,
203+
_class: JClass<'local>,
204+
handle: jlong,
205+
path: JString<'local>,
206+
compression: JString<'local>,
207+
single_file_output_set: jboolean,
208+
single_file_output_value: jboolean,
209+
) {
210+
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
211+
if handle == 0 {
212+
return Err("DataFrame handle is null".into());
213+
}
214+
let df = unsafe { &*(handle as *const DataFrame) }.clone();
215+
let path: String = env.get_string(&path)?.into();
216+
217+
let mut write_opts = DataFrameWriteOptions::new();
218+
if single_file_output_set != 0 {
219+
write_opts = write_opts.with_single_file_output(single_file_output_value != 0);
220+
}
221+
222+
let writer_opts: Option<TableParquetOptions> = if !compression.is_null() {
223+
let c: String = env.get_string(&compression)?.into();
224+
let mut tpo = TableParquetOptions::default();
225+
tpo.global.compression = Some(c);
226+
Some(tpo)
227+
} else {
228+
None
229+
};
230+
231+
runtime().block_on(df.write_parquet(&path, write_opts, writer_opts))?;
232+
Ok(())
233+
})
234+
}
235+
198236
#[no_mangle]
199237
pub extern "system" fn Java_org_apache_datafusion_DataFrame_closeDataFrame<'local>(
200238
mut env: JNIEnv<'local>,

‎src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,36 @@ public DataFrame filter(String predicate) {
116116
return new DataFrame(filterRows(nativeHandle, predicate));
117117
}
118118

119+
/**
120+
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
121+
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
122+
* remains usable and must still be closed independently.
123+
*
124+
* @throws RuntimeException if the write fails.
125+
*/
126+
public void writeParquet(String path) {
127+
writeParquet(path, new ParquetWriteOptions());
128+
}
129+
130+
/**
131+
* Materialize this DataFrame as Parquet at {@code path} with the supplied {@link
132+
* ParquetWriteOptions}. The receiver remains usable and must still be closed independently.
133+
*
134+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
135+
* etc.).
136+
*/
137+
public void writeParquet(String path, ParquetWriteOptions options) {
138+
if (nativeHandle == 0) {
139+
throw new IllegalStateException("DataFrame is closed or already collected");
140+
}
141+
writeParquetWithOptions(
142+
nativeHandle,
143+
path,
144+
options.compression(),
145+
options.singleFileOutput() != null,
146+
options.singleFileOutput() != null && options.singleFileOutput());
147+
}
148+
119149
@Override
120150
public void close() {
121151
if (nativeHandle != 0) {
@@ -137,4 +167,11 @@ public void close() {
137167
private static native long selectColumns(long handle, String[] columnNames);
138168

139169
private static native long filterRows(long handle, String predicate);
170+
171+
private static native void writeParquetWithOptions(
172+
long handle,
173+
String path,
174+
String compression,
175+
boolean singleFileOutputSet,
176+
boolean singleFileOutputValue);
140177
}
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import java.nio.file.Files;
26+
import java.nio.file.Path;
27+
28+
import org.apache.arrow.memory.BufferAllocator;
29+
import org.apache.arrow.memory.RootAllocator;
30+
import org.apache.arrow.vector.BigIntVector;
31+
import org.apache.arrow.vector.VectorSchemaRoot;
32+
import org.apache.arrow.vector.ipc.ArrowReader;
33+
import org.junit.jupiter.api.Assumptions;
34+
import org.junit.jupiter.api.Test;
35+
import org.junit.jupiter.api.io.TempDir;
36+
37+
class DataFrameWriteParquetTest {
38+
39+
private static final Path LINEITEM = Path.of("tpch-data/sf1/lineitem.parquet");
40+
private static final long LINEITEM_ROWS = 6_001_215L;
41+
42+
private static void assumeLineitem() {
43+
Assumptions.assumeTrue(
44+
Files.exists(LINEITEM), "TPC-H SF1 data not found; run `make tpch-data` first");
45+
}
46+
47+
private static long countRowsAt(Path path) throws Exception {
48+
try (BufferAllocator allocator = new RootAllocator();
49+
SessionContext ctx = new SessionContext()) {
50+
ctx.registerParquet("t", path.toAbsolutePath().toString());
51+
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
52+
ArrowReader reader = df.collect(allocator)) {
53+
assertTrue(reader.loadNextBatch());
54+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
55+
assertEquals(1, root.getRowCount());
56+
return ((BigIntVector) root.getVector(0)).get(0);
57+
}
58+
}
59+
}
60+
61+
@Test
62+
void writeParquetRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
63+
assumeLineitem();
64+
Path out = tempDir.resolve("out");
65+
66+
try (SessionContext ctx = new SessionContext();
67+
DataFrame df = ctx.readParquet(LINEITEM.toAbsolutePath().toString())) {
68+
df.writeParquet(out.toString());
69+
}
70+
71+
assertEquals(LINEITEM_ROWS, countRowsAt(out));
72+
}
73+
}

0 commit comments

Comments
 (0)