Skip to content

Commit 9ed8f50

Browse files
committed
feat(dataframe): expose withColumn and unnestColumns
Closes #41 withColumn(name, expr) takes a SQL fragment, parses it via df.parse_sql_expr (the same convention filter(String) uses), and calls DataFusion's with_column. Replaces a column of the same name in place; otherwise appends. unnestColumns(String...) and unnestColumns(UnnestOptions, String...) route to unnest_columns_with_options. UnnestOptions exposes the preserveNulls knob (default true, matching upstream). Recursions are deferred to a follow-up since they need a richer column-pair representation (RecursionUnnestOption with input/output/depth). Tests: 11 new transformation tests, 2 UnnestOptions setter tests, plus the two close/collect tests extended for the new methods. The unnest tests cover both preserveNulls=true (keep null rows) and preserveNulls=false (drop null rows). withColumn tests cover both the append-new and replace-existing branches. make test (90 tests, 0 failures) and cargo clippy/fmt are clean.
1 parent 4a967e4 commit 9ed8f50

5 files changed

Lines changed: 343 additions & 0 deletions

File tree

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,55 @@ public DataFrame withColumnRenamed(String oldName, String newName) {
171171
return new DataFrame(renameColumn(nativeHandle, oldName, newName));
172172
}
173173

174+
/**
175+
* Add a column to this DataFrame computed from a SQL expression. If a column with the given name
176+
* already exists, it is replaced in place; otherwise the new column is appended. The expression
177+
* is parsed against this DataFrame's own schema, matching the convention used by {@link
178+
* #filter(String)}. The receiver remains usable and must still be closed independently.
179+
*
180+
* @throws IllegalArgumentException if {@code name} or {@code expr} is {@code null}.
181+
*/
182+
public DataFrame withColumn(String name, String expr) {
183+
if (nativeHandle == 0) {
184+
throw new IllegalStateException("DataFrame is closed or already collected");
185+
}
186+
if (name == null) {
187+
throw new IllegalArgumentException("withColumn name must be non-null");
188+
}
189+
if (expr == null) {
190+
throw new IllegalArgumentException("withColumn expr must be non-null");
191+
}
192+
return new DataFrame(withColumnExpr(nativeHandle, name, expr));
193+
}
194+
195+
/**
196+
* Expand list or struct columns into rows or fields, with default {@link UnnestOptions} (i.e.
197+
* {@code preserveNulls = true}). The receiver remains usable and must still be closed
198+
* independently.
199+
*/
200+
public DataFrame unnestColumns(String... columns) {
201+
return unnestColumns(new UnnestOptions(), columns);
202+
}
203+
204+
/**
205+
* Expand list or struct columns into rows or fields with the supplied {@link UnnestOptions}. The
206+
* receiver remains usable and must still be closed independently.
207+
*
208+
* @throws IllegalArgumentException if {@code options} or {@code columns} is {@code null}.
209+
*/
210+
public DataFrame unnestColumns(UnnestOptions options, String... columns) {
211+
if (nativeHandle == 0) {
212+
throw new IllegalStateException("DataFrame is closed or already collected");
213+
}
214+
if (options == null) {
215+
throw new IllegalArgumentException("unnestColumns options must be non-null");
216+
}
217+
if (columns == null) {
218+
throw new IllegalArgumentException("unnestColumns columns must be non-null");
219+
}
220+
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
221+
}
222+
174223
/**
175224
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
176225
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -231,6 +280,10 @@ public void close() {
231280

232281
private static native long renameColumn(long handle, String oldName, String newName);
233282

283+
private static native long withColumnExpr(long handle, String name, String expr);
284+
285+
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
286+
234287
private static native void writeParquetWithOptions(
235288
long handle,
236289
String path,
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Configuration knobs for {@link DataFrame#unnestColumns(UnnestOptions, String...)}, mirroring
24+
* DataFusion's {@code UnnestOptions}. Defaults match upstream: {@code preserveNulls = true}.
25+
*
26+
* <p>Per-column recursion (DataFusion's {@code recursions}) is intentionally not exposed yet — it
27+
* needs a richer column-pair representation and is filed separately.
28+
*/
29+
public final class UnnestOptions {
30+
31+
private boolean preserveNulls = true;
32+
33+
/**
34+
* When {@code true} (the default), nulls in the input column are preserved as null rows in the
35+
* output. When {@code false}, nulls and empty lists are dropped.
36+
*/
37+
public UnnestOptions preserveNulls(boolean v) {
38+
this.preserveNulls = v;
39+
return this;
40+
}
41+
42+
/** The current {@code preserveNulls} setting. */
43+
public boolean preserveNulls() {
44+
return preserveNulls;
45+
}
46+
}

‎core/src/test/java/org/apache/datafusion/DataFrameTransformationsTest.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ void methodsThrowAfterClose() {
133133
assertThrows(IllegalStateException.class, df::distinct);
134134
assertThrows(IllegalStateException.class, () -> df.dropColumns("x"));
135135
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
136+
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
137+
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
136138
assertThrows(IllegalStateException.class, df::count);
137139
assertThrows(IllegalStateException.class, df::show);
138140
assertThrows(IllegalStateException.class, () -> df.show(5));
@@ -154,6 +156,8 @@ void methodsThrowAfterCollect() throws Exception {
154156
assertThrows(IllegalStateException.class, df::distinct);
155157
assertThrows(IllegalStateException.class, () -> df.dropColumns("x"));
156158
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
159+
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
160+
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
157161
assertThrows(IllegalStateException.class, df::count);
158162
assertThrows(IllegalStateException.class, df::show);
159163
assertThrows(IllegalStateException.class, () -> df.show(5));
@@ -353,4 +357,152 @@ void withColumnRenamedUnknownColumnIsNoOp() throws Exception {
353357
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
354358
}
355359
}
360+
361+
@Test
362+
void withColumnAppendsNewColumn() throws Exception {
363+
try (BufferAllocator allocator = new RootAllocator();
364+
SessionContext ctx = new SessionContext();
365+
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b");
366+
DataFrame extended = source.withColumn("c", "a + b");
367+
ArrowReader reader = extended.collect(allocator)) {
368+
assertTrue(reader.loadNextBatch());
369+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
370+
assertEquals(1, root.getRowCount());
371+
assertArrayEquals(
372+
new String[] {"a", "b", "c"},
373+
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
374+
assertEquals(3L, ((BigIntVector) root.getVector("c")).get(0));
375+
}
376+
}
377+
378+
@Test
379+
void withColumnReplacesExistingColumn() throws Exception {
380+
try (BufferAllocator allocator = new RootAllocator();
381+
SessionContext ctx = new SessionContext();
382+
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b");
383+
DataFrame replaced = source.withColumn("b", "a * 10");
384+
ArrowReader reader = replaced.collect(allocator)) {
385+
assertTrue(reader.loadNextBatch());
386+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
387+
assertArrayEquals(
388+
new String[] {"a", "b"},
389+
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
390+
assertEquals(10L, ((BigIntVector) root.getVector("b")).get(0));
391+
}
392+
}
393+
394+
@Test
395+
void withColumnIsNonDestructive() {
396+
try (SessionContext ctx = new SessionContext();
397+
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b")) {
398+
try (DataFrame extended = source.withColumn("c", "a + b")) {
399+
assertEquals(1L, extended.count());
400+
}
401+
assertEquals(1L, source.count());
402+
}
403+
}
404+
405+
@Test
406+
void withColumnUnknownColumnRefThrows() {
407+
try (SessionContext ctx = new SessionContext();
408+
DataFrame df = ctx.sql("SELECT 1 AS x")) {
409+
assertThrows(RuntimeException.class, () -> df.withColumn("y", "not_a_column + 1"));
410+
}
411+
}
412+
413+
@Test
414+
void withColumnRejectsNullArgs() {
415+
try (SessionContext ctx = new SessionContext();
416+
DataFrame df = ctx.sql("SELECT 1 AS x")) {
417+
assertThrows(IllegalArgumentException.class, () -> df.withColumn(null, "x + 1"));
418+
assertThrows(IllegalArgumentException.class, () -> df.withColumn("y", null));
419+
}
420+
}
421+
422+
@Test
423+
void unnestColumnsExpandsList() throws Exception {
424+
try (BufferAllocator allocator = new RootAllocator();
425+
SessionContext ctx = new SessionContext();
426+
DataFrame source = ctx.sql("SELECT 1 AS id, [10, 20, 30] AS vals");
427+
DataFrame unnested = source.unnestColumns("vals");
428+
ArrowReader reader = unnested.collect(allocator)) {
429+
int rows = 0;
430+
while (reader.loadNextBatch()) {
431+
rows += reader.getVectorSchemaRoot().getRowCount();
432+
}
433+
assertEquals(3, rows);
434+
}
435+
}
436+
437+
@Test
438+
void unnestColumnsPreserveNullsTrueKeepsNullRow() throws Exception {
439+
String sql =
440+
"SELECT id, vals FROM (VALUES "
441+
+ "(1, [10, 20]), "
442+
+ "(2, CAST(NULL AS BIGINT[])), "
443+
+ "(3, [30])) AS t(id, vals)";
444+
try (BufferAllocator allocator = new RootAllocator();
445+
SessionContext ctx = new SessionContext();
446+
DataFrame source = ctx.sql(sql);
447+
DataFrame unnested = source.unnestColumns(new UnnestOptions().preserveNulls(true), "vals");
448+
ArrowReader reader = unnested.collect(allocator)) {
449+
int rows = 0;
450+
while (reader.loadNextBatch()) {
451+
rows += reader.getVectorSchemaRoot().getRowCount();
452+
}
453+
assertEquals(4, rows);
454+
}
455+
}
456+
457+
@Test
458+
void unnestColumnsPreserveNullsFalseDropsNullRow() throws Exception {
459+
String sql =
460+
"SELECT id, vals FROM (VALUES "
461+
+ "(1, [10, 20]), "
462+
+ "(2, CAST(NULL AS BIGINT[])), "
463+
+ "(3, [30])) AS t(id, vals)";
464+
try (BufferAllocator allocator = new RootAllocator();
465+
SessionContext ctx = new SessionContext();
466+
DataFrame source = ctx.sql(sql);
467+
DataFrame unnested =
468+
source.unnestColumns(new UnnestOptions().preserveNulls(false), "vals");
469+
ArrowReader reader = unnested.collect(allocator)) {
470+
int rows = 0;
471+
while (reader.loadNextBatch()) {
472+
rows += reader.getVectorSchemaRoot().getRowCount();
473+
}
474+
assertEquals(3, rows);
475+
}
476+
}
477+
478+
@Test
479+
void unnestColumnsIsNonDestructive() {
480+
try (SessionContext ctx = new SessionContext();
481+
DataFrame source = ctx.sql("SELECT 1 AS id, [10, 20] AS vals")) {
482+
try (DataFrame unnested = source.unnestColumns("vals")) {
483+
assertEquals(2L, unnested.count());
484+
}
485+
assertEquals(1L, source.count());
486+
}
487+
}
488+
489+
@Test
490+
void unnestColumnsUnknownColumnThrows() {
491+
try (SessionContext ctx = new SessionContext();
492+
DataFrame df = ctx.sql("SELECT 1 AS x")) {
493+
assertThrows(RuntimeException.class, () -> df.unnestColumns("not_a_column"));
494+
}
495+
}
496+
497+
@Test
498+
void unnestColumnsRejectsNullArgs() {
499+
try (SessionContext ctx = new SessionContext();
500+
DataFrame df = ctx.sql("SELECT 1 AS x, [1, 2] AS vals")) {
501+
assertThrows(
502+
IllegalArgumentException.class, () -> df.unnestColumns((UnnestOptions) null, "vals"));
503+
assertThrows(
504+
IllegalArgumentException.class,
505+
() -> df.unnestColumns(new UnnestOptions(), (String[]) null));
506+
}
507+
}
356508
}
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertFalse;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import org.junit.jupiter.api.Test;
26+
27+
class UnnestOptionsTest {
28+
29+
@Test
30+
void defaultPreserveNullsIsTrue() {
31+
assertTrue(new UnnestOptions().preserveNulls());
32+
}
33+
34+
@Test
35+
void preserveNullsSetterRoundTrips() {
36+
UnnestOptions opts = new UnnestOptions().preserveNulls(false);
37+
assertFalse(opts.preserveNulls());
38+
opts.preserveNulls(true);
39+
assertTrue(opts.preserveNulls());
40+
}
41+
}

‎native/src/lib.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use std::sync::{Arc, OnceLock};
3030
use datafusion::arrow::datatypes::SchemaRef;
3131
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
3232
use datafusion::arrow::record_batch::RecordBatchIterator;
33+
use datafusion::common::UnnestOptions;
3334
use datafusion::config::TableParquetOptions;
3435
use datafusion::dataframe::DataFrame;
3536
use datafusion::dataframe::DataFrameWriteOptions;
@@ -358,6 +359,56 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_renameColumn<'local>
358359
})
359360
}
360361

362+
#[no_mangle]
363+
pub extern "system" fn Java_org_apache_datafusion_DataFrame_withColumnExpr<'local>(
364+
mut env: JNIEnv<'local>,
365+
_class: JClass<'local>,
366+
handle: jlong,
367+
name: JString<'local>,
368+
expr: JString<'local>,
369+
) -> jlong {
370+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
371+
if handle == 0 {
372+
return Err("DataFrame handle is null".into());
373+
}
374+
let df = unsafe { &*(handle as *const DataFrame) }.clone();
375+
let name: String = env.get_string(&name)?.into();
376+
let expr: String = env.get_string(&expr)?.into();
377+
let parsed = df.parse_sql_expr(&expr)?;
378+
let new_df = df.with_column(&name, parsed)?;
379+
Ok(Box::into_raw(Box::new(new_df)) as jlong)
380+
})
381+
}
382+
383+
#[no_mangle]
384+
pub extern "system" fn Java_org_apache_datafusion_DataFrame_unnestColumns<'local>(
385+
mut env: JNIEnv<'local>,
386+
_class: JClass<'local>,
387+
handle: jlong,
388+
columns: JObjectArray<'local>,
389+
preserve_nulls: jboolean,
390+
) -> jlong {
391+
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult<jlong> {
392+
if handle == 0 {
393+
return Err("DataFrame handle is null".into());
394+
}
395+
let df = unsafe { &*(handle as *const DataFrame) }.clone();
396+
397+
let len = env.get_array_length(&columns)?;
398+
let mut owned: Vec<String> = Vec::with_capacity(len as usize);
399+
for i in 0..len {
400+
let elem = env.get_object_array_element(&columns, i)?;
401+
let jstr: JString = elem.into();
402+
owned.push(env.get_string(&jstr)?.into());
403+
}
404+
let refs: Vec<&str> = owned.iter().map(String::as_str).collect();
405+
406+
let opts = UnnestOptions::new().with_preserve_nulls(preserve_nulls != 0);
407+
let new_df = df.unnest_columns_with_options(&refs, opts)?;
408+
Ok(Box::into_raw(Box::new(new_df)) as jlong)
409+
})
410+
}
411+
361412
#[no_mangle]
362413
pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeParquetWithOptions<'local>(
363414
mut env: JNIEnv<'local>,

0 commit comments

Comments
 (0)