Skip to content

Commit 24ad482

Browse files
committed
feat(dataframe): expose set operations (union, intersect, except)
Add eight DataFrame methods mirroring DataFusion's set-op family: union/unionDistinct/unionByName/unionByNameDistinct and intersect/intersectDistinct/except/exceptDistinct. Method names map 1:1 to the upstream Rust API; each Javadoc spells out SQL semantics because the *_distinct convention inverts Spark's *All convention. None of the eight methods consume their arguments. The native handler clones both DataFrames (LogicalPlan is Arc-backed, clone is cheap), matching the established non-destructive contract used by every other transform method. DataFusion implements INTERSECT ALL and EXCEPT ALL as left-semi / left-anti joins on equality rather than standard SQL bag operators. The intersect/except Javadocs flag this divergence with a worked example so callers porting from PostgreSQL or Spark know what to expect.
1 parent 333ce48 commit 24ad482

3 files changed

Lines changed: 390 additions & 2 deletions

File tree

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,155 @@ public DataFrame unnestColumns(UnnestOptions options, String... columns) {
256256
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
257257
}
258258

259+
// -- Set operations ------------------------------------------------------
260+
//
261+
// The Java method names mirror DataFusion's Rust API verbatim. SQL semantics:
262+
//
263+
// union = UNION ALL (positional, keeps duplicates)
264+
// unionDistinct = UNION (positional, deduplicated)
265+
// unionByName = UNION ALL by column name; missing columns become NULL
266+
// unionByNameDistinct = UNION by column name; missing columns become NULL
267+
// intersect = INTERSECT ALL (keeps duplicates)
268+
// intersectDistinct = INTERSECT (deduplicated)
269+
// except = EXCEPT ALL (keeps duplicates)
270+
// exceptDistinct = EXCEPT (deduplicated)
271+
//
272+
// Note: the *_distinct variants deduplicate, while the unsuffixed methods keep
273+
// duplicates. This is the inverse of Spark's convention, where `intersect`
274+
// deduplicates and `intersectAll` keeps duplicates -- consult the Javadoc on
275+
// each method to confirm semantics before porting Spark code.
276+
//
277+
// None of these methods consume the receiver or {@code other}; both DataFrames
278+
// remain usable after the call. The native side clones the LogicalPlan on
279+
// each side, which is cheap (LogicalPlan is Arc-backed in DataFusion).
280+
281+
/**
282+
* Concatenate this DataFrame with {@code other} by column position, keeping all duplicates (SQL
283+
* {@code UNION ALL}). The two schemas must match positionally. Both this DataFrame and {@code
284+
* other} remain usable after the call and must still be closed independently.
285+
*
286+
* @throws IllegalArgumentException if {@code other} is {@code null}.
287+
* @throws RuntimeException if the schemas are incompatible.
288+
*/
289+
public DataFrame union(DataFrame other) {
290+
return new DataFrame(unionRows(nativeHandle, otherHandle("union", other)));
291+
}
292+
293+
/**
294+
* Concatenate this DataFrame with {@code other} by column position, removing duplicates (SQL
295+
* {@code UNION DISTINCT} -- equivalent to plain {@code UNION} in standard SQL). Both DataFrames
296+
* remain usable.
297+
*
298+
* @throws IllegalArgumentException if {@code other} is {@code null}.
299+
* @throws RuntimeException if the schemas are incompatible.
300+
*/
301+
public DataFrame unionDistinct(DataFrame other) {
302+
return new DataFrame(unionDistinctRows(nativeHandle, otherHandle("unionDistinct", other)));
303+
}
304+
305+
/**
306+
* Concatenate this DataFrame with {@code other} by column name, keeping all duplicates. Columns
307+
* present in only one side are filled with NULL on the other. Both DataFrames remain usable.
308+
*
309+
* @throws IllegalArgumentException if {@code other} is {@code null}.
310+
* @throws RuntimeException if column types disagree on a shared name.
311+
*/
312+
public DataFrame unionByName(DataFrame other) {
313+
return new DataFrame(unionByNameRows(nativeHandle, otherHandle("unionByName", other)));
314+
}
315+
316+
/**
317+
* Concatenate this DataFrame with {@code other} by column name, removing duplicates. Columns
318+
* present in only one side are filled with NULL on the other. Both DataFrames remain usable.
319+
*
320+
* @throws IllegalArgumentException if {@code other} is {@code null}.
321+
* @throws RuntimeException if column types disagree on a shared name.
322+
*/
323+
public DataFrame unionByNameDistinct(DataFrame other) {
324+
return new DataFrame(
325+
unionByNameDistinctRows(nativeHandle, otherHandle("unionByNameDistinct", other)));
326+
}
327+
328+
/**
329+
* Rows present in both this DataFrame and {@code other}, keeping duplicates from the receiver
330+
* (SQL {@code INTERSECT ALL}). Both schemas must match positionally. Both DataFrames remain
331+
* usable.
332+
*
333+
* <p><strong>Implementation note:</strong> DataFusion implements {@code INTERSECT ALL} as a
334+
* left-semi join on equality, not as standard SQL bag intersection. A left row is kept iff any
335+
* matching row exists in {@code other}. With {@code left = (1, 2, 2, 3)} and {@code right = (2,
336+
* 3)}, the result is {@code (2, 2, 3)} -- both copies of {@code 2} survive because each finds a
337+
* match in {@code right}. PostgreSQL / Spark {@code INTERSECT ALL} would also yield {@code (2, 2,
338+
* 3)} here, but the two engines diverge when {@code other} has fewer copies than {@code this} of
339+
* a row that appears in both.
340+
*
341+
* @throws IllegalArgumentException if {@code other} is {@code null}.
342+
* @throws RuntimeException if the schemas are incompatible.
343+
*/
344+
public DataFrame intersect(DataFrame other) {
345+
return new DataFrame(intersectRows(nativeHandle, otherHandle("intersect", other)));
346+
}
347+
348+
/**
349+
* Rows present in both this DataFrame and {@code other}, deduplicated (SQL {@code INTERSECT}).
350+
* Both schemas must match positionally. Both DataFrames remain usable.
351+
*
352+
* @throws IllegalArgumentException if {@code other} is {@code null}.
353+
* @throws RuntimeException if the schemas are incompatible.
354+
*/
355+
public DataFrame intersectDistinct(DataFrame other) {
356+
return new DataFrame(
357+
intersectDistinctRows(nativeHandle, otherHandle("intersectDistinct", other)));
358+
}
359+
360+
/**
361+
* Rows present in this DataFrame but not in {@code other}, keeping duplicates from the receiver
362+
* (SQL {@code EXCEPT ALL}). Both schemas must match positionally. Both DataFrames remain usable.
363+
*
364+
* <p><strong>Implementation note:</strong> DataFusion implements {@code EXCEPT ALL} as a
365+
* left-anti join on equality, not as standard SQL bag difference. A left row is kept iff
366+
* <em>no</em> matching row exists in {@code other} -- the multiplicity of matches is irrelevant.
367+
* With {@code left = (1, 1, 2, 2, 3)} and {@code right = (1, 3)}, the result is {@code (2, 2)}:
368+
* both copies of {@code 2} survive (no match in {@code right}); both copies of {@code 1} and the
369+
* {@code 3} drop. PostgreSQL / Spark {@code EXCEPT ALL} would yield the same answer here, but the
370+
* two engines diverge when {@code right} contains fewer copies than {@code left} of a row that
371+
* appears in both.
372+
*
373+
* @throws IllegalArgumentException if {@code other} is {@code null}.
374+
* @throws RuntimeException if the schemas are incompatible.
375+
*/
376+
public DataFrame except(DataFrame other) {
377+
return new DataFrame(exceptRows(nativeHandle, otherHandle("except", other)));
378+
}
379+
380+
/**
381+
* Rows present in this DataFrame but not in {@code other}, deduplicated (SQL {@code EXCEPT}).
382+
* Both schemas must match positionally. Both DataFrames remain usable.
383+
*
384+
* @throws IllegalArgumentException if {@code other} is {@code null}.
385+
* @throws RuntimeException if the schemas are incompatible.
386+
*/
387+
public DataFrame exceptDistinct(DataFrame other) {
388+
return new DataFrame(exceptDistinctRows(nativeHandle, otherHandle("exceptDistinct", other)));
389+
}
390+
391+
/**
392+
* Validate the receiver and the other DataFrame and return {@code other.nativeHandle}. Common
393+
* preamble for the eight set-operation methods so the validation logic stays in one place.
394+
*/
395+
private long otherHandle(String op, DataFrame other) {
396+
if (nativeHandle == 0) {
397+
throw new IllegalStateException("DataFrame is closed or already collected");
398+
}
399+
if (other == null) {
400+
throw new IllegalArgumentException(op + " other must be non-null");
401+
}
402+
if (other.nativeHandle == 0) {
403+
throw new IllegalStateException(op + " other DataFrame is closed or already collected");
404+
}
405+
return other.nativeHandle;
406+
}
407+
259408
/**
260409
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
261410
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -354,6 +503,22 @@ public void close() {
354503

355504
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
356505

506+
private static native long unionRows(long handle, long otherHandle);
507+
508+
private static native long unionDistinctRows(long handle, long otherHandle);
509+
510+
private static native long unionByNameRows(long handle, long otherHandle);
511+
512+
private static native long unionByNameDistinctRows(long handle, long otherHandle);
513+
514+
private static native long intersectRows(long handle, long otherHandle);
515+
516+
private static native long intersectDistinctRows(long handle, long otherHandle);
517+
518+
private static native long exceptRows(long handle, long otherHandle);
519+
520+
private static native long exceptDistinctRows(long handle, long otherHandle);
521+
357522
private static native void writeParquetWithOptions(
358523
long handle,
359524
String path,

‎core/src/test/java/org/apache/datafusion/DataFrameTransformationsTest.java‎

Lines changed: 166 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ void chainFilterSelectCount() {
123123

124124
@Test
125125
void methodsThrowAfterClose() {
126-
try (SessionContext ctx = new SessionContext()) {
126+
try (SessionContext ctx = new SessionContext();
127+
DataFrame other = ctx.sql("SELECT 1 AS x")) {
127128
DataFrame df = ctx.sql("SELECT 1 AS x");
128129
df.close();
129130
assertThrows(IllegalStateException.class, () -> df.select("x"));
@@ -135,6 +136,14 @@ void methodsThrowAfterClose() {
135136
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
136137
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
137138
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
139+
assertThrows(IllegalStateException.class, () -> df.union(other));
140+
assertThrows(IllegalStateException.class, () -> df.unionDistinct(other));
141+
assertThrows(IllegalStateException.class, () -> df.unionByName(other));
142+
assertThrows(IllegalStateException.class, () -> df.unionByNameDistinct(other));
143+
assertThrows(IllegalStateException.class, () -> df.intersect(other));
144+
assertThrows(IllegalStateException.class, () -> df.intersectDistinct(other));
145+
assertThrows(IllegalStateException.class, () -> df.except(other));
146+
assertThrows(IllegalStateException.class, () -> df.exceptDistinct(other));
138147
assertThrows(IllegalStateException.class, df::count);
139148
assertThrows(IllegalStateException.class, df::show);
140149
assertThrows(IllegalStateException.class, () -> df.show(5));
@@ -145,7 +154,8 @@ void methodsThrowAfterClose() {
145154
void methodsThrowAfterCollect() throws Exception {
146155
try (BufferAllocator allocator = new RootAllocator();
147156
SessionContext ctx = new SessionContext();
148-
DataFrame df = ctx.sql("SELECT 1 AS x")) {
157+
DataFrame df = ctx.sql("SELECT 1 AS x");
158+
DataFrame other = ctx.sql("SELECT 1 AS x")) {
149159
try (ArrowReader reader = df.collect(allocator)) {
150160
assertTrue(reader.loadNextBatch());
151161
}
@@ -158,6 +168,14 @@ void methodsThrowAfterCollect() throws Exception {
158168
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
159169
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
160170
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
171+
assertThrows(IllegalStateException.class, () -> df.union(other));
172+
assertThrows(IllegalStateException.class, () -> df.unionDistinct(other));
173+
assertThrows(IllegalStateException.class, () -> df.unionByName(other));
174+
assertThrows(IllegalStateException.class, () -> df.unionByNameDistinct(other));
175+
assertThrows(IllegalStateException.class, () -> df.intersect(other));
176+
assertThrows(IllegalStateException.class, () -> df.intersectDistinct(other));
177+
assertThrows(IllegalStateException.class, () -> df.except(other));
178+
assertThrows(IllegalStateException.class, () -> df.exceptDistinct(other));
161179
assertThrows(IllegalStateException.class, df::count);
162180
assertThrows(IllegalStateException.class, df::show);
163181
assertThrows(IllegalStateException.class, () -> df.show(5));
@@ -504,4 +522,150 @@ void unnestColumnsRejectsNullArgs() {
504522
() -> df.unnestColumns(new UnnestOptions(), (String[]) null));
505523
}
506524
}
525+
526+
// -- Set operations -------------------------------------------------------
527+
528+
@Test
529+
void unionKeepsDuplicates() {
530+
try (SessionContext ctx = new SessionContext();
531+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (2), (2)) AS t(x)");
532+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (2), (3)) AS t(x)");
533+
DataFrame combined = left.union(right)) {
534+
// union = UNION ALL: 3 + 2 = 5 rows, duplicates preserved.
535+
assertEquals(5L, combined.count());
536+
}
537+
}
538+
539+
@Test
540+
void unionDistinctRemovesDuplicates() {
541+
try (SessionContext ctx = new SessionContext();
542+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (2), (2)) AS t(x)");
543+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (2), (3)) AS t(x)");
544+
DataFrame combined = left.unionDistinct(right)) {
545+
// unionDistinct = UNION (deduplicated): {1, 2, 3} = 3 rows.
546+
assertEquals(3L, combined.count());
547+
}
548+
}
549+
550+
@Test
551+
void unionByNameAlignsColumnsByName() {
552+
try (SessionContext ctx = new SessionContext();
553+
// Left: (a, b). Right: (b, a) -- same names, swapped column positions.
554+
DataFrame left = ctx.sql("SELECT 1 AS a, 2 AS b");
555+
DataFrame right = ctx.sql("SELECT 4 AS b, 3 AS a");
556+
DataFrame combined = left.unionByName(right)) {
557+
// 2 rows: positional union would mix columns, but unionByName aligns them.
558+
assertEquals(2L, combined.count());
559+
}
560+
}
561+
562+
@Test
563+
void unionByNameDistinctRemovesDuplicates() {
564+
try (SessionContext ctx = new SessionContext();
565+
DataFrame left = ctx.sql("SELECT 1 AS a, 2 AS b");
566+
DataFrame right = ctx.sql("SELECT 2 AS b, 1 AS a");
567+
DataFrame combined = left.unionByNameDistinct(right)) {
568+
// After name-aligning, both rows are (a=1, b=2). Distinct collapses to 1.
569+
assertEquals(1L, combined.count());
570+
}
571+
}
572+
573+
@Test
574+
void intersectKeepsCommonRowsWithDuplicates() {
575+
try (SessionContext ctx = new SessionContext();
576+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (2), (2), (3)) AS t(x)");
577+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (2), (2), (3)) AS t(x)");
578+
DataFrame inter = left.intersect(right)) {
579+
// INTERSECT ALL: row r appears min(count(r) in left, count(r) in right) times.
580+
// 2 appears 2 times in both => 2 rows; 3 appears 1 time in both => 1 row. Total 3.
581+
assertEquals(3L, inter.count());
582+
}
583+
}
584+
585+
@Test
586+
void intersectDistinctReturnsUniqueCommonRows() {
587+
try (SessionContext ctx = new SessionContext();
588+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (2), (2), (3)) AS t(x)");
589+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (2), (2), (3)) AS t(x)");
590+
DataFrame inter = left.intersectDistinct(right)) {
591+
// INTERSECT (deduplicated): {2, 3} = 2 rows.
592+
assertEquals(2L, inter.count());
593+
}
594+
}
595+
596+
@Test
597+
void exceptKeepsLeftMinusRightWithDuplicates() {
598+
try (SessionContext ctx = new SessionContext();
599+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
600+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (1), (3)) AS t(x)");
601+
DataFrame diff = left.except(right)) {
602+
// DataFusion's EXCEPT ALL is implemented as a LeftAnti join, so a left row is kept iff
603+
// its key has no match in right. With left {1, 1, 2, 2, 3} and right {1, 3}: both 1s and
604+
// the 3 are dropped, both 2s are kept. Total 2 rows -- duplicates of 2 preserved.
605+
assertEquals(2L, diff.count());
606+
}
607+
}
608+
609+
@Test
610+
void exceptDistinctReturnsUniqueLeftOnlyRows() {
611+
try (SessionContext ctx = new SessionContext();
612+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
613+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (1), (3)) AS t(x)");
614+
DataFrame diff = left.exceptDistinct(right)) {
615+
// EXCEPT (DISTINCT): left is deduplicated to {1, 2, 3}, then anti-joined against
616+
// right {1, 3}. Result: {2} = 1 row.
617+
assertEquals(1L, diff.count());
618+
}
619+
}
620+
621+
@Test
622+
void setOpsAreNonDestructive() {
623+
try (SessionContext ctx = new SessionContext();
624+
DataFrame left = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)");
625+
DataFrame right = ctx.sql("SELECT * FROM (VALUES (2), (3)) AS t(x)")) {
626+
try (DataFrame combined = left.union(right)) {
627+
assertEquals(4L, combined.count());
628+
}
629+
// Both originals still usable after the set-op call.
630+
assertEquals(2L, left.count());
631+
assertEquals(2L, right.count());
632+
}
633+
}
634+
635+
@Test
636+
void setOpsRejectNullOther() {
637+
try (SessionContext ctx = new SessionContext();
638+
DataFrame df = ctx.sql("SELECT 1 AS x")) {
639+
assertThrows(IllegalArgumentException.class, () -> df.union(null));
640+
assertThrows(IllegalArgumentException.class, () -> df.unionDistinct(null));
641+
assertThrows(IllegalArgumentException.class, () -> df.unionByName(null));
642+
assertThrows(IllegalArgumentException.class, () -> df.unionByNameDistinct(null));
643+
assertThrows(IllegalArgumentException.class, () -> df.intersect(null));
644+
assertThrows(IllegalArgumentException.class, () -> df.intersectDistinct(null));
645+
assertThrows(IllegalArgumentException.class, () -> df.except(null));
646+
assertThrows(IllegalArgumentException.class, () -> df.exceptDistinct(null));
647+
}
648+
}
649+
650+
@Test
651+
void setOpsRejectClosedOther() {
652+
try (SessionContext ctx = new SessionContext();
653+
DataFrame df = ctx.sql("SELECT 1 AS x")) {
654+
DataFrame other = ctx.sql("SELECT 1 AS x");
655+
other.close();
656+
assertThrows(IllegalStateException.class, () -> df.union(other));
657+
assertThrows(IllegalStateException.class, () -> df.intersectDistinct(other));
658+
assertThrows(IllegalStateException.class, () -> df.exceptDistinct(other));
659+
}
660+
}
661+
662+
@Test
663+
void unionWithIncompatibleSchemaThrows() {
664+
// Different column count -- LogicalPlanBuilder rejects at plan-build time.
665+
try (SessionContext ctx = new SessionContext();
666+
DataFrame left = ctx.sql("SELECT 1 AS x, 2 AS y");
667+
DataFrame right = ctx.sql("SELECT 1 AS x")) {
668+
assertThrows(RuntimeException.class, () -> left.union(right));
669+
}
670+
}
507671
}

0 commit comments

Comments
 (0)