Skip to content

Commit 29e2f8c

Browse files
committed
feat(dataframe): add distinct method
1 parent 45b379e commit 29e2f8c

3 files changed

Lines changed: 50 additions & 0 deletions

File tree

‎native/src/lib.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,22 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_limitRows<'local>(
213213
})
214214
}
215215

216+
#[no_mangle]
217+
pub extern "system" fn Java_org_apache_datafusion_DataFrame_distinctRows<'local>(
218+
mut env: JNIEnv<'local>,
219+
_class: JClass<'local>,
220+
handle: jlong,
221+
) -> jlong {
222+
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult<jlong> {
223+
if handle == 0 {
224+
return Err("DataFrame handle is null".into());
225+
}
226+
let df = unsafe { &*(handle as *const DataFrame) }.clone();
227+
let new_df = df.distinct()?;
228+
Ok(Box::into_raw(Box::new(new_df)) as jlong)
229+
})
230+
}
231+
216232
#[no_mangle]
217233
pub extern "system" fn Java_org_apache_datafusion_DataFrame_closeDataFrame<'local>(
218234
mut env: JNIEnv<'local>,

‎src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,17 @@ public DataFrame limit(int skip, int fetch) {
141141
return new DataFrame(limitRows(nativeHandle, skip, fetch));
142142
}
143143

144+
/**
145+
* Deduplicate rows across all columns. The receiver remains usable and must still be closed
146+
* independently.
147+
*/
148+
public DataFrame distinct() {
149+
if (nativeHandle == 0) {
150+
throw new IllegalStateException("DataFrame is closed or already collected");
151+
}
152+
return new DataFrame(distinctRows(nativeHandle));
153+
}
154+
144155
@Override
145156
public void close() {
146157
if (nativeHandle != 0) {
@@ -164,4 +175,6 @@ public void close() {
164175
private static native long filterRows(long handle, String predicate);
165176

166177
private static native long limitRows(long handle, int skip, int fetch);
178+
179+
private static native long distinctRows(long handle);
167180
}

‎src/test/java/org/apache/datafusion/DataFrameTransformationsTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,4 +232,25 @@ void limitRejectsNegativeArgs() {
232232
assertThrows(IllegalArgumentException.class, () -> df.limit(0, -1));
233233
}
234234
}
235+
236+
@Test
237+
void distinctRemovesDuplicates() {
238+
try (SessionContext ctx = new SessionContext();
239+
DataFrame source =
240+
ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
241+
DataFrame deduped = source.distinct()) {
242+
assertEquals(3L, deduped.count());
243+
}
244+
}
245+
246+
@Test
247+
void distinctIsNonDestructive() {
248+
try (SessionContext ctx = new SessionContext();
249+
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (1), (2)) AS t(x)")) {
250+
try (DataFrame deduped = source.distinct()) {
251+
assertEquals(2L, deduped.count());
252+
}
253+
assertEquals(3L, source.count());
254+
}
255+
}
235256
}

0 commit comments

Comments
 (0)