Skip to content

Commit 607a751

Browse files
committed
fix(write): default to directory output when singleFileOutput is unset
Signed-off-by: Lantao Jin <jinlantao@gmail.com>
1 parent 7ef0a47 commit 607a751

4 files changed

Lines changed: 61 additions & 13 deletions

File tree

‎core/src/test/java/org/apache/datafusion/DataFrameWriteCsvTest.java‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,24 @@ void writeCsvRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
7272
assertEquals(3L, countRowsAt(out, new CsvReadOptions()));
7373
}
7474

75+
@Test
76+
void writeCsvDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir) throws Exception {
77+
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
78+
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
79+
// (".csv" here) silently flips the output to a single file. The native handler explicitly
80+
// pins the default to directory mode so this contract holds regardless of path shape.
81+
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
82+
Path out = tempDir.resolve("out.csv");
83+
84+
try (SessionContext ctx = new SessionContext();
85+
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
86+
df.writeCsv(out.toString());
87+
}
88+
89+
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
90+
assertEquals(3L, countRowsAt(out, new CsvReadOptions()));
91+
}
92+
7593
@Test
7694
void writeCsvSingleFileProducesOneFile(@TempDir Path tempDir) throws Exception {
7795
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
@@ -111,8 +129,7 @@ void writeCsvWithGzipCompressionRoundTrips(@TempDir Path tempDir) throws Excepti
111129
try (SessionContext ctx = new SessionContext();
112130
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
113131
df.writeCsv(
114-
out.toString(),
115-
new CsvWriteOptions().fileCompressionType(FileCompressionType.GZIP));
132+
out.toString(), new CsvWriteOptions().fileCompressionType(FileCompressionType.GZIP));
116133
}
117134

118135
try (Stream<Path> stream = Files.walk(out)) {
@@ -126,9 +143,7 @@ void writeCsvWithGzipCompressionRoundTrips(@TempDir Path tempDir) throws Excepti
126143
}
127144

128145
CsvReadOptions readOpts =
129-
new CsvReadOptions()
130-
.fileCompressionType(FileCompressionType.GZIP)
131-
.fileExtension(".csv.gz");
146+
new CsvReadOptions().fileCompressionType(FileCompressionType.GZIP).fileExtension(".csv.gz");
132147
assertEquals(4L, countRowsAt(out, readOpts));
133148
}
134149

‎core/src/test/java/org/apache/datafusion/DataFrameWriteParquetTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,4 +109,27 @@ void writeParquetRetainsDataFrame(@TempDir Path tempDir) throws Exception {
109109
assertEquals(LINEITEM_ROWS, df.count());
110110
}
111111
}
112+
113+
@Test
114+
void writeParquetDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir)
115+
throws Exception {
116+
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
117+
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
118+
// (".parquet" here) silently flips the output to a single file. The native handler explicitly
119+
// pins the default to directory mode so this contract holds regardless of path shape.
120+
//
121+
// Uses an in-memory 3-row source so this regression test runs without TPC-H fixtures.
122+
Path out = tempDir.resolve("out.parquet");
123+
124+
try (SessionContext ctx = new SessionContext();
125+
DataFrame df =
126+
ctx.sql(
127+
"SELECT * FROM (VALUES (CAST(1 AS BIGINT), 'alice'), (CAST(2 AS BIGINT), 'bob'),"
128+
+ " (CAST(3 AS BIGINT), 'carol')) AS t(id, name)")) {
129+
df.writeParquet(out.toString());
130+
}
131+
132+
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
133+
assertEquals(3L, countRowsAt(out));
134+
}
112135
}

‎native/src/csv.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,13 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeCsvWithOptions<
172172
None
173173
};
174174

175-
let mut write_opts = DataFrameWriteOptions::new();
176-
if let Some(v) = p.single_file_output {
177-
write_opts = write_opts.with_single_file_output(v);
178-
}
175+
// When the caller left `singleFileOutput` unset, force directory output (`false`)
176+
// rather than leaving DataFusion in `Automatic` mode. Automatic mode treats paths
177+
// with an extension (e.g. `out.csv`) as single-file targets, which would silently
178+
// contradict the documented "directory unless overridden" default and surprise any
179+
// caller that hands writeCsv a `.csv` path.
180+
let mut write_opts = DataFrameWriteOptions::new()
181+
.with_single_file_output(p.single_file_output.unwrap_or(false));
179182
if !p.partition_cols.is_empty() {
180183
write_opts = write_opts.with_partition_by(p.partition_cols.clone());
181184
}

‎native/src/lib.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -445,10 +445,17 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeParquetWithOpti
445445
let df = unsafe { &*(handle as *const DataFrame) }.clone();
446446
let path: String = env.get_string(&path)?.into();
447447

448-
let mut write_opts = DataFrameWriteOptions::new();
449-
if single_file_output_set != 0 {
450-
write_opts = write_opts.with_single_file_output(single_file_output_value != 0);
451-
}
448+
// When the caller left `singleFileOutput` unset, force directory output (`false`)
449+
// rather than leaving DataFusion in `Automatic` mode. Automatic mode treats paths
450+
// with an extension (e.g. `out.parquet`) as single-file targets, which would silently
451+
// contradict the documented "directory unless overridden" default and surprise any
452+
// caller that hands writeParquet a `.parquet` path.
453+
let single_file = if single_file_output_set != 0 {
454+
single_file_output_value != 0
455+
} else {
456+
false
457+
};
458+
let write_opts = DataFrameWriteOptions::new().with_single_file_output(single_file);
452459

453460
let writer_opts: Option<TableParquetOptions> = if !compression.is_null() {
454461
let c: String = env.get_string(&compression)?.into();

0 commit comments

Comments
 (0)