diff --git a/native/core/src/parquet/file_error_context.rs b/native/core/src/parquet/file_error_context.rs index 09e8692d60c..f9977301e11 100644 --- a/native/core/src/parquet/file_error_context.rs +++ b/native/core/src/parquet/file_error_context.rs @@ -34,6 +34,7 @@ use datafusion::physical_plan::{ DisplayFormatType, SortOrderPushdownResult, }; use datafusion_comet_common::SparkError; +use datafusion_comet_jni_bridge::errors::file_read_error; use datafusion_datasource::{ file_stream::FileOpener, morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer}, @@ -188,7 +189,8 @@ struct FileContext { impl MorselPlanner for FileContext> { fn plan(self: Box) -> Result> { let Self { inner, path } = *self; - let Some(mut plan) = inner.plan()? else { + // Opening the file (reading its footer) can fail here or in the pending planner below. + let Some(mut plan) = inner.plan().map_err(|error| with_file_path(error, &path))? else { return Ok(None); }; let morsels = plan @@ -213,10 +215,10 @@ impl MorselPlanner for FileContext> { .collect(); if let Some(pending) = plan.take_pending_planner() { plan.set_pending_planner(async move { - Ok(Box::new(FileContext { - inner: pending.await?, - path, - }) as Box) + let inner = pending + .await + .map_err(|error| with_file_path(error, &path))?; + Ok(Box::new(FileContext { inner, path }) as Box) }); } Ok(Some(plan.with_morsels(morsels).with_planners(planners))) @@ -228,27 +230,40 @@ impl Morsel for FileContext> { let Self { inner, path } = *self; inner .into_stream() - .map_err(move |error| { - // Conversion runs inside this file's stream, before batches from other files mix. - if let DataFusionError::External(source) = &error { - if let Some(SparkError::ParquetTimestampOverflow { .. }) = - source.downcast_ref::() - { - return SparkError::ParquetTimestampOverflow { - file_path: path.to_string(), - } - .into(); - } - } - error - }) + .map_err(move |error| with_file_path(error, &path)) .boxed() } } +/// Name the file in a file-level error raised while planning or reading it. This runs inside the +/// file's own planner and stream, before batches from other files mix, so `path` is the file that +/// failed. Without it a read failure reaches the JVM with no path, and the JVM falls back to every +/// file of the task. +fn with_file_path(error: DataFusionError, path: &str) -> DataFusionError { + if let DataFusionError::External(source) = &error { + if let Some(SparkError::ParquetTimestampOverflow { .. }) = + source.downcast_ref::() + { + return SparkError::ParquetTimestampOverflow { + file_path: path.to_string(), + } + .into(); + } + } + match file_read_error(&error) { + Some(SparkError::CannotReadFile { message, .. }) => SparkError::CannotReadFile { + file_path: path.to_string(), + message, + } + .into(), + _ => error, + } +} + #[cfg(test)] mod tests { use super::*; + use parquet::errors::ParquetError; #[derive(Debug)] struct FailedMorsel(SparkError); @@ -299,4 +314,67 @@ mod tests { assert!(matches!(error, DataFusionError::External(source) if matches!(source.downcast_ref::(), Some(SparkError::DivideByZero)))); } + + fn cannot_read_path(error: &DataFusionError) -> Option<&str> { + let DataFusionError::External(source) = error else { + return None; + }; + match source.downcast_ref::() { + Some(SparkError::CannotReadFile { file_path, .. }) => Some(file_path), + _ => None, + } + } + + #[test] + fn read_failures_name_the_failed_file() { + let path = "file:///tmp/t/part-00001.parquet"; + let corrupt = DataFusionError::ParquetError(Box::new(ParquetError::General( + "Failed to fetch metadata for file tmp/t/part-00001.parquet: Invalid Parquet file. \ + Corrupt footer" + .to_string(), + ))); + let named = with_file_path(corrupt, path); + assert_eq!(cannot_read_path(&named), Some(path)); + // The native message is kept as the cause. + assert!(named.to_string().contains("Corrupt footer")); + + let wrapped = DataFusionError::Context( + "open".to_string(), + Box::new(DataFusionError::ParquetError(Box::new(ParquetError::EOF( + "unexpected end of file".to_string(), + )))), + ); + assert_eq!(cannot_read_path(&with_file_path(wrapped, path)), Some(path)); + } + + #[test] + fn other_errors_keep_their_type() { + let path = "file:///tmp/t/part-00001.parquet"; + // A Spark error the reader raised itself. + let missing_ids = DataFusionError::ParquetError(Box::new(ParquetError::External( + Box::new(SparkError::ParquetMissingFieldIds { + file_path: "tmp/t/part-00001.parquet".to_string(), + }), + ))); + let error = with_file_path(missing_ids, path); + assert!(matches!(&error, DataFusionError::ParquetError(pe) + if matches!(pe.as_ref(), ParquetError::External(inner) + if matches!(inner.downcast_ref::(), + Some(SparkError::ParquetMissingFieldIds { .. }))))); + + // A missing file stays a file-not-found error. + let not_found = DataFusionError::ParquetError(Box::new(ParquetError::General( + "Failed to fetch metadata for file tmp/t/gone.parquet: Object Store error: \ + Object at location tmp/t/gone.parquet not found: No such file or directory" + .to_string(), + ))); + assert!(cannot_read_path(&with_file_path(not_found, path)).is_none()); + + // An I/O error can come from spill or shuffle, so it is not a file read failure. + let io = DataFusionError::IoError(std::io::Error::other("disk full")); + assert!(matches!( + with_file_path(io, path), + DataFusionError::IoError(_) + )); + } } diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs index 04c20611f3c..59b121ffacd 100644 --- a/native/jni-bridge/src/errors.rs +++ b/native/jni-bridge/src/errors.rs @@ -899,6 +899,20 @@ fn spark_exception(spark_error: &SparkError) -> JvmException<'static> { } } +/// The `CannotReadFile` error a scan should raise for `error`, if it is a per-file read failure, +/// classified the same way the JNI conversion does. A `SparkError` the reader raised itself keeps +/// its own type, so this returns `None` for it. The scan calls this where it still knows which +/// file failed, so that file can be named in the error. +pub fn file_read_error(error: &DataFusionError) -> Option { + if extract_spark_payload(error).is_some() { + return None; + } + match try_classify_file_read_error(error)? { + cannot_read @ SparkError::CannotReadFile { .. } => Some(cannot_read), + _ => None, + } +} + /// Classify a `DataFusionError` as a per-file read failure by TYPED variant (not message text), /// returning `SparkError::CannotReadFile` if so. This is the structured replacement for the /// previous JVM-side substring matching on error prose. @@ -2249,4 +2263,41 @@ mod tests { try_classify_file_read_error(&DataFusionError::Internal("bug".to_string())).is_none() ); } + + /// A `CannotReadFile` the scan attached as `External` keeps its structured exception under the + /// `Context` and `Shared` wrappers a repartition or a hash join build side adds, as it does bare. + #[test] + fn external_cannot_read_file_survives_context_and_shared() { + let external = || { + DataFusionError::External(Box::new(SparkError::CannotReadFile { + file_path: "file:/tmp/part-0.parquet".to_string(), + message: "corrupt footer".to_string(), + })) + }; + let cases = [ + ("bare", external()), + ( + "context", + DataFusionError::Context("repartition".to_string(), Box::new(external())), + ), + ("shared", DataFusionError::Shared(Arc::new(external()))), + ( + "shared context", + DataFusionError::Shared(Arc::new(DataFusionError::Context( + "repartition".to_string(), + Box::new(external()), + ))), + ), + ]; + for (label, source) in cases { + let error = CometError::from(source); + match jvm_exception(&error) { + JvmException::Spark { json } => { + assert!(json.contains("CannotReadFile"), "{label}: {json}"); + assert!(json.contains("part-0.parquet"), "{label}: {json}"); + } + other => panic!("{label}: expected a Spark exception, got {other:?}"), + } + } + } } diff --git a/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala b/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala index dbfb52133aa..8810c2030eb 100644 --- a/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala +++ b/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala @@ -86,10 +86,11 @@ object SparkErrorConverter extends ShimSparkErrorConverter { val json = parse(e.getMessage) val errorJson = json.extract[ErrorJson] val rawParams = errorJson.params.getOrElse(Map.empty) - // Only CannotReadFile falls back to the per-task file list from CometExecIterator: its native - // error carries a path only for object_store NotFound. ParquetTimestampOverflow is also a - // file-read error, but it must stay out of this fallback: its native reader attaches the - // failing file's path, while the joined list would name every file of a multi-file task. + // Only CannotReadFile falls back to the per-task file list from CometExecIterator. The native + // Parquet scan attaches the failing file's path, so the fallback covers read errors raised + // outside the scan, which carry none. ParquetTimestampOverflow is also a file-read error, but + // it must stay out of this fallback: its native reader attaches the failing file's path, while + // the joined list would name every file of a multi-file task. val params = if (errorJson.errorType == "CannotReadFile" && rawParams.get("filePath").forall(p => p == null || p.toString.isEmpty) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala index 4bd78cf7aee..167ab5ff0c5 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala @@ -299,13 +299,22 @@ class CometTopKSuite extends CometTestBase { } } - for (corruption <- Seq("footer", "page")) { - test(s"local TopK preserves the corrupt Parquet $corruption file in read errors") { + for { + corruption <- Seq("footer", "page") + sharedPartition <- Seq(false, true) + } { + test( + s"local TopK preserves the corrupt Parquet $corruption file in read errors: " + + s"sharedPartition=$sharedPartition") { + // With a shared partition, both files land in one scan partition, so the task's file list + // names both of them and only the native reader knows which one failed. + val numPartitions = if (sharedPartition) 1 else 2 withSQLConf( CometConf.COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED.key -> "true", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", - SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "2", - SQLConf.FILES_MAX_PARTITION_BYTES.key -> "4194304", + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> numPartitions.toString, + SQLConf.FILES_MAX_PARTITION_BYTES.key -> + (if (sharedPartition) "134217728" else "4194304"), SQLConf.IGNORE_CORRUPT_FILES.key -> "false") { withTempPath { path => spark @@ -326,7 +335,9 @@ class CometTopKSuite extends CometTestBase { .sortBy(_.getFileName.toString) } finally entries.close() assert(files.size == 2) - val corruptFile = files.last + // The first file holds the smallest keys, so in a shared partition the TopK dynamic + // filter can't prune the corrupt file's row group, whichever file is read first. + val corruptFile = if (sharedPartition) files.head else files.last val bytes = Files.readAllBytes(corruptFile) if (corruption == "footer") { // Keep the file length but destroy the footer magic, failing during reader setup. @@ -353,13 +364,16 @@ class CometTopKSuite extends CometTestBase { val scans = collect(plan) { case scan: CometNativeScanExec => scan } assert(scans.size == 1, s"Expected one native scan:\n$plan") val partitionPaths = scans.head.perPartitionFilePaths - assert(partitionPaths.length == 2 && partitionPaths.forall(_.size == 1)) + assert( + partitionPaths.length == numPartitions && + partitionPaths.forall(_.size == 2 / numPartitions), + s"Unexpected scan partitions: ${partitionPaths.toSeq}") val expectedPath = partitionPaths.flatten .find(file => new URI(file) == corruptFile.toUri) .getOrElse(fail(s"Missing corrupt file $corruptFile in scan partitions")) val topKs = collect(plan) { case topK: CometTakeOrderedAndProjectExec => topK } assert(topKs.size == 1, s"Expected one native TopK:\n$plan") - assert(topKs.head.child.executeColumnar().getNumPartitions == 2) + assert(topKs.head.child.executeColumnar().getNumPartitions == numPartitions) val locals = collect(plan) { case local: CometLocalTopKExec => local } assert(locals.size == (if (fusion) 1 else 0), s"fusion=$fusion:\n$plan") val failure = intercept[Exception](query.collect()) @@ -372,7 +386,7 @@ class CometTopKSuite extends CometTestBase { } .getOrElse(fail(s"Missing $errorClass in ${causeChain(failure)}")) // A lower-level cause can mention the file even when the structured path is lost. - // With one file per partition, the path must name only the corrupt file. + // The path must name only the corrupt file, even when its partition has two. assert(readError.getMessageParameters.get("path") == expectedPath) } }