Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 97 additions & 19 deletions native/core/src/parquet/file_error_context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ use datafusion::physical_plan::{
DisplayFormatType, SortOrderPushdownResult,
};
use datafusion_comet_common::SparkError;
use datafusion_comet_jni_bridge::errors::file_read_error;
use datafusion_datasource::{
file_stream::FileOpener,
morsel::{Morsel, MorselPlan, MorselPlanner, Morselizer},
Expand Down Expand Up @@ -188,7 +189,8 @@ struct FileContext<T> {
impl MorselPlanner for FileContext<Box<dyn MorselPlanner>> {
fn plan(self: Box<Self>) -> Result<Option<MorselPlan>> {
let Self { inner, path } = *self;
let Some(mut plan) = inner.plan()? else {
// Opening the file (reading its footer) can fail here or in the pending planner below.
let Some(mut plan) = inner.plan().map_err(|error| with_file_path(error, &path))? else {
return Ok(None);
};
let morsels = plan
Expand All @@ -213,10 +215,10 @@ impl MorselPlanner for FileContext<Box<dyn MorselPlanner>> {
.collect();
if let Some(pending) = plan.take_pending_planner() {
plan.set_pending_planner(async move {
Ok(Box::new(FileContext {
inner: pending.await?,
path,
}) as Box<dyn MorselPlanner>)
let inner = pending
.await
.map_err(|error| with_file_path(error, &path))?;
Ok(Box::new(FileContext { inner, path }) as Box<dyn MorselPlanner>)
});
}
Ok(Some(plan.with_morsels(morsels).with_planners(planners)))
Expand All @@ -228,27 +230,40 @@ impl Morsel for FileContext<Box<dyn Morsel>> {
let Self { inner, path } = *self;
inner
.into_stream()
.map_err(move |error| {
// Conversion runs inside this file's stream, before batches from other files mix.
if let DataFusionError::External(source) = &error {
if let Some(SparkError::ParquetTimestampOverflow { .. }) =
source.downcast_ref::<SparkError>()
{
return SparkError::ParquetTimestampOverflow {
file_path: path.to_string(),
}
.into();
}
}
error
})
.map_err(move |error| with_file_path(error, &path))
.boxed()
}
}

/// Name the file in a file-level error raised while planning or reading it. This runs inside the
/// file's own planner and stream, before batches from other files mix, so `path` is the file that
/// failed. Without it a read failure reaches the JVM with no path, and the JVM falls back to every
/// file of the task.
fn with_file_path(error: DataFusionError, path: &str) -> DataFusionError {
if let DataFusionError::External(source) = &error {
if let Some(SparkError::ParquetTimestampOverflow { .. }) =
source.downcast_ref::<SparkError>()
{
return SparkError::ParquetTimestampOverflow {
file_path: path.to_string(),
}
.into();
}
}
match file_read_error(&error) {
Some(SparkError::CannotReadFile { message, .. }) => SparkError::CannotReadFile {
file_path: path.to_string(),
message,
}
.into(),
_ => error,
}
}

#[cfg(test)]
mod tests {
use super::*;
use parquet::errors::ParquetError;

#[derive(Debug)]
struct FailedMorsel(SparkError);
Expand Down Expand Up @@ -299,4 +314,67 @@ mod tests {
assert!(matches!(error, DataFusionError::External(source)
if matches!(source.downcast_ref::<SparkError>(), Some(SparkError::DivideByZero))));
}

fn cannot_read_path(error: &DataFusionError) -> Option<&str> {
let DataFusionError::External(source) = error else {
return None;
};
match source.downcast_ref::<SparkError>() {
Some(SparkError::CannotReadFile { file_path, .. }) => Some(file_path),
_ => None,
}
}

#[test]
fn read_failures_name_the_failed_file() {
Comment thread
ump45nose marked this conversation as resolved.
Resolved
let path = "file:///tmp/t/part-00001.parquet";
let corrupt = DataFusionError::ParquetError(Box::new(ParquetError::General(
"Failed to fetch metadata for file tmp/t/part-00001.parquet: Invalid Parquet file. \
Corrupt footer"
.to_string(),
)));
let named = with_file_path(corrupt, path);
assert_eq!(cannot_read_path(&named), Some(path));
// The native message is kept as the cause.
assert!(named.to_string().contains("Corrupt footer"));

let wrapped = DataFusionError::Context(
"open".to_string(),
Box::new(DataFusionError::ParquetError(Box::new(ParquetError::EOF(
"unexpected end of file".to_string(),
)))),
);
assert_eq!(cannot_read_path(&with_file_path(wrapped, path)), Some(path));
}

#[test]
fn other_errors_keep_their_type() {
let path = "file:///tmp/t/part-00001.parquet";
// A Spark error the reader raised itself.
let missing_ids = DataFusionError::ParquetError(Box::new(ParquetError::External(
Box::new(SparkError::ParquetMissingFieldIds {
file_path: "tmp/t/part-00001.parquet".to_string(),
}),
)));
let error = with_file_path(missing_ids, path);
assert!(matches!(&error, DataFusionError::ParquetError(pe)
if matches!(pe.as_ref(), ParquetError::External(inner)
if matches!(inner.downcast_ref::<SparkError>(),
Some(SparkError::ParquetMissingFieldIds { .. })))));

// A missing file stays a file-not-found error.
let not_found = DataFusionError::ParquetError(Box::new(ParquetError::General(
"Failed to fetch metadata for file tmp/t/gone.parquet: Object Store error: \
Object at location tmp/t/gone.parquet not found: No such file or directory"
.to_string(),
)));
assert!(cannot_read_path(&with_file_path(not_found, path)).is_none());

// An I/O error can come from spill or shuffle, so it is not a file read failure.
let io = DataFusionError::IoError(std::io::Error::other("disk full"));
assert!(matches!(
with_file_path(io, path),
DataFusionError::IoError(_)
));
}
}
51 changes: 51 additions & 0 deletions native/jni-bridge/src/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -899,6 +899,20 @@ fn spark_exception(spark_error: &SparkError) -> JvmException<'static> {
}
}

/// The `CannotReadFile` error a scan should raise for `error`, if it is a per-file read failure,
/// classified the same way the JNI conversion does. A `SparkError` the reader raised itself keeps
/// its own type, so this returns `None` for it. The scan calls this where it still knows which
/// file failed, so that file can be named in the error.
pub fn file_read_error(error: &DataFusionError) -> Option<SparkError> {
if extract_spark_payload(error).is_some() {
return None;
}
match try_classify_file_read_error(error)? {
cannot_read @ SparkError::CannotReadFile { .. } => Some(cannot_read),
_ => None,
}
}

/// Classify a `DataFusionError` as a per-file read failure by TYPED variant (not message text),
/// returning `SparkError::CannotReadFile` if so. This is the structured replacement for the
/// previous JVM-side substring matching on error prose.
Expand Down Expand Up @@ -2249,4 +2263,41 @@ mod tests {
try_classify_file_read_error(&DataFusionError::Internal("bug".to_string())).is_none()
);
}

/// A `CannotReadFile` the scan attached as `External` keeps its structured exception under the
/// `Context` and `Shared` wrappers a repartition or a hash join build side adds, as it does bare.
#[test]
fn external_cannot_read_file_survives_context_and_shared() {
let external = || {
DataFusionError::External(Box::new(SparkError::CannotReadFile {
file_path: "file:/tmp/part-0.parquet".to_string(),
message: "corrupt footer".to_string(),
}))
};
let cases = [
("bare", external()),
(
"context",
DataFusionError::Context("repartition".to_string(), Box::new(external())),
),
("shared", DataFusionError::Shared(Arc::new(external()))),
(
"shared context",
DataFusionError::Shared(Arc::new(DataFusionError::Context(
"repartition".to_string(),
Box::new(external()),
))),
),
];
for (label, source) in cases {
let error = CometError::from(source);
match jvm_exception(&error) {
JvmException::Spark { json } => {
assert!(json.contains("CannotReadFile"), "{label}: {json}");
assert!(json.contains("part-0.parquet"), "{label}: {json}");
}
other => panic!("{label}: expected a Spark exception, got {other:?}"),
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,11 @@ object SparkErrorConverter extends ShimSparkErrorConverter {
val json = parse(e.getMessage)
val errorJson = json.extract[ErrorJson]
val rawParams = errorJson.params.getOrElse(Map.empty)
// Only CannotReadFile falls back to the per-task file list from CometExecIterator: its native
// error carries a path only for object_store NotFound. ParquetTimestampOverflow is also a
// file-read error, but it must stay out of this fallback: its native reader attaches the
// failing file's path, while the joined list would name every file of a multi-file task.
// Only CannotReadFile falls back to the per-task file list from CometExecIterator. The native
// Parquet scan attaches the failing file's path, so the fallback covers read errors raised
// outside the scan, which carry none. ParquetTimestampOverflow is also a file-read error, but
// it must stay out of this fallback: its native reader attaches the failing file's path, while
// the joined list would name every file of a multi-file task.
val params =
if (errorJson.errorType == "CannotReadFile"
&& rawParams.get("filePath").forall(p => p == null || p.toString.isEmpty)
Expand Down
30 changes: 22 additions & 8 deletions spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -299,13 +299,22 @@ class CometTopKSuite extends CometTestBase {
}
}

for (corruption <- Seq("footer", "page")) {
test(s"local TopK preserves the corrupt Parquet $corruption file in read errors") {
for {
corruption <- Seq("footer", "page")
sharedPartition <- Seq(false, true)
} {
test(
s"local TopK preserves the corrupt Parquet $corruption file in read errors: " +
s"sharedPartition=$sharedPartition") {
// With a shared partition, both files land in one scan partition, so the task's file list
// names both of them and only the native reader knows which one failed.
val numPartitions = if (sharedPartition) 1 else 2
withSQLConf(
CometConf.COMET_EXEC_TOPK_DYNAMIC_FILTER_ENABLED.key -> "true",
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "2",
SQLConf.FILES_MAX_PARTITION_BYTES.key -> "4194304",
SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> numPartitions.toString,
SQLConf.FILES_MAX_PARTITION_BYTES.key ->
(if (sharedPartition) "134217728" else "4194304"),
SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
withTempPath { path =>
spark
Expand All @@ -326,7 +335,9 @@ class CometTopKSuite extends CometTestBase {
.sortBy(_.getFileName.toString)
} finally entries.close()
assert(files.size == 2)
val corruptFile = files.last
// The first file holds the smallest keys, so in a shared partition the TopK dynamic
// filter can't prune the corrupt file's row group, whichever file is read first.
val corruptFile = if (sharedPartition) files.head else files.last
val bytes = Files.readAllBytes(corruptFile)
if (corruption == "footer") {
// Keep the file length but destroy the footer magic, failing during reader setup.
Expand All @@ -353,13 +364,16 @@ class CometTopKSuite extends CometTestBase {
val scans = collect(plan) { case scan: CometNativeScanExec => scan }
assert(scans.size == 1, s"Expected one native scan:\n$plan")
val partitionPaths = scans.head.perPartitionFilePaths
assert(partitionPaths.length == 2 && partitionPaths.forall(_.size == 1))
assert(
partitionPaths.length == numPartitions &&
partitionPaths.forall(_.size == 2 / numPartitions),
s"Unexpected scan partitions: ${partitionPaths.toSeq}")
val expectedPath = partitionPaths.flatten
.find(file => new URI(file) == corruptFile.toUri)
.getOrElse(fail(s"Missing corrupt file $corruptFile in scan partitions"))
val topKs = collect(plan) { case topK: CometTakeOrderedAndProjectExec => topK }
assert(topKs.size == 1, s"Expected one native TopK:\n$plan")
assert(topKs.head.child.executeColumnar().getNumPartitions == 2)
assert(topKs.head.child.executeColumnar().getNumPartitions == numPartitions)
val locals = collect(plan) { case local: CometLocalTopKExec => local }
assert(locals.size == (if (fusion) 1 else 0), s"fusion=$fusion:\n$plan")
val failure = intercept[Exception](query.collect())
Expand All @@ -372,7 +386,7 @@ class CometTopKSuite extends CometTestBase {
}
.getOrElse(fail(s"Missing $errorClass in ${causeChain(failure)}"))
// A lower-level cause can mention the file even when the structured path is lost.
// With one file per partition, the path must name only the corrupt file.
// The path must name only the corrupt file, even when its partition has two.
assert(readError.getMessageParameters.get("path") == expectedPath)
}
}
Expand Down