diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 121e8bdb2a9..965102dff4c 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -2399,7 +2399,7 @@ index 104b4e416cd..835aaa18e39 100644 case _ => throw new AnalysisException("Can not match ParquetTable in the query.") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala -index 29cb224c878..62e3ab96004 100644 +index 29cb224c878..d76c2e8e588 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala @@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat @@ -2430,17 +2430,7 @@ index 29cb224c878..62e3ab96004 100644 withAllParquetReaders { withTempPath { path => // Repeated values for dictionary encoding. -@@ -1031,7 +1034,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS - testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96") - } - -- test("SPARK-34212 Parquet should read decimals correctly") { -+ test("SPARK-34212 Parquet should read decimals correctly", -+ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) { - def readParquet(schema: String, path: File): DataFrame = { - spark.read.schema(schema).parquet(path.toString) - } -@@ -1047,7 +1051,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1047,7 +1050,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS checkAnswer(readParquet(schema, path), df) } @@ -2450,7 +2440,7 @@ index 29cb224c878..62e3ab96004 100644 val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)" checkAnswer(readParquet(schema1, path), df) val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)" -@@ -1069,7 +1074,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1069,7 +1073,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d") df.write.parquet(path.toString) @@ -2460,7 +2450,7 @@ index 29cb224c878..62e3ab96004 100644 checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00")) checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null)) checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0")) -@@ -1113,7 +1119,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1113,7 +1118,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS } } @@ -2470,15 +2460,6 @@ index 29cb224c878..62e3ab96004 100644 withTempPath { path => Seq(0).toDF("a").write.parquet(path.toString) // The vectorized and non-vectorized readers will produce different exceptions, we don't need -@@ -1128,7 +1135,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS - .where(s"a < ${Long.MaxValue}") - .collect() - } -- assert(exception.getCause.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException]) -+ assert(exception.getMessage.contains("Column: [a], Expected: bigint, Found: INT32")) - } - } - } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala index 240bb4e6dcb..8287ffa03ca 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala diff --git a/dev/diffs/3.5.9.diff b/dev/diffs/3.5.9.diff index c441e922236..5791db9bbf3 100644 --- a/dev/diffs/3.5.9.diff +++ b/dev/diffs/3.5.9.diff @@ -2717,7 +2717,7 @@ index 8e88049f51e..f8e2194a8ac 100644 case _ => throw new AnalysisException("Can not match ParquetTable in the query.") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala -index ecec7d7b217..f6a4c746226 100644 +index ecec7d7b217..7c841068e4a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala @@ -185,7 +185,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS @@ -2740,17 +2740,7 @@ index ecec7d7b217..f6a4c746226 100644 withAllParquetReaders { withTempPath { path => // Repeated values for dictionary encoding. -@@ -1087,7 +1089,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS - testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96") - } - -- test("SPARK-34212 Parquet should read decimals correctly") { -+ test("SPARK-34212 Parquet should read decimals correctly", -+ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) { - def readParquet(schema: String, path: File): DataFrame = { - spark.read.schema(schema).parquet(path.toString) - } -@@ -1103,7 +1106,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1103,7 +1105,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS checkAnswer(readParquet(schema, path), df) } @@ -2760,7 +2750,7 @@ index ecec7d7b217..f6a4c746226 100644 val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)" checkAnswer(readParquet(schema1, path), df) val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)" -@@ -1125,7 +1129,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1125,7 +1128,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d") df.write.parquet(path.toString) @@ -2770,7 +2760,7 @@ index ecec7d7b217..f6a4c746226 100644 checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00")) checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null)) checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0")) -@@ -1169,7 +1174,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1169,7 +1173,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS } } @@ -2780,15 +2770,6 @@ index ecec7d7b217..f6a4c746226 100644 withTempPath { path => Seq(0).toDF("a").write.parquet(path.toString) // The vectorized and non-vectorized readers will produce different exceptions, we don't need -@@ -1184,7 +1190,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS - .where(s"a < ${Long.MaxValue}") - .collect() - } -- assert(exception.getCause.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException]) -+ assert(exception.getMessage.contains("Column: [a], Expected: bigint, Found: INT32")) - } - } - } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala index 4f906411345..6cc69f7e915 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRebaseDatetimeSuite.scala diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index b3f93e18762..a3b0e4b50c1 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -573,8 +573,10 @@ fn spark_has_updater( return false; } - // Decimal-to-decimal narrowing. Spark's `isDecimalTypeMatched` (the - // `DecimalLogicalTypeAnnotation` branch) allows the read only when + // Spark 3.x requires equal decimal scales. Spark 4+ permits scale widening + // when the requested precision grows by at least as much as the scale. + // Its `isDecimalTypeMatched` (the `DecimalLogicalTypeAnnotation` branch) + // allows the read only when // `dst_scale >= src_scale` AND // `dst_precision - dst_scale >= src_precision - src_scale`. // Either failure means silently dropping fractional digits or losing @@ -584,6 +586,12 @@ fn spark_has_updater( { let src_int_precision = i32::from(*src_p) - i32::from(*src_s); let dst_int_precision = i32::from(*dst_p) - i32::from(*dst_s); + if !options.allow_type_promotion && dst_s > src_s { + // Match the vectorized reader: reject while decoding a non-empty row + // group, rather than failing an empty file during schema adaptation. + // This is the Spark 3.x SPARK-34212 failure tracked under Comet #4354. + return reject_on_non_empty(); + } if dst_s < src_s || dst_int_precision < src_int_precision { return false; } @@ -2074,11 +2082,10 @@ pub(crate) mod test { Ok(()) } - /// Sanity check: widening both precision and scale by the same amount is - /// allowed (the cast is lossless). Decimal(5, 2) -> Decimal(7, 4) gives - /// scaleIncrease=2, precisionIncrease=2, so `precisionIncrease >= scaleIncrease`. + /// Spark 4+ allows lossless scale widening; Spark 3.x rejects it only when + /// reading rows. Empty files must remain readable in both versions. #[tokio::test] - async fn parquet_decimal_widening_succeeds() -> Result<(), DataFusionError> { + async fn parquet_decimal_widening_follows_spark_version() -> Result<(), DataFusionError> { let batch = decimal_batch(5, 2)?; let required_schema = Arc::new(Schema::new(vec![Field::new( "a", @@ -2086,7 +2093,36 @@ pub(crate) mod test { false, )])); - let _ = roundtrip(&batch, required_schema).await?; + for allow_type_promotion in [false, true] { + let mut options = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false); + options.allow_type_promotion = allow_type_promotion; + let mut stream = scan_parquet(&batch, Arc::clone(&required_schema), options.clone())?; + let result = stream.next().await.unwrap(); + if allow_type_promotion { + let result = result?; + assert_eq!( + result + .column(0) + .as_primitive::() + .values(), + &[12300, 45600] + ); + } else { + let err = result.expect_err("Spark 3.x must reject decimal scale widening"); + let DataFusionError::External(source) = err else { + panic!("expected a typed schema conversion error, got {err}"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(datafusion_comet_common::SparkError::ParquetSchemaConvert { .. }) + )); + } + let empty = RecordBatch::new_empty(batch.schema()); + let mut stream = scan_parquet(&empty, Arc::clone(&required_schema), options)?; + while let Some(result) = stream.next().await { + assert_eq!(result?.num_rows(), 0); + } + } Ok(()) } diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs index a89742a9e85..cde9b3846d1 100644 --- a/native/jni-bridge/src/errors.rs +++ b/native/jni-bridge/src/errors.rs @@ -718,6 +718,9 @@ fn jvm_exception(error: &CometError) -> JvmException<'_> { message: exception.msg, }; } + if let Some(json) = spark_error_json(error) { + return JvmException::Spark { json }; + } match error { CometError::JavaException { class, @@ -809,6 +812,24 @@ fn typed_jvm_exception(error: &(dyn std::error::Error + 'static)) -> Option DataFusion Context -> the original error (DF #24638). +fn spark_error_json(error: &(dyn std::error::Error + 'static)) -> Option { + let mut cause = Some(error); + while let Some(error) = cause { + if let Some(error) = error.downcast_ref::() { + return Some(error.to_json()); + } + if let Some(error) = error.downcast_ref::() { + return Some(error.to_json()); + } + cause = error.source(); + } + None +} + /// Generic fallback for an error that isn't a structured `SparkError`. Recognises a file-not-found /// arriving through non-typed wrapping paths and duplicate-field errors; otherwise the error's /// natural JVM exception. @@ -1442,6 +1463,26 @@ mod tests { assert_eq!(message, SparkError::DivideByZero.to_json()); } + #[test] + fn native_error_spark_error_survives_parquet_predicate_wrappers() { + let spark_error = SparkError::ParquetSchemaConvert { + file_path: String::new(), + column: "[c]".to_string(), + physical_type: "INT32".to_string(), + spark_type: "bigint".to_string(), + }; + let expected = spark_error.to_json(); + let predicate = DataFusionError::External(Box::new(spark_error)) + .context("Error evaluating filter predicate"); + let arrow = ArrowError::ExternalError(Box::new(predicate)); + let parquet = ParquetError::External(Box::new(arrow)); + let error = CometError::from(DataFusionError::ParquetError(Box::new(parquet))); + + let native_error = NativeError::from_comet_error(&error); + assert_eq!(native_error.exception_class(), QUERY_EXECUTION_EXCEPTION); + assert_eq!(native_error.message(), expected); + } + #[test] fn native_error_read_ancient_datetime_through_datafusion() { let spark_error = SparkError::read_ancient_datetime("d", false); diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index c36d5d69f33..20a64abbeaa 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -22,11 +22,38 @@ package org.apache.comet import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.{SparkException, SparkThrowable} +import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus class SparkErrorConverterSuite extends AnyFunSuite { + test("ParquetSchemaConvert preserves the file error and its typed cause") { + Seq(None, Some(""), Some("file:/tmp/data/part-0.parquet")).foreach { path => + val fileParam = path.map(p => s""", "filePath":"$p"""").getOrElse("") + val json = + s"""{"errorType":"ParquetSchemaConvert","errorClass":"","params":{ + |"column":"[c]","physicalType":"BINARY","sparkType":"decimal(37,1)"$fileParam}} + |""".stripMargin + val error = SparkErrorConverter.convertToSparkException( + new org.apache.comet.exceptions.CometQueryExecutionException(json)) + assert(error.isInstanceOf[SparkException]) + val expectedClass = if (CometSparkSessionExtensions.isSpark40Plus) { + "FAILED_READ_FILE.PARQUET_COLUMN_DATA_TYPE_MISMATCH" + } else { + "_LEGACY_ERROR_TEMP_2063" + } + assert(error.asInstanceOf[SparkException].getErrorClass == expectedClass) + assert(error.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException]) + val cause = error.getCause.asInstanceOf[SchemaColumnConvertNotSupportedException] + assert(cause.getColumn == "[c]") + assert(cause.getPhysicalType == "BINARY") + assert(cause.getLogicalType == "decimal(37,1)") + assert(cause.getCause == null) + path.filter(_.nonEmpty).foreach(p => assert(error.getMessage.contains(p))) + } + } + test("ArithmeticOverflow preserves the suggested function") { for (function <- Seq("try_add", "try_subtract", "try_multiply")) { val error = SparkErrorConverter diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 56743367a4a..1e7d151b8a1 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -1705,20 +1705,98 @@ abstract class ParquetReadSuite extends CometTestBase { // CAST('1.2' AS BINARY) writes BYTE_ARRAY with no decimal annotation. sql("SELECT CAST('1.2' AS BINARY) c").write.parquet(path) Seq("DECIMAL(3, 2)", "DECIMAL(18, 1)", "DECIMAL(37, 1)").foreach { schema => - val outer = intercept[SparkException] { - spark.read.schema(s"c $schema").parquet(path).collect() + val expected = intercept[SparkException] { + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark.read.schema(s"c $schema").parquet(path).collect() + } + } + val df = spark.read.schema(s"c $schema").parquet(path) + assert(collect(df.queryExecution.executedPlan) { case scan: CometNativeScanExec => + scan + }.nonEmpty) + val actual = intercept[SparkException](df.collect()) + // Spark 3.x adds a driver-side SparkException; Spark 4+ passes the structured + // file error through. Compare the entire chain rather than searching for a leaf. + assert(causeChain(actual).map(_.getClass) == causeChain(expected).map(_.getClass)) + assert(causeChain(actual).last.getMessage == causeChain(expected).last.getMessage) + } + } + } + } + + Seq( + ("decimal(2,1)", "decimal(3,2)"), + ("decimal(17,2)", "decimal(18,3)"), + ("decimal(36,2)", "decimal(37,3)"), + ("decimal(5,2)", "decimal(9,2)"), + ("decimal(10,2)", "decimal(10,1)"), + ("int", "bigint")).foreach { case (writeType, readType) => + test(s"Parquet schema conversion matches Spark: $writeType -> $readType") { + withSQLConf( + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true") { + withClue(s"$writeType -> $readType: ") { + withTempPath { dir => + val path = dir.getCanonicalPath + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark.sql(s"SELECT CAST(1 AS $writeType) c").write.parquet(path) + } + var expected: Either[SparkException, Seq[Row]] = Right(Seq.empty) + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + try { + expected = Right(spark.read.schema(s"c $readType").parquet(path).collect().toSeq) + } catch { + case e: SparkException => expected = Left(e) + } + } + val df = spark.read.schema(s"c $readType").parquet(path) + assert(collect(df.queryExecution.executedPlan) { case scan: CometNativeScanExec => + scan + }.nonEmpty) + expected match { + case Right(rows) => checkAnswer(df, rows) + case Left(error) => + val actual = intercept[SparkException](df.collect()) + assert(causeChain(actual).map(_.getClass) == causeChain(error).map(_.getClass)) + assert(causeChain(actual).last.getMessage == causeChain(error).last.getMessage) + } + } + } + } + } + } + + Seq(false, true).foreach { pushdown => + test(s"Parquet schema conversion with a row-group predicate, pushdown=$pushdown") { + withSQLConf( + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true", + CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> pushdown.toString) { + withTempPath { dir => + val path = dir.getCanonicalPath + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + Seq(0).toDF("c").write.parquet(path) + } + def read(): DataFrame = + spark.read.schema("c BIGINT").parquet(path).where(s"c < ${Long.MaxValue}") + val df = read() + assert(collect(df.queryExecution.executedPlan) { case scan: CometNativeScanExec => + scan + }.nonEmpty) + if (isSpark40Plus) { + checkAnswer(df, Seq(Row(0L))) + } else { + val expected = intercept[SparkException] { + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { read().collect() } + } + val actual = intercept[SparkException](df.collect()) + assert(causeChain(actual).map(_.getClass) == causeChain(expected).map(_.getClass)) + assert(causeChain(actual).last.getMessage == causeChain(expected).last.getMessage) } - // Walk the cause chain: Comet's shim adds an extra SparkException - // wrap on Spark 3.x compared to vanilla Spark. - val chain = causeChain(outer) - assert( - chain.exists(_.isInstanceOf[ - org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException]), - s"expected SchemaColumnConvertNotSupportedException for $schema; chain was:\n" + - chain.map(t => s" ${t.getClass.getName}: ${t.getMessage}").mkString("\n")) } } } + } test("native scan rejects incompatible decimal precision/scale") {