You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/19 19:10:48 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6046-f0787bcb4c2de95d5a9bddf9e009585aebe38414 created (now 1c25b492bf)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6046-f0787bcb4c2de95d5a9bddf9e009585aebe38414
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 1c25b492bf fix: preserve Spark row index read errors (#6046)
This branch includes the following new commits:
new 1c25b492bf fix: preserve Spark row index read errors (#6046)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: fix: preserve Spark row index read errors (#6046)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6046-f0787bcb4c2de95d5a9bddf9e009585aebe38414
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 1c25b492bfb5d4615c7838cf24ec4da867263034
Author: liupoyi-1031 <bo...@gmail.com>
AuthorDate: Sat Sep 19 17:37:55 2026 +0000
fix: preserve Spark row index read errors (#6046)
Fall back before validating row index column types so Spark can surface invalid schemas as FAILED_READ_FILE.
Co-authored-by: liupoyi-1031 <24...@users.noreply.github.com>
---
dev/diffs/4.0.4.diff | 10 ----------
dev/diffs/4.1.3.diff | 10 ----------
.../src/main/scala/org/apache/comet/rules/CometScanRule.scala | 6 +++++-
3 files changed, 5 insertions(+), 21 deletions(-)
diff --git a/dev/diffs/4.0.4.diff b/dev/diffs/4.0.4.diff
index 12eaabd1be..1d9978f418 100644
--- a/dev/diffs/4.0.4.diff
+++ b/dev/diffs/4.0.4.diff
@@ -3056,16 +3056,6 @@ index 08fd8a9ecb5..06967aec8e1 100644
case _ =>
}
assert(numPartitions > 0)
-@@ -303,6 +312,9 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
- val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
-
- test(s"invalid row index column type - ${conf.desc}") {
-+ // https://github.com/apache/datafusion-comet/issues/3886
-+ // Comet throws RuntimeException instead of SparkException
-+ assume(false)
- withSQLConf(conf.sqlConfs: _*) {
- withTempPath{ path =>
- val df = spark.range(0, 10, 1, 1).toDF("id")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index 5c0b7def039..151184bc98c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
diff --git a/dev/diffs/4.1.3.diff b/dev/diffs/4.1.3.diff
index bec73bfb98..af819e4a1b 100644
--- a/dev/diffs/4.1.3.diff
+++ b/dev/diffs/4.1.3.diff
@@ -3222,16 +3222,6 @@ index 08fd8a9ecb5..06967aec8e1 100644
case _ =>
}
assert(numPartitions > 0)
-@@ -303,6 +312,9 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
- val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
-
- test(s"invalid row index column type - ${conf.desc}") {
-+ // https://github.com/apache/datafusion-comet/issues/3886
-+ // Comet throws RuntimeException instead of SparkException
-+ assume(false)
- withSQLConf(conf.sqlConfs: _*) {
- withTempPath{ path =>
- val df = spark.range(0, 10, 1, 1).toDF("id")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index 5c0b7def039..151184bc98c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
index d6a5223bee..5f37761a05 100644
--- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
+++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
@@ -346,7 +346,11 @@ case class CometScanRule(session: SparkSession)
"input_file_block_start, or input_file_block_length")
return None
}
- if (ShimFileFormat.findRowIndexColumnIndexInSchema(scanExec.requiredSchema) >= 0) {
+ // Check the name directly instead of calling findRowIndexColumnIndexInSchema, which validates
+ // the temporary column's type and can throw a raw RuntimeException. Falling back lets Spark's
+ // Parquet reader wrap that validation failure as FAILED_READ_FILE.
+ if (scanExec.requiredSchema.fieldNames.contains(
+ ShimFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME)) {
withFallbackReason(scanExec, "Native Parquet scan does not support row index generation")
return None
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org