You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2026/06/29 09:56:27 UTC
(spark) branch master updated: [SPARK-57746][SQL] Assign a name to the error conditions _LEGACY_ERROR_TEMP_3171, 3172
This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 916616903ae7 [SPARK-57746][SQL] Assign a name to the error conditions _LEGACY_ERROR_TEMP_3171, 3172
916616903ae7 is described below
commit 916616903ae731011c85ad5be6eef02c3ab46c7c
Author: Maxim Gekk <ma...@gmail.com>
AuthorDate: Mon Jun 29 11:56:13 2026 +0200
[SPARK-57746][SQL] Assign a name to the error conditions _LEGACY_ERROR_TEMP_3171, 3172
### What changes were proposed in this pull request?
This PR assigns a proper name to the legacy error conditions `_LEGACY_ERROR_TEMP_3171` and `_LEGACY_ERROR_TEMP_3172` as part of the error-class migration (SPARK-37935).
Both are thrown by Parquet aggregate push-down in `ParquetUtils` when the required column-chunk statistics are missing. They share the same parameters (`filePath`, `config`) and remedy, and differ only in which statistic is absent, so they are merged into a single parent condition with two sub-conditions:
- `_LEGACY_ERROR_TEMP_3172` -> `PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_MIN_MAX` (thrown by `getCurrentBlockMaxOrMin` during MIN/MAX push-down when a column chunk has no non-null values).
- `_LEGACY_ERROR_TEMP_3171` -> `PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_NUM_NULLS` (thrown by `getNumNulls` during COUNT push-down when the null count is not set).
The new condition uses SQLSTATE `0A000` (feature not supported).
### Why are the changes needed?
`_LEGACY_ERROR_TEMP_*` are temporary placeholder conditions with no proper name or SQLSTATE. Assigning real names is part of the ongoing error-class migration (SPARK-37935) and improves the user-facing error experience.
### Does this PR introduce _any_ user-facing change?
Yes. The error condition names, messages, and SQLSTATE for the two Parquet aggregate push-down errors change.
Before:
- `[_LEGACY_ERROR_TEMP_3172] No min/max found for Parquet file <filePath>. Set SQLConf <config> to false and execute again.`
- `[_LEGACY_ERROR_TEMP_3171] Number of nulls not set for Parquet file <filePath>. Set SQLConf <config> to false and execute again.`
After (SQLSTATE `0A000`):
- `[PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_MIN_MAX] Aggregate push-down is not supported for Parquet file <filePath>. Set SQLConf <config> to false and execute again. No min/max value found in the column statistics.`
- `[PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_NUM_NULLS] Aggregate push-down is not supported for Parquet file <filePath>. Set SQLConf <config> to false and execute again. The number of nulls is not set in the column statistics.`
### How was this patch tested?
- Added a `checkError` test in `ParquetV2AggregatePushDownSuite` that writes a Parquet file with column statistics disabled (`parquet.column.statistics.enabled=false`) and asserts both sub-conditions via `MIN`/`COUNT` aggregate push-down.
- `build/sbt "core/testOnly org.apache.spark.SparkThrowableSuite"`
- `build/sbt "sql/testOnly *ParquetV2AggregatePushDownSuite *ParquetV1AggregatePushDownSuite"`
- `build/sbt "sql/scalastyle" "sql/Test/scalastyle"`
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Cursor (Claude Opus 4.8)
Closes #56859 from MaxGekk/error-cond_LEGACY_ERROR_TEMP_3172.
Authored-by: Maxim Gekk <ma...@gmail.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../src/main/resources/error/error-conditions.json | 28 +++++++++-----
.../datasources/parquet/ParquetUtils.scala | 4 +-
.../FileSourceAggregatePushDownSuite.scala | 43 +++++++++++++++++++++-
3 files changed, 62 insertions(+), 13 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index b6206653a161..10bae53a5a7b 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -6044,6 +6044,24 @@
],
"sqlState" : "42805"
},
+ "PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED" : {
+ "message" : [
+ "Aggregate push-down is not supported for Parquet file <filePath>. Set SQLConf <config> to false and execute again."
+ ],
+ "subClass" : {
+ "NO_MIN_MAX" : {
+ "message" : [
+ "No min/max value found in the column statistics."
+ ]
+ },
+ "NO_NUM_NULLS" : {
+ "message" : [
+ "The number of nulls is not set in the column statistics."
+ ]
+ }
+ },
+ "sqlState" : "0A000"
+ },
"PARQUET_CONVERSION_FAILURE" : {
"message" : [
"Unable to create a Parquet converter for the data type <dataType> whose Parquet type is <parquetType>."
@@ -11511,16 +11529,6 @@
"SortAggregate code-gen does not support grouping keys"
]
},
- "_LEGACY_ERROR_TEMP_3171" : {
- "message" : [
- "Number of nulls not set for Parquet file <filePath>. Set SQLConf <config> to false and execute again."
- ]
- },
- "_LEGACY_ERROR_TEMP_3172" : {
- "message" : [
- "No min/max found for Parquet file <filePath>. Set SQLConf <config> to false and execute again."
- ]
- },
"_LEGACY_ERROR_TEMP_3173" : {
"message" : [
"Cannot specify 'USING index_type' in 'CREATE INDEX'"
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
index a884b4c5fe23..7e67f14e0ad9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
@@ -407,7 +407,7 @@ object ParquetUtils extends Logging {
val statistics = columnChunkMetaData.get(i).getStatistics
if (!statistics.hasNonNullValue) {
throw new SparkUnsupportedOperationException(
- errorClass = "_LEGACY_ERROR_TEMP_3172",
+ errorClass = "PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_MIN_MAX",
messageParameters = Map(
"filePath" -> filePath,
"config" -> PARQUET_AGGREGATE_PUSHDOWN_ENABLED.key))
@@ -423,7 +423,7 @@ object ParquetUtils extends Logging {
val statistics = columnChunkMetaData.get(i).getStatistics
if (!statistics.isNumNullsSet) {
throw new SparkUnsupportedOperationException(
- errorClass = "_LEGACY_ERROR_TEMP_3171",
+ errorClass = "PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_NUM_NULLS",
messageParameters = Map(
"filePath" -> filePath,
"config" -> PARQUET_AGGREGATE_PUSHDOWN_ENABLED.key))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala
index 706657712763..c95842fc7306 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceAggregatePushDownSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources
import java.sql.{Date, Timestamp}
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, SparkUnsupportedOperationException}
import org.apache.spark.sql.{DataFrame, ExplainSuiteHelper, Row}
import org.apache.spark.sql.catalyst.optimizer.CollapseGroupedSumOfCount
import org.apache.spark.sql.catalyst.plans.logical.Aggregate
@@ -618,6 +618,47 @@ class ParquetV2AggregatePushDownSuite extends ParquetAggregatePushDownSuite {
override protected def sparkConf: SparkConf =
super.sparkConf.set(SQLConf.USE_V1_SOURCE_LIST, "")
+
+ // Aggregate push-down only happens with the DSv2 file source, so this test lives in the V2
+ // suite. It writes a Parquet file with column statistics disabled, which leaves both the
+ // min/max values and the number of nulls unavailable for push-down.
+ test("SPARK-57746: aggregate push-down fails when Parquet statistics are missing") {
+ Seq("false", "true").foreach { enableVectorizedReader =>
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ spark.range(10).selectExpr("CAST(id AS INT) AS i").coalesce(1)
+ .write.option("parquet.column.statistics.enabled", "false").parquet(path)
+ withTempView("t") {
+ spark.read.parquet(path).createOrReplaceTempView("t")
+ withSQLConf(
+ aggPushDownEnabledKey -> "true",
+ vectorizedReaderEnabledKey -> enableVectorizedReader) {
+ checkErrorMatchPVals(
+ exception = interceptAggPushDownError("SELECT min(i) FROM t"),
+ condition = "PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_MIN_MAX",
+ parameters = Map("filePath" -> ".*", "config" -> aggPushDownEnabledKey))
+ checkErrorMatchPVals(
+ exception = interceptAggPushDownError("SELECT count(i) FROM t"),
+ condition = "PARQUET_AGGREGATE_PUSH_DOWN_UNSUPPORTED.NO_NUM_NULLS",
+ parameters = Map("filePath" -> ".*", "config" -> aggPushDownEnabledKey))
+ }
+ }
+ }
+ }
+ }
+
+ // The error originates in the executor-side partition reader, so it may be wrapped in a
+ // higher-level exception. Walk the cause chain to find the structured Spark exception.
+ private def interceptAggPushDownError(query: String): SparkUnsupportedOperationException = {
+ val e = intercept[Exception](spark.sql(query).collect())
+ var cause: Throwable = e
+ while (cause != null && !cause.isInstanceOf[SparkUnsupportedOperationException]) {
+ cause = cause.getCause
+ }
+ assert(cause != null,
+ s"Expected a SparkUnsupportedOperationException but got: $e")
+ cause.asInstanceOf[SparkUnsupportedOperationException]
+ }
}
abstract class OrcAggregatePushDownSuite extends OrcTest with FileSourceAggregatePushDownSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org