You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/10/09 01:12:54 UTC
(spark) branch master updated: [SPARK-49906][SQL] Introduce and use CONFLICTING_DIRECTORY_STRUCTURES error for PartitioningUtils
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 345a2be41320 [SPARK-49906][SQL] Introduce and use CONFLICTING_DIRECTORY_STRUCTURES error for PartitioningUtils
345a2be41320 is described below
commit 345a2be4132040c03920670a1f90624f9ad0f88d
Author: Vladimir Golubev <vl...@databricks.com>
AuthorDate: Wed Oct 9 10:12:40 2024 +0900
[SPARK-49906][SQL] Introduce and use CONFLICTING_DIRECTORY_STRUCTURES error for PartitioningUtils
### What changes were proposed in this pull request?
Improve Spark user experience by introducing a new error type: `CONFLICTING_DIRECTORY_STRUCTURES` for `PartitioningUtils`.
### Why are the changes needed?
`PartitioningUtils.parsePartitions(...)` uses an assertion to if partitions are misconfigured. We should use a proper error type for this case.
### Does this PR introduce _any_ user-facing change?
Yes, the error will be nicer.
### How was this patch tested?
Updated the existing tests.
### Was this patch authored or co-authored using generative AI tooling?
`copilot.vim`.
Closes #48383 from vladimirg-db/vladimirg-db/introduce-conflicting-directory-structures-error.
Authored-by: Vladimir Golubev <vl...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
common/utils/src/main/resources/error/error-conditions.json | 10 ++++++++++
.../org/apache/spark/sql/errors/QueryExecutionErrors.scala | 10 ++++++++++
.../spark/sql/execution/datasources/PartitioningUtils.scala | 11 +++--------
.../spark/sql/execution/datasources/FileIndexSuite.scala | 4 ++--
.../datasources/parquet/ParquetPartitionDiscoverySuite.scala | 8 ++++----
5 files changed, 29 insertions(+), 14 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index e3bffea0b62e..8100f0580b21 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -669,6 +669,16 @@
],
"sqlState" : "40000"
},
+ "CONFLICTING_DIRECTORY_STRUCTURES" : {
+ "message" : [
+ "Conflicting directory structures detected.",
+ "Suspicious paths:",
+ "<discoveredBasePaths>",
+ "If provided paths are partition directories, please set \"basePath\" in the options of the data source to specify the root directory of the table.",
+ "If there are multiple root directories, please load them separately and then union them."
+ ],
+ "sqlState" : "KD009"
+ },
"CONFLICTING_PARTITION_COLUMN_NAMES" : {
"message" : [
"Conflicting partition column names detected:",
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index bc6c7681ea1a..301880f1bfc6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -2845,6 +2845,16 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
)
}
+ def conflictingDirectoryStructuresError(
+ discoveredBasePaths: Seq[String]): SparkRuntimeException = {
+ new SparkRuntimeException(
+ errorClass = "CONFLICTING_DIRECTORY_STRUCTURES",
+ messageParameters = Map(
+ "discoveredBasePaths" -> discoveredBasePaths.distinct.mkString("\n\t", "\n\t", "\n")
+ )
+ )
+ }
+
def conflictingPartitionColumnNamesError(
distinctPartColLists: Seq[String],
suspiciousPaths: Seq[Path]): SparkRuntimeException = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index ffdca6515105..402b70065d8e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -173,14 +173,9 @@ object PartitioningUtils extends SQLConfHelper {
// "hdfs://host:9000/path"
// TODO: Selective case sensitivity.
val discoveredBasePaths = optDiscoveredBasePaths.flatten.map(_.toString.toLowerCase())
- assert(
- ignoreInvalidPartitionPaths || discoveredBasePaths.distinct.size == 1,
- "Conflicting directory structures detected. Suspicious paths:\b" +
- discoveredBasePaths.distinct.mkString("\n\t", "\n\t", "\n\n") +
- "If provided paths are partition directories, please set " +
- "\"basePath\" in the options of the data source to specify the " +
- "root directory of the table. If there are multiple root directories, " +
- "please load them separately and then union them.")
+ if (!ignoreInvalidPartitionPaths && discoveredBasePaths.distinct.size != 1) {
+ throw QueryExecutionErrors.conflictingDirectoryStructuresError(discoveredBasePaths)
+ }
val resolvedPartitionValues = resolvePartitions(pathsWithPartitionValues, caseSensitive)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index 31b738088915..e9f78f9f598e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -566,7 +566,7 @@ class FileIndexSuite extends SharedSparkSession {
new File(directoryPath, "part_col=1").renameTo(new File(directoryPath, "undefined"))
// By default, we expect the invalid path assertion to trigger.
- val ex = intercept[AssertionError] {
+ val ex = intercept[SparkRuntimeException] {
spark.read
.format("parquet")
.load(directoryPath.getCanonicalPath)
@@ -585,7 +585,7 @@ class FileIndexSuite extends SharedSparkSession {
// Data source option override takes precedence.
withSQLConf(SQLConf.IGNORE_INVALID_PARTITION_PATHS.key -> "true") {
- val ex = intercept[AssertionError] {
+ val ex = intercept[SparkRuntimeException] {
spark.read
.format("parquet")
.option(FileIndexOptions.IGNORE_INVALID_PARTITION_PATHS, "false")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index 52d67a095432..eb4618834504 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -111,7 +111,7 @@ abstract class ParquetPartitionDiscoverySuite
"hdfs://host:9000/path/a=10/b=20",
"hdfs://host:9000/path/a=10.5/b=hello")
- var exception = intercept[AssertionError] {
+ var exception = intercept[SparkRuntimeException] {
parsePartitions(
paths.map(new Path(_)), true, Set.empty[Path], None, true, true, timeZoneId, false)
}
@@ -173,7 +173,7 @@ abstract class ParquetPartitionDiscoverySuite
"hdfs://host:9000/path/a=10/b=20",
"hdfs://host:9000/path/path1")
- exception = intercept[AssertionError] {
+ exception = intercept[SparkRuntimeException] {
parsePartitions(
paths.map(new Path(_)),
true,
@@ -197,7 +197,7 @@ abstract class ParquetPartitionDiscoverySuite
"hdfs://host:9000/tmp/tables/nonPartitionedTable1",
"hdfs://host:9000/tmp/tables/nonPartitionedTable2")
- exception = intercept[AssertionError] {
+ exception = intercept[SparkRuntimeException] {
parsePartitions(
paths.map(new Path(_)),
true,
@@ -878,7 +878,7 @@ abstract class ParquetPartitionDiscoverySuite
checkAnswer(twoPartitionsDF, df.filter("b != 3"))
- intercept[AssertionError] {
+ intercept[SparkRuntimeException] {
spark
.read
.parquet(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org