You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/07/26 11:20:04 UTC
[spark] branch branch-3.2 updated: [SPARK-34402][SQL] Group
exception about data format schema
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 07c7a6f [SPARK-34402][SQL] Group exception about data format schema
07c7a6f is described below
commit 07c7a6f739cbbd04bdddaa8091d000769ca9f49f
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Mon Jul 26 19:18:43 2021 +0800
[SPARK-34402][SQL] Group exception about data format schema
### What changes were proposed in this pull request?
Group exception about data format schema of different format, orc/parquet
### Why are the changes needed?
group exception
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Not need
Closes #33296 from AngersZhuuuu/SPARK-34402.
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit a63802f2c69e5fb271e694bfe3e2e15f09d33320)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../execution/datasources/parquet/ParquetSchemaConverter.scala | 8 +++-----
.../scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 8 ++++----
2 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
index a23eebe..f3bfd99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
@@ -586,11 +586,9 @@ private[sql] object ParquetSchemaConverter {
def checkFieldName(name: String): Unit = {
// ,;{}()\n\t= and space are special characters in Parquet schema
- checkConversionRequirement(
- !name.matches(".*[ ,;{}()\n\t=].*"),
- s"""Attribute name "$name" contains invalid character(s) among " ,;{}()\\n\\t=".
- |Please use alias to rename it.
- """.stripMargin.split("\n").mkString(" ").trim)
+ if (name.matches(".*[ ,;{}()\n\t=].*")) {
+ throw QueryCompilationErrors.columnNameContainsInvalidCharactersError(name)
+ }
}
def checkFieldNames(schema: StructType): Unit = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 3e01fcb..b1f348c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -3002,8 +3002,8 @@ class HiveDDLSuite
spark.sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path.getCanonicalPath}' " +
s"STORED AS PARQUET SELECT ID, if(1=1, 1, 0), abs(id), '^-' FROM v")
}.getMessage
- assert(e.contains("Attribute name \"(IF((1 = 1), 1, 0))\" contains" +
- " invalid character(s) among \" ,;{}()\\n\\t=\". Please use alias to rename it."))
+ assert(e.contains("Column name \"(IF((1 = 1), 1, 0))\" contains invalid character(s). " +
+ "Please use alias to rename it."))
}
}
}
@@ -3022,8 +3022,8 @@ class HiveDDLSuite
|FROM v
""".stripMargin)
}.getMessage
- assert(e.contains("Attribute name \"IF(ID=1,ID,0)\" contains" +
- " invalid character(s) among \" ,;{}()\\n\\t=\". Please use alias to rename it."))
+ assert(e.contains("Column name \"IF(ID=1,ID,0)\" contains" +
+ " invalid character(s). Please use alias to rename it."))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org