You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/03/24 05:35:14 UTC
[spark] branch branch-3.1 updated: [SPARK-34763][SQL] col(),
$"" and df("name") should handle quoted column names properly
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8e73cf0 [SPARK-34763][SQL] col(), $"<name>" and df("name") should handle quoted column names properly
8e73cf0 is described below
commit 8e73cf05548fbf3c9d649a4f217170e572b543be
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Wed Mar 24 13:34:10 2021 +0800
[SPARK-34763][SQL] col(), $"<name>" and df("name") should handle quoted column names properly
### What changes were proposed in this pull request?
This PR fixes an issue that `col()`, `$"<name>"` and `df("name")` don't handle quoted column names like ``` `a``b.c` ```properly.
For example, if we have a following DataFrame.
```
val df1 = spark.sql("SELECT 'col1' AS `a``b.c`")
```
For the DataFrame, this query is successfully executed.
```
scala> df1.selectExpr("`a``b.c`").show
+-----+
|a`b.c|
+-----+
| col1|
+-----+
```
But the following query will fail because ``` df1("`a``b.c`") ``` throws an exception.
```
scala> df1.select(df1("`a``b.c`")).show
org.apache.spark.sql.AnalysisException: syntax error in attribute name: `a``b.c`;
at org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute$.e$1(unresolved.scala:152)
at org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute$.parseAttributeName(unresolved.scala:162)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveQuoted(LogicalPlan.scala:121)
at org.apache.spark.sql.Dataset.resolve(Dataset.scala:221)
at org.apache.spark.sql.Dataset.col(Dataset.scala:1274)
at org.apache.spark.sql.Dataset.apply(Dataset.scala:1241)
... 49 elided
```
### Why are the changes needed?
It's a bug.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New tests.
Closes #31854 from sarutak/fix-parseAttributeName.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit f7e9b6efc70b6874587c440725d3af8efa3a316e)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/analysis/unresolved.scala | 11 +++++++---
.../org/apache/spark/sql/DataFrameSuite.scala | 25 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index efc9e97..55eca63 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -190,7 +190,7 @@ object UnresolvedAttribute {
* Used to split attribute name by dot with backticks rule.
* Backticks must appear in pairs, and the quoted string must be a complete name part,
* which means `ab..c`e.f is not allowed.
- * Escape character is not supported now, so we can't use backtick inside name part.
+ * We can use backtick only inside quoted name parts.
*/
def parseAttributeName(name: String): Seq[String] = {
def e = new AnalysisException(s"syntax error in attribute name: $name")
@@ -202,8 +202,13 @@ object UnresolvedAttribute {
val char = name(i)
if (inBacktick) {
if (char == '`') {
- inBacktick = false
- if (i + 1 < name.length && name(i + 1) != '.') throw e
+ if (i + 1 < name.length && name(i + 1) == '`') {
+ tmp += '`'
+ i += 1
+ } else {
+ inBacktick = false
+ if (i + 1 < name.length && name(i + 1) != '.') throw e
+ }
} else {
tmp += char
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 594b142..2fa840c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2586,6 +2586,31 @@ class DataFrameSuite extends QueryTest
checkAnswer(df.select(col2), Row(1) :: Row(2) :: Row(3) :: Nil)
}
+ test("SPARK-34763: col(), $\"<name>\", df(\"name\") should handle quoted column name properly") {
+ val df1 = spark.sql("SELECT 'col1' AS `a``b.c`")
+ checkAnswer(df1.selectExpr("`a``b.c`"), Row("col1"))
+ checkAnswer(df1.select(df1("`a``b.c`")), Row("col1"))
+ checkAnswer(df1.select(col("`a``b.c`")), Row("col1"))
+ checkAnswer(df1.select($"`a``b.c`"), Row("col1"))
+
+ val df2 = df1.as("d.e`f")
+ checkAnswer(df2.selectExpr("`a``b.c`"), Row("col1"))
+ checkAnswer(df2.select(df2("`a``b.c`")), Row("col1"))
+ checkAnswer(df2.select(col("`a``b.c`")), Row("col1"))
+ checkAnswer(df2.select($"`a``b.c`"), Row("col1"))
+
+ checkAnswer(df2.selectExpr("`d.e``f`.`a``b.c`"), Row("col1"))
+ checkAnswer(df2.select(df2("`d.e``f`.`a``b.c`")), Row("col1"))
+ checkAnswer(df2.select(col("`d.e``f`.`a``b.c`")), Row("col1"))
+ checkAnswer(df2.select($"`d.e``f`.`a``b.c`"), Row("col1"))
+
+ val df3 = df1.as("*-#&% ?")
+ checkAnswer(df3.selectExpr("`*-#&% ?`.`a``b.c`"), Row("col1"))
+ checkAnswer(df3.select(df3("*-#&% ?.`a``b.c`")), Row("col1"))
+ checkAnswer(df3.select(col("*-#&% ?.`a``b.c`")), Row("col1"))
+ checkAnswer(df3.select($"*-#&% ?.`a``b.c`"), Row("col1"))
+ }
+
test("SPARK-34776: Nested column pruning should not prune Window produced attributes") {
val df = Seq(
("t1", "123", "bob"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org