You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/06/14 20:21:46 UTC
spark git commit: [SPARK-8358] [SQL] Wait for child resolution when
resolving generators
Repository: spark
Updated Branches:
refs/heads/master ea7fd2ff6 -> 9073a426e
[SPARK-8358] [SQL] Wait for child resolution when resolving generators
Author: Michael Armbrust <mi...@databricks.com>
Closes #6811 from marmbrus/aliasExplodeStar and squashes the following commits:
fbd2065 [Michael Armbrust] more style
806a373 [Michael Armbrust] fix style
7cbb530 [Michael Armbrust] [SPARK-8358][SQL] Wait for child resolution when resolving generatorsa
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9073a426
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9073a426
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9073a426
Branch: refs/heads/master
Commit: 9073a426e444e4bc6efa8608e54e0a986f38a270
Parents: ea7fd2f
Author: Michael Armbrust <mi...@databricks.com>
Authored: Sun Jun 14 11:21:42 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Sun Jun 14 11:21:42 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 6 ++++--
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 8 ++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9073a426/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index cbd8def..4b7fef7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -563,7 +563,9 @@ class Analyzer(
private object AliasedGenerator {
def unapply(e: Expression): Option[(Generator, Seq[String])] = e match {
case Alias(g: Generator, name)
- if g.elementTypes.size > 1 && java.util.regex.Pattern.matches("_c[0-9]+", name) => {
+ if g.resolved &&
+ g.elementTypes.size > 1 &&
+ java.util.regex.Pattern.matches("_c[0-9]+", name) => {
// Assume the default name given by parser is "_c[0-9]+",
// TODO in long term, move the naming logic from Parser to Analyzer.
// In projection, Parser gave default name for TGF as does for normal UDF,
@@ -572,7 +574,7 @@ class Analyzer(
// Let's simply ignore the default given name for this case.
Some((g, Nil))
}
- case Alias(g: Generator, name) if g.elementTypes.size > 1 =>
+ case Alias(g: Generator, name) if g.resolved && g.elementTypes.size > 1 =>
// If not given the default names, and the TGF with multiple output columns
failAnalysis(
s"""Expect multiple names given for ${g.getClass.getName},
http://git-wip-us.apache.org/repos/asf/spark/blob/9073a426/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 84835c0..fa98e23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -134,6 +134,14 @@ class DataFrameSuite extends QueryTest {
)
}
+ test("explode alias and star") {
+ val df = Seq((Array("a"), 1)).toDF("a", "b")
+
+ checkAnswer(
+ df.select(explode($"a").as("a"), $"*"),
+ Row("a", Seq("a"), 1) :: Nil)
+ }
+
test("selectExpr") {
checkAnswer(
testData.selectExpr("abs(key)", "value"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org