You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2017/02/22 14:42:50 UTC
spark git commit: [SPARK-13721][SQL] Make GeneratorOuter unresolved.
Repository: spark
Updated Branches:
refs/heads/master bf7bb4977 -> 10c566cc3
[SPARK-13721][SQL] Make GeneratorOuter unresolved.
## What changes were proposed in this pull request?
This is a small change to make GeneratorOuter always unresolved. It is mostly no-op change but makes it more clear since GeneratorOuter shouldn't survive analysis phase.
This requires also handling in ResolveAliases rule.
## How was this patch tested?
Existing generator tests.
Author: Bogdan Raducanu <bo...@databricks.com>
Author: Reynold Xin <rx...@databricks.com>
Closes #17026 from bogdanrdc/PR16958.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/10c566cc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/10c566cc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/10c566cc
Branch: refs/heads/master
Commit: 10c566cc3b5f93ddd823b1c987d9de8286cdea60
Parents: bf7bb49
Author: Bogdan Raducanu <bo...@databricks.com>
Authored: Wed Feb 22 15:42:40 2017 +0100
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Wed Feb 22 15:42:40 2017 +0100
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../apache/spark/sql/catalyst/expressions/generators.scala | 8 +++++++-
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 39a2762..c477cb4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -222,6 +222,7 @@ class Analyzer(
expr.transformUp { case u @ UnresolvedAlias(child, optGenAliasFunc) =>
child match {
case ne: NamedExpression => ne
+ case go @ GeneratorOuter(g: Generator) if g.resolved => MultiAlias(go, Nil)
case e if !e.resolved => u
case g: Generator => MultiAlias(g, Nil)
case c @ Cast(ne: NamedExpression, _, _) => Alias(c, ne.name)()
@@ -1665,7 +1666,6 @@ class Analyzer(
var resolvedGenerator: Generate = null
val newProjectList = projectList.flatMap {
-
case AliasedGenerator(generator, names, outer) if generator.childrenResolved =>
// It's a sanity check, this should not happen as the previous case will throw
// exception earlier.
http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 1b98c30..e84796f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -173,7 +173,6 @@ case class Stack(children: Seq[Expression]) extends Generator {
}
}
-
/**
* Only support code generation when stack produces 50 rows or less.
*/
@@ -204,6 +203,10 @@ case class Stack(children: Seq[Expression]) extends Generator {
}
}
+/**
+ * Wrapper around another generator to specify outer behavior. This is used to implement functions
+ * such as explode_outer. This expression gets replaced during analysis.
+ */
case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator {
final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] =
throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
@@ -212,7 +215,10 @@ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generat
throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
override def elementSchema: StructType = child.elementSchema
+
+ override lazy val resolved: Boolean = false
}
+
/**
* A base class for [[Explode]] and [[PosExplode]].
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org