You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/03/07 19:32:42 UTC
spark git commit: [SPARK-13694][SQL] QueryPlan.expressions should
always include all expressions
Repository: spark
Updated Branches:
refs/heads/master d7eac9d79 -> 489641117
[SPARK-13694][SQL] QueryPlan.expressions should always include all expressions
## What changes were proposed in this pull request?
It's weird that expressions don't always have all the expressions in it. This PR marks `QueryPlan.expressions` final to forbid sub classes overriding it to exclude some expressions. Currently only `Generate` override it, we can use `producedAttributes` to fix the unresolved attribute problem for it.
Note that this PR doesn't fix the problem in #11497
## How was this patch tested?
existing tests.
Author: Wenchen Fan <we...@databricks.com>
Closes #11532 from cloud-fan/generate.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/48964111
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/48964111
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/48964111
Branch: refs/heads/master
Commit: 489641117651d11806d2773b7ded7c163d0260e5
Parents: d7eac9d
Author: Wenchen Fan <we...@databricks.com>
Authored: Mon Mar 7 10:32:34 2016 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Mar 7 10:32:34 2016 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala | 2 +-
.../apache/spark/sql/catalyst/plans/logical/basicOperators.scala | 4 +---
.../org/apache/spark/sql/catalyst/plans/logical/object.scala | 2 --
.../src/main/scala/org/apache/spark/sql/execution/Generate.scala | 2 +-
4 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/48964111/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 0e0453b..c62d5ea 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -194,7 +194,7 @@ abstract class QueryPlan[PlanType <: TreeNode[PlanType]] extends TreeNode[PlanTy
}
/** Returns all of the expressions present in this query plan operator. */
- def expressions: Seq[Expression] = {
+ final def expressions: Seq[Expression] = {
// Recursively find all expressions from a traversable.
def seqToExpressions(seq: Traversable[Any]): Traversable[Expression] = seq.flatMap {
case e: Expression => e :: Nil
http://git-wip-us.apache.org/repos/asf/spark/blob/48964111/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
index 5223487..411594c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -89,9 +89,7 @@ case class Generate(
generatorOutput.forall(_.resolved)
}
- // we don't want the gOutput to be taken as part of the expressions
- // as that will cause exceptions like unresolved attributes etc.
- override def expressions: Seq[Expression] = generator :: Nil
+ override def producedAttributes: AttributeSet = AttributeSet(generatorOutput)
def output: Seq[Attribute] = {
val qualified = qualifier.map(q =>
http://git-wip-us.apache.org/repos/asf/spark/blob/48964111/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
index 3f97662..da7f81c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala
@@ -208,8 +208,6 @@ case class CoGroup(
left: LogicalPlan,
right: LogicalPlan) extends BinaryNode with ObjectOperator {
- override def producedAttributes: AttributeSet = outputSet
-
override def deserializers: Seq[(Expression, Seq[Attribute])] =
// The `leftGroup` and `rightGroup` are guaranteed te be of same schema, so it's safe to resolve
// the `keyDeserializer` based on either of them, here we pick the left one.
http://git-wip-us.apache.org/repos/asf/spark/blob/48964111/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala
index 6bc4649..9938d21 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Generate.scala
@@ -58,7 +58,7 @@ case class Generate(
private[sql] override lazy val metrics = Map(
"numOutputRows" -> SQLMetrics.createLongMetric(sparkContext, "number of output rows"))
- override def expressions: Seq[Expression] = generator :: Nil
+ override def producedAttributes: AttributeSet = AttributeSet(output)
val boundGenerator = BindReferences.bindReference(generator, child.output)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org