You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by zs...@apache.org on 2016/11/16 00:55:19 UTC
spark git commit: [SPARK-18300][SQL] Fix scala 2.10 build for
FoldablePropagation
Repository: spark
Updated Branches:
refs/heads/master 3ce057d00 -> 4b35d13ba
[SPARK-18300][SQL] Fix scala 2.10 build for FoldablePropagation
## What changes were proposed in this pull request?
Commit https://github.com/apache/spark/commit/f14ae4900ad0ed66ba36108b7792d56cd6767a69 broke the scala 2.10 build. This PR fixes this by simplifying the used pattern match.
## How was this patch tested?
Tested building manually. Ran `build/sbt -Dscala-2.10 -Pscala-2.10 package`.
Author: Herman van Hovell <hv...@databricks.com>
Closes #15891 from hvanhovell/SPARK-18300-scala-2.10.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4b35d13b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4b35d13b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4b35d13b
Branch: refs/heads/master
Commit: 4b35d13baca189a50cdaa2ba435d10a1f953e3f8
Parents: 3ce057d
Author: Herman van Hovell <hv...@databricks.com>
Authored: Tue Nov 15 16:55:02 2016 -0800
Committer: Shixiong Zhu <sh...@databricks.com>
Committed: Tue Nov 15 16:55:02 2016 -0800
----------------------------------------------------------------------
.../sql/catalyst/optimizer/expressions.scala | 33 ++++++++++++++++----
1 file changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4b35d13b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 3a7004e..6958398 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -442,12 +442,9 @@ object FoldablePropagation extends Rule[LogicalPlan] {
case l: LeafNode =>
l
- // Whitelist of all nodes we are allowed to apply this rule to.
- case p @ (_: Project | _: Filter | _: SubqueryAlias | _: Aggregate | _: Window |
- _: Sample | _: GlobalLimit | _: LocalLimit | _: Generate | _: Distinct |
- _: AppendColumns | _: AppendColumnsWithObject | _: BroadcastHint |
- _: RedistributeData | _: Repartition | _: Sort | _: TypedFilter) if !stop =>
- p.transformExpressions(replaceFoldable)
+ // We can only propagate foldables for a subset of unary nodes.
+ case u: UnaryNode if !stop && canPropagateFoldables(u) =>
+ u.transformExpressions(replaceFoldable)
// Allow inner joins. We do not allow outer join, although its output attributes are
// derived from its children, they are actually different attributes: the output of outer
@@ -474,6 +471,30 @@ object FoldablePropagation extends Rule[LogicalPlan] {
})
}
}
+
+ /**
+ * Whitelist of all [[UnaryNode]]s for which allow foldable propagation.
+ */
+ private def canPropagateFoldables(u: UnaryNode): Boolean = u match {
+ case _: Project => true
+ case _: Filter => true
+ case _: SubqueryAlias => true
+ case _: Aggregate => true
+ case _: Window => true
+ case _: Sample => true
+ case _: GlobalLimit => true
+ case _: LocalLimit => true
+ case _: Generate => true
+ case _: Distinct => true
+ case _: AppendColumns => true
+ case _: AppendColumnsWithObject => true
+ case _: BroadcastHint => true
+ case _: RedistributeData => true
+ case _: Repartition => true
+ case _: Sort => true
+ case _: TypedFilter => true
+ case _ => false
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org