You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by gatorsmile <gi...@git.apache.org> on 2018/03/14 19:57:00 UTC
[GitHub] spark pull request #20816: [SPARK-21479][SQL] Outer join filter pushdown in ...
Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/20816#discussion_r174591237
--- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala ---
@@ -673,7 +676,48 @@ object InferFiltersFromConstraints extends Rule[LogicalPlan] with PredicateHelpe
case None =>
additionalConstraints.reduceOption(And)
}
- if (newConditionOpt.isDefined) Join(left, right, joinType, newConditionOpt) else join
+ val j = if (newConditionOpt.isDefined) Join(left, right, joinType, newConditionOpt) else join
+
+ // Infer filter for left/right outer joins
+ joinType match {
+ case RightOuter if j.condition.isDefined =>
+ val rightConstraints = right.constraints.union(
+ splitConjunctivePredicates(j.condition.get).toSet)
+ val inferredConstraints = ExpressionSet(
+ QueryPlanConstraints.inferAdditionalConstraints(rightConstraints))
+ val leftConditions = inferredConstraints
+ .filter(_.deterministic)
+ .filter(_.references.subsetOf(left.outputSet))
+ if (leftConditions.isEmpty) {
+ j
+ } else {
+ // push the predicate down to left side sub query.
+ val newLeft = leftConditions.
+ reduceLeftOption(And).map(Filter(_, left)).getOrElse(left)
+ val newRight = right
+
+ Join(newLeft, newRight, RightOuter, j.condition)
+ }
+ case LeftOuter if j.condition.isDefined =>
+ val leftConstraints = left.constraints.union(
+ splitConjunctivePredicates(j.condition.get).toSet)
+ val inferredConstraints = ExpressionSet(
+ QueryPlanConstraints.inferAdditionalConstraints(leftConstraints))
+ val rightConditions = inferredConstraints
+ .filter(_.deterministic)
+ .filter(_.references.subsetOf(right.outputSet))
+ if (rightConditions.isEmpty) {
+ j
+ } else {
+ // push the predicate down to right side sub query.
+ val newRight = rightConditions.
+ reduceLeftOption(And).map(Filter(_, right)).getOrElse(right)
+ val newLeft = left
+
+ Join(newLeft, newRight, LeftOuter, j.condition)
+ }
+ case _ => j
+ }
--- End diff --
Could you simplify the code to something like?
```Scala
val newConditionOpt = ...
val newRight = joinType match {
...
}
val newLeft = joinType match {
...
}
if (newConditionOpt.isDefined || newLeft.isDefined || newRight.isDefined) {
Join(newLeft.getOrElse(left), newRight.getOrElse(right), joinType,
newConditionOpt.orElse(conditionOpt))
} else {
join
}
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org