You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by gatorsmile <gi...@git.apache.org> on 2018/03/14 19:57:00 UTC

[GitHub] spark pull request #20816: [SPARK-21479][SQL] Outer join filter pushdown in ...

Github user gatorsmile commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20816#discussion_r174591237
  
    --- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala ---
    @@ -673,7 +676,48 @@ object InferFiltersFromConstraints extends Rule[LogicalPlan] with PredicateHelpe
             case None =>
               additionalConstraints.reduceOption(And)
           }
    -      if (newConditionOpt.isDefined) Join(left, right, joinType, newConditionOpt) else join
    +      val j = if (newConditionOpt.isDefined) Join(left, right, joinType, newConditionOpt) else join
    +
    +      // Infer filter for left/right outer joins
    +      joinType match {
    +        case RightOuter if j.condition.isDefined =>
    +          val rightConstraints = right.constraints.union(
    +            splitConjunctivePredicates(j.condition.get).toSet)
    +          val inferredConstraints = ExpressionSet(
    +            QueryPlanConstraints.inferAdditionalConstraints(rightConstraints))
    +          val leftConditions = inferredConstraints
    +            .filter(_.deterministic)
    +            .filter(_.references.subsetOf(left.outputSet))
    +          if (leftConditions.isEmpty) {
    +            j
    +          } else {
    +            // push the predicate down to left side sub query.
    +            val newLeft = leftConditions.
    +              reduceLeftOption(And).map(Filter(_, left)).getOrElse(left)
    +            val newRight = right
    +
    +            Join(newLeft, newRight, RightOuter, j.condition)
    +          }
    +        case LeftOuter if j.condition.isDefined =>
    +          val leftConstraints = left.constraints.union(
    +            splitConjunctivePredicates(j.condition.get).toSet)
    +          val inferredConstraints = ExpressionSet(
    +            QueryPlanConstraints.inferAdditionalConstraints(leftConstraints))
    +          val rightConditions = inferredConstraints
    +            .filter(_.deterministic)
    +            .filter(_.references.subsetOf(right.outputSet))
    +          if (rightConditions.isEmpty) {
    +            j
    +          } else {
    +            // push the predicate down to right side sub query.
    +            val newRight = rightConditions.
    +              reduceLeftOption(And).map(Filter(_, right)).getOrElse(right)
    +            val newLeft = left
    +
    +            Join(newLeft, newRight, LeftOuter, j.condition)
    +          }
    +        case _ => j
    +      }
    --- End diff --
    
    Could you simplify the code to something like?
    
    ```Scala
    
    val newConditionOpt = ... 
    
    val newRight = joinType match {
      ...
    }
    
    val newLeft = joinType match {
      ...
    }
    
    if (newConditionOpt.isDefined || newLeft.isDefined || newRight.isDefined) {
      Join(newLeft.getOrElse(left), newRight.getOrElse(right), joinType,
        newConditionOpt.orElse(conditionOpt))
    } else {
      join
    }
    ```


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org