You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2017/03/20 15:57:00 UTC

[7/7] drill git commit: DRILL-5359: Fix ClassCastException when Drill pushes down filter on the output of flatten operator.

DRILL-5359: Fix ClassCastException when Drill pushes down filter on the output of flatten operator.

- Move findItemOrFlatten as a static method in DrillRelOptUtil.
- Exclude filter conditions if they contain item/flatten operator.

close apache/drill#786


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/adbf363d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/adbf363d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/adbf363d

Branch: refs/heads/master
Commit: adbf363d286d548b324b55aa167b333addad8441
Parents: a8046be
Author: Jinfeng Ni <jn...@apache.org>
Authored: Thu Mar 16 14:44:35 2017 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Mon Mar 20 08:14:24 2017 -0700

----------------------------------------------------------------------
 .../exec/planner/common/DrillRelOptUtil.java    | 54 +++++++++++++++++++
 .../logical/DrillPushFilterPastProjectRule.java | 55 +++-----------------
 .../store/parquet/ParquetPushDownFilter.java    | 24 ++++++++-
 .../parquet/TestParquetFilterPushDown.java      | 14 +++++
 4 files changed, 98 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
index 733577e..b3e261c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
@@ -29,11 +29,15 @@ import org.apache.calcite.rel.rules.ProjectRemoveRule;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.validate.SqlValidatorUtil;
 import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.resolver.TypeCastRules;
@@ -169,4 +173,54 @@ public abstract class DrillRelOptUtil {
     }
     return true;
   }
+
+  /**
+   * Travesal RexNode to find the item/flattern operator. Continue search if RexNode has a
+   * RexInputRef which refers to a RexNode in project expressions.
+   *
+   * @param node : RexNode to search
+   * @param projExprs : the list of project expressions. Empty list means there is No project operator underneath.
+   * @return : Return null if there is NONE; return the first appearance of item/flatten RexCall.
+   */
+  public static RexCall findItemOrFlatten(
+      final RexNode node,
+      final List<RexNode> projExprs) {
+    try {
+      RexVisitor<Void> visitor =
+          new RexVisitorImpl<Void>(true) {
+            public Void visitCall(RexCall call) {
+              if ("item".equals(call.getOperator().getName().toLowerCase()) ||
+                  "flatten".equals(call.getOperator().getName().toLowerCase())) {
+                throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
+                                              other utility methods in RexUtil.java */
+              }
+              return super.visitCall(call);
+            }
+
+            public Void visitInputRef(RexInputRef inputRef) {
+              if (projExprs.size() == 0 ) {
+                return super.visitInputRef(inputRef);
+              } else {
+                final int index = inputRef.getIndex();
+                RexNode n = projExprs.get(index);
+                if (n instanceof RexCall) {
+                  RexCall r = (RexCall) n;
+                  if ("item".equals(r.getOperator().getName().toLowerCase()) ||
+                      "flatten".equals(r.getOperator().getName().toLowerCase())) {
+                    throw new Util.FoundOne(r);
+                  }
+                }
+
+                return super.visitInputRef(inputRef);
+              }
+            }
+          };
+      node.accept(visitor);
+      return null;
+    } catch (Util.FoundOne e) {
+      Util.swallow(e, null);
+      return (RexCall) e.getNode();
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
index 6591bfd..c2dbfb9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
@@ -17,65 +17,24 @@
  */
 package org.apache.drill.exec.planner.logical;
 
-import java.util.List;
-
 import com.google.common.collect.Lists;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.logical.LogicalFilter;
 import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexVisitor;
-import org.apache.calcite.rex.RexVisitorImpl;
-import org.apache.calcite.util.Util;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
+
+import java.util.List;
 
 public class DrillPushFilterPastProjectRule extends RelOptRule {
 
   public final static RelOptRule INSTANCE = new DrillPushFilterPastProjectRule();
 
-  private RexCall findItemOrFlatten(
-      final RexNode node,
-      final List<RexNode> projExprs) {
-    try {
-      RexVisitor<Void> visitor =
-          new RexVisitorImpl<Void>(true) {
-        public Void visitCall(RexCall call) {
-          if ("item".equals(call.getOperator().getName().toLowerCase()) ||
-            "flatten".equals(call.getOperator().getName().toLowerCase())) {
-            throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
-                                              other utility methods in RexUtil.java */
-          }
-          return super.visitCall(call);
-        }
-
-        public Void visitInputRef(RexInputRef inputRef) {
-          final int index = inputRef.getIndex();
-          RexNode n = projExprs.get(index);
-          if (n instanceof RexCall) {
-            RexCall r = (RexCall) n;
-            if ("item".equals(r.getOperator().getName().toLowerCase()) ||
-                "flatten".equals(r.getOperator().getName().toLowerCase())) {
-              throw new Util.FoundOne(r);
-            }
-          }
-
-          return super.visitInputRef(inputRef);
-        }
-      };
-      node.accept(visitor);
-      return null;
-    } catch (Util.FoundOne e) {
-      Util.swallow(e, null);
-      return (RexCall) e.getNode();
-    }
-  }
-
   protected DrillPushFilterPastProjectRule() {
     super(
         operand(
@@ -99,7 +58,7 @@ public class DrillPushFilterPastProjectRule extends RelOptRule {
 
 
     for (final RexNode pred : predList) {
-      if (findItemOrFlatten(pred, projRel.getProjects()) == null) {
+      if (DrillRelOptUtil.findItemOrFlatten(pred, projRel.getProjects()) == null) {
         qualifiedPredList.add(pred);
       } else {
         unqualifiedPredList.add(pred);

http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
index 6f870f7..1ec10d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
@@ -17,16 +17,19 @@ package org.apache.drill.exec.store.parquet;
 
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.RelOptRuleOperand;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.ValueExpressions;
 import org.apache.drill.exec.ops.OptimizerRulesContext;
 import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
 import org.apache.drill.exec.planner.logical.DrillOptiq;
 import org.apache.drill.exec.planner.logical.DrillParseContext;
 import org.apache.drill.exec.planner.logical.RelOptHelper;
@@ -36,6 +39,7 @@ import org.apache.drill.exec.planner.physical.ProjectPrel;
 import org.apache.drill.exec.planner.physical.ScanPrel;
 import org.apache.drill.exec.store.StoragePluginOptimizerRule;
 
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
@@ -116,8 +120,26 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
       return;
     }
 
+    // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
+    // then we could not pushed down. Otherwise, it's qualified to be pushed down.
+    final List<RexNode> predList = RelOptUtil.conjunctions(condition);
+
+    final List<RexNode> qualifiedPredList = Lists.newArrayList();
+
+    for (final RexNode pred : predList) {
+      if (DrillRelOptUtil.findItemOrFlatten(pred, ImmutableList.<RexNode>of()) == null) {
+        qualifiedPredList.add(pred);
+      }
+    }
+
+    final RexNode qualifedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
+
+    if (qualifedPred == null) {
+      return;
+    }
+
     LogicalExpression conditionExp = DrillOptiq.toDrill(
-        new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
+        new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifedPred);
 
     Stopwatch timer = Stopwatch.createStarted();
     final GroupScan newGroupScan = groupScan.applyFilter(conditionExp,optimizerContext,

http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 1ad000e..782973d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -366,6 +366,20 @@ public class TestParquetFilterPushDown extends PlanTestBase {
     testParquetFilterPD(query3, 49, 2, false);
   }
 
+  @Test // DRILL-5359
+  public void testFilterWithItemFlatten() throws  Exception {
+    final String sql = "select n_regionkey\n"
+        + "from (select n_regionkey, \n"
+        + "            flatten(nation.cities) as cities \n"
+        + "      from cp.`tpch/nation.parquet` nation) as flattenedCities \n"
+        + "where flattenedCities.cities.`zip` = '12345'";
+
+    final String[] expectedPlan = {"(?s)Filter.*Flatten"};
+    final String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+
+  }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////
   // Some test helper functions.