You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2017/03/20 15:57:00 UTC
[7/7] drill git commit: DRILL-5359: Fix ClassCastException when Drill
pushes down filter on the output of flatten operator.
DRILL-5359: Fix ClassCastException when Drill pushes down filter on the output of flatten operator.
- Move findItemOrFlatten as a static method in DrillRelOptUtil.
- Exclude filter conditions if they contain item/flatten operator.
close apache/drill#786
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/adbf363d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/adbf363d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/adbf363d
Branch: refs/heads/master
Commit: adbf363d286d548b324b55aa167b333addad8441
Parents: a8046be
Author: Jinfeng Ni <jn...@apache.org>
Authored: Thu Mar 16 14:44:35 2017 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Mon Mar 20 08:14:24 2017 -0700
----------------------------------------------------------------------
.../exec/planner/common/DrillRelOptUtil.java | 54 +++++++++++++++++++
.../logical/DrillPushFilterPastProjectRule.java | 55 +++-----------------
.../store/parquet/ParquetPushDownFilter.java | 24 ++++++++-
.../parquet/TestParquetFilterPushDown.java | 14 +++++
4 files changed, 98 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
index 733577e..b3e261c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
@@ -29,11 +29,15 @@ import org.apache.calcite.rel.rules.ProjectRemoveRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.resolver.TypeCastRules;
@@ -169,4 +173,54 @@ public abstract class DrillRelOptUtil {
}
return true;
}
+
+ /**
+ * Travesal RexNode to find the item/flattern operator. Continue search if RexNode has a
+ * RexInputRef which refers to a RexNode in project expressions.
+ *
+ * @param node : RexNode to search
+ * @param projExprs : the list of project expressions. Empty list means there is No project operator underneath.
+ * @return : Return null if there is NONE; return the first appearance of item/flatten RexCall.
+ */
+ public static RexCall findItemOrFlatten(
+ final RexNode node,
+ final List<RexNode> projExprs) {
+ try {
+ RexVisitor<Void> visitor =
+ new RexVisitorImpl<Void>(true) {
+ public Void visitCall(RexCall call) {
+ if ("item".equals(call.getOperator().getName().toLowerCase()) ||
+ "flatten".equals(call.getOperator().getName().toLowerCase())) {
+ throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
+ other utility methods in RexUtil.java */
+ }
+ return super.visitCall(call);
+ }
+
+ public Void visitInputRef(RexInputRef inputRef) {
+ if (projExprs.size() == 0 ) {
+ return super.visitInputRef(inputRef);
+ } else {
+ final int index = inputRef.getIndex();
+ RexNode n = projExprs.get(index);
+ if (n instanceof RexCall) {
+ RexCall r = (RexCall) n;
+ if ("item".equals(r.getOperator().getName().toLowerCase()) ||
+ "flatten".equals(r.getOperator().getName().toLowerCase())) {
+ throw new Util.FoundOne(r);
+ }
+ }
+
+ return super.visitInputRef(inputRef);
+ }
+ }
+ };
+ node.accept(visitor);
+ return null;
+ } catch (Util.FoundOne e) {
+ Util.swallow(e, null);
+ return (RexCall) e.getNode();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
index 6591bfd..c2dbfb9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
@@ -17,65 +17,24 @@
*/
package org.apache.drill.exec.planner.logical;
-import java.util.List;
-
import com.google.common.collect.Lists;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalProject;
-import org.apache.calcite.plan.RelOptRule;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rex.RexCall;
-import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.rex.RexVisitor;
-import org.apache.calcite.rex.RexVisitorImpl;
-import org.apache.calcite.util.Util;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
+
+import java.util.List;
public class DrillPushFilterPastProjectRule extends RelOptRule {
public final static RelOptRule INSTANCE = new DrillPushFilterPastProjectRule();
- private RexCall findItemOrFlatten(
- final RexNode node,
- final List<RexNode> projExprs) {
- try {
- RexVisitor<Void> visitor =
- new RexVisitorImpl<Void>(true) {
- public Void visitCall(RexCall call) {
- if ("item".equals(call.getOperator().getName().toLowerCase()) ||
- "flatten".equals(call.getOperator().getName().toLowerCase())) {
- throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
- other utility methods in RexUtil.java */
- }
- return super.visitCall(call);
- }
-
- public Void visitInputRef(RexInputRef inputRef) {
- final int index = inputRef.getIndex();
- RexNode n = projExprs.get(index);
- if (n instanceof RexCall) {
- RexCall r = (RexCall) n;
- if ("item".equals(r.getOperator().getName().toLowerCase()) ||
- "flatten".equals(r.getOperator().getName().toLowerCase())) {
- throw new Util.FoundOne(r);
- }
- }
-
- return super.visitInputRef(inputRef);
- }
- };
- node.accept(visitor);
- return null;
- } catch (Util.FoundOne e) {
- Util.swallow(e, null);
- return (RexCall) e.getNode();
- }
- }
-
protected DrillPushFilterPastProjectRule() {
super(
operand(
@@ -99,7 +58,7 @@ public class DrillPushFilterPastProjectRule extends RelOptRule {
for (final RexNode pred : predList) {
- if (findItemOrFlatten(pred, projRel.getProjects()) == null) {
+ if (DrillRelOptUtil.findItemOrFlatten(pred, projRel.getProjects()) == null) {
qualifiedPredList.add(pred);
} else {
unqualifiedPredList.add(pred);
http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
index 6f870f7..1ec10d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
@@ -17,16 +17,19 @@ package org.apache.drill.exec.store.parquet;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.planner.common.DrillRelOptUtil;
import org.apache.drill.exec.planner.logical.DrillOptiq;
import org.apache.drill.exec.planner.logical.DrillParseContext;
import org.apache.drill.exec.planner.logical.RelOptHelper;
@@ -36,6 +39,7 @@ import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
+import java.util.List;
import java.util.concurrent.TimeUnit;
public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
@@ -116,8 +120,26 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
return;
}
+ // get a conjunctions of the filter condition. For each conjunction, if it refers to ITEM or FLATTEN expression
+ // then we could not pushed down. Otherwise, it's qualified to be pushed down.
+ final List<RexNode> predList = RelOptUtil.conjunctions(condition);
+
+ final List<RexNode> qualifiedPredList = Lists.newArrayList();
+
+ for (final RexNode pred : predList) {
+ if (DrillRelOptUtil.findItemOrFlatten(pred, ImmutableList.<RexNode>of()) == null) {
+ qualifiedPredList.add(pred);
+ }
+ }
+
+ final RexNode qualifedPred = RexUtil.composeConjunction(filter.getCluster().getRexBuilder(), qualifiedPredList, true);
+
+ if (qualifedPred == null) {
+ return;
+ }
+
LogicalExpression conditionExp = DrillOptiq.toDrill(
- new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
+ new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, qualifedPred);
Stopwatch timer = Stopwatch.createStarted();
final GroupScan newGroupScan = groupScan.applyFilter(conditionExp,optimizerContext,
http://git-wip-us.apache.org/repos/asf/drill/blob/adbf363d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index 1ad000e..782973d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -366,6 +366,20 @@ public class TestParquetFilterPushDown extends PlanTestBase {
testParquetFilterPD(query3, 49, 2, false);
}
+ @Test // DRILL-5359
+ public void testFilterWithItemFlatten() throws Exception {
+ final String sql = "select n_regionkey\n"
+ + "from (select n_regionkey, \n"
+ + " flatten(nation.cities) as cities \n"
+ + " from cp.`tpch/nation.parquet` nation) as flattenedCities \n"
+ + "where flattenedCities.cities.`zip` = '12345'";
+
+ final String[] expectedPlan = {"(?s)Filter.*Flatten"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+
+ }
//////////////////////////////////////////////////////////////////////////////////////////////////
// Some test helper functions.