You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2017/03/14 16:40:14 UTC
calcite git commit: [CALCITE-1683] Druid Specific Filter Transpose
Rules (Nishant Bangarwa)
Repository: calcite
Updated Branches:
refs/heads/master 0d996daf5 -> cdc1d960a
[CALCITE-1683] Druid Specific Filter Transpose Rules (Nishant Bangarwa)
Close apache/calcite#393
Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/cdc1d960
Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/cdc1d960
Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/cdc1d960
Branch: refs/heads/master
Commit: cdc1d960a2422d14dc7c25c856cb57103006f4a0
Parents: 0d996da
Author: Nishant Bangarwa <ni...@gmail.com>
Authored: Tue Mar 14 16:29:44 2017 +0000
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Mar 14 16:39:54 2017 +0000
----------------------------------------------------------------------
.../rel/rules/AggregateFilterTransposeRule.java | 13 +++-
.../rel/rules/FilterAggregateTransposeRule.java | 10 ++-
.../rel/rules/FilterProjectTransposeRule.java | 17 +++--
.../rel/rules/ProjectFilterTransposeRule.java | 14 ++--
.../calcite/adapter/druid/DruidQuery.java | 6 +-
.../calcite/adapter/druid/DruidRules.java | 75 +++++++++++++++++++-
.../org/apache/calcite/test/DruidAdapterIT.java | 12 +++-
7 files changed, 128 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java
index b2ebf84..943f25a 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateFilterTransposeRule.java
@@ -18,16 +18,19 @@ package org.apache.calcite.rel.rules;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.SubstitutionVisitor;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.mapping.Mappings;
@@ -57,9 +60,15 @@ public class AggregateFilterTransposeRule extends RelOptRule {
new AggregateFilterTransposeRule();
private AggregateFilterTransposeRule() {
- super(
+ this(
operand(Aggregate.class,
- operand(Filter.class, any())));
+ operand(Filter.class, any())),
+ RelFactories.LOGICAL_BUILDER);
+ }
+
+ protected AggregateFilterTransposeRule(RelOptRuleOperand operand,
+ RelBuilderFactory relBuilderFactory) {
+ super(operand, relBuilderFactory, null);
}
public void onMatch(RelOptRuleCall call) {
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java
index 6347cbf..118e637 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterAggregateTransposeRule.java
@@ -19,6 +19,7 @@ package org.apache.calcite.rel.rules;
import org.apache.calcite.plan.Contexts;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
@@ -64,10 +65,15 @@ public class FilterAggregateTransposeRule extends RelOptRule {
Class<? extends Filter> filterClass,
RelBuilderFactory builderFactory,
Class<? extends Aggregate> aggregateClass) {
- super(
+ this(
operand(filterClass,
operand(aggregateClass, any())),
- builderFactory, null);
+ builderFactory);
+ }
+
+ protected FilterAggregateTransposeRule(RelOptRuleOperand operand,
+ RelBuilderFactory builderFactory) {
+ super(operand, builderFactory, null);
}
@Deprecated // to be removed before 2.0
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
index f37ca7b..2c0797b 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/FilterProjectTransposeRule.java
@@ -18,6 +18,7 @@ package org.apache.calcite.rel.rules;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
@@ -62,12 +63,10 @@ public class FilterProjectTransposeRule extends RelOptRule {
Class<? extends Project> projectClass,
boolean copyFilter, boolean copyProject,
RelBuilderFactory relBuilderFactory) {
- super(
+ this(
operand(filterClass,
operand(projectClass, any())),
- relBuilderFactory, null);
- this.copyFilter = copyFilter;
- this.copyProject = copyProject;
+ copyFilter, copyProject, relBuilderFactory);
}
@Deprecated // to be removed before 2.0
@@ -81,6 +80,16 @@ public class FilterProjectTransposeRule extends RelOptRule {
RelBuilder.proto(filterFactory, projectFactory));
}
+ protected FilterProjectTransposeRule(
+ RelOptRuleOperand operand,
+ boolean copyFilter,
+ boolean copyProject,
+ RelBuilderFactory relBuilderFactory) {
+ super(operand, relBuilderFactory, null);
+ this.copyFilter = copyFilter;
+ this.copyProject = copyProject;
+ }
+
//~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java
index 516de3b..0c7cb5c 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/ProjectFilterTransposeRule.java
@@ -18,6 +18,7 @@ package org.apache.calcite.rel.rules;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
@@ -57,11 +58,17 @@ public class ProjectFilterTransposeRule extends RelOptRule {
Class<? extends Filter> filterClass,
RelBuilderFactory relBuilderFactory,
PushProjector.ExprCondition preserveExprCondition) {
- super(
+ this(
operand(
projectClass,
operand(filterClass, any())),
- relBuilderFactory, null);
+ preserveExprCondition, relBuilderFactory);
+ }
+
+ protected ProjectFilterTransposeRule(RelOptRuleOperand operand,
+ PushProjector.ExprCondition preserveExprCondition,
+ RelBuilderFactory relBuilderFactory) {
+ super(operand, relBuilderFactory, null);
this.preserveExprCondition = preserveExprCondition;
}
@@ -71,8 +78,7 @@ public class ProjectFilterTransposeRule extends RelOptRule {
public void onMatch(RelOptRuleCall call) {
Project origProj;
Filter filter;
-
- if (call.rels.length == 2) {
+ if (call.rels.length >= 2) {
origProj = call.rel(0);
filter = call.rel(1);
} else {
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
index 74c0cc0..061c52c 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidQuery.java
@@ -65,7 +65,6 @@ import org.apache.calcite.util.Util;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
-
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -275,8 +274,9 @@ public class DruidQuery extends AbstractRelNode implements BindableRel {
/** Extends a DruidQuery. */
public static DruidQuery extendQuery(DruidQuery query, RelNode r) {
final ImmutableList.Builder<RelNode> builder = ImmutableList.builder();
- return DruidQuery.create(query.getCluster(), r.getTraitSet(), query.getTable(),
- query.druidTable, query.intervals, builder.addAll(query.rels).add(r).build());
+ return DruidQuery.create(query.getCluster(), r.getTraitSet().replace(query.getConvention()),
+ query.getTable(), query.druidTable, query.intervals,
+ builder.addAll(query.rels).add(r).build());
}
/** Extends a DruidQuery. */
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
----------------------------------------------------------------------
diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
index 0a81c47..91a29ab 100644
--- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
+++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidRules.java
@@ -27,8 +27,14 @@ import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.rules.AggregateFilterTransposeRule;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.ProjectFilterTransposeRule;
import org.apache.calcite.rel.rules.ProjectSortTransposeRule;
+import org.apache.calcite.rel.rules.PushProjector;
import org.apache.calcite.rel.rules.SortProjectTransposeRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -72,8 +78,22 @@ public class DruidRules {
public static final DruidProjectSortRule PROJECT_SORT = new DruidProjectSortRule();
public static final DruidSortProjectRule SORT_PROJECT = new DruidSortProjectRule();
- public static final List<RelOptRule> RULES = ImmutableList.of(FILTER, PROJECT_AGGREGATE,
- PROJECT, AGGREGATE, PROJECT_SORT, SORT, SORT_PROJECT);
+ public static final DruidProjectFilterTransposeRule PROJECT_FILTER_TRANSPOSE_RULE =
+ new DruidProjectFilterTransposeRule();
+
+ public static final DruidFilterProjectTransposeRule FILTER_PROJECT_TRANSPOSE_RULE =
+ new DruidFilterProjectTransposeRule();
+
+ public static final DruidAggregateFilterTransposeRule AGGREGATE_FILTER_TRANSPOSE_RULE =
+ new DruidAggregateFilterTransposeRule();
+
+ public static final DruidFilterAggregateTransposeRule FILTER_AGGREGATE_TRANSPOSE_RULE =
+ new DruidFilterAggregateTransposeRule();
+
+ public static final List<RelOptRule> RULES = ImmutableList.of(FILTER,
+ PROJECT_FILTER_TRANSPOSE_RULE, AGGREGATE_FILTER_TRANSPOSE_RULE, PROJECT_AGGREGATE,
+ PROJECT, AGGREGATE, FILTER_AGGREGATE_TRANSPOSE_RULE, FILTER_PROJECT_TRANSPOSE_RULE,
+ PROJECT_SORT, SORT, SORT_PROJECT);
/** Predicate that returns whether Druid can not handle an aggregate. */
private static final Predicate<Aggregate> BAD_AGG =
@@ -555,6 +575,57 @@ public class DruidRules {
return false;
}
+
+ /**
+ * Rule to push an {@link org.apache.calcite.rel.core.Project} past a
+ * {@link org.apache.calcite.rel.core.Filter} when filter is on top of {@link org.apache.calcite.adapter.druid.DruidQuery}
+ */
+ private static class DruidProjectFilterTransposeRule extends ProjectFilterTransposeRule {
+ private DruidProjectFilterTransposeRule() {
+ super(
+ operand(Project.class, operand(Filter.class, any())), PushProjector.ExprCondition.FALSE,
+ RelFactories.LOGICAL_BUILDER);
+ }
+ }
+
+ /**
+ * Rule to push an {@link org.apache.calcite.rel.core.Filter} past a
+ * {@link org.apache.calcite.rel.core.Project} when project is on top of {@link org.apache.calcite.adapter.druid.DruidQuery}
+ */
+ private static class DruidFilterProjectTransposeRule extends FilterProjectTransposeRule {
+ private DruidFilterProjectTransposeRule() {
+ super(
+ operand(Filter.class, operand(Project.class, operand(DruidQuery.class, none()))), true,
+ true, RelFactories.LOGICAL_BUILDER
+ );
+ }
+ }
+
+ /**
+ * Rule to push an {@link org.apache.calcite.rel.core.Filter} past a
+ * {@link org.apache.calcite.rel.core.Project} when project is on top of {@link org.apache.calcite.adapter.druid.DruidQuery}
+ */
+ private static class DruidAggregateFilterTransposeRule extends AggregateFilterTransposeRule {
+ private DruidAggregateFilterTransposeRule() {
+ super(
+ operand(Aggregate.class, operand(Filter.class, operand(DruidQuery.class, none()))),
+ RelFactories.LOGICAL_BUILDER);
+ }
+ }
+
+ /**
+ * Rule to push an {@link org.apache.calcite.rel.core.Aggregate} past a
+ * {@link org.apache.calcite.rel.core.Filter} when filter is on top of {@link org.apache.calcite.adapter.druid.DruidQuery}
+ */
+ private static class DruidFilterAggregateTransposeRule extends FilterAggregateTransposeRule {
+ private DruidFilterAggregateTransposeRule() {
+ super(
+ operand(Filter.class, operand(Aggregate.class, operand(DruidQuery.class, none()))),
+ RelFactories.LOGICAL_BUILDER
+ );
+ }
+ }
+
}
// End DruidRules.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/cdc1d960/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
----------------------------------------------------------------------
diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
index cd28362..099d0c9 100644
--- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
+++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java
@@ -1207,10 +1207,18 @@ public class DruidAdapterIT {
+ "EnumerableInterpreter\n"
+ " BindableSort(sort0=[$1], dir0=[DESC], fetch=[2])\n"
+ " BindableProject(state_province=[$0], CDC=[FLOOR($1)])\n"
- + " BindableAggregate(group=[{1}], agg#0=[COUNT($0)])\n"
- + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{29, 30}], aggs=[[]])";
+ + " DruidQuery(table=[[foodmart, foodmart]], intervals=[[1900-01-09T00:00:00.000/2992-01-10T00:00:00.000]], groups=[{30}], aggs=[[COUNT(DISTINCT $29)]])\n";
+
+ final String druidQuery = "{\"queryType\":\"groupBy\","
+ + "\"dataSource\":\"foodmart\","
+ + "\"granularity\":\"all\","
+ + "\"dimensions\":[\"state_province\"],"
+ + "\"limitSpec\":{\"type\":\"default\"},"
+ + "\"aggregations\":[{\"type\":\"cardinality\",\"name\":\"$f1\",\"fieldNames\":[\"city\"]}],"
+ + "\"intervals\":[\"1900-01-09T00:00:00.000/2992-01-10T00:00:00.000\"]}";
sql(sql)
.explainContains(explain)
+ .queryContains(druidChecker(druidQuery))
.returnsUnordered("state_province=CA; CDC=45",
"state_province=WA; CDC=22");
}