You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/12/15 05:33:58 UTC
drill git commit: DRILL-4198: Enhance StoragePlugin interface to
expose logical space rules for planning purpose
Repository: drill
Updated Branches:
refs/heads/master 1e45f9fa1 -> 2953fe587
DRILL-4198: Enhance StoragePlugin interface to expose logical space rules for planning purpose
Also move Hive partition pruning rules to logical storage plugin rulesets.
this closes #300
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2953fe58
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2953fe58
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2953fe58
Branch: refs/heads/master
Commit: 2953fe587b73f0f6677b70e2ccfceff1910ed2aa
Parents: 1e45f9f
Author: vkorukanti <ve...@gmail.com>
Authored: Mon Dec 14 16:02:55 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Mon Dec 14 18:17:23 2015 -0800
----------------------------------------------------------------------
.../exec/store/hbase/HBaseStoragePlugin.java | 2 +-
.../exec/store/hive/HiveStoragePlugin.java | 16 ++++++++------
.../exec/store/jdbc/JdbcStoragePlugin.java | 2 +-
.../exec/store/mongo/MongoStoragePlugin.java | 2 +-
.../drill/exec/planner/sql/DrillSqlWorker.java | 14 +++++++------
.../drill/exec/store/AbstractStoragePlugin.java | 7 ++++++-
.../apache/drill/exec/store/StoragePlugin.java | 12 ++++++++---
.../drill/exec/store/StoragePluginRegistry.java | 22 +++++++++++++++-----
.../drill/exec/store/dfs/FileSystemPlugin.java | 2 +-
.../store/ischema/InfoSchemaStoragePlugin.java | 2 +-
10 files changed, 55 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
index ffdd12b..08fd8e1 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
@@ -77,7 +77,7 @@ public class HBaseStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
return ImmutableSet.of(HBasePushFilterIntoScan.FILTER_ON_SCAN, HBasePushFilterIntoScan.FILTER_ON_PROJECT);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
index 191a253..c1b6a0c 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveStoragePlugin.java
@@ -91,20 +91,24 @@ public class HiveStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<StoragePluginOptimizerRule> getLogicalOptimizerRules(OptimizerRulesContext optimizerContext) {
final String defaultPartitionValue = HiveUtilities.getDefaultPartitionValue(config.getHiveConfigOverride());
ImmutableSet.Builder<StoragePluginOptimizerRule> ruleBuilder = ImmutableSet.builder();
- ruleBuilder.add(HivePushPartitionFilterIntoScan.getFilterOnProject(optimizerRulesContext, defaultPartitionValue));
- ruleBuilder.add(HivePushPartitionFilterIntoScan.getFilterOnScan(optimizerRulesContext, defaultPartitionValue));
+ ruleBuilder.add(HivePushPartitionFilterIntoScan.getFilterOnProject(optimizerContext, defaultPartitionValue));
+ ruleBuilder.add(HivePushPartitionFilterIntoScan.getFilterOnScan(optimizerContext, defaultPartitionValue));
+ return ruleBuilder.build();
+ }
+
+ @Override
+ public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
if(optimizerRulesContext.getPlannerSettings().getOptions()
.getOption(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS).bool_val) {
- ruleBuilder.add(ConvertHiveParquetScanToDrillParquetScan.INSTANCE);
+ return ImmutableSet.<StoragePluginOptimizerRule>of(ConvertHiveParquetScanToDrillParquetScan.INSTANCE);
}
- return ruleBuilder.build();
+ return ImmutableSet.of();
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java
index c22979f..9efb7af 100755
--- a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java
+++ b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcStoragePlugin.java
@@ -466,7 +466,7 @@ public class JdbcStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Set<RelOptRule> getOptimizerRules(OptimizerRulesContext context) {
+ public Set<RelOptRule> getPhysicalOptimizerRules(OptimizerRulesContext context) {
return convention.getRules();
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
index b69ea47..7b3fa17 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePlugin.java
@@ -98,7 +98,7 @@ public class MongoStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
return ImmutableSet.of(MongoPushDownFilterForScan.INSTANCE);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 92e59ec..614ad2b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -116,31 +116,33 @@ public class DrillSqlWorker {
}
private RuleSet[] getRules(QueryContext context) {
- StoragePluginRegistry storagePluginRegistry = context.getStorage();
+ final RuleSet[] storagePluginRules = context.getStorage().getStoragePluginRuleSet(context);
// Ruleset for the case where VolcanoPlanner is used for everything : join, filter/project pushdown, partition pruning.
RuleSet drillLogicalVolOnlyRules = DrillRuleSets.mergedRuleSets(
DrillRuleSets.getDrillBasicRules(context),
DrillRuleSets.getPruneScanRules(context),
DrillRuleSets.getJoinPermRules(context),
- DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+ DrillRuleSets.getDrillUserConfigurableLogicalRules(context),
+ storagePluginRules[0]);
// Ruleset for the case where join planning is done in Hep-LOPT, filter/project pushdown and parttion pruning are done in VolcanoPlanner
RuleSet drillLogicalHepJoinRules = DrillRuleSets.mergedRuleSets(
DrillRuleSets.getDrillBasicRules(context),
DrillRuleSets.getPruneScanRules(context),
- DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+ DrillRuleSets.getDrillUserConfigurableLogicalRules(context),
+ storagePluginRules[0]);
// Ruleset for the case where join planning and partition pruning is done in Hep, filter/project pushdown are done in VolcanoPlanner
RuleSet drillLogicalHepJoinPPRules = DrillRuleSets.mergedRuleSets(
DrillRuleSets.getDrillBasicRules(context),
- DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+ DrillRuleSets.getDrillUserConfigurableLogicalRules(context),
+ storagePluginRules[0]);
// Ruleset for physical planning rules
RuleSet drillPhysicalMem = DrillRuleSets.mergedRuleSets(
DrillRuleSets.getPhysicalRules(context),
- storagePluginRegistry.getStoragePluginRuleSet(context));
-
+ storagePluginRules[1]);
RuleSet[] allRules = new RuleSet[] {drillLogicalVolOnlyRules, drillPhysicalMem, drillLogicalHepJoinRules, drillLogicalHepJoinPPRules};
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
index ecebd15..4c428e1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractStoragePlugin.java
@@ -49,7 +49,12 @@ public abstract class AbstractStoragePlugin implements StoragePlugin{
}
@Override
- public Set<? extends RelOptRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<? extends RelOptRule> getLogicalOptimizerRules(OptimizerRulesContext optimizerContext) {
+ return ImmutableSet.of();
+ }
+
+ @Override
+ public Set<? extends RelOptRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
return ImmutableSet.of();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
index 5113a18..d0fbb34 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePlugin.java
@@ -43,10 +43,16 @@ public interface StoragePlugin extends SchemaFactory, AutoCloseable {
public boolean supportsWrite();
/** An implementation of this method will return one or more specialized rules that Drill query
- * optimizer can leverage. Otherwise, it should return an empty set.
- * @return an empty set or a set of plugin specific optimizer rules.
+ * optimizer can leverage in <i>logical</i> space. Otherwise, it should return an empty set.
+ * @return an empty set or a set of plugin specific logical optimizer rules.
*/
- public Set<? extends RelOptRule> getOptimizerRules(OptimizerRulesContext optimizerContext);
+ public Set<? extends RelOptRule> getLogicalOptimizerRules(OptimizerRulesContext optimizerContext);
+
+ /** An implementation of this method will return one or more specialized rules that Drill query
+ * optimizer can leverage in <i>physical</i> space. Otherwise, it should return an empty set.
+ * @return an empty set or a set of plugin specific physical optimizer rules.
+ */
+ public Set<? extends RelOptRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerContext);
/**
* Get the physical scan operator for the particular GroupScan (read) node.
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
index 6e11084..9d7dc57 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistry.java
@@ -356,17 +356,29 @@ public class StoragePluginRegistry implements Iterable<Map.Entry<String, Storage
return plugins.iterator();
}
- public RuleSet getStoragePluginRuleSet(OptimizerRulesContext optimizerRulesContext) {
+ /**
+ * Return StoragePlugin rule sets.
+ * @param optimizerRulesContext
+ * @return Array of logical and physical rule sets.
+ */
+ public RuleSet[] getStoragePluginRuleSet(OptimizerRulesContext optimizerRulesContext) {
// query registered engines for optimizer rules and build the storage plugin RuleSet
- Builder<RelOptRule> setBuilder = ImmutableSet.builder();
+ Builder<RelOptRule> logicalRulesBuilder = ImmutableSet.builder();
+ Builder<RelOptRule> physicalRulesBuilder = ImmutableSet.builder();
for (StoragePlugin plugin : this.plugins.plugins()) {
- Set<? extends RelOptRule> rules = plugin.getOptimizerRules(optimizerRulesContext);
+ Set<? extends RelOptRule> rules = plugin.getLogicalOptimizerRules(optimizerRulesContext);
if (rules != null && rules.size() > 0) {
- setBuilder.addAll(rules);
+ logicalRulesBuilder.addAll(rules);
+ }
+ rules = plugin.getPhysicalOptimizerRules(optimizerRulesContext);
+ if (rules != null && rules.size() > 0) {
+ physicalRulesBuilder.addAll(rules);
}
}
- return DrillRuleSets.create(setBuilder.build());
+ return new RuleSet[] {
+ DrillRuleSets.create(logicalRulesBuilder.build()),
+ DrillRuleSets.create(physicalRulesBuilder.build()) };
}
public DrillSchemaFactory getSchemaFactory() {
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 5e46d4a..ac30410 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -144,7 +144,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
}
@Override
- public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
Builder<StoragePluginOptimizerRule> setBuilder = ImmutableSet.builder();
for(FormatPlugin plugin : formatCreator.getConfiguredFormatPlugins()){
Set<StoragePluginOptimizerRule> rules = plugin.getOptimizerRules();
http://git-wip-us.apache.org/repos/asf/drill/blob/2953fe58/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
index 385c99f..27648c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaStoragePlugin.java
@@ -108,7 +108,7 @@ public class InfoSchemaStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Set<StoragePluginOptimizerRule> getOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
+ public Set<StoragePluginOptimizerRule> getPhysicalOptimizerRules(OptimizerRulesContext optimizerRulesContext) {
return ImmutableSet.of(
InfoSchemaPushFilterIntoRecordGenerator.IS_FILTER_ON_PROJECT,
InfoSchemaPushFilterIntoRecordGenerator.IS_FILTER_ON_SCAN);