You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ad...@apache.org on 2016/09/13 01:32:06 UTC
[19/50] [abbrv] drill git commit: Filter push-down support for JSON
tables.
Filter push-down support for JSON tables.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d1adebd8
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d1adebd8
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d1adebd8
Branch: refs/heads/master
Commit: d1adebd836e9c7c08bfd9abce11e638b582864aa
Parents: aa63121
Author: Smidth Panchamia <sp...@mapr.com>
Authored: Sat Oct 24 15:48:35 2015 -0700
Committer: Aditya Kishore <ad...@apache.org>
Committed: Fri Sep 9 10:08:32 2016 -0700
----------------------------------------------------------------------
.../exec/store/maprdb/MapRDBFormatPlugin.java | 4 +-
.../store/maprdb/MapRDBPushFilterIntoScan.java | 199 ++++++++++++
.../maprdb/binary/MapRDBFilterBuilder.java | 2 +-
.../maprdb/binary/MapRDBPushFilterIntoScan.java | 141 ---------
.../maprdb/json/CompareFunctionsProcessor.java | 201 ++++++++++++
.../store/maprdb/json/JsonConditionBuilder.java | 231 ++++++++++++++
.../exec/store/maprdb/json/JsonScanSpec.java | 94 ++++++
.../exec/store/maprdb/json/JsonSubScanSpec.java | 94 ++++++
.../store/maprdb/json/JsonTableGroupScan.java | 57 ++--
.../maprdb/json/MaprDBJsonRecordReader.java | 21 +-
.../tests/binary/TestMapRDBCFAsJSONString.java | 47 +++
.../tests/binary/TestMapRDBProjectPushDown.java | 47 +++
.../maprdb/tests/binary/TestMapRDBQueries.java | 47 +++
.../drill/maprdb/tests/json/TestSimpleJson.java | 309 +++++++++++++++++++
.../src/test/resources/json/business.json | 20 +-
15 files changed, 1319 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java
index d22434d..0694f5b 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBFormatPlugin.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.store.dfs.FormatMatcher;
import org.apache.drill.exec.store.dfs.FormatPlugin;
import org.apache.drill.exec.store.hbase.HBaseScanSpec;
import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan;
-import org.apache.drill.exec.store.maprdb.binary.MapRDBPushFilterIntoScan;
+import org.apache.drill.exec.store.maprdb.json.JsonScanSpec;
import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -131,7 +131,7 @@ public class MapRDBFormatPlugin implements FormatPlugin {
TableProperties props = maprfs.getTableProperties(new Path(tableName));
if (props.getAttr().getJson()) {
- MapRDBSubScanSpec scanSpec = new MapRDBSubScanSpec().setTableName(tableName);
+ JsonScanSpec scanSpec = new JsonScanSpec(tableName, null/*condition*/);
return new JsonTableGroupScan(userName, getStoragePlugin(), this, scanSpec, columns);
} else {
HBaseScanSpec scanSpec = new HBaseScanSpec(tableName);
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
new file mode 100644
index 0000000..714221f
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/MapRDBPushFilterIntoScan.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.maprdb;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.exec.planner.logical.DrillOptiq;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.logical.RelOptHelper;
+import org.apache.drill.exec.planner.physical.FilterPrel;
+import org.apache.drill.exec.planner.physical.PrelUtil;
+import org.apache.drill.exec.planner.physical.ProjectPrel;
+import org.apache.drill.exec.planner.physical.ScanPrel;
+import org.apache.drill.exec.store.StoragePluginOptimizerRule;
+import org.apache.drill.exec.store.hbase.HBaseScanSpec;
+import org.apache.drill.exec.store.maprdb.binary.BinaryTableGroupScan;
+import org.apache.drill.exec.store.maprdb.binary.MapRDBFilterBuilder;
+import org.apache.drill.exec.store.maprdb.json.JsonConditionBuilder;
+import org.apache.drill.exec.store.maprdb.json.JsonScanSpec;
+import org.apache.drill.exec.store.maprdb.json.JsonTableGroupScan;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rex.RexNode;
+import org.ojai.store.QueryCondition;
+
+import com.google.common.collect.ImmutableList;
+
+public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBPushFilterIntoScan.class);
+
+ private MapRDBPushFilterIntoScan(RelOptRuleOperand operand, String description) {
+ super(operand, description);
+ }
+
+ public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan:Filter_On_Scan") {
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final ScanPrel scan = (ScanPrel) call.rel(1);
+ final FilterPrel filter = (FilterPrel) call.rel(0);
+ final RexNode condition = filter.getCondition();
+
+ if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
+ BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
+ doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition);
+ } else {
+ assert(scan.getGroupScan() instanceof JsonTableGroupScan);
+ JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan();
+ doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition);
+ }
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final ScanPrel scan = (ScanPrel) call.rel(1);
+ if (scan.getGroupScan() instanceof BinaryTableGroupScan ||
+ scan.getGroupScan() instanceof JsonTableGroupScan) {
+ return super.matches(call);
+ }
+ return false;
+ }
+ };
+
+ public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), "MapRDBPushFilterIntoScan:Filter_On_Project") {
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ final ScanPrel scan = (ScanPrel) call.rel(2);
+ final ProjectPrel project = (ProjectPrel) call.rel(1);
+ final FilterPrel filter = (FilterPrel) call.rel(0);
+
+ // convert the filter to one that references the child of the project
+ final RexNode condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project);
+
+ if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
+ BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
+ doPushFilterIntoBinaryGroupScan(call, filter, null, scan, groupScan, condition);
+ } else {
+ assert(scan.getGroupScan() instanceof JsonTableGroupScan);
+ JsonTableGroupScan groupScan = (JsonTableGroupScan)scan.getGroupScan();
+ doPushFilterIntoJsonGroupScan(call, filter, null, scan, groupScan, condition);
+ }
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final ScanPrel scan = (ScanPrel) call.rel(2);
+ if (scan.getGroupScan() instanceof BinaryTableGroupScan ||
+ scan.getGroupScan() instanceof JsonTableGroupScan) {
+ return super.matches(call);
+ }
+ return false;
+ }
+ };
+
+ protected void doPushFilterIntoJsonGroupScan(RelOptRuleCall call,
+ FilterPrel filter, final ProjectPrel project, ScanPrel scan,
+ JsonTableGroupScan groupScan, RexNode condition) {
+
+ if (groupScan.isFilterPushedDown()) {
+ /*
+ * The rule can get triggered again due to the transformed "scan => filter" sequence
+ * created by the earlier execution of this rule when we could not do a complete
+ * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon
+ * this flag to not do a re-processing of the rule on the already transformed call.
+ */
+ return;
+ }
+
+ final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
+ final JsonConditionBuilder jsonConditionBuilder = new JsonConditionBuilder(groupScan, conditionExp);
+ final JsonScanSpec newScanSpec = jsonConditionBuilder.parseTree();
+ if (newScanSpec == null) {
+ return; //no filter pushdown ==> No transformation.
+ }
+
+ final JsonTableGroupScan newGroupsScan = new JsonTableGroupScan(groupScan.getUserName(),
+ groupScan.getStoragePlugin(),
+ groupScan.getFormatPlugin(),
+ newScanSpec,
+ groupScan.getColumns());
+ newGroupsScan.setFilterPushedDown(true);
+
+ final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
+
+ // Depending on whether is a project in the middle, assign either scan or copy of project to childRel.
+ final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));;
+
+ if (jsonConditionBuilder.isAllExpressionsConverted()) {
+ /*
+ * Since we could convert the entire filter condition expression into an HBase filter,
+ * we can eliminate the filter operator altogether.
+ */
+ call.transformTo(childRel);
+ } else {
+ call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel)));
+ }
+ }
+
+ protected void doPushFilterIntoBinaryGroupScan(final RelOptRuleCall call,
+ final FilterPrel filter,
+ final ProjectPrel project,
+ final ScanPrel scan,
+ final BinaryTableGroupScan groupScan,
+ final RexNode condition) {
+
+ if (groupScan.isFilterPushedDown()) {
+ /*
+ * The rule can get triggered again due to the transformed "scan => filter" sequence
+ * created by the earlier execution of this rule when we could not do a complete
+ * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon
+ * this flag to not do a re-processing of the rule on the already transformed call.
+ */
+ return;
+ }
+
+ final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
+ final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp);
+ final HBaseScanSpec newScanSpec = maprdbFilterBuilder.parseTree();
+ if (newScanSpec == null) {
+ return; //no filter pushdown ==> No transformation.
+ }
+
+ final BinaryTableGroupScan newGroupsScan = new BinaryTableGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(),
+ groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns());
+ newGroupsScan.setFilterPushedDown(true);
+
+ final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
+
+ // Depending on whether is a project in the middle, assign either scan or copy of project to childRel.
+ final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));;
+
+ if (maprdbFilterBuilder.isAllExpressionsConverted()) {
+ /*
+ * Since we could convert the entire filter condition expression into an HBase filter,
+ * we can eliminate the filter operator altogether.
+ */
+ call.transformTo(childRel);
+ } else {
+ call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java
index 800d155..07c3364 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBFilterBuilder.java
@@ -52,7 +52,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
private static Boolean nullComparatorSupported = null;
- MapRDBFilterBuilder(BinaryTableGroupScan groupScan, LogicalExpression le) {
+ public MapRDBFilterBuilder(BinaryTableGroupScan groupScan, LogicalExpression le) {
this.groupScan = groupScan;
this.le = le;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java
deleted file mode 100644
index 5adff38..0000000
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/binary/MapRDBPushFilterIntoScan.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.maprdb.binary;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.exec.planner.logical.DrillOptiq;
-import org.apache.drill.exec.planner.logical.DrillParseContext;
-import org.apache.drill.exec.planner.logical.RelOptHelper;
-import org.apache.drill.exec.planner.physical.FilterPrel;
-import org.apache.drill.exec.planner.physical.PrelUtil;
-import org.apache.drill.exec.planner.physical.ProjectPrel;
-import org.apache.drill.exec.planner.physical.ScanPrel;
-import org.apache.drill.exec.store.StoragePluginOptimizerRule;
-import org.apache.drill.exec.store.hbase.HBaseScanSpec;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptRuleOperand;
-import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rex.RexNode;
-
-import com.google.common.collect.ImmutableList;
-
-public abstract class MapRDBPushFilterIntoScan extends StoragePluginOptimizerRule {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapRDBPushFilterIntoScan.class);
-
- private MapRDBPushFilterIntoScan(RelOptRuleOperand operand, String description) {
- super(operand, description);
- }
-
- public static final StoragePluginOptimizerRule FILTER_ON_SCAN = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.any(ScanPrel.class)), "MapRDBPushFilterIntoScan:Filter_On_Scan") {
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final ScanPrel scan = (ScanPrel) call.rel(1);
- final FilterPrel filter = (FilterPrel) call.rel(0);
- final RexNode condition = filter.getCondition();
-
- BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
- if (groupScan.isFilterPushedDown()) {
- /*
- * The rule can get triggered again due to the transformed "scan => filter" sequence
- * created by the earlier execution of this rule when we could not do a complete
- * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon
- * this flag to not do a re-processing of the rule on the already transformed call.
- */
- return;
- }
-
- doPushFilterToScan(call, filter, null, scan, groupScan, condition);
- }
-
- @Override
- public boolean matches(RelOptRuleCall call) {
- final ScanPrel scan = (ScanPrel) call.rel(1);
- if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
- return super.matches(call);
- }
- return false;
- }
- };
-
- public static final StoragePluginOptimizerRule FILTER_ON_PROJECT = new MapRDBPushFilterIntoScan(RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))), "MapRDBPushFilterIntoScan:Filter_On_Project") {
-
- @Override
- public void onMatch(RelOptRuleCall call) {
- final ScanPrel scan = (ScanPrel) call.rel(2);
- final ProjectPrel project = (ProjectPrel) call.rel(1);
- final FilterPrel filter = (FilterPrel) call.rel(0);
-
- BinaryTableGroupScan groupScan = (BinaryTableGroupScan)scan.getGroupScan();
- if (groupScan.isFilterPushedDown()) {
- /*
- * The rule can get triggered again due to the transformed "scan => filter" sequence
- * created by the earlier execution of this rule when we could not do a complete
- * conversion of Optiq Filter's condition to HBase Filter. In such cases, we rely upon
- * this flag to not do a re-processing of the rule on the already transformed call.
- */
- return;
- }
-
- // convert the filter to one that references the child of the project
- final RexNode condition = RelOptUtil.pushFilterPastProject(filter.getCondition(), project);
-
- doPushFilterToScan(call, filter, project, scan, groupScan, condition);
- }
-
- @Override
- public boolean matches(RelOptRuleCall call) {
- final ScanPrel scan = (ScanPrel) call.rel(2);
- if (scan.getGroupScan() instanceof BinaryTableGroupScan) {
- return super.matches(call);
- }
- return false;
- }
- };
-
- protected void doPushFilterToScan(final RelOptRuleCall call, final FilterPrel filter, final ProjectPrel project, final ScanPrel scan, final BinaryTableGroupScan groupScan, final RexNode condition) {
-
- final LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
- final MapRDBFilterBuilder maprdbFilterBuilder = new MapRDBFilterBuilder(groupScan, conditionExp);
- final HBaseScanSpec newScanSpec = maprdbFilterBuilder.parseTree();
- if (newScanSpec == null) {
- return; //no filter pushdown ==> No transformation.
- }
-
- final BinaryTableGroupScan newGroupsScan = new BinaryTableGroupScan(groupScan.getUserName(), groupScan.getStoragePlugin(),
- groupScan.getFormatPlugin(), newScanSpec, groupScan.getColumns());
- newGroupsScan.setFilterPushedDown(true);
-
- final ScanPrel newScanPrel = ScanPrel.create(scan, filter.getTraitSet(), newGroupsScan, scan.getRowType());
-
- // Depending on whether is a project in the middle, assign either scan or copy of project to childRel.
- final RelNode childRel = project == null ? newScanPrel : project.copy(project.getTraitSet(), ImmutableList.of((RelNode)newScanPrel));;
-
- if (maprdbFilterBuilder.isAllExpressionsConverted()) {
- /*
- * Since we could convert the entire filter condition expression into an HBase filter,
- * we can eliminate the filter operator altogether.
- */
- call.transformTo(childRel);
- } else {
- call.transformTo(filter.copy(filter.getTraitSet(), ImmutableList.of(childRel)));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java
new file mode 100644
index 0000000..924c93f
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/CompareFunctionsProcessor.java
@@ -0,0 +1,201 @@
+package org.apache.drill.exec.store.maprdb.json;
+
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Calendar;
+import java.util.TimeZone;
+
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DateExpression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal28Expression;
+import org.apache.drill.common.expression.ValueExpressions.Decimal38Expression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalDayExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntervalYearExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
+import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
+import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.ojai.Value;
+
+import static org.ojai.util.Constants.MILLISECONDSPERDAY;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.mapr.db.rowcol.KeyValueBuilder;
+
+class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> {
+
+ private String functionName;
+ private Boolean success;
+ private Value value;
+ private SchemaPath path;
+
+ public CompareFunctionsProcessor(String functionName) {
+ this.functionName = functionName;
+ this.success = false;
+ this.value = null;
+ }
+
+ public static boolean isCompareFunction(String functionName) {
+ return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName);
+ }
+
+ public static CompareFunctionsProcessor process(FunctionCall call) {
+ String functionName = call.getName();
+ LogicalExpression nameArg = call.args.get(0);
+ LogicalExpression valueArg = call.args.size() >= 2? call.args.get(1) : null;
+ CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName);
+
+ //if (valueArg != null) {
+ if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
+ LogicalExpression swapArg = valueArg;
+ valueArg = nameArg;
+ nameArg = swapArg;
+ evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName);
+ }
+ evaluator.success = nameArg.accept(evaluator, valueArg);
+ //}
+
+ return evaluator;
+ }
+
+ public boolean isSuccess() {
+ // TODO Auto-generated method stub
+ return success;
+ }
+
+ public SchemaPath getPath() {
+ return path;
+ }
+
+ public Value getValue() {
+ return value;
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
+
+ @Override
+ public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException {
+ // If valueArg is null, this might be a IS NULL/IS NOT NULL type of query
+ if (valueArg == null) {
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof QuotedString) {
+ this.value = KeyValueBuilder.initFrom(((QuotedString) valueArg).value);
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof IntExpression) {
+ this.value = KeyValueBuilder.initFrom(((IntExpression)valueArg).getInt());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof FloatExpression) {
+ this.value = KeyValueBuilder.initFrom(((FloatExpression)valueArg).getFloat());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof BooleanExpression) {
+ this.value = KeyValueBuilder.initFrom(((BooleanExpression)valueArg).getBoolean());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof Decimal28Expression) {
+ this.value = KeyValueBuilder.initFrom(((Decimal28Expression)valueArg).getBigDecimal());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof Decimal38Expression) {
+ this.value = KeyValueBuilder.initFrom(((Decimal38Expression)valueArg).getBigDecimal());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof DoubleExpression) {
+ this.value = KeyValueBuilder.initFrom(((DoubleExpression)valueArg).getDouble());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof LongExpression) {
+ this.value = KeyValueBuilder.initFrom(((LongExpression)valueArg).getLong());
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof DateExpression) {
+ this.value = KeyValueBuilder.initFrom(new Date(((DateExpression)valueArg).getDate()));
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof TimeExpression) {
+ this.value = KeyValueBuilder.initFrom(new Time(((TimeExpression)valueArg).getTime()));
+ this.path = path;
+ return true;
+ }
+
+ if (valueArg instanceof TimeStampExpression) {
+ this.value = KeyValueBuilder.initFrom(new Timestamp(((TimeStampExpression)valueArg).getTimeStamp()));
+ this.path = path;
+ return true;
+ }
+
+ return false;
+ }
+
+ private static final ImmutableSet<Class<? extends LogicalExpression>> VALUE_EXPRESSION_CLASSES;
+ static {
+ ImmutableSet.Builder<Class<? extends LogicalExpression>> builder = ImmutableSet.builder();
+ VALUE_EXPRESSION_CLASSES = builder
+ .add(BooleanExpression.class)
+ .add(DateExpression.class)
+ .add(DoubleExpression.class)
+ .add(FloatExpression.class)
+ .add(IntExpression.class)
+ .add(LongExpression.class)
+ .add(QuotedString.class)
+ .add(TimeExpression.class)
+ .build();
+ }
+
+ private static final ImmutableMap<String, String> COMPARE_FUNCTIONS_TRANSPOSE_MAP;
+ static {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
+ // unary functions
+ .put("isnotnull", "isnotnull")
+ .put("isNotNull", "isNotNull")
+ .put("is not null", "is not null")
+ .put("isnull", "isnull")
+ .put("isNull", "isNull")
+ .put("is null", "is null")
+ // binary functions
+ .put("like", "like")
+ .put("equal", "equal")
+ .put("not_equal", "not_equal")
+ .put("greater_than_or_equal_to", "less_than_or_equal_to")
+ .put("greater_than", "less_than")
+ .put("less_than_or_equal_to", "greater_than_or_equal_to")
+ .put("less_than", "greater_than")
+ .build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java
new file mode 100644
index 0000000..a48d784
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonConditionBuilder.java
@@ -0,0 +1,231 @@
+package org.apache.drill.exec.store.maprdb.json;
+
+import org.apache.drill.common.expression.BooleanOperator;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
+import org.apache.hadoop.hbase.HConstants;
+import org.bouncycastle.util.Arrays;
+import org.ojai.Value;
+
+import static org.ojai.DocumentConstants.ID_KEY;
+
+import org.ojai.store.QueryCondition;
+import org.ojai.store.QueryCondition.Op;
+
+import com.google.common.collect.ImmutableList;
+import com.mapr.db.MapRDB;
+import com.mapr.db.impl.IdCodec;
+
+public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void, RuntimeException> implements DrillHBaseConstants {
+
+ final private JsonTableGroupScan groupScan;
+
+ final private LogicalExpression le;
+
+ private boolean allExpressionsConverted = true;
+
+ public JsonConditionBuilder(JsonTableGroupScan groupScan,
+ LogicalExpression conditionExp) {
+ this.groupScan = groupScan;
+ this.le = conditionExp;
+ }
+
+ public JsonScanSpec parseTree() {
+ JsonScanSpec parsedSpec = le.accept(this, null);
+ if (parsedSpec != null) {
+ parsedSpec.mergeScanSpec("booleanAnd", this.groupScan.getScanSpec());
+ }
+ return parsedSpec;
+ }
+
+ public boolean isAllExpressionsConverted() {
+ // TODO Auto-generated method stub
+ return allExpressionsConverted;
+ }
+
+ @Override
+ public JsonScanSpec visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
+ allExpressionsConverted = false;
+ return null;
+ }
+
+ @Override
+ public JsonScanSpec visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException {
+ return visitFunctionCall(op, value);
+ }
+
+ @Override
+ public JsonScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
+ JsonScanSpec nodeScanSpec = null;
+ String functionName = call.getName();
+ ImmutableList<LogicalExpression> args = call.args;
+
+ if (CompareFunctionsProcessor.isCompareFunction(functionName)) {
+ CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call);
+ if (processor.isSuccess()) {
+ nodeScanSpec = createJsonScanSpec(call, processor);
+ }
+ } else {
+ switch(functionName) {
+ case "booleanAnd":
+ case "booleanOr":
+ nodeScanSpec = args.get(0).accept(this, null);
+ for (int i = 1; i < args.size(); ++i) {
+ JsonScanSpec nextScanSpec = args.get(i).accept(this, null);
+ if (nodeScanSpec != null && nextScanSpec != null) {
+ nodeScanSpec.mergeScanSpec(functionName, nextScanSpec);
+ } else {
+ allExpressionsConverted = false;
+ if ("booleanAnd".equals(functionName)) {
+ nodeScanSpec = nodeScanSpec == null ? nextScanSpec : nodeScanSpec;
+ }
+ }
+ }
+ break;
+ }
+ }
+
+ if (nodeScanSpec == null) {
+ allExpressionsConverted = false;
+ }
+
+ return nodeScanSpec;
+ }
+
+ private void setIsCondition(QueryCondition c,
+ String str,
+ QueryCondition.Op op,
+ Value v) {
+ switch (v.getType()) {
+ case BOOLEAN:
+ c.is(str, op, v.getBoolean());
+ break;
+ case STRING:
+ c.is(str, op, v.getString());
+ break;
+ case BYTE:
+ c.is(str, op, v.getByte());
+ break;
+ case SHORT:
+ c.is(str, op, v.getShort());
+ break;
+ case INT:
+ c.is(str, op, v.getInt());
+ break;
+ case LONG:
+ c.is(str, op, v.getLong());
+ break;
+ case FLOAT:
+ c.is(str, op, v.getFloat());
+ break;
+ case DOUBLE:
+ c.is(str, op, v.getDouble());
+ break;
+ case DECIMAL:
+ c.is(str, op, v.getDecimal());
+ break;
+ case DATE:
+ c.is(str, op, v.getDate());
+ break;
+ case TIME:
+ c.is(str, op, v.getTime());
+ break;
+ case TIMESTAMP:
+ c.is(str, op, v.getTimestamp());
+ break;
+ case BINARY:
+ c.is(str, op, v.getBinary());
+ break;
+ // XXX/TODO: Map, Array?
+ default:
+ break;
+ }
+ }
+
+ private JsonScanSpec createJsonScanSpec(FunctionCall call,
+ CompareFunctionsProcessor processor) {
+ String functionName = processor.getFunctionName();
+ SchemaPath field = processor.getPath();
+ Value fieldValue = processor.getValue();
+
+ boolean isRowKey = field.getAsUnescapedPath().equals(ID_KEY);
+
+ QueryCondition cond = null;
+ switch (functionName) {
+ case "equal":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.EQUAL, fieldValue);
+ cond.build();
+ break;
+
+ case "not_equal":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.NOT_EQUAL, fieldValue);
+ cond.build();
+ break;
+
+ case "less_than":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS, fieldValue);
+ cond.build();
+ break;
+
+ case "less_than_or_equal_to":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.LESS_OR_EQUAL, fieldValue);
+ cond.build();
+ break;
+
+ case "greater_than":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER, fieldValue);
+ cond.build();
+ break;
+
+ case "greater_than_or_equal_to":
+ cond = MapRDB.newCondition();
+ setIsCondition(cond, field.getAsUnescapedPath(), Op.GREATER_OR_EQUAL, fieldValue);
+ cond.build();
+ break;
+
+ case "isnull":
+ cond = MapRDB.newCondition().notExists(field.getAsUnescapedPath()).build();
+ break;
+
+ case "isnotnull":
+ cond = MapRDB.newCondition().exists(field.getAsUnescapedPath()).build();
+ break;
+
+ case "istrue":
+ cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, true).build();
+ break;
+
+ case "isnotfalse":
+ cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, false).build();
+ break;
+
+ case "isfalse":
+ cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.EQUAL, false).build();
+ break;
+
+ case "isnottrue":
+ cond = MapRDB.newCondition().is(field.getAsUnescapedPath(), Op.NOT_EQUAL, true).build();
+ break;
+
+ case "like":
+ cond = MapRDB.newCondition().like(field.getAsUnescapedPath(), fieldValue.getString()).build();
+ break;
+
+ default:
+ }
+
+ if (cond != null) {
+ return new JsonScanSpec(groupScan.getTableName(), cond);
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java
new file mode 100644
index 0000000..f278dd4
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonScanSpec.java
@@ -0,0 +1,94 @@
+package org.apache.drill.exec.store.maprdb.json;
+
+import org.apache.drill.exec.store.hbase.HBaseUtils;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.ojai.store.QueryCondition;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.mapr.db.MapRDB;
+import com.mapr.db.impl.ConditionImpl;
+
+public class JsonScanSpec {
+ protected String tableName;
+ protected QueryCondition condition;
+
+ @JsonCreator
+ public JsonScanSpec(@JsonProperty("tableName") String tableName,
+ @JsonProperty("condition") QueryCondition condition) {
+ this.tableName = tableName;
+ this.condition = condition;
+ }
+
+ public String getTableName() {
+ return this.tableName;
+ }
+
+ public byte[] getStartRow() {
+ if (condition == null) {
+ return HConstants.EMPTY_START_ROW;
+ }
+ return ((ConditionImpl)this.condition).getRowkeyRanges().get(0).getStartRow();
+ }
+
+ public byte[] getStopRow() {
+ if (condition == null) {
+ return HConstants.EMPTY_END_ROW;
+ }
+
+ return ((ConditionImpl)this.condition).getRowkeyRanges().get(0).getStopRow();
+ }
+
+ public Object getSerializedFilter() {
+ if (this.condition != null) {
+ return ((ConditionImpl)this.condition).getDescriptor().getSerialized();
+ }
+
+ return null;
+ }
+
+ public void setCondition(QueryCondition condition) {
+ this.condition = condition;
+ }
+
+ @JsonIgnore
+ public QueryCondition getCondition() {
+ return this.condition;
+ }
+
+ public void mergeScanSpec(String functionName, JsonScanSpec scanSpec) {
+
+ if (this.condition != null && scanSpec.getCondition() != null) {
+ QueryCondition newCond = MapRDB.newCondition();
+ switch (functionName) {
+ case "booleanAnd":
+ newCond.and();
+ break;
+ case "booleanOr":
+ newCond.or();
+ break;
+ default:
+ assert(false);
+ }
+
+ newCond.condition(this.condition)
+ .condition(scanSpec.getCondition())
+ .close()
+ .build();
+
+ this.condition = newCond;
+ } else if (scanSpec.getCondition() != null){
+ this.condition = scanSpec.getCondition();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "JsonScanSpec [tableName=" + tableName
+ + ", condition=" + (condition == null ? null : condition.toString())
+ + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java
new file mode 100644
index 0000000..936002d
--- /dev/null
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonSubScanSpec.java
@@ -0,0 +1,94 @@
+package org.apache.drill.exec.store.maprdb.json;
+
+import java.nio.ByteBuffer;
+
+import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec;
+import org.apache.hadoop.hbase.HConstants;
+import org.bouncycastle.util.Arrays;
+import org.ojai.DocumentConstants;
+import org.ojai.Value;
+import org.ojai.store.QueryCondition;
+import org.ojai.store.QueryCondition.Op;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.mapr.db.MapRDB;
+import com.mapr.db.impl.ConditionImpl;
+import com.mapr.db.impl.IdCodec;
+
+public class JsonSubScanSpec extends MapRDBSubScanSpec {
+
+ protected QueryCondition condition;
+
+ @JsonCreator
+ public JsonSubScanSpec(@JsonProperty("tableName") String tableName,
+ @JsonProperty("regionServer") String regionServer,
+ @JsonProperty("startRow") byte[] startRow,
+ @JsonProperty("stopRow") byte[] stopRow,
+ @JsonProperty("cond") QueryCondition cond) {
+ super(tableName, regionServer, null, null, null, null);
+
+ this.condition = MapRDB.newCondition().and();
+
+ if (cond != null) {
+ this.condition.condition(cond);
+ }
+
+ if (startRow != null &&
+ Arrays.areEqual(startRow, HConstants.EMPTY_START_ROW) == false) {
+ Value startVal = IdCodec.decode(startRow);
+
+ switch(startVal.getType()) {
+ case BINARY:
+ this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, startVal.getBinary());
+ break;
+ case STRING:
+ this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, startVal.getString());
+ break;
+ default:
+ throw new IllegalStateException("Encountered an unsupported type " + startVal.getType()
+ + " for _id");
+ }
+ }
+
+ if (stopRow != null &&
+ Arrays.areEqual(stopRow, HConstants.EMPTY_END_ROW) == false) {
+ Value stopVal = IdCodec.decode(stopRow);
+
+ switch(stopVal.getType()) {
+ case BINARY:
+ this.condition.is(DocumentConstants.ID_FIELD, Op.GREATER_OR_EQUAL, stopVal.getBinary());
+ break;
+ case STRING:
+ this.condition.is(DocumentConstants.ID_FIELD, Op.LESS, stopVal.getString());
+ break;
+ default:
+ throw new IllegalStateException("Encountered an unsupported type " + stopVal.getType()
+ + " for _id");
+ }
+ }
+
+ this.condition.close().build();
+ }
+
+ public void setCondition(QueryCondition cond) {
+ condition = cond;
+ }
+
+ @JsonIgnore
+ public QueryCondition getCondition() {
+ return this.condition;
+ }
+
+ public byte[] getSerializedFilter() {
+ if (this.condition != null) {
+ ByteBuffer bbuf = ((ConditionImpl)this.condition).getDescriptor().getSerialized();
+ byte[] serFilter = new byte[bbuf.limit() - bbuf.position()];
+ bbuf.get(serFilter);
+ return serFilter;
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java
index e798c52..e723179 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/JsonTableGroupScan.java
@@ -21,15 +21,20 @@ import static org.apache.drill.exec.store.maprdb.util.CommonFns.isNullOrEmpty;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
import java.util.TreeMap;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
+import org.apache.drill.exec.physical.base.AbstractGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.ScanStats;
import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.FileSystemConfig;
import org.apache.drill.exec.store.dfs.FileSystemPlugin;
@@ -37,9 +42,10 @@ import org.apache.drill.exec.store.maprdb.MapRDBFormatPlugin;
import org.apache.drill.exec.store.maprdb.MapRDBFormatPluginConfig;
import org.apache.drill.exec.store.maprdb.MapRDBGroupScan;
import org.apache.drill.exec.store.maprdb.MapRDBSubScan;
-import org.apache.drill.exec.store.maprdb.MapRDBSubScanSpec;
import org.apache.drill.exec.store.maprdb.MapRDBTableStats;
import org.apache.drill.exec.store.maprdb.TabletFragmentInfo;
+import org.apache.drill.exec.store.maprdb.json.JsonScanSpec;
+import org.apache.drill.exec.store.maprdb.json.JsonSubScanSpec;
import org.apache.hadoop.conf.Configuration;
import org.codehaus.jackson.annotate.JsonCreator;
@@ -61,11 +67,11 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
private MapRDBTableStats tableStats;
- private MapRDBSubScanSpec subscanSpec;
+ private JsonScanSpec scanSpec;
@JsonCreator
public JsonTableGroupScan(@JsonProperty("userName") final String userName,
- @JsonProperty("subscanSpec") MapRDBSubScanSpec subscanSpec,
+ @JsonProperty("scanSpec") JsonScanSpec scanSpec,
@JsonProperty("storage") FileSystemConfig storagePluginConfig,
@JsonProperty("format") MapRDBFormatPluginConfig formatPluginConfig,
@JsonProperty("columns") List<SchemaPath> columns,
@@ -73,13 +79,13 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
this (userName,
(FileSystemPlugin) pluginRegistry.getPlugin(storagePluginConfig),
(MapRDBFormatPlugin) pluginRegistry.getFormatPlugin(storagePluginConfig, formatPluginConfig),
- subscanSpec, columns);
+ scanSpec, columns);
}
public JsonTableGroupScan(String userName, FileSystemPlugin storagePlugin,
- MapRDBFormatPlugin formatPlugin, MapRDBSubScanSpec subscanSpec, List<SchemaPath> columns) {
+ MapRDBFormatPlugin formatPlugin, JsonScanSpec scanSpec, List<SchemaPath> columns) {
super(storagePlugin, formatPlugin, columns, userName);
- this.subscanSpec = subscanSpec;
+ this.scanSpec = scanSpec;
init();
}
@@ -89,7 +95,7 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
*/
private JsonTableGroupScan(JsonTableGroupScan that) {
super(that);
- this.subscanSpec = that.subscanSpec;
+ this.scanSpec = that.scanSpec;
this.endpointFragmentMapping = that.endpointFragmentMapping;
this.tableStats = that.tableStats;
}
@@ -105,40 +111,40 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
logger.debug("Getting tablet locations");
try {
Configuration conf = new Configuration();
- Table t = MapRDB.getTable(subscanSpec.getTableName());
- TabletInfo[] tabletInfos = t.getTabletInfos();
- tableStats = new MapRDBTableStats(conf, subscanSpec.getTableName());
+ Table t = MapRDB.getTable(scanSpec.getTableName());
+ TabletInfo[] tabletInfos = t.getTabletInfos(scanSpec.getCondition());
+ tableStats = new MapRDBTableStats(conf, scanSpec.getTableName());
boolean foundStartRegion = false;
regionsToScan = new TreeMap<TabletFragmentInfo, String>();
for (TabletInfo tabletInfo : tabletInfos) {
TabletInfoImpl tabletInfoImpl = (TabletInfoImpl) tabletInfo;
if (!foundStartRegion
- && !isNullOrEmpty(subscanSpec.getStartRow())
- && !tabletInfoImpl.containsRow(subscanSpec.getStartRow())) {
+ && !isNullOrEmpty(scanSpec.getStartRow())
+ && !tabletInfoImpl.containsRow(scanSpec.getStartRow())) {
continue;
}
foundStartRegion = true;
regionsToScan.put(new TabletFragmentInfo(tabletInfoImpl), tabletInfo.getLocations()[0]);
- if (!isNullOrEmpty(subscanSpec.getStopRow())
- && tabletInfoImpl.containsRow(subscanSpec.getStopRow())) {
+ if (!isNullOrEmpty(scanSpec.getStopRow())
+ && tabletInfoImpl.containsRow(scanSpec.getStopRow())) {
break;
}
}
} catch (Exception e) {
- throw new DrillRuntimeException("Error getting region info for table: " + subscanSpec.getTableName(), e);
+ throw new DrillRuntimeException("Error getting region info for table: " + scanSpec.getTableName(), e);
}
}
- protected MapRDBSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) {
- MapRDBSubScanSpec spec = subscanSpec;
- MapRDBSubScanSpec subScanSpec = new MapRDBSubScanSpec(
+ protected JsonSubScanSpec getSubScanSpec(TabletFragmentInfo tfi) {
+ // XXX/TODO check filter/Condition
+ JsonScanSpec spec = scanSpec;
+ JsonSubScanSpec subScanSpec = new JsonSubScanSpec(
spec.getTableName(),
regionsToScan.get(tfi),
(!isNullOrEmpty(spec.getStartRow()) && tfi.containsRow(spec.getStartRow())) ? spec.getStartRow() : tfi.getStartKey(),
(!isNullOrEmpty(spec.getStopRow()) && tfi.containsRow(spec.getStopRow())) ? spec.getStopRow() : tfi.getEndKey(),
- spec.getSerializedFilter(),
- null);
+ spec.getCondition());
return subScanSpec;
}
@@ -154,7 +160,7 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
@Override
public ScanStats getScanStats() {
//TODO: look at stats for this.
- long rowCount = (long) ((subscanSpec.getSerializedFilter() != null ? .5 : 1) * tableStats.getNumRows());
+ long rowCount = (long) ((scanSpec.getSerializedFilter() != null ? .5 : 1) * tableStats.getNumRows());
int avgColumnSize = 10;
int numColumns = (columns == null || columns.isEmpty()) ? 100 : columns.size();
return new ScanStats(GroupScanProperty.NO_EXACT_ROW_COUNT, rowCount, 1, avgColumnSize * numColumns * rowCount);
@@ -169,18 +175,17 @@ public class JsonTableGroupScan extends MapRDBGroupScan {
@JsonIgnore
public String getTableName() {
- return subscanSpec.getTableName();
+ return scanSpec.getTableName();
}
@Override
public String toString() {
return "JsonTableGroupScan [ScanSpec="
- + subscanSpec + ", columns="
+ + scanSpec + ", columns="
+ columns + "]";
}
- public MapRDBSubScanSpec getSubscanSpec() {
- return subscanSpec;
+ public JsonScanSpec getScanSpec() {
+ return scanSpec;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
index 590c6e3..8044e40 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/maprdb/json/MaprDBJsonRecordReader.java
@@ -63,6 +63,7 @@ import com.mapr.db.Table;
import com.mapr.db.Table.TableOption;
import com.mapr.db.exceptions.DBException;
import com.mapr.db.impl.IdCodec;
+import com.mapr.db.ojai.DBDocumentReader;
import com.mapr.db.ojai.DBDocumentReaderBase;
import com.mapr.db.util.ByteBufs;
import com.mapr.org.apache.hadoop.hbase.util.Bytes;
@@ -80,9 +81,6 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
private OperatorContext operatorContext;
private VectorContainerWriter writer;
- @SuppressWarnings("unused")
- private boolean idOnly;
-
private DrillBuf buffer;
private DocumentStream<DBDocument> documentStream;
@@ -93,13 +91,7 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
List<SchemaPath> projectedColumns, FragmentContext context) {
buffer = context.getManagedBuffer();
tableName = Preconditions.checkNotNull(subScanSpec, "MapRDB reader needs a sub-scan spec").getTableName();
- condition = MapRDB.newCondition().and();
- addKeyCondition(condition, Op.GREATER_OR_EQUAL, subScanSpec.getStartRow());
- addKeyCondition(condition, Op.LESS, subScanSpec.getStopRow());
- if (subScanSpec.getSerializedFilter() != null) {
- condition.condition(com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter())));
- }
- condition.close().build();
+ condition = com.mapr.db.impl.ConditionImpl.parseFrom(ByteBufs.wrap(subScanSpec.getSerializedFilter()));
setColumns(projectedColumns);
}
@@ -122,20 +114,19 @@ public class MaprDBJsonRecordReader extends AbstractRecordReader {
@Override
protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) {
Set<SchemaPath> transformed = Sets.newLinkedHashSet();
- idOnly = true; // TODO: handle the case when only ID is requested.
if (!isStarQuery()) {
ArrayList<Object> projectedFieldsList = Lists.newArrayList();
for (SchemaPath column : columns) {
if (column.getRootSegment().getPath().equalsIgnoreCase(ID_KEY)) {
transformed.add(ID_PATH);
- continue;
+ projectedFieldsList.add(ID_FIELD);
+ } else {
+ transformed.add(SchemaPath.getSimplePath(column.getRootSegment().getPath()));
+ projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
}
- idOnly = false;
- projectedFieldsList.add(FieldPath.parseFrom(column.getAsUnescapedPath()));
}
projectedFields = projectedFieldsList.toArray(new FieldPath[projectedFieldsList.size()]);
} else {
- idOnly = false;
transformed.add(ID_PATH);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java
new file mode 100644
index 0000000..525b034
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBCFAsJSONString.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.binary;
+
+import org.apache.drill.hbase.TestHBaseCFAsJSONString;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+/**
+ * This class does not define any test method but includes all test methods
+ * defined in the parent class, all of which are tested against MapRDB instead
+ * of HBase.
+ */
+@Category(ClusterTest.class)
+public class TestMapRDBCFAsJSONString extends TestHBaseCFAsJSONString {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java
new file mode 100644
index 0000000..59d7a51
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBProjectPushDown.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.binary;
+
+import org.apache.drill.hbase.TestHBaseProjectPushDown;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+/**
+ * This class does not define any test method but includes all test methods
+ * defined in the parent class, all of which are tested against MapRDB instead
+ * of HBase.
+ */
+@Category(ClusterTest.class)
+public class TestMapRDBProjectPushDown extends TestHBaseProjectPushDown {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java
new file mode 100644
index 0000000..69e04a5
--- /dev/null
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/binary/TestMapRDBQueries.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.mapr.drill.maprdb.tests.binary;
+
+import org.apache.drill.hbase.TestHBaseQueries;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+import com.mapr.drill.maprdb.tests.MaprDBTestsSuite;
+import com.mapr.tests.annotations.ClusterTest;
+
+/**
+ * This class does not define any test method but includes all test methods
+ * defined in the parent class, all of which are tested against MapRDB instead
+ * of HBase.
+ */
+@Category(ClusterTest.class)
+public class TestMapRDBQueries extends TestHBaseQueries {
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ MaprDBTestsSuite.setupTests();
+ conf = MaprDBTestsSuite.createPluginAndGetConf(getDrillbitContext());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ MaprDBTestsSuite.cleanupTests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/d1adebd8/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
----------------------------------------------------------------------
diff --git a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
index c92fc44..f4c7e89 100644
--- a/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
+++ b/contrib/format-maprdb/src/test/java/com/mapr/drill/maprdb/tests/json/TestSimpleJson.java
@@ -20,6 +20,7 @@ package com.mapr.drill.maprdb.tests.json;
import java.util.List;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.PlanTestBase;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.AfterClass;
@@ -55,6 +56,314 @@ public class TestSimpleJson extends BaseTestQuery {
runSQLAndVerifyCount(sql, 10);
}
+ @Test
+ public void testPushdownStringEqual() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " name = 'Sprint'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(name = \"Sprint\"\\)"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushdownStringLike() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " name LIKE 'S%'"
+ ;
+ runSQLAndVerifyCount(sql, 3);
+
+ final String[] expectedPlan = {"condition=\\(name MATCHES \"\\^\\\\\\\\QS\\\\\\\\E\\.\\*\\$\"\\)"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushdownStringNotEqual() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " name <> 'Sprint'"
+ ;
+ runSQLAndVerifyCount(sql, 9);
+
+ final String[] expectedPlan = {"condition=\\(name != \"Sprint\"\\)"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushdownLongEqual() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " zip = 85260"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(zip = \\{\"\\$numberLong\":85260\\}\\)"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testCompositePredicate() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " zip = 85260\n"
+ + " OR\n"
+ + " city = 'Las Vegas'"
+ ;
+ runSQLAndVerifyCount(sql, 4);
+
+ final String[] expectedPlan = {"condition=\\(\\(zip = \\{\"\\$numberLong\":85260\\}\\) or \\(city = \"Las Vegas\"\\)\\)"};
+ final String[] excludedPlan = {};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPruneScanRange() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " _id = 'jFTZmywe7StuZ2hEjxyA'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPruneScanRangeAndPushDownCondition() throws Exception {
+ // XXX/TODO:
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, categories, full_address\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " _id = 'jFTZmywe7StuZ2hEjxyA' AND\n"
+ + " name = 'Subway'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(\\(_id = \"jFTZmywe7StuZ2hEjxyA\"\\) and \\(name = \"Subway\"\\)\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownOnSubField1() throws Exception {
+ setColumnWidths(new int[] {25, 120, 20});
+ final String sql = "SELECT\n"
+ + " _id, name, b.attributes.Ambience.touristy attributes\n"
+ + "FROM\n"
+ + " hbase.`business` b\n"
+ + "WHERE\n"
+ + " b.`attributes.Ambience.casual` = false"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = false\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownOnSubField2() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, b.attributes.Attire attributes\n"
+ + "FROM\n"
+ + " hbase.`business` b\n"
+ + "WHERE\n"
+ + " b.`attributes.Attire` = 'casual'"
+ ;
+ runSQLAndVerifyCount(sql, 4);
+
+ final String[] expectedPlan = {"condition=\\(attributes.Attire = \"casual\"\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+ @Test
+ public void testPushDownIsNull() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+
+ final String sql = "SELECT\n"
+ + " _id, name, attributes\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`attributes.Ambience.casual` IS NULL"
+ ;
+ runSQLAndVerifyCount(sql, 7);
+
+ final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual = null\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownIsNotNull() throws Exception {
+ setColumnWidths(new int[] {25, 75, 75, 50});
+
+ final String sql = "SELECT\n"
+ + " _id, name, b.attributes.Parking\n"
+ + "FROM\n"
+ + " hbase.`business` b\n"
+ + "WHERE\n"
+ + " b.`attributes.Ambience.casual` IS NOT NULL"
+ ;
+ runSQLAndVerifyCount(sql, 3);
+
+ final String[] expectedPlan = {"condition=\\(attributes.Ambience.casual != null\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownOnSubField3() throws Exception {
+ setColumnWidths(new int[] {25, 40, 40, 40});
+ final String sql = "SELECT\n"
+ + " _id, name, b.attributes.`Accepts Credit Cards` attributes\n"
+ + "FROM\n"
+ + " hbase.`business` b\n"
+ + "WHERE\n"
+ + " b.`attributes.Accepts Credit Cards` IS NULL"
+ ;
+ runSQLAndVerifyCount(sql, 3);
+
+ final String[] expectedPlan = {"condition=\\(attributes.Accepts Credit Cards = null\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownLong() throws Exception {
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " stars > 4.0"
+ ;
+ runSQLAndVerifyCount(sql, 2);
+
+ final String[] expectedPlan = {"condition=\\(stars > 4\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ @Test
+ public void testPushDownSubField4() throws Exception {
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`attributes.Good For.lunch` = true AND"
+ + " stars > 4.1"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+
+ final String[] expectedPlan = {"condition=\\(\\(attributes.Good For.lunch = true\\) and \\(stars > 4.1\\)\\)"};
+ final String[] excludedPlan ={};
+
+ PlanTestBase.testPlanMatchingPatterns(sql, expectedPlan, excludedPlan);
+ }
+
+ /*
+ @Test
+ public void testPushDownSubField5() throws Exception {
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`hours.Tuesday.open` < TIME '10:30:00'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+ }
+
+ @Test
+ public void testPushDownSubField6() throws Exception {
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`hours.Sunday.close` > TIME '20:30:00'"
+ ;
+ runSQLAndVerifyCount(sql, 4);
+ }
+
+ @Test
+ public void testPushDownSubField7() throws Exception {
+ setColumnWidths(new int[] {25, 40, 25, 45});
+ final String sql = "SELECT\n"
+ + " _id, name, start_date, last_update\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`start_date` = DATE '2012-07-14'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+ }
+
+ @Test
+ public void testPushDownSubField8() throws Exception {
+ setColumnWidths(new int[] {25, 40, 25, 45});
+ final String sql = "SELECT\n"
+ + " _id, name, start_date, last_update\n"
+ + "FROM\n"
+ + " hbase.`business` business\n"
+ + "WHERE\n"
+ + " business.`last_update` = TIMESTAMP '2012-10-20 07:42:46'"
+ ;
+ runSQLAndVerifyCount(sql, 1);
+ }
+ */
+
protected List<QueryDataBatch> runHBaseSQLlWithResults(String sql) throws Exception {
System.out.println("Running query:\n" + sql);
return testSqlWithResults(sql);