You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ad...@apache.org on 2014/10/23 04:00:20 UTC
[1/3] git commit: DRILL-1567 - Pushdown fails with WHERE clause with
more than one AND or OR operator of same type
Repository: incubator-drill
Updated Branches:
refs/heads/master ae2790ed8 -> 71f5ad447
DRILL-1567 - Pushdown fails with WHERE clause with more than one AND or OR operator of same type
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/67df8cfa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/67df8cfa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/67df8cfa
Branch: refs/heads/master
Commit: 67df8cfa27041c9696255ee96e2d066e034f6f58
Parents: 958de3d
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue Oct 21 20:53:39 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Wed Oct 22 18:20:00 2014 -0700
----------------------------------------------------------------------
.../exec/store/hbase/HBaseFilterBuilder.java | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/67df8cfa/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
index ee011a5..9d74b69 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
@@ -100,15 +100,18 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
switch (functionName) {
case "booleanAnd":
case "booleanOr":
- HBaseScanSpec leftScanSpec = args.get(0).accept(this, null);
- HBaseScanSpec rightScanSpec = args.get(1).accept(this, null);
- if (leftScanSpec != null && rightScanSpec != null) {
- nodeScanSpec = mergeScanSpecs(functionName, leftScanSpec, rightScanSpec);
- } else {
- allExpressionsConverted = false;
- if ("booleanAnd".equals(functionName)) {
- nodeScanSpec = leftScanSpec == null ? rightScanSpec : leftScanSpec;
+ HBaseScanSpec firstScanSpec = args.get(0).accept(this, null);
+ for (int i = 1; i < args.size(); ++i) {
+ HBaseScanSpec nextScanSpec = args.get(i).accept(this, null);
+ if (firstScanSpec != null && nextScanSpec != null) {
+ nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec);
+ } else {
+ allExpressionsConverted = false;
+ if ("booleanAnd".equals(functionName)) {
+ nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec;
+ }
}
+ firstScanSpec = nodeScanSpec;
}
break;
}
[3/3] git commit: DRILL-1508: Implement pushdown for LIKE operator in
HBase storage engine
Posted by ad...@apache.org.
DRILL-1508: Implement pushdown for LIKE operator in HBase storage engine
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/71f5ad44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/71f5ad44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/71f5ad44
Branch: refs/heads/master
Commit: 71f5ad447823cba96cf73fb6709f8ed6cc0b63e9
Parents: 67df8cf
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Tue Oct 21 23:54:01 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Wed Oct 22 18:20:08 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/test/DrillTest.java | 9 +
.../store/hbase/CompareFunctionsProcessor.java | 6 +-
.../exec/store/hbase/HBaseFilterBuilder.java | 68 +++++++-
.../exec/store/hbase/HBaseRegexParser.java | 165 +++++++++++++++++++
.../org/apache/drill/hbase/BaseHBaseTest.java | 10 --
.../org/apache/drill/hbase/HBaseTestsSuite.java | 7 +-
.../drill/hbase/TestHBaseFilterPushDown.java | 36 ++++
.../drill/hbase/TestHBaseRegexParser.java | 65 ++++++++
.../apache/drill/hbase/TestTableGenerator.java | 45 +++++
9 files changed, 393 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/common/src/test/java/org/apache/drill/test/DrillTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/test/DrillTest.java b/common/src/test/java/org/apache/drill/test/DrillTest.java
index 55f89b3..8abcb6a 100644
--- a/common/src/test/java/org/apache/drill/test/DrillTest.java
+++ b/common/src/test/java/org/apache/drill/test/DrillTest.java
@@ -25,8 +25,10 @@ import java.util.List;
import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.common.util.TestTools;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
+import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
@@ -51,6 +53,13 @@ public class DrillTest {
@Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(50000);
@Rule public final TestLogReporter logOutcome = LOG_OUTCOME;
+ @Rule public TestName TEST_NAME = new TestName();
+
+ @Before
+ public void printID() throws Exception {
+ System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName());
+ }
+
@BeforeClass
public static void initDrillTest() throws Exception {
memWatcher = new MemWatcher();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
index 6810f81..1635c5d 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
@@ -37,6 +37,7 @@ import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -54,7 +55,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
public static CompareFunctionsProcessor process(FunctionCall call, boolean nullComparatorSupported) {
String functionName = call.getName();
LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() == 2 ? call.args.get(1) : null;
+ LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName);
if (valueArg != null) { // binary function
@@ -186,7 +187,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
@Override
public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException {
if (valueArg instanceof QuotedString) {
- this.value = ((QuotedString) valueArg).value.getBytes();
+ this.value = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8);
this.path = path;
return true;
}
@@ -220,6 +221,7 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
.put("isNull", "isNull")
.put("is null", "is null")
// binary functions
+ .put("like", "like")
.put("equal", "equal")
.put("not_equal", "not_equal")
.put("greater_than_or_equal_to", "less_than_or_equal_to")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
index 9d74b69..5c49f18 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
@@ -29,10 +29,12 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.NullComparator;
+import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
+import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void, RuntimeException> implements DrillHBaseConstants {
@@ -43,6 +45,8 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
private boolean allExpressionsConverted = true;
+ private static Boolean nullComparatorSupported = null;
+
HBaseFilterBuilder(HBaseGroupScan groupScan, LogicalExpression le) {
this.groupScan = groupScan;
this.le = le;
@@ -56,7 +60,8 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
* If RowFilter is THE filter attached to the scan specification,
* remove it since its effect is also achieved through startRow and stopRow.
*/
- if (parsedSpec.filter instanceof RowFilter) {
+ if (parsedSpec.filter instanceof RowFilter &&
+ ((RowFilter)parsedSpec.filter).getComparator() instanceof BinaryComparator) {
parsedSpec.filter = null;
}
}
@@ -90,11 +95,13 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
* causes a filter with NullComparator to fail. Enable only if specified in
* the configuration (after ensuring that the HBase cluster has the fix).
*/
- boolean nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false);
+ if (nullComparatorSupported == null) {
+ nullComparatorSupported = groupScan.getHBaseConf().getBoolean("drill.hbase.supports.null.comparator", false);
+ }
CompareFunctionsProcessor processor = CompareFunctionsProcessor.process(call, nullComparatorSupported);
if (processor.isSuccess()) {
- nodeScanSpec = createHBaseScanSpec(processor.getFunctionName(), processor.getPath(), processor.getValue());
+ nodeScanSpec = createHBaseScanSpec(call, processor);
}
} else {
switch (functionName) {
@@ -143,10 +150,17 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter);
}
- private HBaseScanSpec createHBaseScanSpec(String functionName, SchemaPath field, byte[] fieldValue) {
+ private HBaseScanSpec createHBaseScanSpec(FunctionCall call, CompareFunctionsProcessor processor) {
+ String functionName = processor.getFunctionName();
+ SchemaPath field = processor.getPath();
+ byte[] fieldValue = processor.getValue();
boolean isRowKey = field.getAsUnescapedPath().equals(ROW_KEY);
if (!(isRowKey
- || (field.getRootSegment().getChild() != null && field.getRootSegment().getChild().isNamed()))) {
+ || (!field.getRootSegment().isLastPath()
+ && field.getRootSegment().getChild().isLastPath()
+ && field.getRootSegment().getChild().isNamed())
+ )
+ ) {
/*
* if the field in this function is neither the row_key nor a qualified HBase column, return.
*/
@@ -163,6 +177,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
compareOp = CompareOp.EQUAL;
if (isRowKey) {
startRow = stopRow = fieldValue;
+ compareOp = null;
}
break;
case "not_equal":
@@ -213,6 +228,49 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
compareOp = CompareOp.NOT_EQUAL;
comparator = new NullComparator();
break;
+ case "like":
+ /*
+ * Convert the LIKE operand to Regular Expression pattern so that we can
+ * apply RegexStringComparator()
+ */
+ HBaseRegexParser parser = new HBaseRegexParser(call).parse();
+ compareOp = CompareOp.EQUAL;
+ comparator = new RegexStringComparator(parser.getRegexString());
+
+ /*
+ * We can possibly do better if the LIKE operator is on the row_key
+ */
+ if (isRowKey) {
+ String prefix = parser.getPrefixString();
+ if (prefix != null) { // group 3 is literal
+ /*
+ * If there is a literal prefix, it can help us prune the scan to a sub range
+ */
+ if (prefix.equals(parser.getLikeString())) {
+ /* The operand value is literal. This turns the LIKE operator to EQUAL operator */
+ startRow = stopRow = fieldValue;
+ compareOp = null;
+ } else {
+ startRow = prefix.getBytes(Charsets.UTF_8);
+ stopRow = startRow.clone();
+ boolean isMaxVal = true;
+ for (int i = stopRow.length - 1; i >= 0 ; --i) {
+ int nextByteValue = (0xff & stopRow[i]) + 1;
+ if (nextByteValue < 0xff) {
+ stopRow[i] = (byte) nextByteValue;
+ isMaxVal = false;
+ break;
+ } else {
+ stopRow[i] = 0;
+ }
+ }
+ if (isMaxVal) {
+ stopRow = HConstants.EMPTY_END_ROW;
+ }
+ }
+ }
+ }
+ break;
}
if (compareOp != null || startRow != HConstants.EMPTY_START_ROW || stopRow != HConstants.EMPTY_END_ROW) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
new file mode 100644
index 0000000..c05baaa
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+
+public class HBaseRegexParser {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseRegexParser.class);
+
+ /**
+ * Regular expression pattern to parse the value operand of the SQL LIKE operator.
+ * The tokens could be one of the 3 types.<br/>
+ * <ol>
+ * <li>Wildcards, i.e. "%" or "_" ==> first regex group ([%_])</li>
+ * <li>Character ranges, i.e. "[]" or "[^]" ==> second regex group (\[[^]]*\])</li>
+ * <li>Literals ==> third regex group ([^%_\[]+)</li>
+ * </ol>
+ */
+ private static final Pattern SQL_LIKE_REGEX = Pattern.compile("([%_])|(\\[[^]]*\\])|([^%_\\[]+)");
+
+ private static final String SQL_LIKE_ESCAPE_REGEX_STR = "(%s.?)|([%%_])|(\\[[^]]*\\])|([^%%_\\[%s]+)";
+
+ private static final String JAVA_REGEX_SPECIALS = ".()[]{}<>|^-+=*?!$\\";
+
+ private final String likeString_;
+
+ private final String escapeChar_;
+
+ private String regexString_ = null;
+
+ private String prefixString_ = null;
+
+ public HBaseRegexParser(FunctionCall call) {
+ this(likeString(call), escapeString(call));
+ }
+
+ public HBaseRegexParser(String likeString) {
+ this(likeString, null);
+ }
+
+ public HBaseRegexParser(String likeString, Character escapeChar) {
+ likeString_ = likeString;
+ if (escapeChar == null) {
+ escapeChar_ = null;
+ } else {
+ escapeChar_ = JAVA_REGEX_SPECIALS.indexOf(escapeChar) == -1
+ ? String.valueOf(escapeChar) : ("\\" + escapeChar);
+ }
+ }
+
+ /**
+ * Convert a SQL LIKE operator Value to a Regular Expression.
+ */
+ public HBaseRegexParser parse() {
+ if (regexString_ != null) {
+ return this;
+ }
+
+ Matcher matcher = null;
+ StringBuilder prefixSB = new StringBuilder();
+ StringBuilder regexSB = new StringBuilder("^"); // starts with
+ if (escapeChar_ == null) {
+ matcher = SQL_LIKE_REGEX.matcher(likeString_);
+ } else {
+ /*
+ * When an escape character is specified, add another capturing group
+ * with the escape character in the front for the escape sequence and
+ * add the escape character to the exclusion list of literals
+ */
+ matcher = Pattern.compile(
+ String.format(SQL_LIKE_ESCAPE_REGEX_STR, escapeChar_, escapeChar_))
+ .matcher(likeString_);
+ }
+ String fragment = null;
+ boolean literalsSoFar = true;
+ while (matcher.find()) {
+ if (escapeChar_ != null && matcher.group(1) != null) {
+ fragment = matcher.group(1);
+ if (fragment.length() != 2) {
+ throw new IllegalArgumentException("Invalid fragment '"
+ + fragment + "' at index " + matcher.start()
+ + " in the LIKE operand '" + likeString_ + "'");
+ }
+ String escapedChar = fragment.substring(1);
+ if (literalsSoFar) {
+ prefixSB.append(escapedChar);
+ }
+ regexSB.append(Pattern.quote(escapedChar));
+ } else {
+ fragment = matcher.group();
+ switch (fragment) {
+ case "_": // LIKE('_') => REGEX('.')
+ literalsSoFar = false;
+ regexSB.append(".");
+ break;
+ case "%": // LIKE('%') => REGEX('.*')
+ literalsSoFar = false;
+ regexSB.append(".*");
+ break;
+ default: // ALL other including character ranges
+ if (fragment.startsWith("[") && fragment.endsWith("]")) {
+ literalsSoFar = false;
+ regexSB.append(fragment);
+ } else {
+ if (literalsSoFar) {
+ prefixSB.append(fragment);
+ }
+ // found literal, just quote it.
+ regexSB.append(Pattern.quote(fragment));
+ }
+ break;
+ }
+ }
+ }
+ prefixString_ = prefixSB.toString();
+ regexString_ = regexSB.append("$") // ends with
+ .toString();
+
+ logger.debug("Converted LIKE string '{}' to REGEX string '{}'.", likeString_, regexString_);
+ return this;
+ }
+
+ public String getRegexString() {
+ return regexString_;
+ }
+
+ public String getPrefixString() {
+ return prefixString_;
+ }
+
+ public String getLikeString() {
+ return likeString_;
+ }
+
+ private static String likeString(FunctionCall call) {
+ return ((QuotedString) call.args.get(1)).value;
+ }
+
+ private static Character escapeString(FunctionCall call) {
+ if (call.args.size() > 2) {
+ return ((QuotedString) call.args.get(2)).value.charAt(0);
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index 086900a..1152b7b 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -30,10 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.TestName;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
@@ -48,13 +45,6 @@ public class BaseHBaseTest extends BaseTestQuery {
protected static HBaseStoragePluginConfig storagePluginConfig;
- @Rule public TestName TEST_NAME = new TestName();
-
- @Before
- public void printID() throws Exception {
- System.out.printf("Running %s#%s\n", getClass().getName(), TEST_NAME.getMethodName());
- }
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
/*
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
index 18cf87c..888a9f5 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java
@@ -34,6 +34,7 @@ import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
+ TestHBaseRegexParser.class,
HBaseRecordReaderTest.class,
TestHBaseFilterPushDown.class,
TestHBaseProjectPushDown.class,
@@ -47,6 +48,7 @@ public class HBaseTestsSuite {
private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
protected static final String TEST_TABLE_1 = "TestTable1";
+ protected static final String TEST_TABLE_3 = "TestTable3";
private static Configuration conf;
@@ -128,7 +130,7 @@ public class HBaseTestsSuite {
}
private static boolean tablesExist() throws IOException {
- return admin.tableExists(TEST_TABLE_1);
+ return admin.tableExists(TEST_TABLE_1) && admin.tableExists(TEST_TABLE_3);
}
private static void createTestTables() throws Exception {
@@ -138,11 +140,14 @@ public class HBaseTestsSuite {
* Will revert to multiple region once the issue is resolved.
*/
TestTableGenerator.generateHBaseDataset1(admin, TEST_TABLE_1, 1);
+ TestTableGenerator.generateHBaseDataset3(admin, TEST_TABLE_3, 1);
}
private static void cleanupTestTables() throws IOException {
admin.disableTable(TEST_TABLE_1);
admin.deleteTable(TEST_TABLE_1);
+ admin.disableTable(TEST_TABLE_3);
+ admin.deleteTable(TEST_TABLE_3);
}
public static int getZookeeperPort() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 29e7033..4e63a3d 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -34,6 +34,42 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
}
@Test
+ public void testFilterPushDownRowKeyLike() throws Exception {
+ setColumnWidths(new int[] {8, 22});
+ runHBaseSQLVerifyCount("SELECT\n"
+ + " row_key, convert_from(tableName.f.c, 'UTF8') `f.c`\n"
+ + "FROM\n"
+ + " hbase.`TestTable3` tableName\n"
+ + "WHERE\n"
+ + " row_key LIKE '08%0' OR row_key LIKE '%70'"
+ , 21);
+ }
+
+ @Test
+ public void testFilterPushDownRowKeyLikeWithEscape() throws Exception {
+ setColumnWidths(new int[] {8, 22});
+ runHBaseSQLVerifyCount("SELECT\n"
+ + " row_key, convert_from(tableName.f.c, 'UTF8') `f.c`\n"
+ + "FROM\n"
+ + " hbase.`TestTable3` tableName\n"
+ + "WHERE\n"
+ + " row_key LIKE '!%!_AS!_PREFIX!_%' ESCAPE '!'"
+ , 2);
+ }
+
+ @Test
+ public void testFilterPushDownRowKeyRangeAndColumnValueLike() throws Exception {
+ setColumnWidths(new int[] {8, 22});
+ runHBaseSQLVerifyCount("SELECT\n"
+ + " row_key, convert_from(tableName.f.c, 'UTF8') `f.c`\n"
+ + "FROM\n"
+ + " hbase.`TestTable3` tableName\n"
+ + "WHERE\n"
+ + " row_key >= '07' AND row_key < '09' AND tableName.f.c LIKE 'value 0%9'"
+ , 22);
+ }
+
+ @Test
public void testFilterPushDownRowKeyGreaterThan() throws Exception {
setColumnWidths(new int[] {8, 38, 38});
runHBaseSQLVerifyCount("SELECT\n"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java
new file mode 100644
index 0000000..a925d0e
--- /dev/null
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseRegexParser.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.regex.Pattern;
+
+import org.apache.drill.exec.store.hbase.HBaseRegexParser;
+import org.apache.drill.test.DrillTest;
+import org.junit.Test;
+
+public class TestHBaseRegexParser extends DrillTest {
+
+ @Test
+ public void testLikeExprToRegex() throws Exception {
+ HBaseRegexParser parser = new HBaseRegexParser("ABC%[0-7][0-9A-Fa-f]").parse();
+ assertEquals("^\\QABC\\E.*[0-7][0-9A-Fa-f]$", parser.getRegexString());
+ assertEquals("ABC", parser.getPrefixString());
+ Pattern pattern = Pattern.compile(parser.getRegexString(), Pattern.DOTALL);
+ assertTrue(pattern.matcher("ABC79").matches());
+ assertTrue(pattern.matcher("ABCxxxxxxx79").matches());
+
+ parser = new HBaseRegexParser("ABC%[0-8]%_").parse();
+ assertEquals("^\\QABC\\E.*[0-8].*.$", parser.getRegexString());
+ assertEquals("ABC", parser.getPrefixString());
+ pattern = Pattern.compile(parser.getRegexString(), Pattern.DOTALL);
+ assertTrue(pattern.matcher("ABC79").matches());
+ assertTrue(pattern.matcher("ABCxxxx79").matches());
+ assertTrue(pattern.matcher("ABCxxxx7xxxxx9").matches());
+ assertTrue(pattern.matcher("ABC[0-8]_").matches());
+
+ parser = new HBaseRegexParser("ABC%[0-8]%_", '%').parse();
+ assertEquals("^\\QABC\\E\\Q[\\E\\Q0-8]\\E\\Q_\\E$", parser.getRegexString());
+ assertEquals("ABC[0-8]_", parser.getPrefixString());
+ pattern = Pattern.compile(parser.getRegexString(), Pattern.DOTALL);
+ assertFalse(pattern.matcher("ABC79").matches());
+ assertTrue(pattern.matcher("ABC[0-8]_").matches());
+
+ try {
+ parser = new HBaseRegexParser("ABC%[0-8][^a-f]%", '%').parse();
+ fail("Parsed an illegal LIKE expression.");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/71f5ad44/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
index ccf5ad5..c244f9e 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
@@ -34,6 +34,9 @@ public class TestTableGenerator {
{'r'}, {'s'}, {'t'}, {'u'}, {'v'}, {'w'}, {'x'}, {'y'}, {'z'}
};
+ static final byte[] FAMILY_F = {'f'};
+ static final byte[] COLUMN_C = {'c'};
+
public static void generateHBaseDataset1(HBaseAdmin admin, String tableName, int numberRegions) throws Exception {
if (admin.tableExists(tableName)) {
admin.disableTable(tableName);
@@ -152,4 +155,46 @@ public class TestTableGenerator {
admin.flush(tableName);
}
+ public static void generateHBaseDataset3(HBaseAdmin admin, String tableName, int numberRegions) throws Exception {
+ if (admin.tableExists(tableName)) {
+ admin.disableTable(tableName);
+ admin.deleteTable(tableName);
+ }
+
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(new HColumnDescriptor(FAMILY_F));
+
+ if (numberRegions > 1) {
+ admin.createTable(desc, Arrays.copyOfRange(SPLIT_KEYS, 0, numberRegions-1));
+ } else {
+ admin.createTable(desc);
+ }
+
+ HTable table = new HTable(admin.getConfiguration(), tableName);
+
+ for (int i = 0; i <= 100; ++i) {
+ Put p = new Put((String.format("%03d", i)).getBytes());
+ p.add(FAMILY_F, COLUMN_C, String.format("value %03d", i).getBytes());
+ table.put(p);
+ }
+ for (int i = 0; i <= 1000; ++i) {
+ Put p = new Put((String.format("%04d", i)).getBytes());
+ p.add(FAMILY_F, COLUMN_C, String.format("value %04d", i).getBytes());
+ table.put(p);
+ }
+
+ Put p = new Put("%_AS_PREFIX_ROW1".getBytes());
+ p.add(FAMILY_F, COLUMN_C, "dummy".getBytes());
+ table.put(p);
+
+ p = new Put("%_AS_PREFIX_ROW2".getBytes());
+ p.add(FAMILY_F, COLUMN_C, "dummy".getBytes());
+ table.put(p);
+
+ table.flushCommits();
+ table.close();
+
+ admin.flush(tableName);
+ }
+
}
[2/3] git commit: DRILL-1540: Provide a fallback data format in case
Drill is unable to determine one
Posted by ad...@apache.org.
DRILL-1540: Provide a fallback data format in case Drill is unable to determine one
+ added "storageformat" to ignored list to avoid failure in existing cluster.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/958de3d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/958de3d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/958de3d6
Branch: refs/heads/master
Commit: 958de3d62082ac1a70fd2a29c8f4b5eca3a78f91
Parents: ae2790e
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Wed Oct 15 19:23:30 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Wed Oct 22 18:20:00 2014 -0700
----------------------------------------------------------------------
.../drill/exec/store/dfs/FileSystemPlugin.java | 33 +++++++++-----------
.../drill/exec/store/dfs/WorkspaceConfig.java | 15 ++++-----
.../exec/store/dfs/WorkspaceSchemaFactory.java | 26 ++++++++++-----
.../resources/bootstrap-storage-plugins.json | 3 +-
.../resources/bootstrap-storage-plugins.json | 3 +-
.../largefiles/merging_receiver_large_data.json | 6 ++--
.../resources/writer/simple_csv_writer.json | 6 ++--
7 files changed, 47 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index b0855c8..0449db3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -18,7 +18,6 @@
package org.apache.drill.exec.store.dfs;
import java.io.IOException;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -53,7 +52,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemPlugin.class);
private final FileSystemSchemaFactory schemaFactory;
- private Map<String, FormatPlugin> formatsByName;
+ private Map<String, FormatPlugin> formatPluginsByName;
private Map<FormatPluginConfig, FormatPlugin> formatPluginsByConfig;
private FileSystemConfig config;
private DrillbitContext context;
@@ -68,30 +67,28 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());
- this.fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf);
- this.formatsByName = FormatCreator.getFormatPlugins(context, fs, config);
+ fs = FileSystemCreator.getFileSystem(context.getConfig(), fsConf);
+ formatPluginsByName = FormatCreator.getFormatPlugins(context, fs, config);
List<FormatMatcher> matchers = Lists.newArrayList();
formatPluginsByConfig = Maps.newHashMap();
- for (FormatPlugin p : formatsByName.values()) {
+ for (FormatPlugin p : formatPluginsByName.values()) {
matchers.add(p.getMatcher());
formatPluginsByConfig.put(p.getConfig(), p);
}
- List<WorkspaceSchemaFactory> factories;
- if (config.workspaces == null || config.workspaces.isEmpty()) {
- factories = Collections.singletonList(
- new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, "default", name, fs, WorkspaceConfig.DEFAULT, matchers));
- } else {
- factories = Lists.newArrayList();
+ boolean noWorkspace = config.workspaces == null || config.workspaces.isEmpty();
+ List<WorkspaceSchemaFactory> factories = Lists.newArrayList();
+ if (!noWorkspace) {
for (Map.Entry<String, WorkspaceConfig> space : config.workspaces.entrySet()) {
factories.add(new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, space.getKey(), name, fs, space.getValue(), matchers));
}
+ }
- // if the "default" workspace is not given add one.
- if (!config.workspaces.containsKey("default")) {
- factories.add(new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, "default", name, fs, WorkspaceConfig.DEFAULT, matchers));
- }
+ // if the "default" workspace is not given add one.
+ if (noWorkspace || !config.workspaces.containsKey("default")) {
+ factories.add(new WorkspaceSchemaFactory(context.getConfig(), context.getPersistentStoreProvider(), this, "default", name, fs, WorkspaceConfig.DEFAULT, matchers));
}
+
this.schemaFactory = new FileSystemSchemaFactory(name, factories);
} catch (IOException e) {
throw new ExecutionSetupException("Failure setting up file system plugin.", e);
@@ -113,7 +110,7 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
FormatSelection formatSelection = selection.getWith(context.getConfig(), FormatSelection.class);
FormatPlugin plugin;
if (formatSelection.getFormat() instanceof NamedFormatPluginConfig) {
- plugin = formatsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name);
+ plugin = formatPluginsByName.get( ((NamedFormatPluginConfig) formatSelection.getFormat()).name);
} else {
plugin = formatPluginsByConfig.get(formatSelection.getFormat());
}
@@ -129,12 +126,12 @@ public class FileSystemPlugin extends AbstractStoragePlugin{
}
public FormatPlugin getFormatPlugin(String name) {
- return formatsByName.get(name);
+ return formatPluginsByName.get(name);
}
public FormatPlugin getFormatPlugin(FormatPluginConfig config) {
if (config instanceof NamedFormatPluginConfig) {
- return formatsByName.get(((NamedFormatPluginConfig) config).name);
+ return formatPluginsByName.get(((NamedFormatPluginConfig) config).name);
} else {
return formatPluginsByConfig.get(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java
index 2103a96..a6ee545 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceConfig.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.dfs;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
@@ -25,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* - writable flag to indicate whether the location supports creating new tables.
* - default storage format for new tables created in this workspace.
*/
+@JsonIgnoreProperties(value = {"storageformat"})
public class WorkspaceConfig {
/** Default workspace is a root directory which supports read, but not write. */
@@ -32,14 +34,14 @@ public class WorkspaceConfig {
private final String location;
private final boolean writable;
- private final String storageformat;
+ private final String defaultInputFormat;
public WorkspaceConfig(@JsonProperty("location") String location,
@JsonProperty("writable") boolean writable,
- @JsonProperty("storageformat") String storageformat) {
+ @JsonProperty("defaultInputFormat") String defaultInputFormat) {
this.location = location;
this.writable = writable;
- this.storageformat = storageformat;
+ this.defaultInputFormat = defaultInputFormat;
}
public String getLocation() {
@@ -50,9 +52,8 @@ public class WorkspaceConfig {
return writable;
}
- @JsonProperty("storageformat")
- public String getStorageFormat() {
- return storageformat;
+ public String getDefaultInputFormat() {
+ return defaultInputFormat;
}
@Override
@@ -68,6 +69,6 @@ public class WorkspaceConfig {
WorkspaceConfig that = (WorkspaceConfig) obj;
return ((this.location == null && that.location == null) || this.location.equals(that.location)) &&
this.writable == that.writable &&
- ((this.storageformat == null && that.storageformat == null) || this.storageformat.equals(that.storageformat));
+ ((this.defaultInputFormat == null && that.defaultInputFormat == null) || this.defaultInputFormat.equals(that.defaultInputFormat));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
index 68f380c..acd8fcb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java
@@ -21,10 +21,14 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Pattern;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
import net.hydromatic.optiq.Table;
import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.dotdrill.DotDrillFile;
@@ -63,7 +67,6 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
private final String storageEngineName;
private final String schemaName;
private final FileSystemPlugin plugin;
-// private final PTable<String> knownPaths;
private final PStore<String> knownViews;
private final ObjectMapper mapper;
@@ -84,17 +87,11 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
// setup cache
if (storageEngineName == null) {
this.knownViews = null;
-// this.knownPaths = null;
} else {
this.knownViews = provider.getPStore(PStoreConfig //
.newJacksonBuilder(drillConfig.getMapper(), String.class) //
.name(Joiner.on('.').join("storage.views", storageEngineName, schemaName)) //
.build());
-
-// this.knownPaths = provider.getPTable(PTableConfig //
-// .newJacksonBuilder(drillConfig.getMapper(), String.class) //
-// .name(Joiner.on('.').join("storage.cache", storageEngineName, schemaName)) //
-// .build());
}
@@ -105,6 +102,19 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
fileMatchers.add(m);
}
+ // NOTE: Add fallback format matcher if given in the configuration. Make sure fileMatchers is an order-preserving list.
+ final String defaultInputFormat = config.getDefaultInputFormat();
+ if (!Strings.isNullOrEmpty(defaultInputFormat)) {
+ final FormatPlugin formatPlugin = plugin.getFormatPlugin(defaultInputFormat);
+ if (formatPlugin == null) {
+ final String message = String.format("Unable to find default input format[%s] for workspace[%s.%s]",
+ defaultInputFormat, storageEngineName, schemaName);
+ throw new ExecutionSetupException(message);
+ }
+ final FormatMatcher fallbackMatcher = new BasicFormatMatcher(formatPlugin, fs,
+ ImmutableList.of(Pattern.compile(".*")), ImmutableList.<MagicString>of());
+ fileMatchers.add(fallbackMatcher);
+ }
}
private Path getViewPath(String name) {
@@ -283,7 +293,7 @@ public class WorkspaceSchemaFactory implements ExpandingConcurrentMap.MapValueFa
FormatPlugin formatPlugin = plugin.getFormatPlugin(storage);
if (formatPlugin == null) {
throw new UnsupportedOperationException(
- String.format("Unsupported format '%s' in workspace '%s'", config.getStorageFormat(),
+ String.format("Unsupported format '%s' in workspace '%s'", config.getDefaultInputFormat(),
Joiner.on(".").join(getSchemaPath())));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index 31df303..4a20bea 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -10,8 +10,7 @@
},
"tmp" : {
location: "/tmp",
- writable: true,
- storageformat: "csv"
+ writable: true
}
},
formats: {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
index 35ee717..e9772cf 100644
--- a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
@@ -10,8 +10,7 @@
},
"tmp" : {
location: "/tmp/drilltest",
- writable: true,
- storageformat: "csv"
+ writable: true
}
},
formats: {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/test/resources/largefiles/merging_receiver_large_data.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/largefiles/merging_receiver_large_data.json b/exec/java-exec/src/test/resources/largefiles/merging_receiver_large_data.json
index 902a59e..265486d 100644
--- a/exec/java-exec/src/test/resources/largefiles/merging_receiver_large_data.json
+++ b/exec/java-exec/src/test/resources/largefiles/merging_receiver_large_data.json
@@ -16,13 +16,11 @@
"workspaces" : {
"root" : {
"location" : "/",
- "writable" : false,
- "storageformat" : null
+ "writable" : false
},
"tmp" : {
"location" : "/tmp",
- "writable" : true,
- "storageformat" : "csv"
+ "writable" : true
}
},
"formats" : {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/958de3d6/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/writer/simple_csv_writer.json b/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
index ff670d5..7980eb5 100644
--- a/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
+++ b/exec/java-exec/src/test/resources/writer/simple_csv_writer.json
@@ -34,13 +34,11 @@
"workspaces" : {
"root" : {
"location" : "/",
- "writable" : false,
- "storageformat" : null
+ "writable" : false
},
"tmp" : {
"location" : "/tmp",
- "writable" : true,
- "storageformat" : "csv"
+ "writable" : true
}
},
"formats" : {