You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2015/08/14 02:36:57 UTC
drill git commit: DRILL-3621: Fix incorrect result if HBase filter
contains row_key "or" filter or in list filter
Repository: drill
Updated Branches:
refs/heads/master c0060f7f5 -> e3fbc6a19
DRILL-3621: Fix incorrect result if HBase filter contains row_key "or" filter or in list filter
Add unit test for row_key "or" filter and row_key in list filter.
Modify expected results for couple of existing unit tests, by specifying more strict regex pattern.
Add one row in Hbase test table, per review comment.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e3fbc6a1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e3fbc6a1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e3fbc6a1
Branch: refs/heads/master
Commit: e3fbc6a19ee77a9f3c71c39eaaa3424a991969ca
Parents: c0060f7
Author: Jinfeng Ni <jn...@apache.org>
Authored: Tue Aug 11 16:35:37 2015 -0700
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Aug 13 14:15:53 2015 -0700
----------------------------------------------------------------------
.../exec/store/hbase/HBaseFilterBuilder.java | 6 +-
.../drill/hbase/HBaseRecordReaderTest.java | 6 +-
.../drill/hbase/TestHBaseFilterPushDown.java | 70 ++++++++++++++++++--
.../drill/hbase/TestHBaseProjectPushDown.java | 6 +-
.../apache/drill/hbase/TestTableGenerator.java | 9 +++
5 files changed, 83 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e3fbc6a1/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
index b10415d..245d77d 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
@@ -176,8 +176,10 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
case "equal":
compareOp = CompareOp.EQUAL;
if (isRowKey) {
- startRow = stopRow = fieldValue;
- compareOp = null;
+ startRow = fieldValue;
+ /* stopRow should be just greater than 'value'*/
+ stopRow = Arrays.copyOf(fieldValue, fieldValue.length+1);
+ compareOp = CompareOp.EQUAL;
}
break;
case "not_equal":
http://git-wip-us.apache.org/repos/asf/drill/blob/e3fbc6a1/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
index 60db266..79db8b6 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseRecordReaderTest.java
@@ -24,19 +24,19 @@ public class HBaseRecordReaderTest extends BaseHBaseTest {
@Test
public void testLocalDistributed() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical.json";
- runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 6);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 7);
}
@Test
public void testLocalDistributedColumnSelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_column_select.json";
- runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 2);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
}
@Test
public void testLocalDistributedFamilySelect() throws Exception {
String planName = "/hbase/hbase_scan_screen_physical_family_select.json";
- runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 3);
+ runHBasePhysicalVerifyCount(planName, HBaseTestsSuite.TEST_TABLE_1, 4);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e3fbc6a1/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 7abcb29..00de0ba 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -34,7 +34,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
runHBaseSQLVerifyCount(sql, 1);
- final String[] expectedPlan = {".*startRow=b4, stopRow=b4.*"};
+ final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
@@ -52,7 +52,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
runHBaseSQLVerifyCount(sql, 1);
- final String[] expectedPlan = {".*startRow=b4, stopRow=b4.*"};
+ final String[] expectedPlan = {".*startRow=b4, stopRow=b4\\\\x00, filter=null.*"};
final String[] excludedPlan ={".*startRow=null, stopRow=null.*"};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
@@ -122,9 +122,9 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
+ "WHERE\n"
+ " row_key > 'b4'";
- runHBaseSQLVerifyCount(sql, 2);
+ runHBaseSQLVerifyCount(sql, 3);
- final String[] expectedPlan = {".*startRow=b4.*stopRow=,.*"};
+ final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=,.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
@@ -142,7 +142,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
runHBaseSQLVerifyCount(sql, 2);
- final String[] expectedPlan = {".*startRow=b4.*stopRow=,.*"};
+ final String[] expectedPlan = {".*startRow=b4\\\\x00.*stopRow=, filter=null.*"};
final String[] excludedPlan ={};
final String sqlHBase = canonizeHBaseSQL(sql);
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
@@ -228,7 +228,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
+ "WHERE\n"
+ " convert_from(row_key, 'UTF8') > 'b4'";
- runHBaseSQLVerifyCount(sql, 2);
+ runHBaseSQLVerifyCount(sql, 3);
final String[] expectedPlan = {".*startRow=b4\\\\x00, stopRow=,.*"};
final String[] excludedPlan ={};
@@ -303,4 +303,62 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
}
+ @Test
+ public void testFilterPushDownOrRowKeyEqual() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`[TABLE_NAME]` tableName\n"
+ + "WHERE\n"
+ + " row_key = 'b4' or row_key = 'a2'";
+
+ runHBaseSQLVerifyCount(sql, 2);
+
+ final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"};
+ final String[] excludedPlan ={};
+ final String sqlHBase = canonizeHBaseSQL(sql);
+ PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
+
+ }
+
+ @Test
+ public void testFilterPushDownOrRowKeyInPred() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`[TABLE_NAME]` tableName\n"
+ + "WHERE\n"
+ + " row_key in ('b4', 'a2')";
+
+ runHBaseSQLVerifyCount(sql, 2);
+
+ final String[] expectedPlan = {".*startRow=a2, stopRow=b4\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, b4\\), RowFilter \\(EQUAL, a2\\).*"};
+ final String[] excludedPlan ={};
+ final String sqlHBase = canonizeHBaseSQL(sql);
+ PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
+
+ }
+
+ @Test
+ public void testFilterPushDownOrRowKeyEqualRangePred() throws Exception {
+ setColumnWidths(new int[] {8, 38, 38});
+ final String sql = "SELECT\n"
+ + " *\n"
+ + "FROM\n"
+ + " hbase.`[TABLE_NAME]` tableName\n"
+ + "WHERE\n"
+ + " row_key = 'a2' or row_key between 'b5' and 'b6'";
+
+ runHBaseSQLVerifyCount(sql, 3);
+
+ final String[] expectedPlan = {".*startRow=a2, stopRow=b6\\\\x00, filter=FilterList OR \\(2/2\\): \\[RowFilter \\(EQUAL, a2\\), FilterList AND \\(2/2\\): \\[RowFilter \\(GREATER_OR_EQUAL, b5\\), RowFilter \\(LESS_OR_EQUAL, b6.*"};
+ final String[] excludedPlan ={};
+ final String sqlHBase = canonizeHBaseSQL(sql);
+ PlanTestBase.testPlanMatchingPatterns(sqlHBase, expectedPlan, excludedPlan);
+
+ }
+
}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/e3fbc6a1/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index 6efe905..b27b2a0 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -28,7 +28,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
+ "row_key\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
- , 6);
+ , 7);
}
@Test
@@ -48,7 +48,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
+ "row_key, t.f.c1*31 as `t.f.c1*31`, t.f.c2 as `t.f.c2`, 5 as `5`, 'abc' as `'abc'`\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` t"
- , 6);
+ , 7);
}
@Test
@@ -58,7 +58,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
+ "row_key, f, f2\n"
+ "FROM\n"
+ " hbase.`[TABLE_NAME]` tableName"
- , 6);
+ , 7);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/e3fbc6a1/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
index 097947c..5cb9aa5 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestTableGenerator.java
@@ -81,6 +81,15 @@ public class TestTableGenerator {
p.add("f".getBytes(), "c9".getBytes(), "6".getBytes());
table.put(p);
+ p = new Put(new byte[]{'b', '4', 0});
+ p.add("f".getBytes(), "c1".getBytes(), "1".getBytes());
+ p.add("f2".getBytes(), "c2".getBytes(), "2".getBytes());
+ p.add("f".getBytes(), "c3".getBytes(), "3".getBytes());
+ p.add("f2".getBytes(), "c4".getBytes(), "4".getBytes());
+ p.add("f".getBytes(), "c5".getBytes(), "5".getBytes());
+ p.add("f2".getBytes(), "c6".getBytes(), "6".getBytes());
+ table.put(p);
+
p = new Put("b4".getBytes());
p.add("f".getBytes(), "c1".getBytes(), "1".getBytes());
p.add("f2".getBytes(), "c2".getBytes(), "2".getBytes());