You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2015/11/19 04:28:39 UTC
hbase git commit: HBASE-14782 FuzzyRowFilter skips valid rows
(Vladimir Rodionov)
Repository: hbase
Updated Branches:
refs/heads/master 3aa3fae13 -> ebe5801e0
HBASE-14782 FuzzyRowFilter skips valid rows (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ebe5801e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ebe5801e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ebe5801e
Branch: refs/heads/master
Commit: ebe5801e0081dcce7f3eb918b39161fcd2298087
Parents: 3aa3fae
Author: chenheng <ch...@apache.org>
Authored: Thu Nov 19 10:56:18 2015 +0800
Committer: chenheng <ch...@apache.org>
Committed: Thu Nov 19 11:28:18 2015 +0800
----------------------------------------------------------------------
.../hadoop/hbase/filter/FuzzyRowFilter.java | 27 ++++++++++-
.../hadoop/hbase/filter/TestFuzzyRowFilter.java | 8 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 51 ++++++++++++++++++++
3 files changed, 81 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index a9dd596..a4c6c3b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -154,6 +154,7 @@ public class FuzzyRowFilter extends FilterBase {
}
// NOT FOUND -> seek next using hint
lastFoundIndex = -1;
+
return ReturnCode.SEEK_NEXT_USING_HINT;
}
@@ -572,7 +573,31 @@ public class FuzzyRowFilter extends FilterBase {
}
}
- return result;
+ return reverse? result: trimTrailingZeroes(result, fuzzyKeyMeta, toInc);
+ }
+
+ /**
+ * For forward scanner, next cell hint should not contain any trailing zeroes
+ * unless they are part of fuzzyKeyMeta
+ * hint = '\x01\x01\x01\x00\x00'
+ * will skip valid row '\x01\x01\x01'
+ *
+ * @param result
+ * @param fuzzyKeyMeta
+ * @param toInc - position of incremented byte
+ * @return trimmed version of result
+ */
+
+ private static byte[] trimTrailingZeroes(byte[] result, byte[] fuzzyKeyMeta, int toInc) {
+ int off = fuzzyKeyMeta.length >= result.length? result.length -1:
+ fuzzyKeyMeta.length -1;
+ for( ; off >= 0; off--){
+ if(fuzzyKeyMeta[off] != 0) break;
+ }
+ if (off < toInc) off = toInc;
+ byte[] retValue = new byte[off+1];
+ System.arraycopy(result, 0, retValue, 0, retValue.length);
+ return retValue;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
index d420959..0d48088 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java
@@ -231,7 +231,7 @@ public class TestFuzzyRowFilter {
new byte[]{0, 1, 2}, // fuzzy row
new byte[]{0, -1, -1}, // mask
new byte[]{1, 2, 1, 0, 1}, // current
- new byte[]{2, 1, 2, 0, 0}); // expected next
+ new byte[]{2, 1, 2}); // expected next
assertNext(false,
new byte[]{0, 1, 2}, // fuzzy row
@@ -243,13 +243,13 @@ public class TestFuzzyRowFilter {
new byte[]{0, 1, 0, 2, 0}, // fuzzy row
new byte[]{0, -1, 0, -1, 0}, // mask
new byte[]{1, 0, 2, 0, 1}, // current
- new byte[]{1, 1, 0, 2, 0}); // expected next
+ new byte[]{1, 1, 0, 2}); // expected next
assertNext(false,
new byte[]{1, 0, 1},
new byte[]{-1, 0, -1},
new byte[]{1, (byte) 128, 2, 0, 1},
- new byte[]{1, (byte) 129, 1, 0, 0});
+ new byte[]{1, (byte) 129, 1});
assertNext(false,
new byte[]{0, 1, 0, 1},
@@ -315,7 +315,7 @@ public class TestFuzzyRowFilter {
new byte[]{1, 1, 0, 0},
new byte[]{-1, -1, 0, 0},
new byte[]{0, 1, 3, 2},
- new byte[]{1, 1, 0, 0});
+ new byte[]{1, 1});
// No next for this one
Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(
http://git-wip-us.apache.org/repos/asf/hbase/blob/ebe5801e/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
index ba1d2a1..cc09042 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java
@@ -110,6 +110,57 @@ public class TestFuzzyRowFilterEndToEnd {
// Nothing to do.
}
+ @Test
+ public void testHBASE14782() throws IOException
+ {
+ String cf = "f";
+ String cq = "q";
+ String table = "HBASE14872";
+
+ Table ht =
+ TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+ // Load data
+ String[] rows = new String[]{
+ "\\x9C\\x00\\x044\\x00\\x00\\x00\\x00",
+ "\\x9C\\x00\\x044\\x01\\x00\\x00\\x00",
+ "\\x9C\\x00\\x044\\x00\\x01\\x00\\x00",
+ "\\x9C\\x00\\x044\\x00\\x00\\x01\\x00",
+ "\\x9C\\x00\\x044\\x00\\x01\\x00\\x01",
+ "\\x9B\\x00\\x044e\\xBB\\xB2\\xBB",
+ };
+
+ String badRow = "\\x9C\\x00\\x03\\xE9e\\xBB{X\\x1Fwts\\x1F\\x15vRX";
+
+ for(int i=0; i < rows.length; i++){
+ Put p = new Put(Bytes.toBytesBinary(rows[i]));
+ p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
+ ht.put(p);
+ }
+
+ Put p = new Put(Bytes.toBytesBinary(badRow));
+ p.addColumn(cf.getBytes(), cq.getBytes(), "value".getBytes());
+ ht.put(p);
+
+ TEST_UTIL.flush();
+
+ List<Pair<byte[], byte[]>> data = new ArrayList<Pair<byte[], byte[]>>();
+ byte[] fuzzyKey = Bytes.toBytesBinary("\\x00\\x00\\x044");
+ byte[] mask = new byte[] { 1,0,0,0};
+ data.add(new Pair<byte[], byte[]>(fuzzyKey, mask));
+ FuzzyRowFilter filter = new FuzzyRowFilter(data);
+
+ Scan scan = new Scan();
+ scan.setFilter(filter);
+
+ ResultScanner scanner = ht.getScanner(scan);
+ int total = 0;
+ while(scanner.next() != null){
+ total++;
+ }
+ assertEquals(rows.length, total);
+ TEST_UTIL.deleteTable(TableName.valueOf(table));
+ }
+
@Test
public void testEndToEnd() throws Exception {
String cf = "f";