You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/01/08 03:45:22 UTC
hbase git commit: HBASE-12817 Data missing while scanning using
PREFIX_TREE data block encoding (Duo Zhang)
Repository: hbase
Updated Branches:
refs/heads/master 7dce0d5c4 -> 645fbd7d8
HBASE-12817 Data missing while scanning using PREFIX_TREE data block encoding (Duo Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/645fbd7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/645fbd7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/645fbd7d
Branch: refs/heads/master
Commit: 645fbd7d87450b31b67b1e535cdb9c1cf50ffd16
Parents: 7dce0d5
Author: tedyu <yu...@gmail.com>
Authored: Wed Jan 7 18:45:09 2015 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jan 7 18:45:09 2015 -0800
----------------------------------------------------------------------
.../codec/prefixtree/PrefixTreeSeeker.java | 9 +-
.../PrefixTreeArrayReversibleScanner.java | 1 +
.../decode/PrefixTreeArrayScanner.java | 21 +-
.../hbase/io/encoding/TestPrefixTree.java | 258 ++++++++++---------
4 files changed, 136 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/645fbd7d/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
index 2db0f29..b95055c 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
@@ -130,14 +130,9 @@ public class PrefixTreeSeeker implements EncodedSeeker {
@Override
public boolean next() {
- boolean advance = ptSearcher.advance();
- if (ptSearcher.hasMovedToPreviousAsPartOfSeek()) {
- ptSearcher.setMovedToPreviousAsPartOfSeek(false);
- }
- return advance;
+ return ptSearcher.advance();
}
-// @Override
public boolean advance() {
return ptSearcher.advance();
}
@@ -222,7 +217,6 @@ public class PrefixTreeSeeker implements EncodedSeeker {
if (CellScannerPosition.AT == position) {
if (seekBefore) {
- // We need not set movedToPrevious because the intention is to seekBefore
ptSearcher.previous();
return 1;
}
@@ -233,7 +227,6 @@ public class PrefixTreeSeeker implements EncodedSeeker {
if (CellScannerPosition.AFTER == position) {
if (!ptSearcher.isBeforeFirst()) {
ptSearcher.previous();
- ptSearcher.setMovedToPreviousAsPartOfSeek(true);
}
return 1;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/645fbd7d/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
index 584a8fb..5a817cb 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayReversibleScanner.java
@@ -115,6 +115,7 @@ public class PrefixTreeArrayReversibleScanner extends PrefixTreeArrayScanner imp
}
}
if (currentRowNode.hasOccurrences()) {// escape clause
+ currentRowNode.resetFanIndex();
return true;// found some values
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/645fbd7d/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
index a68f8f8..cb7eeea 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java
@@ -60,7 +60,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
protected boolean nubCellsRemain;
protected int currentCellIndex;
- protected boolean movedToPrevious;
/*********************** construct ******************************/
@@ -261,12 +260,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
return true;
}
} else {
- if (movedToPrevious && currentRowNode.hasOccurrences()
- && currentRowNode.getFanIndex() == getNextFanIndex()) {
- followFan(getNextFanIndex());
- } else {
- discardCurrentRowNode(true);
- }
+ discardCurrentRowNode(true);
}
}
return false;// went past the end
@@ -303,10 +297,6 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
followFan(currentFanPosition);
}
- protected int getNextFanIndex() {
- return rowNodes[rowNodeStackIndex + 1].getFanIndex();
- }
-
protected void followNextFan() {
int nextFanPosition = currentRowNode.getFanIndex() + 1;
followFan(nextFanPosition);
@@ -537,13 +527,4 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne
public int getTagBufferLength() {
return tagsBuffer.length;
}
-
- public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) {
- this.movedToPrevious = movedToPrevious;
- }
-
- public boolean hasMovedToPreviousAsPartOfSeek() {
- return this.movedToPrevious;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/645fbd7d/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
index 2562b10..9b87c37 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
@@ -18,34 +18,38 @@
package org.apache.hadoop.hbase.io.encoding;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-@Category({IOTests.class, MediumTests.class})
+@Category({ IOTests.class, SmallTests.class })
public class TestPrefixTree {
+
private static final String row4 = "a-b-B-2-1402397300-1402416535";
private static final byte[] row4_bytes = Bytes.toBytes(row4);
private static final String row3 = "a-b-A-1-1402397227-1402415999";
@@ -54,131 +58,135 @@ public class TestPrefixTree {
private static final byte[] row2_bytes = Bytes.toBytes(row2);
private static final String row1 = "a-b-A-1";
private static final byte[] row1_bytes = Bytes.toBytes(row1);
- public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
private final static byte[] fam = Bytes.toBytes("cf_1");
private final static byte[] qual1 = Bytes.toBytes("qf_1");
private final static byte[] qual2 = Bytes.toBytes("qf_2");
- public static Configuration conf;
-
- @Rule
- public final TestName TEST_NAME = new TestName();
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- // setup configuration
- conf = TEST_UTIL.getConfiguration();
- conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false);
- conf.setBoolean("hbase.online.schema.update.enable", true);
- conf.setInt("hbase.client.scanner.timeout.period", 600000);
- TEST_UTIL.startMiniCluster(2);
+
+ private final HBaseTestingUtility testUtil = new HBaseTestingUtility();
+
+ private HRegion region;
+
+ @Before
+ public void setUp() throws Exception {
+ TableName tableName = TableName.valueOf(getClass().getSimpleName());
+ HTableDescriptor htd = new HTableDescriptor(tableName);
+ htd.addFamily(new HColumnDescriptor(fam).setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE));
+ HRegionInfo info = new HRegionInfo(tableName, null, null, false);
+ Path path = testUtil.getDataTestDir(getClass().getSimpleName());
+ region = HRegion.createHRegion(info, path, testUtil.getConfiguration(), htd);
}
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
+ @After
+ public void tearDown() throws Exception {
+ region.close(true);
+ testUtil.cleanupTestDir();
}
@Test
public void testHBASE11728() throws Exception {
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
- Table table = null;
- try {
- Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
- HColumnDescriptor colDesc = new HColumnDescriptor(fam);
- HTableDescriptor desc = new HTableDescriptor(tableName);
- colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE);
- colDesc.setTimeToLive(15552000);
- desc.addFamily(colDesc);
- hBaseAdmin.createTable(desc);
- table = new HTable(conf, tableName);
- Put put = new Put(Bytes.toBytes("a-b-0-0"));
- put.add(fam, qual1, Bytes.toBytes("c1-value"));
- table.put(put);
- put = new Put(row1_bytes);
- put.add(fam, qual1, Bytes.toBytes("c1-value"));
- table.put(put);
- put = new Put(row2_bytes);
- put.add(fam, qual2, Bytes.toBytes("c2-value"));
- table.put(put);
- put = new Put(row3_bytes);
- put.add(fam, qual2, Bytes.toBytes("c2-value-2"));
- table.put(put);
- put = new Put(row4_bytes);
- put.add(fam, qual2, Bytes.toBytes("c2-value-3"));
- table.put(put);
- hBaseAdmin.flush(tableName);
- String[] rows = new String[3];
- rows[0] = row1;
- rows[1] = row2;
- rows[2] = row3;
- byte[][] val = new byte[3][];
- val[0] = Bytes.toBytes("c1-value");
- val[1] = Bytes.toBytes("c2-value");
- val[2] = Bytes.toBytes("c2-value-2");
- Scan scan = new Scan();
- scan.setStartRow(row1_bytes);
- scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
- ResultScanner scanner = table.getScanner(scan);
- Result[] next = scanner.next(10);
- assertEquals(3, next.length);
- int i = 0;
- for (Result res : next) {
- CellScanner cellScanner = res.cellScanner();
- while (cellScanner.advance()) {
- assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
- .current().getRowOffset(), cellScanner.current().getRowLength()));
- assertEquals(Bytes.toString(val[i]), Bytes.toString(
- cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(),
- cellScanner.current().getValueLength()));
- }
- i++;
+ Put put = new Put(Bytes.toBytes("a-b-0-0"));
+ put.add(fam, qual1, Bytes.toBytes("c1-value"));
+ region.put(put);
+ put = new Put(row1_bytes);
+ put.add(fam, qual1, Bytes.toBytes("c1-value"));
+ region.put(put);
+ put = new Put(row2_bytes);
+ put.add(fam, qual2, Bytes.toBytes("c2-value"));
+ region.put(put);
+ put = new Put(row3_bytes);
+ put.add(fam, qual2, Bytes.toBytes("c2-value-2"));
+ region.put(put);
+ put = new Put(row4_bytes);
+ put.add(fam, qual2, Bytes.toBytes("c2-value-3"));
+ region.put(put);
+ region.flushcache(true);
+ String[] rows = new String[3];
+ rows[0] = row1;
+ rows[1] = row2;
+ rows[2] = row3;
+ byte[][] val = new byte[3][];
+ val[0] = Bytes.toBytes("c1-value");
+ val[1] = Bytes.toBytes("c2-value");
+ val[2] = Bytes.toBytes("c2-value-2");
+ Scan scan = new Scan();
+ scan.setStartRow(row1_bytes);
+ scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> cells = new ArrayList<Cell>();
+ for (int i = 0; i < 3; i++) {
+ assertEquals(i < 2, scanner.next(cells));
+ CellScanner cellScanner = Result.create(cells).cellScanner();
+ while (cellScanner.advance()) {
+ assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+ .current().getRowOffset(), cellScanner.current().getRowLength()));
+ assertEquals(Bytes.toString(val[i]), Bytes.toString(cellScanner.current().getValueArray(),
+ cellScanner.current().getValueOffset(), cellScanner.current().getValueLength()));
}
- scanner.close();
- // Add column
- scan = new Scan();
- scan.addColumn(fam, qual2);
- scan.setStartRow(row1_bytes);
- scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
- scanner = table.getScanner(scan);
- next = scanner.next(10);
- assertEquals(2, next.length);
- i = 1;
- for (Result res : next) {
- CellScanner cellScanner = res.cellScanner();
- while (cellScanner.advance()) {
- assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
- .current().getRowOffset(), cellScanner.current().getRowLength()));
- }
- i++;
+ cells.clear();
+ }
+ scanner.close();
+
+ // Add column
+ scan = new Scan();
+ scan.addColumn(fam, qual2);
+ scan.setStartRow(row1_bytes);
+ scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+ scanner = region.getScanner(scan);
+ for (int i = 1; i < 3; i++) {
+ // assertEquals(i < 2, scanner.next(cells));
+ scanner.next(cells);
+ System.out.println(Result.create(cells));
+ CellScanner cellScanner = Result.create(cells).cellScanner();
+ while (cellScanner.advance()) {
+ assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+ .current().getRowOffset(), cellScanner.current().getRowLength()));
}
- scanner.close();
- i = 1;
- scan = new Scan();
- scan.addColumn(fam, qual2);
- scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
- scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
- scanner = table.getScanner(scan);
- next = scanner.next(10);
- assertEquals(2, next.length);
- for (Result res : next) {
- CellScanner cellScanner = res.cellScanner();
- while (cellScanner.advance()) {
- assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
- .current().getRowOffset(), cellScanner.current().getRowLength()));
- }
- i++;
+ cells.clear();
+ }
+ scanner.close();
+
+ scan = new Scan();
+ scan.addColumn(fam, qual2);
+ scan.setStartRow(Bytes.toBytes("a-b-A-1-"));
+ scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+ scanner = region.getScanner(scan);
+ for (int i = 1; i < 3; i++) {
+ assertEquals(i < 2, scanner.next(cells));
+ CellScanner cellScanner = Result.create(cells).cellScanner();
+ while (cellScanner.advance()) {
+ assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
+ .current().getRowOffset(), cellScanner.current().getRowLength()));
}
- scanner.close();
- scan = new Scan();
- scan.addColumn(fam, qual2);
- scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
- scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
- scanner = table.getScanner(scan);
- next = scanner.next(10);
- assertEquals(1, next.length);
- scanner.close();
- } finally {
- table.close();
+ cells.clear();
+ }
+ scanner.close();
+
+ scan = new Scan();
+ scan.addColumn(fam, qual2);
+ scan.setStartRow(Bytes.toBytes("a-b-A-1-140239"));
+ scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
+ scanner = region.getScanner(scan);
+ assertFalse(scanner.next(cells));
+ assertFalse(cells.isEmpty());
+ scanner.close();
+ }
+
+ @Test
+ public void testHBASE12817() throws IOException {
+ for (int i = 0; i < 100; i++) {
+ region.put(new Put(Bytes.toBytes("obj" + (2900 + i))).add(fam, qual1, Bytes.toBytes(i)));
}
+ region.put(new Put(Bytes.toBytes("obj299")).add(fam, qual1, Bytes.toBytes("whatever")));
+ region.put(new Put(Bytes.toBytes("obj29")).add(fam, qual1, Bytes.toBytes("whatever")));
+ region.put(new Put(Bytes.toBytes("obj2")).add(fam, qual1, Bytes.toBytes("whatever")));
+ region.put(new Put(Bytes.toBytes("obj3")).add(fam, qual1, Bytes.toBytes("whatever")));
+ region.flushcache(true);
+ Scan scan = new Scan(Bytes.toBytes("obj29995"));
+ RegionScanner scanner = region.getScanner(scan);
+ List<Cell> cells = new ArrayList<Cell>();
+ assertFalse(scanner.next(cells));
+ assertArrayEquals(Bytes.toBytes("obj3"), Result.create(cells).getRow());
}
}