You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2015/06/22 07:41:42 UTC
hbase git commit: HBASE-13933 - DBE's seekBefore with tags corrupts
the tag's offset information thus leading to incorrect results(Ram)
Repository: hbase
Updated Branches:
refs/heads/0.98 bdaf1e326 -> 213458b77
HBASE-13933 - DBE's seekBefore with tags corrupts the tag's offset
information thus leading to incorrect results(Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/213458b7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/213458b7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/213458b7
Branch: refs/heads/0.98
Commit: 213458b771fb6b91357034c826cc6030ccc0e02d
Parents: bdaf1e3
Author: ramkrishna <ra...@gmail.com>
Authored: Mon Jun 22 11:10:53 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Mon Jun 22 11:10:53 2015 +0530
----------------------------------------------------------------------
.../io/encoding/BufferedDataBlockEncoder.java | 2 +
.../hadoop/hbase/io/hfile/TestSeekTo.java | 95 ++++++++++++++++++--
2 files changed, 90 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/213458b7/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 445b897..3958c7c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -151,6 +151,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
lastCommonPrefix = nextState.lastCommonPrefix;
nextKvOffset = nextState.nextKvOffset;
memstoreTS = nextState.memstoreTS;
+ tagsOffset = nextState.tagsOffset;
+ tagsLength = nextState.tagsLength;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/213458b7/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
index c407635..9149b2a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
@@ -19,25 +19,69 @@
package org.apache.hadoop.hbase.io.hfile;
import java.io.IOException;
-
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
/**
* Test {@link HFileScanner#seekTo(byte[])} and its variants.
*/
@Category(SmallTests.class)
-public class TestSeekTo extends HBaseTestCase {
+@RunWith(Parameterized.class)
+public class TestSeekTo {
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final DataBlockEncoding encoding;
+
+ @Parameters
+ public static Collection<Object[]> parameters() {
+ List<Object[]> paramList = new ArrayList<Object[]>();
+ for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+ // Remove after HBASE-13939
+ if (encoding != DataBlockEncoding.PREFIX_TREE) {
+ paramList.add(new Object[] { encoding });
+ }
+ }
+ return paramList;
+ }
static boolean switchKVs = false;
+ public TestSeekTo(DataBlockEncoding encoding) {
+ this.encoding = encoding;
+ }
+
+ @Before
+ public void setUp() {
+ // reset
+ switchKVs = false;
+ }
+
+
static KeyValue toKV(String row, TagUsage tagUsage) {
if (tagUsage == TagUsage.NO_TAG) {
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
@@ -52,7 +96,7 @@ public class TestSeekTo extends HBaseTestCase {
if (!switchKVs) {
switchKVs = true;
return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
- Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
+ Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
} else {
switchKVs = false;
Tag t = new Tag((byte) 1, "myTag1");
@@ -68,15 +112,17 @@ public class TestSeekTo extends HBaseTestCase {
}
Path makeNewFile(TagUsage tagUsage) throws IOException {
- Path ncTFile = new Path(this.testDir, "basic.hfile");
+ Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
+ FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
+ Configuration conf = TEST_UTIL.getConfiguration();
if (tagUsage != TagUsage.NO_TAG) {
conf.setInt("hfile.format.version", 3);
} else {
conf.setInt("hfile.format.version", 2);
}
- FSDataOutputStream fout = this.fs.create(ncTFile);
int blocksize = toKV("a", tagUsage).getLength() * 3;
HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
+ .withDataBlockEncoding(encoding)
.withIncludesTags(true).build();
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
.withFileContext(context)
@@ -96,6 +142,7 @@ public class TestSeekTo extends HBaseTestCase {
return ncTFile;
}
+ @Test
public void testSeekBefore() throws Exception {
testSeekBeforeInternals(TagUsage.NO_TAG);
testSeekBeforeInternals(TagUsage.ONLY_TAG);
@@ -104,6 +151,8 @@ public class TestSeekTo extends HBaseTestCase {
protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
Path p = makeNewFile(tagUsage);
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Configuration conf = TEST_UTIL.getConfiguration();
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
reader.loadFileInfo();
HFileScanner scanner = reader.getScanner(false, true);
@@ -129,6 +178,15 @@ public class TestSeekTo extends HBaseTestCase {
assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("j", tagUsage).getKey()));
assertEquals("i", toRowStr(scanner.getKeyValue()));
+ Cell cell = scanner.getKeyValue();
+ if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
+ Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ while (tagsIterator.hasNext()) {
+ Tag next = tagsIterator.next();
+ assertEquals("myTag1", Bytes.toString(next.getValue()));
+ }
+ }
assertEquals(true, scanner.seekBefore(toKV("k", tagUsage).getKey()));
assertEquals("i", toRowStr(scanner.getKeyValue()));
assertEquals(true, scanner.seekBefore(toKV("l", tagUsage).getKey()));
@@ -137,6 +195,7 @@ public class TestSeekTo extends HBaseTestCase {
reader.close();
}
+ @Test
public void testSeekBeforeWithReSeekTo() throws Exception {
testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
@@ -145,6 +204,8 @@ public class TestSeekTo extends HBaseTestCase {
protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException {
Path p = makeNewFile(tagUsage);
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Configuration conf = TEST_UTIL.getConfiguration();
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
reader.loadFileInfo();
HFileScanner scanner = reader.getScanner(false, true);
@@ -224,8 +285,16 @@ public class TestSeekTo extends HBaseTestCase {
// reseekTo k
assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
assertEquals("k", toRowStr(scanner.getKeyValue()));
+ deleteTestDir(fs);
}
+ protected void deleteTestDir(FileSystem fs) throws IOException {
+ Path dataTestDir = TEST_UTIL.getDataTestDir();
+ if (fs.exists(dataTestDir)) {
+ fs.delete(dataTestDir, true);
+ }
+ }
+ @Test
public void testSeekTo() throws Exception {
testSeekToInternals(TagUsage.NO_TAG);
testSeekToInternals(TagUsage.ONLY_TAG);
@@ -234,6 +303,8 @@ public class TestSeekTo extends HBaseTestCase {
protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
Path p = makeNewFile(tagUsage);
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Configuration conf = TEST_UTIL.getConfiguration();
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
reader.loadFileInfo();
assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
@@ -249,10 +320,17 @@ public class TestSeekTo extends HBaseTestCase {
assertEquals("g", toRowStr(scanner.getKeyValue()));
assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey()));
- assertEquals("k", toRowStr(scanner.getKeyValue()));
+ if (encoding == DataBlockEncoding.PREFIX_TREE) {
+ // TODO : Fix this
+ assertEquals(null, scanner.getKeyValue());
+ } else {
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
+ }
reader.close();
}
+
+ @Test
public void testBlockContainingKey() throws Exception {
testBlockContainingKeyInternals(TagUsage.NO_TAG);
testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
@@ -261,6 +339,8 @@ public class TestSeekTo extends HBaseTestCase {
protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
Path p = makeNewFile(tagUsage);
+ FileSystem fs = TEST_UTIL.getTestFileSystem();
+ Configuration conf = TEST_UTIL.getConfiguration();
HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
reader.loadFileInfo();
HFileBlockIndex.BlockIndexReader blockIndexReader =
@@ -290,6 +370,7 @@ public class TestSeekTo extends HBaseTestCase {
toKV("l", tagUsage).getKey(), 0, klen));
reader.close();
+ deleteTestDir(fs);
}