You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/05/17 14:50:44 UTC
svn commit: r1339581 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/
main/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/regio...
Author: mbautin
Date: Thu May 17 12:50:44 2012
New Revision: 1339581
URL: http://svn.apache.org/viewvc?rev=1339581&view=rev
Log:
[jira][89-fb] [HBASE-5987] HFileBlockIndex improvement
Author: liyintang
Summary:
Recently we find out a performance problem that it is quite slow when multiple requests are reading the same block of data or index.
>From the profiling, one of the causes is the IdLock contention which has been addressed in HBASE-5898.
Another issue is that the HFileScanner will keep asking the HFileBlockIndex about the data block location for each target key value during the scan proces
To solve this issue, we propose to lookahead for one more block index so that the HFileScanner would know the start key value of next data block. So if th
Test Plan: Unit tests
Reviewers: Kannan, mbautin
Reviewed By: mbautin
CC: JIRA, todd, tedyu
Differential Revision: https://reviews.facebook.net/D3237
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu May 17 12:50:44 2012
@@ -541,6 +541,12 @@ public final class HConstants {
/** Configuration key suffix for Thrift server port */
public static final String THRIFT_PORT_SUFFIX = "port";
+
+ /**
+ * The byte array represents for NO_NEXT_INDEXED_KEY;
+ * The actual value is irrelevant because this is always compared by reference.
+ */
+ public static final byte [] NO_NEXT_INDEXED_KEY = Bytes.toBytes("NO_NEXT_INDEXED_KEY");
private HConstants() {
// Can't be instantiated with this ctor.
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java?rev=1339581&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockWithScanInfo.java Thu May 17 12:50:44 2012
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2012 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+/**
+ * BlockWithScanInfo is wrapper class for HFileBlock with other attributes. These attributes are
+ * supposed to be much cheaper to be maintained in each caller thread than in HFileBlock itself.
+ */
+public class BlockWithScanInfo {
+ private final HFileBlock hFileBlock;
+ /**
+ * The first key in the next block following this one in the HFile.
+ * If this key is unknown, this is reference-equal with HConstants.NO_NEXT_INDEXED_KEY
+ */
+ private final byte[] nextIndexedKey;
+
+ public BlockWithScanInfo(HFileBlock hFileBlock, byte[] nextIndexedKey) {
+ this.hFileBlock = hFileBlock;
+ this.nextIndexedKey = nextIndexedKey;
+ }
+
+ public HFileBlock getHFileBlock() {
+ return hFileBlock;
+ }
+
+ public byte[] getNextIndexedKey() {
+ return nextIndexedKey;
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java Thu May 17 12:50:44 2012
@@ -36,6 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -182,17 +183,57 @@ public class HFileBlockIndex {
int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
boolean pread, boolean isCompaction)
throws IOException {
+ BlockWithScanInfo blockWithScanInfo = loadDataBlockWithScanInfo(key, keyOffset, keyLength,
+ currentBlock, cacheBlocks, pread, isCompaction);
+ if (blockWithScanInfo == null) {
+ return null;
+ } else {
+ return blockWithScanInfo.getHFileBlock();
+ }
+ }
+
+ /**
+ * Return the BlockWithScanInfo which contains the DataBlock with other scan info
+ * such as nextIndexedKey.
+ * This function will only be called when the HFile version is larger than 1.
+ *
+ * @param key the key we are looking for
+ * @param keyOffset the offset of the key in its byte array
+ * @param keyLength the length of the key
+ * @param currentBlock the current block, to avoid re-reading the same
+ * block
+ * @param cacheBlocks
+ * @param pread
+ * @param isCompaction
+ * @return the BlockWithScanInfo which contains the DataBlock with other scan info
+ * such as nextIndexedKey.
+ * @throws IOException
+ */
+ public BlockWithScanInfo loadDataBlockWithScanInfo(final byte[] key, int keyOffset,
+ int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
+ boolean pread, boolean isCompaction)
+ throws IOException {
int rootLevelIndex = rootBlockContainingKey(key, keyOffset, keyLength);
if (rootLevelIndex < 0 || rootLevelIndex >= blockOffsets.length) {
return null;
}
-
+
+ // the next indexed key
+ byte[] nextIndexedKey = null;
+
// Read the next-level (intermediate or leaf) index block.
long currentOffset = blockOffsets[rootLevelIndex];
int currentOnDiskSize = blockDataSizes[rootLevelIndex];
+
+ if (rootLevelIndex < blockKeys.length - 1) {
+ nextIndexedKey = blockKeys[rootLevelIndex + 1];
+ } else {
+ nextIndexedKey = HConstants.NO_NEXT_INDEXED_KEY;
+ }
int lookupLevel = 1; // How many levels deep we are in our lookup.
-
+ int index = -1;
+
HFileBlock block;
while (true) {
@@ -242,8 +283,8 @@ public class HFileBlockIndex {
// Locate the entry corresponding to the given key in the non-root
// (leaf or intermediate-level) index block.
ByteBuffer buffer = block.getBufferWithoutHeader();
- if (!locateNonRootIndexEntry(buffer, key, keyOffset, keyLength,
- comparator)) {
+ index = locateNonRootIndexEntry(buffer, key, keyOffset, keyLength, comparator);
+ if (index == -1) {
throw new IOException("The key "
+ Bytes.toStringBinary(key, keyOffset, keyLength)
+ " is before the" + " first key of the non-root index block "
@@ -252,14 +293,22 @@ public class HFileBlockIndex {
currentOffset = buffer.getLong();
currentOnDiskSize = buffer.getInt();
+
+ // Only update next indexed key if there is a next indexed key in the current level
+ byte[] tmpNextIndexedKey = getNonRootIndexedKey(buffer, index + 1);
+ if (tmpNextIndexedKey != null) {
+ nextIndexedKey = tmpNextIndexedKey;
+ }
}
if (lookupLevel != searchTreeLevel) {
throw new IOException("Reached a data block at level " + lookupLevel +
" but the number of levels is " + searchTreeLevel);
}
-
- return block;
+
+ // set the next indexed key for the current block.
+ BlockWithScanInfo blockWithScanInfo = new BlockWithScanInfo(block, nextIndexedKey);
+ return blockWithScanInfo;
}
/**
@@ -385,6 +434,41 @@ public class HFileBlockIndex {
}
/**
+ * The indexed key at the ith position in the nonRootIndex. The position starts at 0.
+ * @param nonRootIndex
+ * @param i the ith position
+ * @return The indexed key at the ith position in the nonRootIndex.
+ */
+ private byte[] getNonRootIndexedKey(ByteBuffer nonRootIndex, int i) {
+ int numEntries = nonRootIndex.getInt(0);
+ if (i < 0 || i >= numEntries) {
+ return null;
+ }
+
+ // Entries start after the number of entries and the secondary index.
+ // The secondary index takes numEntries + 1 ints.
+ int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+ // Targetkey's offset relative to the end of secondary index
+ int targetKeyRelOffset = nonRootIndex.getInt(
+ Bytes.SIZEOF_INT * (i + 1));
+
+ // The offset of the target key in the blockIndex buffer
+ int targetKeyOffset = entriesOffset // Skip secondary index
+ + targetKeyRelOffset // Skip all entries until mid
+ + SECONDARY_INDEX_ENTRY_OVERHEAD; // Skip offset and on-disk-size
+
+ // We subtract the two consecutive secondary index elements, which
+ // gives us the size of the whole (offset, onDiskSize, key) tuple. We
+ // then need to subtract the overhead of offset and onDiskSize.
+ int targetKeyLength = nonRootIndex.getInt(Bytes.SIZEOF_INT * (i + 2)) -
+ targetKeyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
+
+ int from = nonRootIndex.arrayOffset() + targetKeyOffset;
+ int to = from + targetKeyLength;
+ return Arrays.copyOfRange(nonRootIndex.array(), from, to);
+ }
+
+ /**
* Performs a binary search over a non-root level index block. Utilizes the
* secondary index, which records the offsets of (offset, onDiskSize,
* firstKey) tuples of all entries.
@@ -484,31 +568,30 @@ public class HFileBlockIndex {
* @param key the byte array containing the key
* @param keyOffset the offset of the key in its byte array
* @param keyLength the length of the key
- * @return true in the case the index entry containing the given key was
- * found, false in the case the given key is before the first key
+ * @return the index position where the given key was found,
+ * otherwise return -1 in the case the given key is before the first key.
*
*/
- static boolean locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key,
+ static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key,
int keyOffset, int keyLength, RawComparator<byte[]> comparator) {
int entryIndex = binarySearchNonRootIndex(key, keyOffset, keyLength,
nonRootBlock, comparator);
- if (entryIndex == -1) {
- return false;
+ if (entryIndex != -1) {
+ int numEntries = nonRootBlock.getInt(0);
+
+ // The end of secondary index and the beginning of entries themselves.
+ int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
+
+ // The offset of the entry we are interested in relative to the end of
+ // the secondary index.
+ int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT
+ * (1 + entryIndex));
+
+ nonRootBlock.position(entriesOffset + entryRelOffset);
}
-
- int numEntries = nonRootBlock.getInt(0);
-
- // The end of secondary index and the beginning of entries themselves.
- int entriesOffset = Bytes.SIZEOF_INT * (numEntries + 2);
-
- // The offset of the entry we are interested in relative to the end of
- // the secondary index.
- int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT
- * (1 + entryIndex));
-
- nonRootBlock.position(entriesOffset + entryRelOffset);
- return true;
+
+ return entryIndex;
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Thu May 17 12:50:44 2012
@@ -23,12 +23,14 @@ import java.io.DataInput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
@@ -233,7 +235,7 @@ public class HFileReaderV2 extends Abstr
return metaBlock.getBufferWithoutHeader();
}
}
-
+
/**
* Read in a file block.
* @param dataBlockOffset offset to read.
@@ -406,7 +408,16 @@ public class HFileReaderV2 extends Abstr
protected abstract static class AbstractScannerV2
extends AbstractHFileReader.Scanner {
protected HFileBlock block;
-
+
+ /**
+ * The next indexed key is to keep track of the indexed key of the next data block.
+ * If the nextIndexedKey is HConstants.NO_NEXT_INDEXED_KEY, it means that the
+ * current data block is the last data block.
+ *
+ * If the nextIndexedKey is null, it means the nextIndexedKey has not been loaded yet.
+ */
+ protected byte[] nextIndexedKey;
+
public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
final boolean pread, final boolean isCompaction) {
super(r, cacheBlocks, pread, isCompaction);
@@ -430,19 +441,20 @@ public class HFileReaderV2 extends Abstr
throws IOException {
HFileBlockIndex.BlockIndexReader indexReader =
reader.getDataBlockIndexReader();
- HFileBlock seekToBlock = indexReader.seekToDataBlock(key, offset, length,
- block, cacheBlocks, pread, isCompaction);
- if (seekToBlock == null) {
+ BlockWithScanInfo blockWithScanInfo =
+ indexReader.loadDataBlockWithScanInfo(key, offset, length, block,
+ cacheBlocks, pread, isCompaction);
+ if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) {
// This happens if the key e.g. falls before the beginning of the file.
return -1;
}
- return loadBlockAndSeekToKey(seekToBlock, rewind, key, offset, length,
- false);
+ return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(),
+ blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false);
}
protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
- protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock,
+ protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
throws IOException;
@@ -455,17 +467,28 @@ public class HFileReaderV2 extends Abstr
@Override
public int reseekTo(byte[] key, int offset, int length) throws IOException {
+ int compared;
if (isSeeked()) {
ByteBuffer bb = getKey();
- int compared = reader.getComparator().compare(key, offset,
+ compared = reader.getComparator().compare(key, offset,
length, bb.array(), bb.arrayOffset(), bb.limit());
if (compared < 1) {
// If the required key is less than or equal to current key, then
// don't do anything.
return compared;
+ } else {
+ if (this.nextIndexedKey != null &&
+ (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY ||
+ reader.getComparator().compare(key, offset, length,
+ nextIndexedKey, 0, nextIndexedKey.length) < 0)) {
+ // The reader shall continue to scan the current data block instead of querying the
+ // block index as long as it knows the target key is strictly smaller than
+ // the next indexed key or the current data block is the last data block.
+ return loadBlockAndSeekToKey(this.block, this.nextIndexedKey,
+ false, key, offset, length, false);
+ }
}
}
-
// Don't rewind on a reseek operation, because reseek implies that we are
// always going forward in the file.
return seekTo(key, offset, length, false);
@@ -481,6 +504,7 @@ public class HFileReaderV2 extends Abstr
return false;
}
ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
+
if (reader.getComparator().compare(firstKey.array(),
firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
{
@@ -497,11 +521,11 @@ public class HFileReaderV2 extends Abstr
seekToBlock = reader.readBlock(previousBlockOffset,
seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
pread, isCompaction, BlockType.DATA);
-
// TODO shortcut: seek forward in this block to the last key of the
// block.
}
- loadBlockAndSeekToKey(seekToBlock, true, key, offset, length, true);
+ byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey);
+ loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true);
return true;
}
@@ -677,17 +701,20 @@ public class HFileReaderV2 extends Abstr
}
@Override
- protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
- byte[] key, int offset, int length, boolean seekBefore)
+ protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+ boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
throws IOException {
if (block == null || block.getOffset() != seekToBlock.getOffset()) {
updateCurrBlock(seekToBlock);
} else if (rewind) {
blockBuffer.rewind();
}
+
+ // Update the nextIndexedKey
+ this.nextIndexedKey = nextIndexedKey;
return blockSeek(key, offset, length, seekBefore);
}
-
+
/**
* Updates the current block to be the given {@link HFileBlock}. Seeks to
* the the first key/value pair.
@@ -709,6 +736,9 @@ public class HFileReaderV2 extends Abstr
blockBuffer = block.getBufferWithoutHeader();
readKeyValueLen();
blockFetches++;
+
+ // Reset the next indexed key
+ this.nextIndexedKey = null;
}
private final void readKeyValueLen() {
@@ -994,14 +1024,15 @@ public class HFileReaderV2 extends Abstr
}
@Override
- protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
- byte[] key, int offset, int length, boolean seekBefore)
+ protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey,
+ boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
throws IOException {
if (block == null || block.getOffset() != seekToBlock.getOffset()) {
updateCurrentBlock(seekToBlock);
} else if (rewind) {
seeker.rewind();
}
+ this.nextIndexedKey = nextIndexedKey;
return seeker.seekToKeyInBlock(key, offset, length, seekBefore);
}
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Thu May 17 12:50:44 2012
@@ -30,6 +30,7 @@ import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Delete;
@@ -144,9 +145,15 @@ public abstract class HBaseTestCase exte
return testUtil.getTestDir(testName);
}
- protected HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
+ public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
byte [] endKey)
throws IOException {
+ return createNewHRegion(desc, startKey, endKey, this.conf);
+ }
+
+ public static HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
+ byte [] endKey, Configuration conf)
+ throws IOException {
FileSystem filesystem = FileSystem.get(conf);
Path rootdir = filesystem.makeQualified(
new Path(conf.get(HConstants.HBASE_DIR)));
@@ -198,10 +205,11 @@ public abstract class HBaseTestCase exte
* Adds data of the from 'aaa', 'aab', etc where key and value are the same.
* @param r
* @param columnFamily
+ * @param column
* @throws IOException
* @return count of what we added.
*/
- protected static long addContent(final HRegion r, final byte [] columnFamily)
+ public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
throws IOException {
byte [] startKey = r.getRegionInfo().getStartKey();
byte [] endKey = r.getRegionInfo().getEndKey();
@@ -209,9 +217,23 @@ public abstract class HBaseTestCase exte
if (startKeyBytes == null || startKeyBytes.length == 0) {
startKeyBytes = START_KEY_BYTES;
}
- return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
+ return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
startKeyBytes, endKey, -1);
}
+
+ /**
+ * Add content to region <code>r</code> on the passed column
+ * <code>column</code>.
+ * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
+ * @param r
+ * @param columnFamily
+ * @throws IOException
+ * @return count of what we added.
+ */
+ protected static long addContent(final HRegion r, final byte [] columnFamily)
+ throws IOException {
+ return addContent(r, columnFamily, null);
+ }
/**
* Add content to region <code>r</code> on the passed column
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java Thu May 17 12:50:44 2012
@@ -377,8 +377,8 @@ public class TestHFileBlockIndex {
// Now test we can get the offset and the on-disk-size using a
// higher-level API function.s
boolean locateBlockResult =
- BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey,
- searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR);
+ (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey,
+ searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR) != -1);
if (i == 0) {
assertFalse(locateBlockResult);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java Thu May 17 12:50:44 2012
@@ -24,6 +24,8 @@ import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.List;
+import junit.framework.Assert;
+
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -82,7 +84,7 @@ public class TestReseekTo {
Integer key = keyList.get(i);
String value = valueList.get(i);
scanner.reseekTo(Bytes.toBytes(key));
- assertEquals(value, scanner.getValueString());
+ assertEquals("i is " + i, value, scanner.getValueString());
}
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java?rev=1339581&r1=1339580&r2=1339581&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java Thu May 17 12:50:44 2012
@@ -94,6 +94,89 @@ public class TestSeekTo extends HBaseTes
assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
assertEquals("k", toRowStr(scanner.getKeyValue()));
}
+
+ public void testSeekBeforeWithReSeekTo() throws Exception {
+ Path p = makeNewFile();
+ HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf));
+ reader.loadFileInfo();
+ HFileScanner scanner = reader.getScanner(false, true);
+ assertEquals(false, scanner.seekBefore(toKV("a").getKey()));
+ assertEquals(false, scanner.seekBefore(toKV("b").getKey()));
+ assertEquals(false, scanner.seekBefore(toKV("c").getKey()));
+
+ // seekBefore d, so the scanner points to c
+ assertEquals(true, scanner.seekBefore(toKV("d").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
+ // reseekTo c and g
+ assertEquals(0, scanner.reseekTo(toKV("c").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore e, so the scanner points to c
+ assertEquals(true, scanner.seekBefore(toKV("e").getKey()));
+ assertEquals("c", toRowStr(scanner.getKeyValue()));
+ // reseekTo e and g
+ assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore f, so the scanner points to e
+ assertEquals(true, scanner.seekBefore(toKV("f").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+ // reseekTo e and g
+ assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore g, so the scanner points to e
+ assertEquals(true, scanner.seekBefore(toKV("g").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+ // reseekTo e and g again
+ assertEquals(0, scanner.reseekTo(toKV("e").getKey()));
+ assertEquals("e", toRowStr(scanner.getKeyValue()));
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore h, so the scanner points to g
+ assertEquals(true, scanner.seekBefore(toKV("h").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+ // reseekTo g
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore i, so the scanner points to g
+ assertEquals(true, scanner.seekBefore(toKV("i").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+ // reseekTo g
+ assertEquals(0, scanner.reseekTo(toKV("g").getKey()));
+ assertEquals("g", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore j, so the scanner points to i
+ assertEquals(true, scanner.seekBefore(toKV("j").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+ // reseekTo i
+ assertEquals(0, scanner.reseekTo(toKV("i").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore k, so the scanner points to i
+ assertEquals(true, scanner.seekBefore(toKV("k").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+ // reseekTo i and k
+ assertEquals(0, scanner.reseekTo(toKV("i").getKey()));
+ assertEquals("i", toRowStr(scanner.getKeyValue()));
+ assertEquals(0, scanner.reseekTo(toKV("k").getKey()));
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
+
+ // seekBefore l, so the scanner points to k
+ assertEquals(true, scanner.seekBefore(toKV("l").getKey()));
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
+ // reseekTo k
+ assertEquals(0, scanner.reseekTo(toKV("k").getKey()));
+ assertEquals("k", toRowStr(scanner.getKeyValue()));
+ }
public void testSeekTo() throws Exception {
Path p = makeNewFile();
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java?rev=1339581&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java Thu May 17 12:50:44 2012
@@ -0,0 +1,116 @@
+/**
+ * Copyright 2012 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+@SuppressWarnings("deprecation")
+public class TestBlocksScanned {
+ private static byte [] TABLE = Bytes.toBytes("TestBlocksScanned");
+ private static byte [] FAMILY = Bytes.toBytes("family");
+ private static byte [] COL = Bytes.toBytes("col");
+ private static byte [] START_KEY = Bytes.toBytes("aaa");
+ private static byte [] END_KEY = Bytes.toBytes("zzz");
+ private static int BLOCK_SIZE = 70;
+
+ private static HBaseTestingUtility TEST_UTIL = null;
+ private static HTableDescriptor TESTTABLEDESC = null;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ SchemaMetrics.setUseTableNameInTest(true);
+ TEST_UTIL = new HBaseTestingUtility();
+ TESTTABLEDESC = new HTableDescriptor(TABLE);
+
+ TESTTABLEDESC.addFamily(
+ new HColumnDescriptor(FAMILY)
+ .setMaxVersions(10)
+ .setBlockCacheEnabled(true)
+ .setBlocksize(BLOCK_SIZE)
+ .setCompressionType(Compression.Algorithm.NONE)
+ );
+ }
+
+ @Test
+ public void testBlocksScanned() throws Exception {
+ HRegion r = HBaseTestCase.createNewHRegion(TESTTABLEDESC, START_KEY, END_KEY,
+ TEST_UTIL.getConfiguration());
+ HBaseTestCase.addContent(r, FAMILY, COL);
+ r.flushcache();
+
+ // Get the per-cf metrics
+ SchemaMetrics schemaMetrics =
+ SchemaMetrics.getInstance(Bytes.toString(TABLE), Bytes.toString(FAMILY));
+ Map<String, Long> schemaMetricSnapshot = SchemaMetrics.getMetricsSnapshot();
+
+ // Do simple test of getting one row only first.
+ Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz"));
+ scan.addColumn(FAMILY, COL);
+ scan.setMaxVersions(1);
+
+ InternalScanner s = r.getScanner(scan);
+ List<KeyValue> results = new ArrayList<KeyValue>();
+ while (s.next(results));
+ s.close();
+
+ int expectResultSize = 'z' - 'a';
+ Assert.assertEquals(expectResultSize, results.size());
+
+ int kvPerBlock = (int) Math.ceil(BLOCK_SIZE / (double) results.get(0).getLength());
+ Assert.assertEquals(2, kvPerBlock);
+
+ long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
+ long expectIndexBlockRead = expectDataBlockRead;
+
+ verifyDataAndIndexBlockRead(schemaMetricSnapshot, schemaMetrics,
+ expectDataBlockRead, expectIndexBlockRead);
+ }
+
+ private void verifyDataAndIndexBlockRead(Map<String, Long> previousMetricSnapshot,
+ SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){
+ Map<String, Long> currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot();
+ Map<String, Long> diffs =
+ SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot);
+
+ long dataBlockRead = SchemaMetrics.getLong(diffs,
+ schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT));
+ long indexBlockRead = SchemaMetrics.getLong(diffs,
+ schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT));
+
+ Assert.assertEquals(expectDataBlockRead, dataBlockRead);
+ Assert.assertEquals(expectedIndexBlockRead, indexBlockRead);
+ }
+}
\ No newline at end of file