You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/08/02 01:03:17 UTC
[1/5] hbase git commit: HBASE-16296 Reverse scan performance degrades
when using filter lists (Ted Yu and Lars Hofhansl)
Repository: hbase
Updated Branches:
refs/heads/0.98 903e7d5f7 -> e3e7525e7
refs/heads/branch-1.1 6ff6a62a0 -> dd57e9439
refs/heads/branch-1.2 09d1a278c -> ead083f3b
refs/heads/branch-1.3 6dfaed98f -> b0b4592f1
HBASE-16296 Reverse scan performance degrades when using filter lists (Ted Yu and Lars Hofhansl)
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dd57e943
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dd57e943
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dd57e943
Branch: refs/heads/branch-1.1
Commit: dd57e943924c958e6fdf90a6e0da8789eee1e483
Parents: 6ff6a62
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Aug 1 17:31:40 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 17:41:07 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dd57e943/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 051123e..f033177 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5696,7 +5696,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Check if rowkey filter wants to exclude this row. If so, loop to next.
// Technically, if we hit limits before on this row, we don't need this call.
if (filterRowKey(currentRow, offset, length)) {
- boolean moreRows = nextRow(currentRow, offset, length);
+ boolean moreRows = !isFilterDoneInternal() && nextRow(currentRow, offset, length);
if (!moreRows) {
return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
}
[5/5] hbase git commit: HBASE-16296 Reverse scan performance degrades
when using filter lists (Ted Yu and Lars Hofhansl)
Posted by ap...@apache.org.
HBASE-16296 Reverse scan performance degrades when using filter lists (Ted Yu and Lars Hofhansl)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e3e7525e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e3e7525e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e3e7525e
Branch: refs/heads/0.98
Commit: e3e7525e716fa67a13ed062de488062edca7e4e4
Parents: 7f139a6
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Aug 1 18:01:39 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 18:01:39 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e3e7525e/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index bd2b7e3..ec84ba9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -4333,7 +4333,7 @@ public class HRegion implements HeapSize { // , Writable{
// Check if rowkey filter wants to exclude this row. If so, loop to next.
// Technically, if we hit limits before on this row, we don't need this call.
if (filterRowKey(currentRow, offset, length)) {
- boolean moreRows = nextRow(currentRow, offset, length);
+ boolean moreRows = !isFilterDoneInternal() && nextRow(currentRow, offset, length);
if (!moreRows) return false;
results.clear();
continue;
[4/5] hbase git commit: HBASE-16288 HFile intermediate block level
indexes might recurse forever creating multi TB files
Posted by ap...@apache.org.
HBASE-16288 HFile intermediate block level indexes might recurse forever creating multi TB files
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
Amending-Author: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7f139a64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7f139a64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7f139a64
Branch: refs/heads/0.98
Commit: 7f139a64398e91a64b8af9525334c3ddc22f7841
Parents: 903e7d5
Author: Enis Soztutar <en...@apache.org>
Authored: Mon Aug 1 10:57:09 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 18:01:28 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 39 +++++++++++++++--
.../hadoop/hbase/io/hfile/HFileWriterV2.java | 2 +
.../hbase/io/hfile/TestHFileBlockIndex.java | 44 ++++++++++++++++++++
3 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f139a64/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index 09660cf..bae16c0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -71,6 +71,16 @@ public class HFileBlockIndex {
public static final String MAX_CHUNK_SIZE_KEY = "hfile.index.block.max.size";
/**
+ * Minimum number of entries in a single index block. Even if we are above the
+ * hfile.index.block.max.size we will keep writing to the same block unless we have that many
+ * entries. We should have at least a few entries so that we don't have too many levels in the
+ * multi-level index. This should be at least 2 to make sure there is no infinite recursion.
+ */
+ public static final String MIN_INDEX_NUM_ENTRIES_KEY = "hfile.index.block.min.entries";
+
+ static final int DEFAULT_MIN_INDEX_NUM_ENTRIES = 16;
+
+ /**
* The number of bytes stored in each "secondary index" entry in addition to
* key bytes in the non-root index block format. The first long is the file
* offset of the deeper-level block the entry points to, and the int that
@@ -763,6 +773,9 @@ public class HFileBlockIndex {
/** The maximum size guideline of all multi-level index blocks. */
private int maxChunkSize;
+ /** The maximum level of multi-level index blocks */
+ private int minIndexNumEntries;
+
/** Whether we require this block index to always be single-level. */
private boolean singleLevelOnly;
@@ -795,15 +808,23 @@ public class HFileBlockIndex {
this.cacheConf = cacheConf;
this.nameForCaching = nameForCaching;
this.maxChunkSize = HFileBlockIndex.DEFAULT_MAX_CHUNK_SIZE;
+ this.minIndexNumEntries = HFileBlockIndex.DEFAULT_MIN_INDEX_NUM_ENTRIES;
}
public void setMaxChunkSize(int maxChunkSize) {
if (maxChunkSize <= 0) {
- throw new IllegalArgumentException("Invald maximum index block size");
+ throw new IllegalArgumentException("Invalid maximum index block size");
}
this.maxChunkSize = maxChunkSize;
}
+ public void setMinIndexNumEntries(int minIndexNumEntries) {
+ if (minIndexNumEntries <= 1) {
+ throw new IllegalArgumentException("Invalid maximum index level, should be >= 2");
+ }
+ this.minIndexNumEntries = minIndexNumEntries;
+ }
+
/**
* Writes the root level and intermediate levels of the block index into
* the output stream, generating the tree from bottom up. Assumes that the
@@ -835,7 +856,11 @@ public class HFileBlockIndex {
: null;
if (curInlineChunk != null) {
- while (rootChunk.getRootSize() > maxChunkSize) {
+ while (rootChunk.getRootSize() > maxChunkSize
+ // HBASE-16288: if firstKey is larger than maxChunkSize we will loop indefinitely
+ && rootChunk.getNumEntries() > minIndexNumEntries
+ // Sanity check. We will not hit this (minIndexNumEntries ^ 16) blocks can be addressed
+ && numLevels < 16) {
rootChunk = writeIntermediateLevel(out, rootChunk);
numLevels += 1;
}
@@ -926,8 +951,12 @@ public class HFileBlockIndex {
curChunk.add(currentLevel.getBlockKey(i),
currentLevel.getBlockOffset(i), currentLevel.getOnDiskDataSize(i));
- if (curChunk.getRootSize() >= maxChunkSize)
+ // HBASE-16288: We have to have at least minIndexNumEntries(16) items in the index so that
+ // we won't end up with too-many levels for a index with very large rowKeys. Also, if the
+ // first key is larger than maxChunkSize this will cause infinite recursion.
+ if (i >= minIndexNumEntries && curChunk.getRootSize() >= maxChunkSize) {
writeIntermediateBlock(out, parent, curChunk);
+ }
}
if (curChunk.getNumEntries() > 0) {
@@ -1422,4 +1451,8 @@ public class HFileBlockIndex {
public static int getMaxChunkSize(Configuration conf) {
return conf.getInt(MAX_CHUNK_SIZE_KEY, DEFAULT_MAX_CHUNK_SIZE);
}
+
+ public static int getMinIndexNumEntries(Configuration conf) {
+ return conf.getInt(MIN_INDEX_NUM_ENTRIES_KEY, DEFAULT_MIN_INDEX_NUM_ENTRIES);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f139a64/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index 9c1c942..9a99db7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -123,6 +123,8 @@ public class HFileWriterV2 extends AbstractHFileWriter {
cacheIndexesOnWrite ? name : null);
dataBlockIndexWriter.setMaxChunkSize(
HFileBlockIndex.getMaxChunkSize(conf));
+ dataBlockIndexWriter.setMinIndexNumEntries(
+ HFileBlockIndex.getMinIndexNumEntries(conf));
inlineBlockWriters.add(dataBlockIndexWriter);
// Meta data block index writer
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f139a64/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index ef2917e..ec0ed05 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.compress.Compression;
@@ -647,6 +648,49 @@ public class TestHFileBlockIndex {
valueRead);
}
+ @Test(timeout=10000)
+ public void testIntermediateLevelIndicesWithLargeKeys() throws IOException {
+ testIntermediateLevelIndicesWithLargeKeys(16);
+ }
+
+ @Test(timeout=10000)
+ public void testIntermediateLevelIndicesWithLargeKeysWithMinNumEntries() throws IOException {
+ // because of the large rowKeys, we will end up with a 50-level block index without sanity check
+ testIntermediateLevelIndicesWithLargeKeys(2);
+ }
+
+ public void testIntermediateLevelIndicesWithLargeKeys(int minNumEntries) throws IOException {
+ Path hfPath = new Path(TEST_UTIL.getDataTestDir(),
+ "testIntermediateLevelIndicesWithLargeKeys.hfile");
+ int maxChunkSize = 1024;
+ FileSystem fs = FileSystem.get(conf);
+ CacheConfig cacheConf = new CacheConfig(conf);
+ conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize);
+ conf.setInt(HFileBlockIndex.MIN_INDEX_NUM_ENTRIES_KEY, minNumEntries);
+ HFileContext context = new HFileContextBuilder().withBlockSize(16).build();
+ HFileWriterV2 hfw =
+ (HFileWriterV2) new HFileWriterV2.WriterFactoryV2(conf, cacheConf)
+ .withFileContext(context)
+ .withPath(fs, hfPath).create();
+ List<byte[]> keys = new ArrayList<byte[]>();
+
+ // This should result in leaf-level indices and a root level index
+ for (int i=0; i < 100; i++) {
+ byte[] rowkey = new byte[maxChunkSize + 1];
+ byte[] b = Bytes.toBytes(i);
+ System.arraycopy(b, 0, rowkey, rowkey.length - b.length, b.length);
+ keys.add(rowkey);
+ hfw.append(KeyValueUtil.createFirstOnRow(rowkey));
+ }
+ hfw.close();
+ HFile.Reader reader = HFile.createReader(fs, hfPath, cacheConf, conf);
+ // Scanner doesn't do Cells yet. Fix.
+ HFileScanner scanner = reader.getScanner(true, true);
+ for (int i = 0; i < keys.size(); ++i) {
+ scanner.seekTo(keys.get(i));
+ }
+ reader.close();
+ }
}
[3/5] hbase git commit: HBASE-16296 Reverse scan performance degrades
when using filter lists (Ted Yu)
Posted by ap...@apache.org.
HBASE-16296 Reverse scan performance degrades when using filter lists (Ted Yu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0b4592f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0b4592f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0b4592f
Branch: refs/heads/branch-1.3
Commit: b0b4592f1b4eaa87a94d4af09ae7bd3681a39ad1
Parents: 6dfaed9
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Aug 1 11:13:47 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 17:41:09 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b0b4592f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a3610d1..91917de 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -6022,6 +6022,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Technically, if we hit limits before on this row, we don't need this call.
if (filterRowKey(currentRow, offset, length)) {
incrementCountOfRowsFilteredMetric(scannerContext);
+ // early check, see HBASE-16296
+ if (isFilterDoneInternal()) {
+ return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
+ }
// Typically the count of rows scanned is incremented inside #populateResult. However,
// here we are filtering a row based purely on its row key, preventing us from calling
// #populateResult. Thus, perform the necessary increment here to rows scanned metric
[2/5] hbase git commit: HBASE-16296 Reverse scan performance degrades
when using filter lists (Ted Yu)
Posted by ap...@apache.org.
HBASE-16296 Reverse scan performance degrades when using filter lists (Ted Yu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ead083f3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ead083f3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ead083f3
Branch: refs/heads/branch-1.2
Commit: ead083f3b6f75b31bfead11bf24cfd2c896dcd32
Parents: 09d1a27
Author: Andrew Purtell <ap...@apache.org>
Authored: Mon Aug 1 11:13:47 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Mon Aug 1 17:41:08 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hbase/regionserver/HRegion.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ead083f3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 37a4be1..73fbf78 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -5859,6 +5859,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Technically, if we hit limits before on this row, we don't need this call.
if (filterRowKey(currentRow, offset, length)) {
incrementCountOfRowsFilteredMetric(scannerContext);
+ // early check, see HBASE-16296
+ if (isFilterDoneInternal()) {
+ return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues();
+ }
// Typically the count of rows scanned is incremented inside #populateResult. However,
// here we are filtering a row based purely on its row key, preventing us from calling
// #populateResult. Thus, perform the necessary increment here to rows scanned metric