You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2011/12/15 00:02:42 UTC
svn commit: r1214532 - in /hbase/branches/0.92: ./
src/main/java/org/apache/hadoop/hbase/io/hfile/
Author: jdcryans
Date: Wed Dec 14 23:02:42 2011
New Revision: 1214532
URL: http://svn.apache.org/viewvc?rev=1214532&view=rev
Log:
HBASE-4683 Always cache index and bloom blocks
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1214532&r1=1214531&r2=1214532&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Wed Dec 14 23:02:42 2011
@@ -789,6 +789,7 @@ Release 0.92.0 - Unreleased
HBASE-4903 Return a result from RegionObserver.preIncrement
(Daniel Gómez Ferro via Lars H)
HBASE-4944 Optionally verify bulk loaded HFiles
+ HBASE-4683 Always cache index and bloom blocks
TASKS
HBASE-3559 Move report of split to master OFF the heartbeat channel
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java?rev=1214532&r1=1214531&r2=1214532&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java Wed Dec 14 23:02:42 2011
@@ -37,49 +37,55 @@ public enum BlockType {
// Scanned block section
/** Data block, both versions */
- DATA("DATABLK*"),
+ DATA("DATABLK*", BlockCategory.DATA),
/** Version 2 leaf index block. Appears in the data block section */
- LEAF_INDEX("IDXLEAF2"),
+ LEAF_INDEX("IDXLEAF2", BlockCategory.INDEX),
/** Bloom filter block, version 2 */
- BLOOM_CHUNK("BLMFBLK2"),
+ BLOOM_CHUNK("BLMFBLK2", BlockCategory.BLOOM),
// Non-scanned block section
/** Meta blocks */
- META("METABLKc"),
+ META("METABLKc", BlockCategory.META),
/** Intermediate-level version 2 index in the non-data block section */
- INTERMEDIATE_INDEX("IDXINTE2"),
+ INTERMEDIATE_INDEX("IDXINTE2", BlockCategory.INDEX),
// Load-on-open section.
/** Root index block, also used for the single-level meta index, version 2 */
- ROOT_INDEX("IDXROOT2"),
+ ROOT_INDEX("IDXROOT2", BlockCategory.INDEX),
/** File info, version 2 */
- FILE_INFO("FILEINF2"),
+ FILE_INFO("FILEINF2", BlockCategory.META),
/** Bloom filter metadata, version 2 */
- BLOOM_META("BLMFMET2"),
+ BLOOM_META("BLMFMET2", BlockCategory.BLOOM),
// Trailer
/** Fixed file trailer, both versions (always just a magic string) */
- TRAILER("TRABLK\"$"),
+ TRAILER("TRABLK\"$", BlockCategory.META),
// Legacy blocks
/** Block index magic string in version 1 */
- INDEX_V1("IDXBLK)+");
+ INDEX_V1("IDXBLK)+", BlockCategory.INDEX);
+
+ public enum BlockCategory {
+ DATA, META, INDEX, BLOOM
+ }
public static final int MAGIC_LENGTH = 8;
private final byte[] magic;
+ private final BlockCategory metricCat;
- private BlockType(String magicStr) {
+ private BlockType(String magicStr, BlockCategory metricCat) {
magic = Bytes.toBytes(magicStr);
+ this.metricCat = metricCat;
assert magic.length == MAGIC_LENGTH;
}
@@ -95,6 +101,14 @@ public enum BlockType {
buf.put(magic);
}
+ public String getMetricName(){
+ return metricCat.toString();
+ }
+
+ public BlockCategory getCategory(){
+ return metricCat;
+ }
+
public static BlockType parse(byte[] buf, int offset, int length)
throws IOException {
if (length != MAGIC_LENGTH) {
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java?rev=1214532&r1=1214531&r2=1214532&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java Wed Dec 14 23:02:42 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.DirectMemoryUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
/**
* Stores all of the cache objects and configuration for a single HFile.
@@ -208,6 +209,19 @@ public class CacheConfig {
}
/**
+ * Should we cache a block of a particular category? We always cache
+ * important blocks such as index blocks, as long as the block cache is
+ * available.
+ */
+ public boolean shouldCacheBlockOnRead(BlockCategory category) {
+ boolean shouldCache = isBlockCacheEnabled()
+ && (cacheDataOnRead ||
+ category == BlockCategory.INDEX ||
+ category == BlockCategory.BLOOM);
+ return shouldCache;
+ }
+
+ /**
* @return true if blocks in this file should be flagged as in-memory
*/
public boolean isInMemory() {
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1214532&r1=1214531&r2=1214532&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java Wed Dec 14 23:02:42 2011
@@ -30,6 +30,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
@@ -212,6 +213,12 @@ public class HFileReaderV1 extends Abstr
String cacheKey = HFile.getBlockCacheKey(name, offset);
+ BlockCategory effectiveCategory = BlockCategory.META;
+ if (metaBlockName.equals(HFileWriterV1.BLOOM_FILTER_META_KEY) ||
+ metaBlockName.equals(HFileWriterV1.BLOOM_FILTER_DATA_KEY)) {
+ effectiveCategory = BlockCategory.BLOOM;
+ }
+
// Per meta key from any given file, synchronize reads for said block
synchronized (metaBlockIndexReader.getRootBlockKey(block)) {
metaLoads.incrementAndGet();
@@ -219,7 +226,7 @@ public class HFileReaderV1 extends Abstr
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
(HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey,
- cacheConf.shouldCacheDataOnRead());
+ cacheConf.shouldCacheBlockOnRead(effectiveCategory));
if (cachedBlock != null) {
cacheHits.incrementAndGet();
return cachedBlock.getBufferWithoutHeader();
@@ -236,7 +243,7 @@ public class HFileReaderV1 extends Abstr
HFile.readOps.incrementAndGet();
// Cache the block
- if (cacheConf.shouldCacheDataOnRead() && cacheBlock) {
+ if (cacheBlock && cacheConf.shouldCacheBlockOnRead(effectiveCategory)) {
cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
cacheConf.isInMemory());
}
@@ -310,7 +317,8 @@ public class HFileReaderV1 extends Abstr
HFile.readOps.incrementAndGet();
// Cache the block
- if (cacheConf.shouldCacheDataOnRead() && cacheBlock) {
+ if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+ hfileBlock.getBlockType().getCategory())) {
cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
cacheConf.isInMemory());
}
Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1214532&r1=1214531&r2=1214532&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Wed Dec 14 23:02:42 2011
@@ -19,9 +19,7 @@
*/
package org.apache.hadoop.hbase.io.hfile;
-import java.io.ByteArrayInputStream;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -32,6 +30,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.IdLock;
@@ -229,7 +228,7 @@ public class HFileReaderV2 extends Abstr
*/
@Override
public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
- boolean cacheBlock, boolean pread, final boolean isCompaction)
+ final boolean cacheBlock, boolean pread, final boolean isCompaction)
throws IOException {
if (dataBlockIndexReader == null) {
throw new IOException("Block index not loaded");
@@ -251,8 +250,6 @@ public class HFileReaderV2 extends Abstr
try {
blockLoads.incrementAndGet();
- // Check cache for block. If found return.
- cacheBlock &= cacheConf.shouldCacheDataOnRead();
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
(HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
@@ -266,19 +263,21 @@ public class HFileReaderV2 extends Abstr
// Load block from filesystem.
long startTimeNs = System.nanoTime();
- HFileBlock dataBlock = fsBlockReader.readBlockData(dataBlockOffset,
+ HFileBlock hfileBlock = fsBlockReader.readBlockData(dataBlockOffset,
onDiskBlockSize, -1, pread);
+ BlockCategory blockCategory = hfileBlock.getBlockType().getCategory();
HFile.readTimeNano.addAndGet(System.nanoTime() - startTimeNs);
HFile.readOps.incrementAndGet();
// Cache the block
- if (cacheBlock) {
- cacheConf.getBlockCache().cacheBlock(cacheKey, dataBlock,
+ if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
+ hfileBlock.getBlockType().getCategory())) {
+ cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
cacheConf.isInMemory());
}
- return dataBlock;
+ return hfileBlock;
} finally {
offsetLock.releaseLockEntry(lockEntry);
}