You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/10/26 06:08:51 UTC
svn commit: r1402385 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/io/hfile/
Author: larsh
Date: Fri Oct 26 04:08:50 2012
New Revision: 1402385
URL: http://svn.apache.org/viewvc?rev=1402385&view=rev
Log:
HBASE-7053 port blockcache configurability (part of HBASE-6312, and HBASE-7033) to 0.94 (Sergey Shelukhin)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java?rev=1402385&r1=1402384&r2=1402385&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java Fri Oct 26 04:08:50 2012
@@ -349,7 +349,7 @@ public class CacheConfig {
StringUtils.humanReadableInt(cacheSize));
if (offHeapCacheSize <= 0) {
globalBlockCache = new LruBlockCache(cacheSize,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL);
+ StoreFile.DEFAULT_BLOCKSIZE_SMALL, conf);
} else {
globalBlockCache = new DoubleBlockCache(cacheSize, offHeapCacheSize,
StoreFile.DEFAULT_BLOCKSIZE_SMALL, blockSize, conf);
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java?rev=1402385&r1=1402384&r2=1402385&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java Fri Oct 26 04:08:50 2012
@@ -65,7 +65,7 @@ public class DoubleBlockCache implements
+ StringUtils.humanReadableInt(onHeapSize)
+ "bytes with an average block size of "
+ StringUtils.humanReadableInt(onHeapBlockSize) + " bytes.");
- onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize);
+ onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize, conf);
LOG.info("Creating off-heap cache of size "
+ StringUtils.humanReadableInt(offHeapSize)
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1402385&r1=1402384&r2=1402385&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Fri Oct 26 04:08:50 2012
@@ -96,6 +96,9 @@ public class LruBlockCache implements Bl
static final Log LOG = LogFactory.getLog(LruBlockCache.class);
+ static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
+ static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
+
/** Default Configuration Parameters*/
/** Backing Concurrent Map Configuration */
@@ -178,23 +181,28 @@ public class LruBlockCache implements Bl
* this class.
* @param maxSize maximum size of cache, in bytes
* @param blockSize approximate size of each block, in bytes
+ * @param conf configuration
*/
- public LruBlockCache(long maxSize, long blockSize) {
- this(maxSize, blockSize, true);
+ public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
+ this(maxSize, blockSize, true, conf);
}
/**
* Constructor used for testing. Allows disabling of the eviction thread.
*/
- public LruBlockCache(long maxSize, long blockSize, boolean evictionThread) {
+ public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
this(maxSize, blockSize, evictionThread,
(int)Math.ceil(1.2*maxSize/blockSize),
- DEFAULT_LOAD_FACTOR, DEFAULT_CONCURRENCY_LEVEL,
- DEFAULT_MIN_FACTOR, DEFAULT_ACCEPTABLE_FACTOR,
- DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR,
+ DEFAULT_LOAD_FACTOR,
+ DEFAULT_CONCURRENCY_LEVEL,
+ conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
+ conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
+ DEFAULT_SINGLE_FACTOR,
+ DEFAULT_MULTI_FACTOR,
DEFAULT_MEMORY_FACTOR);
}
+
/**
* Configurable constructor. Use this constructor if not using defaults.
* @param maxSize maximum size of this cache, in bytes
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1402385&r1=1402384&r2=1402385&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Fri Oct 26 04:08:50 2012
@@ -93,7 +93,7 @@ public class TestHFileDataBlockEncoder {
public void testEncodingWithCache() {
HFileBlock block = getSampleHFileBlock();
LruBlockCache blockCache =
- new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
+ new LruBlockCache(8 * 1024 * 1024, 32 * 1024, TEST_UTIL.getConfiguration());
HFileBlock cacheBlock = blockEncoder.diskToCacheFormat(block, false);
BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
blockCache.cacheBlock(cacheKey, cacheBlock);
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java?rev=1402385&r1=1402384&r2=1402385&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java Fri Oct 26 04:08:50 2012
@@ -24,6 +24,8 @@ import java.util.Collection;
import java.util.Map;
import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
@@ -51,6 +53,8 @@ import static org.junit.Assert.*;
public class TestLruBlockCache {
private Map<String, Long> startingMetrics;
+ private final HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
public TestLruBlockCache(boolean useTableName) {
SchemaMetrics.setUseTableNameInTest(useTableName);
@@ -77,7 +81,7 @@ public class TestLruBlockCache {
long maxSize = 100000;
long blockSize = calculateBlockSizeDefault(maxSize, 9); // room for 9, will evict
- LruBlockCache cache = new LruBlockCache(maxSize,blockSize);
+ LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration());
CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
@@ -104,7 +108,7 @@ public class TestLruBlockCache {
long maxSize = 1000000;
long blockSize = calculateBlockSizeDefault(maxSize, 101);
- LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
+ LruBlockCache cache = new LruBlockCache(maxSize, blockSize, TEST_UTIL.getConfiguration());
CachedItem [] blocks = generateRandomBlocks(100, blockSize);
@@ -164,7 +168,7 @@ public class TestLruBlockCache {
long maxSize = 100000;
long blockSize = calculateBlockSizeDefault(maxSize, 10);
- LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
+ LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, TEST_UTIL.getConfiguration());
CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
@@ -205,7 +209,7 @@ public class TestLruBlockCache {
long maxSize = 100000;
long blockSize = calculateBlockSizeDefault(maxSize, 10);
- LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
+ LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, TEST_UTIL.getConfiguration());
CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");