svn commit: r1181962 - in /hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase: io/hfile/LruBlockCache.java regionserver/StoreFile.java

[ns...@apache.org]

Author: nspiegelberg
Date: Tue Oct 11 17:44:28 2011
New Revision: 1181962

URL: http://svn.apache.org/viewvc?rev=1181962&view=rev
Log:
Making block cache bucket sizes configurable in hbase-89

Summary: Porting part of D191289 from hbase-90 to hbase-89 to make block
cache's per-priority bucket sizes (single-accessed blocks, multiply-accessed
blocks, and in-memory blocks) configurable. Also replacing exact floating point
comparison with an approximate one when checking that bucket sizes sum up to
one.

Test Plan: Unit tests, dark launch

Reviewers: kannan, jgray, kranganathan

Reviewed By: kannan

CC: , hbase@lists, kannan

Differential Revision: 315950

Revert Plan: OK

Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1181962&r1=1181961&r2=1181962&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Tue Oct 11 17:44:28 2011
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockInfo;
 import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -174,6 +175,32 @@ public class LruBlockCache implements Bl
   }
 
   /**
+   * Constructor that takes parameters from configuration.
+   * @param maxSize
+   * @param blockSize
+   * @param conf
+   */
+  public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
+    this(maxSize, blockSize,
+        conf.getBoolean("hbase.rs.blockcache.lru.evictionthread", true),
+        (int)Math.ceil(1.2*maxSize/blockSize),
+        conf.getFloat("hbase.rs.blockcache.lru.map.factor",
+            DEFAULT_LOAD_FACTOR),
+        conf.getInt("hbase.rs.blockcache.lru.map.concurrency",
+            DEFAULT_CONCURRENCY_LEVEL),
+        conf.getFloat("hbase.rs.blockcache.lru.watermark.low",
+            DEFAULT_MIN_FACTOR),
+        conf.getFloat("hbase.rs.blockcache.lru.watermark.high",
+            DEFAULT_ACCEPTABLE_FACTOR),
+        conf.getFloat("hbase.rs.blockcache.lru.bucket.single",
+            DEFAULT_SINGLE_FACTOR),
+        conf.getFloat("hbase.rs.blockcache.lru.bucket.multi",
+            DEFAULT_MULTI_FACTOR),
+        conf.getFloat("hbase.rs.blockcache.lru.bucket.inmemory",
+            DEFAULT_MEMORY_FACTOR));
+  }
+
+  /**
    * Configurable constructor.  Use this constructor if not using defaults.
    * @param maxSize maximum size of this cache, in bytes
    * @param blockSize expected average size of blocks, in bytes
@@ -191,9 +218,10 @@ public class LruBlockCache implements Bl
       int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
       float minFactor, float acceptableFactor,
       float singleFactor, float multiFactor, float memoryFactor) {
-    if(singleFactor + multiFactor + memoryFactor != 1) {
-      throw new IllegalArgumentException("Single, multi, and memory factors " +
-          " should total 1.0");
+    if (Math.abs(singleFactor + multiFactor + memoryFactor - 1.0) > 1e-8) {
+      throw new IllegalArgumentException("Single, multi, and memory factors "
+          + " should total 1.0: singleFactor=" + singleFactor
+          + ", multiFactor=" + multiFactor + ", memoryFactor=" + memoryFactor);
     }
     if(minFactor >= acceptableFactor) {
       throw new IllegalArgumentException("minFactor must be smaller than acceptableFactor");

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1181962&r1=1181961&r2=1181962&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Oct 11 17:44:28 2011
@@ -382,7 +382,8 @@ public class StoreFile {
     long cacheSize = (long)(mu.getMax() * cachePercentage);
     LOG.info("Allocating LruBlockCache with maximum size " +
       StringUtils.humanReadableInt(cacheSize));
-    hfileBlockCache = new LruBlockCache(cacheSize, DEFAULT_BLOCKSIZE_SMALL);
+    hfileBlockCache = new LruBlockCache(cacheSize, DEFAULT_BLOCKSIZE_SMALL,
+        conf);
     return hfileBlockCache;
   }