You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jd...@apache.org on 2011/12/13 22:55:37 UTC
svn commit: r1213950 - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/io/hfile/
src/main/java/org/apache/hadoop/hbase/master/
src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/resources/
Author: jdcryans
Date: Tue Dec 13 21:55:36 2011
New Revision: 1213950
URL: http://svn.apache.org/viewvc?rev=1213950&view=rev
Log:
HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/src/main/resources/hbase-default.xml
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue Dec 13 21:55:36 2011
@@ -30,6 +30,7 @@ Release 0.92.0 - Unreleased
HBASE-4648 Bytes.toBigDecimal() doesn't use offset (Bryan Keller via Lars H)
HBASE-4715 Remove stale broke .rb scripts from bin dir
HBASE-3433 Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (Lars H)
+ HBASE-5017 Bump the default hfile.block.cache.size because of HFileV2
BUG FIXES
HBASE-3280 YouAreDeadException being swallowed in HRS getMaster
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java Tue Dec 13 21:55:36 2011
@@ -74,7 +74,9 @@ public class HBaseConfiguration extends
private static void checkForClusterFreeMemoryLimit(Configuration conf) {
float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f);
int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE);
- float blockCacheUpperLimit = conf.getFloat("hfile.block.cache.size", 0.2f);
+ float blockCacheUpperLimit =
+ conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+ HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul)
< (int)(CONVERT_TO_PERCENTAGE *
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Dec 13 21:55:36 2011
@@ -540,6 +540,14 @@ public final class HConstants {
public static final String HBASE_REGION_SPLIT_POLICY_KEY =
"hbase.regionserver.region.split.policy";
+ /**
+ * Configuration key for the size of the block cache
+ */
+ public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
+ "hfile.block.cache.size";
+
+ public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.25f;
+
/*
* Minimum percentage of free heap necessary for a successful cluster startup.
*/
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java Tue Dec 13 21:55:36 2011
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.DirectMemoryUtils;
import org.apache.hadoop.util.StringUtils;
@@ -35,12 +36,6 @@ public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
- * Configuration key for the size of the block cache, in bytes.
- */
- public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
- "hfile.block.cache.size";
-
- /**
* Configuration key to cache data blocks on write. There are separate
* switches for bloom blocks and non-root index blocks.
*/
@@ -312,13 +307,14 @@ public class CacheConfig {
if (globalBlockCache != null) return globalBlockCache;
if (blockCacheDisabled) return null;
- float cachePercentage = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, 0.2f);
+ float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+ HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
if (cachePercentage == 0L) {
blockCacheDisabled = true;
return null;
}
if (cachePercentage > 1.0) {
- throw new IllegalArgumentException(HFILE_BLOCK_CACHE_SIZE_KEY +
+ throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
" must be between 0.0 and 1.0, not > 1.0");
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Dec 13 21:55:36 2011
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KeyComparator;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
@@ -278,7 +279,7 @@ public class HFile {
public static final WriterFactory getWriterFactoryNoCache(Configuration
conf) {
Configuration tempConf = new Configuration(conf);
- tempConf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+ tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
return HFile.getWriterFactory(conf, new CacheConfig(tempConf));
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Tue Dec 13 21:55:36 2011
@@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.coprocess
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.HBaseRPC;
import org.apache.hadoop.hbase.ipc.HBaseServer;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
@@ -217,7 +216,7 @@ implements HMasterInterface, HMasterRegi
throws IOException, KeeperException, InterruptedException {
this.conf = new Configuration(conf);
// Disable the block cache on the master
- this.conf.setFloat(CacheConfig.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
+ this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
// Set how many times to retry talking to another server over HConnection.
HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
// Server to handle client requests.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Tue Dec 13 21:55:36 2011
@@ -85,8 +85,6 @@ import com.google.common.collect.Orderin
public class StoreFile {
static final Log LOG = LogFactory.getLog(StoreFile.class.getName());
- static final String HFILE_BLOCK_CACHE_SIZE_KEY = "hfile.block.cache.size";
-
public static enum BloomType {
/**
* Bloomfilters disabled
Modified: hbase/trunk/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/resources/hbase-default.xml?rev=1213950&r1=1213949&r2=1213950&view=diff
==============================================================================
--- hbase/trunk/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/src/main/resources/hbase-default.xml Tue Dec 13 21:55:36 2011
@@ -445,13 +445,13 @@
</description>
</property>
<property>
- <name>hfile.block.cache.size</name>
- <value>0.2</value>
- <description>
- Percentage of maximum heap (-Xmx setting) to allocate to block cache
- used by HFile/StoreFile. Default of 0.2 means allocate 20%.
- Set to 0 to disable.
- </description>
+ <name>hfile.block.cache.size</name>
+ <value>0.25</value>
+ <description>
+ Percentage of maximum heap (-Xmx setting) to allocate to block cache
+ used by HFile/StoreFile. Default of 0.25 means allocate 25%.
+ Set to 0 to disable but it's not recommended.
+ </description>
</property>
<property>
<name>hbase.hash.type</name>