You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by yu...@apache.org on 2012/06/26 03:46:24 UTC
svn commit: r1353794 - in /incubator/ambari/trunk: CHANGES.txt
hmc/db/hdp_upgrade_to_1.0.1.php hmc/db/schema.dump
hmc/puppet/modules/hdp-hbase/manifests/params.pp
hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb
Author: yusaku
Date: Tue Jun 26 01:46:22 2012
New Revision: 1353794
URL: http://svn.apache.org/viewvc?rev=1353794&view=rev
Log:
AMBARI-411. The HBase puppet templates could include some more config knobs (Contributed by Jaimin Jetly)
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/hmc/db/hdp_upgrade_to_1.0.1.php
incubator/ambari/trunk/hmc/db/schema.dump
incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/manifests/params.pp
incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1353794&r1=1353793&r2=1353794&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Tue Jun 26 01:46:22 2012
@@ -6,6 +6,9 @@ characters wide.
Release 1.0.0 - unreleased
+ AMBARI-411. The HBase puppet templates could include some more config knobs
+ (Jaimin Jetly via yusaku)
+
AMBARI-577. Document the steps to build and install monitoring RPMs for Ambari
(vgogate)
Modified: incubator/ambari/trunk/hmc/db/hdp_upgrade_to_1.0.1.php
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/db/hdp_upgrade_to_1.0.1.php?rev=1353794&r1=1353793&r2=1353794&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/db/hdp_upgrade_to_1.0.1.php (original)
+++ incubator/ambari/trunk/hmc/db/hdp_upgrade_to_1.0.1.php Tue Jun 26 01:46:22 2012
@@ -15,8 +15,58 @@
# limitations under the License.
//The script takes the first argument as the path of sqlite database
-
$db = new PDO("sqlite:".$argv[1]);
+
+
+/***
+ * Adding additional parameters on UI front for HBase
+ */
+
+// maps to hstore_compactionthreshold
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hstore_compactionthreshold\", \"3\", \"HBase HStore compaction threshold\", \"If more than this number of HStoreFiles in any one HStore then a compaction is run to rewrite all HStoreFiles files as one.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hfile_blockcache_size
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hfile_blockcache_size\", \"0.25\", \"HFile block cache size \", \"Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. Set to 0 to disable but it's not recommended.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hstorefile_maxsize.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hstorefile_maxsize\", \"1073741824\", \"Maximum HStoreFile Size\", \"If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to regionserver_handlers.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"regionserver_handlers\", \"30\", \"HBase Region Server Handler\", \"Count of RPC Listener instances spun up on RegionServers\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_majorcompaction.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_majorcompaction\", \"86400000\", \"HBase Region Major Compaction\", \"The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"ms\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_blockmultiplier.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_blockmultiplier\", \"2\", \"HBase Region Block Multiplier\", \"Block updates if memstore has \"\"Multiplier * HBase Region Memstore Flush Size\"\" bytes. Useful preventing runaway memstore during spikes in update traffic\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hregion_memstoreflushsize.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hregion_memstoreflushsize\", \"134217728\", \"HBase Region Memstore Flush Size\", \"Memstore will be flushed to disk if size of the memstore exceeds this number of bytes.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to client_scannercaching.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"client_scannercaching\", \"100\", \"HBase Client Scanner Caching\", \"Number of rows that will be fetched when calling next on a scanner if it is not served from (local, client) memory. Do not set this value such that the time between invocations is greater than the scanner timeout\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"int\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to zookeeper_sessiontimeout.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"zookeeper_sessiontimeout\", \"60000\", \"Zookeeper timeout for HBase Session\", \"HBase passes this to the zk quorum as suggested maximum time for a session\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"ms\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+// maps to hfile_max_keyvalue_size.
+$query = "INSERT OR REPLACE INTO \"ConfigProperties\" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( \"hfile_max_keyvalue_size\", \"10485760\", \"HBase Client Maximum key-value Size\", \"Specifies the combined maximum allowed size of a KeyValue instance. It should be set to a fraction of the maximum region size.\", \"HBASE\" , \"\", '{ \"isPassword\": false, \"noDisplay\": false, \"reconfigurable\": true, \"displayType\": \"text\", \"unit\":\"bytes\" }' )";
+$db->exec($query) or die(print_r($db->errorInfo(), true));
+
+
+/***
+ * changes the display on UI from "HIVE SERVER to HIVE METASTORE"
+ */
+
$query = "UPDATE ServiceComponents SET display_name = 'Hive Metastore' WHERE service_name = 'HIVE' AND component_name = 'HIVE_SERVER'";
$db->exec($query) or die(print_r($db->errorInfo(), true));
?>
Modified: incubator/ambari/trunk/hmc/db/schema.dump
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/db/schema.dump?rev=1353794&r1=1353793&r2=1353794&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/db/schema.dump (original)
+++ incubator/ambari/trunk/hmc/db/schema.dump Tue Jun 26 01:46:22 2012
@@ -338,12 +338,43 @@ INSERT OR REPLACE INTO "ConfigProperties
-- maps to clientPort in gscluster.
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "clientPort", "2181", "Port for running ZK Server", "Port for running ZooKeeper server", "ZOOKEEPER" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+-- maps to hbase_regionserver_heapsize in gscluster in MB.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_regionserver_heapsize", "1024", "HBase Region Servers maximum Java heap size", "Maximum Java heap size for HBase Region Servers (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
-- maps to hbase_master_heapsize in gscluster in MB.
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_master_heapsize", "1024", "HBase Master Maximum Java heap size", "Maximum Java heap size for HBase master (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
--- maps to hbase_regionserver_heapsize in gscluster in MB.
-INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hbase_regionserver_heapsize", "1024", "HBase Region Servers maximum Java heap size", "Maximum Java heap size for HBase Region Servers (Java option -Xmx)", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"MB" }' );
+--maps to hstore_compactionthreshold
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hstore_compactionthreshold", "3", "HBase HStore compaction threshold", "If more than this number of HStoreFiles in any one HStore then a compaction is run to rewrite all HStoreFiles files as one.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+--maps to hfile_blockcache_size
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hfile_blockcache_size", "0.25", "HFile block cache size ", "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. Set to 0 to disable but it's not recommended.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hstorefile_maxsize.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hstorefile_maxsize", "1073741824", "Maximum HStoreFile Size", "If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+-- maps to regionserver_handlers.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "regionserver_handlers", "30", "HBase Region Server Handler", "Count of RPC Listener instances spun up on RegionServers", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hregion_majorcompaction.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_majorcompaction", "86400000", "HBase Region Major Compaction", "The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"ms" }' );
+
+-- maps to hregion_blockmultiplier.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_blockmultiplier", "2", "HBase Region Block Multiplier", "Block updates if memstore has ""Multiplier * HBase Region Memstore Flush Size"" bytes. Useful preventing runaway memstore during spikes in update traffic", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to hregion_memstoreflushsize.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hregion_memstoreflushsize", "134217728", "HBase Region Memstore Flush Size", "Memstore will be flushed to disk if size of the memstore exceeds this number of bytes.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+-- maps to client_scannercaching.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "client_scannercaching", "100", "HBase Client Scanner Caching", "Number of rows that will be fetched when calling next on a scanner if it is not served from (local, client) memory. Do not set this value such that the time between invocations is greater than the scanner timeout", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"int" }' );
+
+-- maps to zookeeper_sessiontimeout.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "zookeeper_sessiontimeout", "60000", "Zookeeper timeout for HBase Session", "HBase passes this to the zk quorum as suggested maximum time for a session", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"ms" }' );
+
+-- maps to hfile_max_keyvalue_size.
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "hfile_max_keyvalue_size", "10485760", "HBase Client Maximum key-value Size", "Specifies the combined maximum allowed size of a KeyValue instance. It should be set to a fraction of the maximum region size.", "HBASE" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text", "unit":"bytes" }' );
+
+
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "lzo_enabled", "false", "LZO compression", "LZO compression enabled", "MAPREDUCE" , "ONOFF", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text" }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "snappy_enabled", "true", "Snappy compression", "Snappy compression enabled", "MAPREDUCE" , "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
Modified: incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/manifests/params.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/manifests/params.pp?rev=1353794&r1=1353793&r2=1353794&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/manifests/params.pp (original)
+++ incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/manifests/params.pp Tue Jun 26 01:46:22 2012
@@ -32,11 +32,25 @@ class hdp-hbase::params() inherits hdp::
$hfile_blockcache_size = hdp_default("hadoop/hbase-site/hfile_blockcache_size","0.25")
+ $hfile_max_keyvalue_size = hdp_default("hadoop/hbase-site/hfile_max_keyvalue_size",10485760)
+
+ $zookeeper_sessiontimeout = hdp_default("hadoop/hbase-site/zookeeper_sessiontimeout",60000)
+
+ $client_scannercaching = hdp_default("hadoop/hbase-site/client_scannercaching",100)
+
$hstore_blockingstorefiles = hdp_default("hadoop/hbase-site/hstore_blockingstorefiles",7)
$hstore_compactionthreshold = hdp_default("hadoop/hbase-site/hstore_compactionthreshold",3)
- $hstorefile_maxsize = hdp_default("hadoop/hbase-site/hstorefile_maxsize",268435456)
+ $hstorefile_maxsize = hdp_default("hadoop/hbase-site/hstorefile_maxsize",1073741824)
+
+ $hregion_blockmultiplier = hdp_default("hadoop/hbase-site/hregion_blockmultiplier",2)
+
+ $hregion_memstoreflushsize = hdp_default("hadoop/hbase-site/hregion_memstoreflushsize",134217728)
+
+ $regionserver_handlers = hdp_default("hadoop/hbase-site/regionserver_handlers", 30)
+
+ $hregion_majorcompaction = hdp_default("hadoop/hbase-site/hregion_majorcompaction", 86400000)
$preloaded_mastercoprocessor_classes = hdp_default("hadoop/hbase-site/preloaded_mastercoprocessor_classes")
Modified: incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb?rev=1353794&r1=1353793&r2=1353794&view=diff
==============================================================================
--- incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb (original)
+++ incubator/ambari/trunk/hmc/puppet/modules/hdp-hbase/templates/hbase-site.xml.erb Tue Jun 26 01:46:22 2012
@@ -65,6 +65,22 @@
</description>
</property>
<property>
+ <name>hbase.regionserver.handler.count</name>
+ <value><%=scope.function_hdp_template_var("regionserver_handlers")%></value>
+ <description>Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ Default is 10.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value><%=scope.function_hdp_template_var("hregion_majorcompaction")%></value>
+ <description>The time (in miliseconds) between 'major' compactions of all
+ HStoreFiles in a region. Default: 1 day.
+ Set to 0 to disable automated major compactions.
+ </description>
+ </property>
+ <property>
<name>hbase.regionserver.global.memstore.lowerLimit</name>
<value><%=scope.function_hdp_template_var("regionserver_memstore_lowerlimit")%></value>
<description>When memstores are being forced to flush to make room in
@@ -75,6 +91,26 @@
</description>
</property>
<property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value><%=scope.function_hdp_template_var("hregion_blockmultiplier")%></value>
+ <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ time hbase.hregion.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value><%=scope.function_hdp_template_var("hregion_memstoreflushsize")%></value>
+ <description>
+ Memstore will be flushed to disk if size of the memstore
+ exceeds this number of bytes. Value is checked by a thread that runs
+ every hbase.server.thread.wakefrequency.
+ </description>
+ </property>
+ <property>
<name>hbase.hregion.memstore.mslab.enabled</name>
<value><%=scope.function_hdp_template_var("regionserver_memstore_lab")%></value>
<description>
@@ -94,6 +130,39 @@
</description>
</property>
<property>
+ <name>hbase.client.scanner.caching</name>
+ <value><%=scope.function_hdp_template_var("client_scannercaching")%></value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value><%=scope.function_hdp_template_var("zookeeper_sessiontimeout")%></value>
+ <description>ZooKeeper session timeout.
+ HBase passes this to the zk quorum as suggested maximum time for a
+ session (This setting becomes zookeeper's 'maxSessionTimeout'). See
+ http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
+ "The client sends a requested timeout, the server responds with the
+ timeout that it can give the client. " In milliseconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value><%=scope.function_hdp_template_var("hfile_max_keyvalue_size")%></value>
+ <description>Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ </property>
+ <property>
<name>hbase.hstore.compactionThreshold</name>
<value><%=scope.function_hdp_template_var("hstore_compactionthreshold")%></value>
<description>