You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vi...@apache.org on 2012/06/07 03:34:56 UTC
svn commit: r1347269 - in /incubator/ambari/branches/ambari-186: ./ hmc/db/
hmc/php/frontend/ hmc/php/puppet/genmanifest/
hmc/puppet/modules/hdp-hadoop/templates/
Author: vikram
Date: Thu Jun 7 01:34:55 2012
New Revision: 1347269
URL: http://svn.apache.org/viewvc?rev=1347269&view=rev
Log:
AMBARI-458. Support configuration of checkpointing-related variables (Contributed by Hitesh)
Modified:
incubator/ambari/branches/ambari-186/CHANGES.txt
incubator/ambari/branches/ambari-186/hmc/db/schema.dump
incubator/ambari/branches/ambari-186/hmc/php/frontend/assignMasters.php
incubator/ambari/branches/ambari-186/hmc/php/frontend/configUtils.php
incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/hostsConfig.php
incubator/ambari/branches/ambari-186/hmc/puppet/modules/hdp-hadoop/templates/core-site.xml.erb
Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Thu Jun 7 01:34:55 2012
@@ -6,6 +6,8 @@ characters wide.
Release 0.1.x - unreleased
+ AMBARI-458. Support configuration of checkpointing-related variables (Hitesh via Vikram)
+
AMBARI-439. Fixes issues in previous commit of 439. (Ramya via Vikram)
AMBARI-457. Create template for SNN checkpoint dir (Ramya via Vikram)
Modified: incubator/ambari/branches/ambari-186/hmc/db/schema.dump
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/db/schema.dump?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/db/schema.dump (original)
+++ incubator/ambari/branches/ambari-186/hmc/db/schema.dump Thu Jun 7 01:34:55 2012
@@ -213,6 +213,7 @@ CREATE TABLE ConfigProperties (
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "dfs_name_dir", "", "NameNode directories", "NameNode directories for HDFS to store the file system image", "HDFS", "TEXT", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "dfs_data_dir", "", "DataNode directories", "DataNode directories for HDFS to store the data blocks", "HDFS", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "fs_checkpoint_dir", "", "SecondaryNameNode Checkpoint directory", "Directory on the local filesystem where the Secondary NameNode should store the temporary images to merge", "HDFS", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "mapred_local_dir", "", "MapReduce local directories", "Directories for MapReduce to store intermediate data files", "MAPREDUCE", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "oozie_data_dir", "", "Oozie DB directory", "Data directory in which the Oozie DB exists", "OOZIE", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "zk_data_dir", "", "ZooKeeper directory", "Data directory for ZooKeeper", "ZOOKEEPER", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "editable": false }' );
@@ -348,6 +349,10 @@ INSERT OR REPLACE INTO "ConfigProperties
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "lzo_enabled", "false", "LZO compression", "LZO compression enabled", "MAPREDUCE" , "ONOFF", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text" }' );
INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "snappy_enabled", "true", "Snappy compression", "Snappy compression enabled", "MAPREDUCE" , "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "fs_checkpoint_period", "21600", "HDFS Maximum Checkpoint Delay", "Maximum delay between two consecutive checkpoints for HDFS", "HDFS" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "unit":"seconds", "editable": false }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "fs_checkpoint_size", "0.5", "HDFS Maximum Edit Log Size for Checkpointing", "Maximum size of the edits log file that forces an urgent checkpoint even if the maximum checkpoint delay is not reached", "HDFS" , "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": false, "displayType": "text", "unit":"GB", "editable": false }' );
+
+
-- Done with gsCluster.properties
CREATE TABLE ServiceConfig (
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/assignMasters.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/assignMasters.php?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/assignMasters.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/assignMasters.php Thu Jun 7 01:34:55 2012
@@ -117,6 +117,11 @@ $outjson = array(
"maxDirectoriesNeeded" => -1,
"suffix" => "hadoop/hdfs/data"
),
+ "fs_checkpoint_dir" => array(
+ "displayName" => "SecondaryNameNode Checkpoint directory",
+ "maxDirectoriesNeeded" => 1,
+ "suffix" => "hadoop/hdfs/namesecondary"
+ )
),
"MAPREDUCE" => array(
"mapred_local_dir" => array(
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/configUtils.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/configUtils.php?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/configUtils.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/configUtils.php Thu Jun 7 01:34:55 2012
@@ -69,6 +69,11 @@ function validateConfigs($svcConfigs) {
if ($check["error"] != "") {
$errors[$key] = $check;
}
+ } else if ($key == "fs_checkpoint_dir") {
+ $check = validateDirList($val);
+ if ($check["error"] != "") {
+ $errors[$key] = $check;
+ }
} else if ($key == "mapred_local_dir") {
$check = validateDirList($val);
if ($check["error"] != "") {
@@ -473,7 +478,8 @@ function sanitizeConfigs($requestObjFrom
|| $key == "dfs_data_dir"
|| $key == "mapred_local_dir"
|| $key == "oozie_data_dir"
- || $key == "zk_data_dir") {
+ || $key == "zk_data_dir"
+ || $key == "fs_checkpoint_dir" ) {
$finalProperties[$key] = trimDirList($val);
} else if ($key == "lzo_enabled"
|| $key == "snappy_enabled") {
Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/hostsConfig.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/hostsConfig.php?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/hostsConfig.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/hostsConfig.php Thu Jun 7 01:34:55 2012
@@ -78,9 +78,10 @@ class HostsConfigManifest {
($key == "snappy_enabled") ||
($key == "wipeoff_data") ) {
$manifest = $manifest . $value . "\n";
- } else if ($key == "datanode_du_reserved") {
+ } else if ($key == "datanode_du_reserved"
+ || $key == "fs_checkpoint_size") {
// convert GB to bytes and round off to whole number
- $newVal = round(intval($value) * 1024 * 1024 * 1024);
+ $newVal = round($value * 1024 * 1024 * 1024);
$manifest = $manifest . "\"" . $newVal . "\"\n";
} else {
$manifest = $manifest . "\"" . $value . "\"\n";
Modified: incubator/ambari/branches/ambari-186/hmc/puppet/modules/hdp-hadoop/templates/core-site.xml.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/modules/hdp-hadoop/templates/core-site.xml.erb?rev=1347269&r1=1347268&r2=1347269&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/modules/hdp-hadoop/templates/core-site.xml.erb (original)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/modules/hdp-hadoop/templates/core-site.xml.erb Thu Jun 7 01:34:55 2012
@@ -76,14 +76,14 @@
<property>
<name>fs.checkpoint.period</name>
- <value>86400</value>
+ <value>21600</value>
<description>The number of seconds between two periodic checkpoints.
</description>
</property>
<property>
<name>fs.checkpoint.size</name>
- <value>2048000000</value>
+ <value>536870912</value>
<description>The size of the current edit log (in bytes) that triggers
a periodic checkpoint even if the fs.checkpoint.period hasn't expired.
</description>