You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vi...@apache.org on 2012/06/06 01:20:24 UTC
svn commit: r1346656 - in /incubator/ambari/branches/ambari-186: CHANGES.txt
hmc/php/puppet/genmanifest/generateManifest.php
hmc/php/puppet/genmanifest/nodeManifest.php
hmc/puppet/manifestloader/site.pp hmc/puppet/modules/puppetApply.sh
Author: vikram
Date: Tue Jun 5 23:20:23 2012
New Revision: 1346656
URL: http://svn.apache.org/viewvc?rev=1346656&view=rev
Log:
AMBARI-353. Passing packages in hdp class for package install optimization. (Contributed by Jitendra)
Added:
incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
Modified:
incubator/ambari/branches/ambari-186/CHANGES.txt
incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php
incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php
incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp
Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Tue Jun 5 23:20:23 2012
@@ -6,6 +6,8 @@ characters wide.
Release 0.1.x - unreleased
+ AMBARI-353. Passing packages in hdp class for package install optimization. (Jitendra via Vikram)
+
AMBARI-344. Fix TxnProgressWidget To Not Hide Previously Pending States (Varun via Vikram)
AMBARI-352. Add flow control - force redirects to appropriate pages based on cluster configuration status for better usability (Yusaku via Vikram)
Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/generateManifest.php Tue Jun 5 23:20:23 2012
@@ -60,12 +60,93 @@ include "RoleDependencies.php";
}
}
}
+ ManifestGenerator::optimizePackageInstall($n, $nm, $rolesStates, $configInfo, $roleStages);
$nodeManifestString = $nm->generateNodeManifest();
fwrite($fh, $nodeManifestString . "\n");
}
fclose($fh);
}
+ static function optimizePackageInstall($node, &$manifestObj, $rolesStatesDs,
+ $configInfo, $roleStages) {
+ //Figure out the state
+ $serviceState = SERVICE_STATE_INSTALLED_AND_CONFIGURED;
+ foreach ($rolesStatesDs as $r => $stateList) {
+ if ((isset($stateList[SERVICE_STATE_KEY])) &&
+ ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+ if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_INSTALLED_AND_CONFIGURED) {
+ $serviceState = NULL;
+ break;
+ }
+ }
+ }
+ if (!isset($serviceState)) {
+ $serviceState = SERVICE_STATE_UNINSTALLED;
+ //See if it is uninstalled
+ foreach ($rolesStatesDs as $r => $stateList) {
+ if ( (isset($stateList[SERVICE_STATE_KEY])) &&
+ ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+ if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_UNINSTALLED) {
+ $serviceState = NULL;
+ break;
+ }
+ }
+ }
+ }
+ if (!isset($serviceState)) {
+ //No optimization needed
+ return;
+ }
+ //get list of packages
+ $stages = array();
+ foreach($roleStages as $roleName => $val) {
+ $stages[$val] = $roleName;
+ }
+ ksort($stages, SORT_NUMERIC);
+ $packageList = array();
+ foreach($stages as $theStage => $r) {
+ if (!isset($rolesStatesDs[$r])) {
+ continue;
+ }
+ //Add in the order of the stages
+ $stateList = $rolesStatesDs[$r];
+ if ($stateList[SERVICE_STATE_KEY] != $serviceState) {
+ continue;
+ }
+ if (isset(self::$rolesToPackageMap[$r])) {
+ $p = self::$rolesToPackageMap[$r];
+ if (!in_array($p, $packageList)) {
+ $packageList[] = $p;
+ }
+ }
+ }
+
+ if (empty($packageList)) {
+ //No packages don't bother
+ return;
+ }
+
+ //lzo and snappy
+ $packageList[] = self::$rolesToPackageMap["snappy"];
+ if ($configInfo["lzo_enabled"] == "true") {
+ $packageList[] = self::$rolesToPackageMap["lzo"];
+ }
+
+ $firstP = true;
+ $pList = "\"";
+ foreach ($packageList as $p) {
+ if ($firstP) {
+ $firstP = false;
+ } else {
+ $pList = $pList . " ";
+ }
+ $pList = $pList . $p;
+ }
+ $pList = $pList . "\"";
+ $manifestObj->setRoleState("hdp", SERVICE_STATE_KEY, $serviceState);
+ $manifestObj->setRoleState("hdp", "pre_installed_pkgs", $pList);
+ }
+
private static function getAllImports($modulesDir) {
$importString = "";
$importString = $importString . "import \"" . $modulesDir . "/hdp/manifests/*.pp" ."\"\n";
@@ -75,7 +156,6 @@ include "RoleDependencies.php";
$importString = $importString . "import \"" . $modulesDir . "/hdp-oozie/manifests/*.pp" ."\"\n";
$importString = $importString . "import \"" . $modulesDir . "/hdp-pig/manifests/*.pp" ."\"\n";
$importString = $importString . "import \"" . $modulesDir . "/hdp-sqoop/manifests/*.pp" ."\"\n";
- #$importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp" ."\"\n";
$importString = $importString . "import \"" . $modulesDir . "/hdp-templeton/manifests/*.pp" ."\"\n";
$importString = $importString . "import \"" . $modulesDir . "/hdp-hive/manifests/*.pp" ."\"\n";
$importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp" ."\"\n";
@@ -83,6 +163,36 @@ include "RoleDependencies.php";
$importString = $importString . "import \"" . $modulesDir . "/hdp-monitor-webserver/manifests/*.pp" ."\"\n";
return $importString;
}
+
+ private static $rolesToPackageMap = array (
+ "hdp-hadoop::namenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+ "hdp-hadoop::snamenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+ "hdp-hadoop::jobtracker" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+ "hdp-hadoop::client" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+ "hdp-hadoop::datanode" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+ "hdp-hadoop::tasktracker" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+ "hdp-hadoop::zookeeper" => "zookeeper",
+ "hdp-hadoop::zookeeper::client" => "zookeeper",
+ "hdp-hbase::master" => "hbase",
+ "hdp-hbase::regionserver" => "hbase",
+ "hdp-hbase::client" => "hbase",
+ "hdp-pig" => "pig.noarch",
+ "hdp-sqoop" => "sqoop",
+ "hdp-hive::server" => "hive",
+ "hdp-hive::client" => "hive",
+ "hdp-hcat" => "hcatalog",
+ "hdp-oozie::server" => "oozie.noarch",
+ "hdp-oozie::client" => "oozie-client.noarch",
+ "hdp-mysql::mysql" => "mysql mysql-server",
+ "hdp-templeton::server" => "templeton.i386",
+ "hdp-templeton::client" => "templeton.i386",
+ "lzo" => "lzo lzo.i386 lzo-devel lzo-devel.i386",
+ "snappy" => "snappy snappy-devel",
+ "hdp-ganglia::monitor_and_server" => "ganglia-gmetad-3.2.0 ganglia-gmond-3.2.0 gweb hdp_mon_ganglia_addons",
+ "hdp-ganglia::monitor" => "ganglia-gmond-3.2.0 gweb hdp_mon_ganglia_addons",
+ "hdp-nagios::server" => "hdp_mon_nagios_addons nagios-3.2.3 nagios-plugins-1.4.9 fping net-snmp-utils",
+ "hdp-dashboard" => "hdp_mon_dashboard",
+ );
}
?>
Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/genmanifest/nodeManifest.php Tue Jun 5 23:20:23 2012
@@ -186,20 +186,4 @@
}
}
-/*** TEST CODE ********
- $nm = new NodeManifest("x.y.z");
- $nm->setRoleStage("namenode", "1");
- $nm->setRoleStage("datanode", "2");
- $nm->setRoleStage("namenode", "3");
- $nm->setRoleState("namenode", "service_state", "running");
- $nm->setRoleOpt("namenode", "deploy", "yes");
- $nm->setRoleOpt("namenode", "deploy", "no");
- $nm->setRoleOpt("namenode", "wait", "10");
- $nm->setRoleOpt("datanode", "running", "yes");
- $nm->setNamenodeRoleState("stopped", 4);
- $nm->setDatanodeRoleState("running", 5);
- $ret = $nm->generateNodeManifest();
- echo "$ret\n";
-***/
-
?>
Modified: incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp?rev=1346656&r1=1346655&r2=1346656&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp (original)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp Tue Jun 5 23:20:23 2012
@@ -1,38 +1,23 @@
class manifestloader () {
- anchor { 'manifestloader::begin': }
file { '/etc/puppet/agent/site.pp':
ensure => present,
source => "puppet:///modules/catalog/site.pp",
mode => '0755',
- require => Anchor['manifestloader::begin'],
- notify => Exec['rm_puppet_apply_log']
}
exec {'rm_puppet_apply_log':
command => "rm -f /var/log/puppet_apply.log",
path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- notify => Exec['puppet_apply'],
- require => File['/etc/puppet/agent/site.pp'],
- logoutput => "true"
}
exec { 'puppet_apply':
- command => "puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush /etc/puppet/agent/site.pp",
+ command => "sh /etc/puppet/agent/modules/puppetApply.sh",
timeout => 1200,
path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- require => Exec['rm_puppet_apply_log'],
- notify => Exec['append'],
logoutput => "true"
}
- exec { 'append':
- command => "cat /var/log/puppet_apply.log",
- path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- require => Exec['puppet_apply'],
- notify => Anchor['manifestloader::end'],
- logoutput => "true"
- }
- anchor { 'manifestloader::end': }
+ File['/etc/puppet/agent/site.pp'] -> Exec['rm_puppet_apply_log'] -> Exec['puppet_apply']
}
node default {
Added: incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh?rev=1346656&view=auto
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh (added)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh Tue Jun 5 23:20:23 2012
@@ -0,0 +1,7 @@
+puppet apply --confdir=/etc/puppet/agent --logdest=console --debug --autoflush --detailed-exitcodes /etc/puppet/agent/site.pp
+ret=$?
+if [ "$ret" == "0" ] || [ "$ret" == "2" ]; then
+ exit 0
+else
+ exit 1
+fi