You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vi...@apache.org on 2012/06/07 00:27:48 UTC
svn commit: r1347179 - in /incubator/ambari/branches/ambari-186: ./
hmc/ShellScripts/ hmc/package/rpm/SPECS/ hmc/php/conf/ hmc/php/puppet/
hmc/puppet/manifestloader/ hmc/puppet/modules/
Author: vikram
Date: Wed Jun 6 22:27:48 2012
New Revision: 1347179
URL: http://svn.apache.org/viewvc?rev=1347179&view=rev
Log:
AMBARI-401. Manual config changes for nn get reset on stop/start from hmc (Contributed by Jitendra)
Modified:
incubator/ambari/branches/ambari-186/CHANGES.txt
incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh
incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec
incubator/ambari/branches/ambari-186/hmc/php/conf/Config.inc
incubator/ambari/branches/ambari-186/hmc/php/puppet/PuppetInvoker.php
incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp
incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Wed Jun 6 22:27:48 2012
@@ -6,6 +6,8 @@ characters wide.
Release 0.1.x - unreleased
+ AMBARI-401. Manual config changes for nn get reset on stop/start from hmc (Jitendra via Vikram)
+
AMBARI-399. Cannot uninstall - the page hangs with the spinning icon (Yusaku via Vikram)
AMBARI-398. if hbase is not installed, nagios sends alerts for "ganglia collector for hbasemaster" being down (Suhas via Vikram)
Modified: incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh (original)
+++ incubator/ambari/branches/ambari-186/hmc/ShellScripts/puppet_agent_install.sh Wed Jun 6 22:27:48 2012
@@ -166,15 +166,6 @@ if [[ "$ret" != "0" ]]; then
exit 1
fi
-#Download modules and untar
-out=`curl -o /etc/puppet/agent/modules.tgz $master/hmc/modules.tgz && cd /etc/puppet/agent/ && tar zxf modules.tgz --strip-components 1 && cd -`
-ret=$?
-if [[ "$ret" != "0" ]]; then
- echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
- exit 1
-fi
-out=`rm /etc/puppet/agent/modules.tgz`
-
#TODO clean this up for better fix. For now make sure we stop puppet agent. The issue here is we do not know if we started this puppet agent during our run or not.
echo "Stopping puppet agent using service stop command"
out=`service puppet stop`
Modified: incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec (original)
+++ incubator/ambari/branches/ambari-186/hmc/package/rpm/SPECS/hmc.spec Wed Jun 6 22:27:48 2012
@@ -69,7 +69,6 @@ fi
%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/
%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/manifests
-%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/modules/catalog/files
%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
%__install -d "%{buildroot}%{hmc_db_dir}"
%__install -d "%{buildroot}%{hmc_log_dir}"
@@ -92,11 +91,11 @@ fi
%__cp -f yuiCombinator.php $RPM_BUILD_ROOT/%{web_prefixdir}/
%__cp -rf conf $RPM_BUILD_ROOT/%{web_prefixdir}/
%__cp -rf puppet/manifestloader $RPM_BUILD_ROOT/%{puppet_master_dir}
-%__cp -rf puppet/modules/stdlib $RPM_BUILD_ROOT/%{puppet_master_dir}/modules
+%__cp -rf puppet/modules $RPM_BUILD_ROOT/%{puppet_master_dir}
+%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/modules/catalog/files
%__cp -f "%{SOURCE2}" $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
%__install -D -m0755 puppet/reports/get_revision $RPM_BUILD_ROOT/%{web_prefixdir}/bin
%__cp -rf puppet/reports/hmcreport.rb $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
-%__tar czf $RPM_BUILD_ROOT/%{web_prefixdir}/modules.tgz puppet/modules
echo "Alias /hdp %{_prefix}/share/hdp" > $RPM_BUILD_ROOT/%{httpd_confdir}/hdp_mon_dashboard.conf
%post
Modified: incubator/ambari/branches/ambari-186/hmc/php/conf/Config.inc
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/conf/Config.inc?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/conf/Config.inc (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/conf/Config.inc Wed Jun 6 22:27:48 2012
@@ -9,12 +9,12 @@ $GLOBALS["CLUSTERMAIN_PATH"] = $INSTALL_
$GLOBALS["DB_PATH"] = "/var/db/hmc/data/data.db";
$GLOBALS["puppetMaxParallelKicks"] = 10;
-$GLOBALS["puppetManifestDir"] = "/etc/puppet/master/modules/catalog/files/";
+$GLOBALS["puppetMasterModulesDirectory"] = "/etc/puppet/master/modules";
$GLOBALS["puppetKickVersionFile"] = "/var/lib/puppet/puppet_kick_version.txt";
$GLOBALS["puppetReportsDir"] = "/var/lib/puppet/reports";
$GLOBALS["puppetKickTimeout"] = 120;
-$GLOBALS["puppetModulesDirectory"] = "/etc/puppet/agent/modules";
-$GLOBALS["manifestloaderFile"] = "/etc/puppet/master/manifestloader/site.pp";
+$GLOBALS["puppetAgentModulesDirectory"] = "/etc/puppet/agent/modules";
+$GLOBALS["manifestloaderDir"] = "/etc/puppet/master/manifestloader";
$GLOBALS["manifestloaderDestinationDir"] = "/etc/puppet/master/manifests";
$GLOBALS["puppet_install"] = "/usr/share/hmc/ShellScripts/puppet_agent_install.sh";
$GLOBALS["DRYRUN"] = FALSE;
Modified: incubator/ambari/branches/ambari-186/hmc/php/puppet/PuppetInvoker.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/puppet/PuppetInvoker.php?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/puppet/PuppetInvoker.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/puppet/PuppetInvoker.php Wed Jun 6 22:27:48 2012
@@ -41,22 +41,33 @@
private function sendKick($nodes, $txnId, &$failedNodes,
&$successNodes, &$prevKickRunningNodes) {
- $cmd = "";
- $cmd = $cmd . "puppet kick ";
- foreach ($nodes as $n) {
- $cmd = $cmd . " --host " . $n;
+ //Some nodes may already be done in the meanwhile
+ $doneNodes = array();
+ $this->checkForReportFiles($nodes, $txnId, $doneNodes);
+ if (count($doneNodes) < count($nodes)) {
+ $cmd = "";
+ $cmd = $cmd . "puppet kick ";
+ foreach ($nodes as $n) {
+ $cmd = $cmd . " --host " . $n;
+ }
+ $p = 10;
+ if (count($nodes) < $p) {
+ $p = count($nodes);
+ }
+ $cmd = $cmd . " --parallel " . $p;
+ $this->logger->log_trace("Kick command: " . $cmd);
+ $output = $this->executeAndGetOutput($cmd);
+ $this->logger->log_trace("Kick response begins ===========");
+ $this->logger->log_trace($output);
+ $this->logger->log_trace("Kick response ends ===========");
}
- $p = 10;
- if (count($nodes) < $p) {
- $p = count($nodes);
- }
- $cmd = $cmd . " --parallel " . $p;
- $this->logger->log_trace("Kick command: " . $cmd);
- $output = $this->executeAndGetOutput($cmd);
- $this->logger->log_trace("Kick response begins ===========");
- $this->logger->log_trace($output);
- $this->logger->log_trace("Kick response ends ===========");
foreach ($nodes as $kNode) {
+ if (isset($doneNodes[$kNode])) {
+ $this->logger->log_debug($kNode . "previous kick has returned, no need to kick again");
+ //Mark as previous kick running, reports will be discovered in waitForResults.
+ $prevKickRunningNodes[] = $kNode;
+ continue;
+ }
$regExSuccess = "/". $kNode . " .* exit code 0/";
$regExRunning = "/". $kNode . ".* is already running/";
if (preg_match($regExSuccess, $output)>0) {
@@ -120,8 +131,9 @@
}
}
+ $manifestDir = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files";
$response = $this->genKickWait($nodes, $txnId, $clusterName, $hostInfo,
- $configInfo, $hostRolesStates, $hostAttributes, $GLOBALS["puppetManifestDir"],
+ $configInfo, $hostRolesStates, $hostAttributes, $manifestDir,
$GLOBALS["puppetKickVersionFile"], $GLOBALS["DRYRUN"]);
return $response;
}
@@ -150,8 +162,9 @@
$hostInfo = $dbReader->getHostNames($clusterName);
$configInfo = $dbReader->getAllConfigs($clusterName);
$hostAttributes = $dbReader->getAllHostAttributes($clusterName);
+ $manifestDir = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files";
$response = $this->genKickWait($nodesToKick, $txnId, $clusterName, $hostInfo,
- $configInfo, $hostRolesStates, $hostAttributes, $GLOBALS["puppetManifestDir"],
+ $configInfo, $hostRolesStates, $hostAttributes, $manifestDir,
$GLOBALS["puppetKickVersionFile"], $GLOBALS["DRYRUN"]);
return $response;
}
@@ -201,12 +214,24 @@
}
//Add manifest loader
- copy($GLOBALS["manifestloaderFile"], $GLOBALS["manifestloaderDestinationDir"] . "/site.pp");
+ copy($GLOBALS["manifestloaderDir"] . "/site.pp", $GLOBALS["manifestloaderDestinationDir"] . "/site.pp");
//Generate manifest
- $modulesDir = $GLOBALS["puppetModulesDirectory"];
+ $agentModulesDir = $GLOBALS["puppetAgentModulesDirectory"];
ManifestGenerator::generateManifest($manifestDir, $hostInfo,
- $configInfo, $hostRolesStates, $hostAttributes, $modulesDir);
+ $configInfo, $hostRolesStates, $hostAttributes, $agentModulesDir);
+
+ //Tar the modules and catalog
+ $tarCmd = "tar zcf ". $GLOBALS["manifestloaderDir"] . "/modules.tgz" . " " . $GLOBALS["puppetMasterModulesDirectory"];
+ $this->logger->log_info($tarCmd);
+ exec($tarCmd);
+ $tarLocation = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files" ;
+ $removeOldTar = "rm -f " . $tarLocation . "/modules.tgz";
+ $this->logger->log_info($removeOldTar);
+ exec($removeOldTar);
+ $placeNewTarCmd = "mv " . $GLOBALS["manifestloaderDir"] . "/modules.tgz" . " " . $tarLocation;
+ $this->logger->log_info($placeNewTarCmd);
+ exec($placeNewTarCmd);
//Write version file
$this->writeVersionFile($versionFile, $txnId);
@@ -237,13 +262,25 @@
sleep(1);
}
$sitePPFile = $manifestDir . "/site.pp";
- system("mv " . $sitePPFile . " " . $sitePPFile ."-".$txnId);
+ system("mv " . $sitePPFile . " " . $GLOBALS["manifestloaderDir"] . "/site.pp-" . $txnId);
// Delete version file, it will be generated next time.
unlink($versionFile);
$response = $this->createGenKickWaitResponse($kickFailedNodes, $failureResponseNodes,
$timedoutNodes, $successfullNodes, $nodes);
return $response;
}
+
+ private function checkForReportFiles($nodes, $txnId, &$doneNodes) {
+ foreach ($nodes as $n) {
+ if (isset($doneNodes[$n])) {
+ continue;
+ }
+ $fileName = $this->getReportFilePattern($n, $txnId);
+ if (file_exists($fileName)) {
+ $doneNodes[$n] = 1;
+ }
+ }
+ }
private function waitForResults($nodes, $txnId, &$successfullNodes,
&$failureResponseNodes, &$timedoutNodes) {
@@ -252,15 +289,7 @@
$this->logger->log_info("Waiting for results from "
. implode(",", $nodes));
while (true) {
- foreach ($nodes as $n) {
- if (isset($doneNodes[$n])) {
- continue;
- }
- $fileName = $this->getReportFilePattern($n, $txnId);
- if (file_exists($fileName)) {
- $doneNodes[$n] = 1;
- }
- }
+ $this->checkForReportFiles($nodes, $txnId, $doneNodes);
$this->logger->log_info(count($doneNodes) . " out of " . count($nodes)
. " nodes have reported");
if (count($doneNodes) >= count($nodes)) {
Modified: incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp (original)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/manifestloader/site.pp Wed Jun 6 22:27:48 2012
@@ -1,14 +1,14 @@
class manifestloader () {
- file { '/etc/puppet/agent/site.pp':
+ file { '/etc/puppet/agent/modules.tgz':
ensure => present,
- source => "puppet:///modules/catalog/site.pp",
+ source => "puppet:///modules/catalog/modules.tgz",
mode => '0755',
}
- exec {'rm_puppet_apply_log':
- command => "rm -f /var/log/puppet_apply.log",
- path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- }
+ exec { 'untar_modules':
+ command => "rm -rf /etc/puppet/agent/modules ; tar zxf /etc/puppet/agent/modules.tgz -C /etc/puppet/agent/ --strip-components 3",
+ path => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+ }
exec { 'puppet_apply':
command => "sh /etc/puppet/agent/modules/puppetApply.sh",
@@ -17,7 +17,7 @@ class manifestloader () {
logoutput => "true"
}
- File['/etc/puppet/agent/site.pp'] -> Exec['rm_puppet_apply_log'] -> Exec['puppet_apply']
+ File['/etc/puppet/agent/modules.tgz'] -> Exec['untar_modules'] -> Exec['puppet_apply']
}
node default {
Modified: incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh?rev=1347179&r1=1347178&r2=1347179&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh (original)
+++ incubator/ambari/branches/ambari-186/hmc/puppet/modules/puppetApply.sh Wed Jun 6 22:27:48 2012
@@ -1,4 +1,5 @@
-puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush --detailed-exitcodes /etc/puppet/agent/site.pp
+rm -f /var/log/puppet_apply.log
+puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush --detailed-exitcodes /etc/puppet/agent/modules/catalog/files/site.pp
ret=$?
cat /var/log/puppet_apply.log
if [ "$ret" == "0" ] || [ "$ret" == "2" ]; then