You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vi...@apache.org on 2012/06/06 23:01:42 UTC
svn commit: r1347121 - in /incubator/ambari/branches/ambari-186: CHANGES.txt
hmc/php/frontend/addNodes/finalizeNodes.php
Author: vikram
Date: Wed Jun 6 21:01:41 2012
New Revision: 1347121
URL: http://svn.apache.org/viewvc?rev=1347121&view=rev
Log:
AMBARI-364. Retry puppet kick --ping on bootstrap to handle intermittent failures (Contributed by Hitesh)
Modified:
incubator/ambari/branches/ambari-186/CHANGES.txt
incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php
Modified: incubator/ambari/branches/ambari-186/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/CHANGES.txt?rev=1347121&r1=1347120&r2=1347121&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/CHANGES.txt (original)
+++ incubator/ambari/branches/ambari-186/CHANGES.txt Wed Jun 6 21:01:41 2012
@@ -6,6 +6,8 @@ characters wide.
Release 0.1.x - unreleased
+ AMBARI-364. Retry puppet kick --ping on bootstrap to handle intermittent failures (Hitesh via Vikram)
+
AMBARI-370. Uninstall needs to clear Puppet directories (Vikram)
AMBARI-368. Server restart kills puppet agent. (Mahadev via Vikram)
Modified: incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php?rev=1347121&r1=1347120&r2=1347121&view=diff
==============================================================================
--- incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php (original)
+++ incubator/ambari/branches/ambari-186/hmc/php/frontend/addNodes/finalizeNodes.php Wed Jun 6 21:01:41 2012
@@ -139,101 +139,125 @@ function sign_and_verify_agent ($hosts,
sleep(5);
-
- // Run kick ping in batches of 10 hosts
- $hostsToKick = array();
- $index = 0;
- $counter = 0;
+ // run multiple attempts for pings to handle intermittent failures
+ $pendingNodes = array();
foreach ($origHosts as $i => $host) {
$host = trim($host);
if (array_key_exists ($host , $output)) {
continue;
}
- $counter++;
- if (!isset($hostsToKick[$index])) {
- $hostsToKick[$index] = array();
- }
- $hostsToKick[$index][] = $host;
-
- if ($counter == 10) {
- $index++;
- $counter = 0;
- }
+ array_push($pendingNodes, $host);
}
- foreach ($hostsToKick as $idx => $hostKickList) {
+ $retryAttempt = 0;
+ do {
+ $retryAttempt++;
+ if ($retryAttempt > 1) {
+ // keep a small sleep between retries
+ // no sleep on first loop
+ sleep(3);
+ }
- $hostList = implode(",", $hostKickList);
+ $logger->log_debug("Puppet kick --ping retry attempt " . $retryAttempt
+ . ", pendingHoststoCheck=" . implode(",", $pendingNodes));
- /* Give puppet kick --ping to check if agent is working */
- $logger->log_debug("Puppet kick --ping for batch $idx , hosts=".$hostList."\n");
+ // Run kick ping in batches of 10 hosts
+ $hostsToKick = array();
+ $index = 0;
+ $counter = 0;
+ foreach ($pendingNodes as $i => $host) {
+ $counter++;
+ if (!isset($hostsToKick[$index])) {
+ $hostsToKick[$index] = array();
+ }
+ $hostsToKick[$index][] = $host;
- $hostListStr = "";
- foreach ($hostKickList as $hostToKick) {
- $hostListStr .= " --host " . $hostToKick;
+ if ($counter == 10) {
+ $index++;
+ $counter = 0;
+ }
}
- $out_arr = array();
- $cmd = "puppet kick -f --parallel 10 --ping $hostListStr 2>/dev/null";
- exec ($cmd, $out_arr, $err);
+ $failedNodes = array();
- // TODO do we need to check $err ?
+ foreach ($hostsToKick as $idx => $hostKickList) {
- $pHostOutput = array();
- $pHostResponse = array();
- foreach ($out_arr as $line) {
- foreach ($hostKickList as $host) {
- if (preg_match ("/$host/", $line)) {
- if (!isset($pHostOutput[$host])) {
- $pHostOutput[$host] = array();
- }
- $pHostOutput[$host][] = $line;
- $pattern = $host." finished with exit code (\d+)";
- $matches = array();
- if (preg_match("/$pattern/", $line, $matches) > 0) {
- $retCode = (int)$matches[1];
- $pHostResponse[$host] = $retCode;
+ $hostList = implode(",", $hostKickList);
+
+ /* Give puppet kick --ping to check if agent is working */
+ $logger->log_debug("Puppet kick --ping for batch $idx , hosts=".$hostList);
+
+ $hostListStr = "";
+ foreach ($hostKickList as $hostToKick) {
+ $hostListStr .= " --host " . $hostToKick;
+ }
+
+ $out_arr = array();
+ $cmd = "puppet kick -f --parallel 10 --ping $hostListStr 2>/dev/null";
+ exec ($cmd, $out_arr, $err);
+
+ // TODO do we need to check $err ?
+
+ $pHostOutput = array();
+ $pHostResponse = array();
+ foreach ($out_arr as $line) {
+ foreach ($hostKickList as $host) {
+ if (preg_match ("/$host/", $line)) {
+ if (!isset($pHostOutput[$host])) {
+ $pHostOutput[$host] = array();
+ }
+ $pHostOutput[$host][] = $line;
+ $pattern = $host." finished with exit code (\d+)";
+ $matches = array();
+ if (preg_match("/$pattern/", $line, $matches) > 0) {
+ $retCode = (int)$matches[1];
+ $pHostResponse[$host] = $retCode;
+ }
}
}
}
- }
- $logger->log_debug("Output for batch $idx, outputLogs="
- . print_r($pHostOutput, true) . " , errorCodes="
- . print_r($pHostResponse, true) );
-
- foreach ($hostKickList as $host) {
- if (isset($pHostResponse[$host])
- && $pHostResponse[$host] == 0) {
- $logger->log_info("Puppet kick succeeded for host " . $host);
- $hostsState[$host] = TRUE;
- if (isset($output[$host])) {
- unset($output[$host]);
- }
- } else {
- $logger->log_error("Failed to do puppet kick -ping on host " . $host);
+ $logger->log_debug("Output for batch $idx, outputLogs="
+ . print_r($pHostOutput, true) . " , errorCodes="
+ . print_r($pHostResponse, true) );
- $errorCode = -1;
- if (isset($pHostResponse[$host])) {
- $errorCode = $pHostResponse[$host];
- }
+ foreach ($hostKickList as $host) {
+ if (isset($pHostResponse[$host])
+ && $pHostResponse[$host] == 0) {
+ $logger->log_info("Puppet kick succeeded for host " . $host);
+ $hostsState[$host] = TRUE;
+ if (isset($output[$host])) {
+ unset($output[$host]);
+ }
+ } else {
+ $logger->log_error("Failed to do puppet kick -ping on host " . $host);
- $errorLogs = "Puppet kick failed";
- if (isset($pHostOutput[$host])) {
- $errorLogs = implode(";", $pHostOutput[$host]);
- }
+ $errorCode = -1;
+
+ $failedNodes[] = $host;
- if (!isset($output[$host])) {
- $output[$host] =
- array ( "discoveryStatus" => "FAILED",
- "badHealthReason" => "Puppet kick failed: "
- . ", error=" . $errorCode
- . ", outputLogs=" . $errorLogs);
+ if (isset($pHostResponse[$host])) {
+ $errorCode = $pHostResponse[$host];
+ }
+
+ $errorLogs = "Puppet kick failed";
+ if (isset($pHostOutput[$host])) {
+ $errorLogs = implode(";", $pHostOutput[$host]);
+ }
+
+ if (!isset($output[$host])) {
+ $output[$host] =
+ array ( "discoveryStatus" => "FAILED",
+ "badHealthReason" => "Puppet kick failed: "
+ . ", error=" . $errorCode
+ . ", outputLogs=" . $errorLogs);
+ }
+ $hostsState[$host] = FALSE;
}
- $hostsState[$host] = FALSE;
}
}
- }
+ $pendingNodes = $failedNodes;
+ } while (!empty($pendingNodes) && $retryAttempt < 3);
$countFailed = 0;
$countSucceeded = 0;