You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2013/11/18 16:02:38 UTC
git commit: AMBARI-3792. Unable to connect to:
https://server:8441/agent/v1/heartbeat/agent_hostname due to No JSON object
could be decoded" when ping_port is not set (Dmytro Sen via dlysnichenko)
Updated Branches:
refs/heads/trunk f44fb1050 -> 015130d44
AMBARI-3792. Unable to connect to: https://server:8441/agent/v1/heartbeat/agent_hostname due to No JSON object could be decoded" when ping_port is not set (Dmytro Sen via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/015130d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/015130d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/015130d4
Branch: refs/heads/trunk
Commit: 015130d44166b1e8dbe02369eb69a38abbedd84e
Parents: f44fb10
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Nov 18 17:00:33 2013 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Nov 18 17:00:33 2013 +0200
----------------------------------------------------------------------
.../main/python/ambari_agent/PuppetExecutor.py | 10 ++-
.../python/ambari_agent/manifestGenerator.py | 90 +++++++++++---------
.../src/test/python/TestManifestGenerator.py | 18 +++-
.../src/test/python/TestPuppetExecutor.py | 18 ++--
.../apache/ambari/server/utils/StageUtils.java | 4 +-
.../ambari/server/utils/TestStageUtils.java | 12 ++-
6 files changed, 98 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py b/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
index 6daa270..804a984 100644
--- a/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PuppetExecutor.py
@@ -26,7 +26,7 @@ import threading
from threading import Thread
from shell import shellRunner
-from manifestGenerator import generateManifest
+import manifestGenerator
from RepoInstaller import RepoInstaller
from Grep import Grep
import shell
@@ -170,8 +170,12 @@ class PuppetExecutor:
if command.has_key("taskId"):
taskId = command['taskId']
siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp")
- generateManifest(command, siteppFileName, self.modulesdir, self.config)
- result = self.run_manifest(command, siteppFileName, tmpoutfile, tmperrfile)
+ errMsg = manifestGenerator.generateManifest(command, siteppFileName,
+ self.modulesdir, self.config)
+ if not errMsg:
+ result = self.run_manifest(command, siteppFileName, tmpoutfile, tmperrfile)
+ else:
+ result = {'stdout': '', 'stderr': errMsg, 'exitcode': 1}
return result
def runPuppetFile(self, puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
index aaebb1b..fdafc12 100644
--- a/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
+++ b/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
@@ -73,49 +73,59 @@ def generateManifest(parsedJson, fileName, modulesdir, ambariconfig):
roles = [{'role' : parsedJson['role'],
'cmd' : parsedJson['roleCommand'],
'roleParams' : roleParams}]
- #writing manifest
- manifest = open(fileName, 'w')
- #Change mode to make site.pp files readable to owner and group only
- os.chmod(fileName, 0660)
+ errMsg = ''
+ try:
+ #writing manifest
+ manifest = open(fileName, 'w')
+ #Change mode to make site.pp files readable to owner and group only
+ os.chmod(fileName, 0660)
- #Check for Ambari Config and make sure you pick the right imports file
-
- #writing imports from external static file
- writeImports(outputFile=manifest, modulesdir=modulesdir, importsList=AmbariConfig.imports)
+ #Check for Ambari Config and make sure you pick the right imports file
- #writing hostname
- writeHostnames(manifest)
+ #writing imports from external static file
+ writeImports(outputFile=manifest, modulesdir=modulesdir, importsList=AmbariConfig.imports)
- #writing nodes
- writeNodes(manifest, clusterHostInfo)
-
- #writing params from map
- writeParams(manifest, params, modulesdir)
-
- nonGlobalConfigurations = {}
- flatConfigurations = {}
-
- if configurations:
- for configKey in configurations.iterkeys():
- if configKey in nonGlobalConfigurationsKeys:
- nonGlobalConfigurations[configKey] = configurations[configKey]
- else:
- flatConfigurations[configKey] = configurations[configKey]
-
- #writing config maps
- if (nonGlobalConfigurations):
- writeNonGlobalConfigurations(manifest, nonGlobalConfigurations)
- if (flatConfigurations):
- writeFlatConfigurations(manifest, flatConfigurations)
-
- #writing host attributes
- #writeHostAttributes(manifest, hostAttributes)
-
- #writing task definitions
- writeTasks(manifest, roles, ambariconfig, clusterHostInfo, hostname)
-
- manifest.close()
-
+ #writing hostname
+ writeHostnames(manifest)
+
+ #writing nodes
+ writeNodes(manifest, clusterHostInfo)
+
+ #writing params from map
+ writeParams(manifest, params, modulesdir)
+
+ nonGlobalConfigurations = {}
+ flatConfigurations = {}
+
+ if configurations:
+ for configKey in configurations.iterkeys():
+ if configKey in nonGlobalConfigurationsKeys:
+ nonGlobalConfigurations[configKey] = configurations[configKey]
+ else:
+ flatConfigurations[configKey] = configurations[configKey]
+
+ #writing config maps
+ if (nonGlobalConfigurations):
+ writeNonGlobalConfigurations(manifest, nonGlobalConfigurations)
+ if (flatConfigurations):
+ writeFlatConfigurations(manifest, flatConfigurations)
+
+ #writing host attributes
+ #writeHostAttributes(manifest, hostAttributes)
+
+ #writing task definitions
+ writeTasks(manifest, roles, ambariconfig, clusterHostInfo, hostname)
+
+
+ except TypeError:
+ errMsg = 'Manifest can\'t be generated from the JSON \n' + \
+ json.dumps(parsedJson, sort_keys=True, indent=4)
+
+ logger.error(errMsg)
+ finally:
+ manifest.close()
+
+ return errMsg
def writeHostnames(outputFile):
fqdn = hostname.hostname()
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-agent/src/test/python/TestManifestGenerator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestManifestGenerator.py b/ambari-agent/src/test/python/TestManifestGenerator.py
index 4007f8f..f4ea6dc 100644
--- a/ambari-agent/src/test/python/TestManifestGenerator.py
+++ b/ambari-agent/src/test/python/TestManifestGenerator.py
@@ -78,6 +78,10 @@ class TestManifestGenerator(TestCase):
print file(tmpFileName).read()
+ def raiseTypeError():
+ raise TypeError()
+ writeNodesMock.side_effect = raiseTypeError
+ manifestGenerator.generateManifest(self.parsedJson, tmpFileName, '../../main/puppet/modules', self.config.getConfig())
pass
def testEscape(self):
@@ -98,6 +102,18 @@ class TestManifestGenerator(TestCase):
tmpFile.close()
os.remove(tmpFileName)
+ def test_writeNodes_failed(self):
+ tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
+ tmpFile = file(tmpFileName, 'r+')
+
+ clusterHostInfo = self.parsedJson['clusterHostInfo']
+ clusterHostInfo.update({u'ZOOKEEPER':[None]})
+ clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
+ self.assertRaises(TypeError, manifestGenerator.writeNodes, tmpFile, clusterHostInfo)
+ tmpFile.seek(0)
+ print tmpFile.read()
+ tmpFile.close()
+ os.remove(tmpFileName)
def test_writeHostAttributes(self):
tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
@@ -123,4 +139,4 @@ class TestManifestGenerator(TestCase):
tmpFile.seek(0)
print tmpFile.read()
tmpFile.close()
- os.remove(tmpFileName)
\ No newline at end of file
+ os.remove(tmpFileName)
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-agent/src/test/python/TestPuppetExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/TestPuppetExecutor.py b/ambari-agent/src/test/python/TestPuppetExecutor.py
index 25bd3a8..42cfe38 100644
--- a/ambari-agent/src/test/python/TestPuppetExecutor.py
+++ b/ambari-agent/src/test/python/TestPuppetExecutor.py
@@ -31,6 +31,7 @@ from AmbariConfig import AmbariConfig
from mock.mock import patch, MagicMock, call
from threading import Thread
from shell import shellRunner
+import manifestGenerator
class TestPuppetExecutor(TestCase):
@@ -55,11 +56,11 @@ class TestPuppetExecutor(TestCase):
cmdrun_mock.return_value = {'exitCode': 0, 'output': 'OK', 'error': ''}
self.assertEquals(puppetInstance.isJavaAvailable(command), True)
-
-
+
+ @patch.object(manifestGenerator, 'generateManifest')
@patch.object(PuppetExecutor, 'isJavaAvailable')
@patch.object(PuppetExecutor, 'runPuppetFile')
- def test_run_command(self, runPuppetFileMock, isJavaAvailableMock):
+ def test_run_command(self, runPuppetFileMock, isJavaAvailableMock, generateManifestMock):
tmpdir = tempfile.gettempdir()
puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig())
jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
@@ -69,6 +70,7 @@ class TestPuppetExecutor(TestCase):
def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect1
+ generateManifestMock.return_value = ''
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = True
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
@@ -83,8 +85,13 @@ class TestPuppetExecutor(TestCase):
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 999)
self.assertFalse(puppetInstance.reposInstalled)
- os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) + '.pp')
-
+
+ generateManifestMock.return_value = 'error during manifest generation'
+ res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt')
+ self.assertTrue(generateManifestMock.called)
+ self.assertEquals(res["exitcode"], 1)
+ generateManifestMock.return_value = ''
+
def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect2
@@ -105,6 +112,7 @@ class TestPuppetExecutor(TestCase):
self.assertEquals(res["exitcode"], 1)
self.assertEquals(res["stderr"], "Cannot access JDK! Make sure java64_home is specified in global config")
+
@patch.object(PuppetExecutor, 'isJavaAvailable')
@patch.object(RepoInstaller, 'generate_repo_manifests')
@patch.object(PuppetExecutor, 'runPuppetFile')
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index a9c73bd..4a41a15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -60,6 +60,7 @@ public class StageUtils {
new HashMap<String, String>();
private volatile static Gson gson;
+ private static final String DEFAULT_PING_PORT = "8670";
public static void setGson(Gson gson) {
if (gson==null) {
@@ -233,7 +234,8 @@ public class StageUtils {
List<String> allHostPingPorts = new ArrayList<String>();
for (Host host : allHosts.values()) {
allHostNames.add(host.getHostName());
- allHostPingPorts.add(host.getCurrentPingPort() == null ? null : host.getCurrentPingPort().toString());
+ allHostPingPorts.add(host.getCurrentPingPort() == null ?
+ DEFAULT_PING_PORT : host.getCurrentPingPort().toString());
}
info.put("all_hosts", allHostNames);
info.put("all_ping_ports", allHostPingPorts);
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/015130d4/ambari-server/src/test/java/org/apache/ambari/server/utils/TestStageUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestStageUtils.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestStageUtils.java
index fbb382f..8eebb22 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestStageUtils.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestStageUtils.java
@@ -194,10 +194,10 @@ public class TestStageUtils {
fsm.getHost("h2").setOsType("centos5");
fsm.getHost("h3").setOsType("centos5");
fsm.getHost("h4").setOsType("centos5");
- fsm.getHost("h1").setCurrentPingPort(1024);
- fsm.getHost("h2").setCurrentPingPort(1024);
- fsm.getHost("h3").setCurrentPingPort(1024);
- fsm.getHost("h4").setCurrentPingPort(1024);
+ fsm.getHost("h1").setCurrentPingPort(8670);
+ fsm.getHost("h2").setCurrentPingPort(null);
+ fsm.getHost("h3").setCurrentPingPort(null);
+ fsm.getHost("h4").setCurrentPingPort(8670);
fsm.getHost("h1").persist();
fsm.getHost("h2").persist();
fsm.getHost("h3").persist();
@@ -220,6 +220,10 @@ public class TestStageUtils {
assertEquals(4, info.get("all_hosts").size());
assertEquals(4, info.get("all_ping_ports").size());
assertEquals("h1", info.get("hbase_master_hosts").get(0));
+ assertEquals("8670", info.get("all_ping_ports").get(0));
+ assertEquals("8670", info.get("all_ping_ports").get(1));
+ assertEquals("8670", info.get("all_ping_ports").get(2));
+ assertEquals("8670", info.get("all_ping_ports").get(3));
assertFalse(info.get("ambari_db_rca_url").get(0).contains(Configuration.HOSTNAME_MACRO));
String address = InetAddress.getLocalHost().getCanonicalHostName();