You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2014/05/25 20:05:05 UTC
git commit: AMBARI-5885. Usability: check that hostnames are
resolveable on each host (Jonathan Hurley via ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 2234623b9 -> 790217c66
AMBARI-5885. Usability: check that hostnames are resolveable on each host (Jonathan Hurley via ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/790217c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/790217c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/790217c6
Branch: refs/heads/trunk
Commit: 790217c667a07865f666a1f3cc5de2f3ad2fe4cb
Parents: 2234623
Author: Nate Cole <nc...@hortonworks.com>
Authored: Sun May 25 13:34:25 2014 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Sun May 25 13:34:25 2014 -0400
----------------------------------------------------------------------
.../main/resources/custom_actions/check_host.py | 262 ++++++++++++-------
ambari-server/src/test/python/TestCheckHost.py | 68 +++++
ambari-server/src/test/python/unitTests.py | 3 +-
.../custom_actions/check_host_ip_addresses.json | 51 ++++
4 files changed, 285 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/790217c6/ambari-server/src/main/resources/custom_actions/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/check_host.py b/ambari-server/src/main/resources/custom_actions/check_host.py
index e5aa920..6ed0b31 100644
--- a/ambari-server/src/main/resources/custom_actions/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/check_host.py
@@ -20,12 +20,15 @@ Ambari Agent
"""
-import os, sys, subprocess
-from resource_management import *
+import os
+import subprocess
+import socket
+from resource_management import Script, Execute
CHECK_JAVA_HOME = "java_home_check"
CHECK_DB_CONNECTION = "db_connection_check"
+CHECK_HOST_RESOLUTION = "host_resolution_check"
DB_NAME_MYSQL = "mysql"
DB_NAME_ORACLE = "oracle"
@@ -45,111 +48,174 @@ class CheckHost(Script):
check_execute_list = config['commandParams']['check_execute_list']
structured_output = {}
+ # check each of the commands; if an unknown exception wasn't handled
+ # by the functions, then produce a generic exit_code : 1
if CHECK_JAVA_HOME in check_execute_list:
- java_home_check_structured_output = execute_java_home_available_check(config)
- structured_output[CHECK_JAVA_HOME] = java_home_check_structured_output
+ try :
+ java_home_check_structured_output = self.execute_java_home_available_check(config)
+ structured_output[CHECK_JAVA_HOME] = java_home_check_structured_output
+ except Exception, exception:
+ print "There was an unexpected error while checking for the Java home location: " + str(exception)
+ structured_output[CHECK_JAVA_HOME] = {"exit_code" : "1", "message": str(exception)}
if CHECK_DB_CONNECTION in check_execute_list:
- db_connection_check_structured_output = execute_db_connection_check(config)
- structured_output[CHECK_DB_CONNECTION] = db_connection_check_structured_output
+ try :
+ db_connection_check_structured_output = self.execute_db_connection_check(config)
+ structured_output[CHECK_DB_CONNECTION] = db_connection_check_structured_output
+ except Exception, exception:
+ print "There was an unknown error while checking database connectivity: " + str(exception)
+ structured_output[CHECK_DB_CONNECTION] = {"exit_code" : "1", "message": str(exception)}
+
+ if CHECK_HOST_RESOLUTION in check_execute_list:
+ try :
+ host_resolution_structured_output = self.execute_host_resolution_check(config)
+ structured_output[CHECK_HOST_RESOLUTION] = host_resolution_structured_output
+ except Exception, exception :
+ print "There was an unknown error while checking IP address lookups: " + str(exception)
+ structured_output[CHECK_HOST_RESOLUTION] = {"exit_code" : "1", "message": str(exception)}
self.put_structured_out(structured_output)
-def execute_java_home_available_check(config):
- print "Java home check started."
- java64_home = config['hostLevelParams']['java_home']
-
- if not os.path.isfile(os.path.join(java64_home, "bin", "java")):
- print "Java home doesn't exist!"
- java_home_check_structured_output = {"exit_code" : "1", "message": "Java home doesn't exist!"}
- else:
- print "Java home exists!"
- java_home_check_structured_output = {"exit_code" : "0", "message": "Java home exists!"}
-
- return java_home_check_structured_output
-
-
-def execute_db_connection_check(config):
- print "DB connection check started."
-
- # initialize needed data
-
- ambari_server_hostname = config['clusterHostInfo']['ambari_server_host']
- check_db_connection_jar_name = "DBConnectionVerification.jar"
- jdk_location = config['hostLevelParams']['jdk_location']
- java64_home = config['hostLevelParams']['java_home']
- db_name = config['commandParams']['db_name']
-
- if db_name == DB_NAME_MYSQL:
- jdbc_url = config['hostLevelParams']['mysql_jdbc_url']
- jdbc_driver = JDBC_DRIVER_MYSQL
- elif db_name == DB_NAME_ORACLE:
- jdbc_url = config['hostLevelParams']['oracle_jdbc_url']
- jdbc_driver = JDBC_DRIVER_ORACLE
- elif db_name == DB_NAME_POSTGRESQL:
- jdbc_url = config['hostLevelParams']['postgresql_jdbc_url']
- jdbc_driver = JDBC_DRIVER_POSTGRESQL
-
- path, jdbc_name = os.path.split(jdbc_url)
- db_connection_url = config['commandParams']['db_connection_url']
- user_name = config['commandParams']['user_name']
- user_passwd = config['commandParams']['user_passwd']
-
- environment = { "no_proxy": format("{ambari_server_hostname}") }
-
- # download DBConnectionVerification.jar from ambari-server resources
-
-
- try:
- cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
- "--retry 5 {jdk_location}{check_db_connection_jar_name} "
- "-o {check_db_connection_jar_name}'")
- Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"), environment = environment)
- except Exception, e:
- message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
- "Ambari Server.\n" + str(e)
- print message
- db_connection_check_structured_output = {"exit_code" : "1", "message": message}
+ def execute_java_home_available_check(self, config):
+ print "Java home check started."
+ java64_home = config['hostLevelParams']['java_home']
+
+ if not os.path.isfile(os.path.join(java64_home, "bin", "java")):
+ print "Java home doesn't exist!"
+ java_home_check_structured_output = {"exit_code" : "1", "message": "Java home doesn't exist!"}
+ else:
+ print "Java home exists!"
+ java_home_check_structured_output = {"exit_code" : "0", "message": "Java home exists!"}
+
+ return java_home_check_structured_output
+
+
+ def execute_db_connection_check(self, config):
+ print "DB connection check started."
+
+ # initialize needed data
+
+ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host']
+ check_db_connection_jar_name = "DBConnectionVerification.jar"
+ jdk_location = config['hostLevelParams']['jdk_location']
+ java64_home = config['hostLevelParams']['java_home']
+ db_name = config['commandParams']['db_name']
+
+ if db_name == DB_NAME_MYSQL:
+ jdbc_url = config['hostLevelParams']['mysql_jdbc_url']
+ jdbc_driver = JDBC_DRIVER_MYSQL
+ elif db_name == DB_NAME_ORACLE:
+ jdbc_url = config['hostLevelParams']['oracle_jdbc_url']
+ jdbc_driver = JDBC_DRIVER_ORACLE
+ elif db_name == DB_NAME_POSTGRESQL:
+ jdbc_url = config['hostLevelParams']['postgresql_jdbc_url']
+ jdbc_driver = JDBC_DRIVER_POSTGRESQL
+
+ path, jdbc_name = os.path.split(jdbc_url)
+ db_connection_url = config['commandParams']['db_connection_url']
+ user_name = config['commandParams']['user_name']
+ user_passwd = config['commandParams']['user_passwd']
+
+ environment = { "no_proxy": format("{ambari_server_hostname}") }
+
+ # download DBConnectionVerification.jar from ambari-server resources
+
+ try:
+ cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
+ "--retry 5 {jdk_location}{check_db_connection_jar_name} "
+ "-o {check_db_connection_jar_name}'")
+ Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{check_db_connection_jar_name}]"), environment = environment)
+ except Exception, e:
+ message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \
+ "Ambari Server.\n" + str(e)
+ print message
+ db_connection_check_structured_output = {"exit_code" : "1", "message": message}
+ return db_connection_check_structured_output
+
+ # download jdbc driver from ambari-server resources
+
+ try:
+ cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
+ "--retry 5 {jdbc_url} -o {jdbc_name}'")
+ Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{jdbc_name}]"), environment = environment)
+ except Exception, e:
+ message = "Error downloading JDBC connector from Ambari Server resources. Check network access to " \
+ "Ambari Server.\n" + str(e)
+ print message
+ db_connection_check_structured_output = {"exit_code" : "1", "message": message}
+ return db_connection_check_structured_output
+
+
+ # try to connect to db
+
+ db_connection_check_command = format("{java64_home}/bin/java -cp /usr/lib/ambari-agent/{check_db_connection_jar_name}:" \
+ "/usr/lib/ambari-agent/{jdbc_name} org.apache.ambari.server.DBConnectionVerification {db_connection_url} " \
+ "{user_name} {user_passwd!p} {jdbc_driver}")
+
+ process = subprocess.Popen(db_connection_check_command,
+ stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=True)
+ (stdoutdata, stderrdata) = process.communicate()
+ print "INFO stdoutdata: " + stdoutdata
+ print "INFO stderrdata: " + stderrdata
+ print "INFO returncode: " + str(process.returncode)
+
+ if process.returncode == 0:
+ db_connection_check_structured_output = {"exit_code" : "0", "message": "DB connection check completed successfully!" }
+ else:
+ db_connection_check_structured_output = {"exit_code" : "1", "message": stdoutdata }
+
return db_connection_check_structured_output
-
- # download jdbc driver from ambari-server resources
-
- try:
- cmd = format("/bin/sh -c 'cd /usr/lib/ambari-agent/ && curl -kf "
- "--retry 5 {jdbc_url} -o {jdbc_name}'")
- Execute(cmd, not_if=format("[ -f /usr/lib/ambari-agent/{jdbc_name}]"), environment = environment)
- except Exception, e:
- message = "Error downloading JDBC connector from Ambari Server resources. Check network access to " \
- "Ambari Server.\n" + str(e)
+
+ # check whether each host in the command can be resolved to an IP address
+ def execute_host_resolution_check(self, config):
+ print "IP address forward resolution check started."
+
+ FORWARD_LOOKUP_REASON = "FORWARD_LOOKUP"
+
+ failedCount = 0
+ failures = []
+
+ if config['commandParams']['hosts'] is not None :
+ hosts = config['commandParams']['hosts'].split(",")
+ successCount = len(hosts)
+ else :
+ successCount = 0
+ hosts = ""
+
+ socket.setdefaulttimeout(3)
+ for host in hosts:
+ try:
+ host = host.strip()
+ socket.gethostbyname(host)
+ except socket.error,exception:
+ successCount -= 1
+ failedCount += 1
+
+ failure = { "host": host, "type": FORWARD_LOOKUP_REASON,
+ "cause": exception.args }
+
+ failures.append(failure)
+
+ if failedCount > 0 :
+ message = "There were " + str(failedCount) + " host(s) that could not resolve to an IP address."
+ else :
+ message = "All hosts resolved to an IP address."
+
print message
- db_connection_check_structured_output = {"exit_code" : "1", "message": message}
- return db_connection_check_structured_output
-
-
- # try to connect to db
-
- db_connection_check_command = format("{java64_home}/bin/java -cp /usr/lib/ambari-agent/{check_db_connection_jar_name}:" \
- "/usr/lib/ambari-agent/{jdbc_name} org.apache.ambari.server.DBConnectionVerification {db_connection_url} " \
- "{user_name} {user_passwd!p} {jdbc_driver}")
-
- process = subprocess.Popen(db_connection_check_command,
- stdout=subprocess.PIPE,
- stdin=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=True)
- (stdoutdata, stderrdata) = process.communicate()
- print "INFO stdoutdata: " + stdoutdata
- print "INFO stderrdata: " + stderrdata
- print "INFO returncode: " + str(process.returncode)
-
- if process.returncode == 0:
- db_connection_check_structured_output = {"exit_code" : "0", "message": "DB connection check completed successfully!" }
- else:
- db_connection_check_structured_output = {"exit_code" : "1", "message": stdoutdata }
-
- return db_connection_check_structured_output
-
+
+ host_resolution_check_structured_output = {
+ "exit_code" : "0",
+ "message" : message,
+ "failed_count" : failedCount,
+ "success_count" : successCount,
+ "failures" : failures
+ }
+
+ return host_resolution_check_structured_output
if __name__ == "__main__":
CheckHost().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/790217c6/ambari-server/src/test/python/TestCheckHost.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestCheckHost.py b/ambari-server/src/test/python/TestCheckHost.py
new file mode 100644
index 0000000..e40dee3
--- /dev/null
+++ b/ambari-server/src/test/python/TestCheckHost.py
@@ -0,0 +1,68 @@
+# !/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+import os
+import socket
+from resource_management import Script,ConfigDictionary
+from mock.mock import patch
+from unittest import TestCase
+
+check_host = __import__('check_host')
+from check_host import CheckHost
+
+class TestCheckHost(TestCase):
+
+ @patch("socket.gethostbyname")
+ @patch.object(Script, 'get_config')
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ def testHostResolution(self, structured_out_mock, mock_config, mock_socket):
+ mock_socket.return_value = "192.168.1.1"
+ jsonFilePath = os.path.join("../resources/custom_actions", "check_host_ip_addresses.json")
+
+ with open(jsonFilePath, "r") as jsonFile:
+ jsonPayload = json.load(jsonFile)
+
+ mock_config.return_value = ConfigDictionary(jsonPayload)
+
+ checkHost = CheckHost()
+ checkHost.actionexecute(None)
+
+ # ensure the correct function was called
+ self.assertTrue(structured_out_mock.called)
+ structured_out_mock.assert_called_with({'host_resolution_check':
+ {'failures': [],
+ 'message': 'All hosts resolved to an IP address.',
+ 'failed_count': 0,
+ 'success_count': 5,
+ 'exit_code': '0'}})
+
+ # try it now with errors
+ mock_socket.side_effect = socket.error
+ checkHost.actionexecute(None)
+
+ structured_out_mock.assert_called_with({'host_resolution_check':
+ {'failures': [
+ {'cause': (), 'host': u'c6401.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
+ {'cause': (), 'host': u'c6402.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
+ {'cause': (), 'host': u'c6403.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
+ {'cause': (), 'host': u'foobar', 'type': 'FORWARD_LOOKUP'},
+ {'cause': (), 'host': u'!!!', 'type': 'FORWARD_LOOKUP'}],
+ 'message': 'There were 5 host(s) that could not resolve to an IP address.',
+ 'failed_count': 5, 'success_count': 0, 'exit_code': '0'}})
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/790217c6/ambari-server/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/unitTests.py b/ambari-server/src/test/python/unitTests.py
index afd68c0..ab99cce 100644
--- a/ambari-server/src/test/python/unitTests.py
+++ b/ambari-server/src/test/python/unitTests.py
@@ -124,7 +124,8 @@ def main():
sys.path.append(ambari_server_folder + "/src/test/python")
sys.path.append(ambari_server_folder + "/src/main/python")
sys.path.append(ambari_server_folder + "/src/main/resources/scripts")
-
+ sys.path.append(ambari_server_folder + "/src/main/resources/custom_actions")
+
stacks_folder = pwd+'/stacks'
#generate test variants(path, service, stack)
test_variants = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/790217c6/ambari-server/src/test/resources/custom_actions/check_host_ip_addresses.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/custom_actions/check_host_ip_addresses.json b/ambari-server/src/test/resources/custom_actions/check_host_ip_addresses.json
new file mode 100644
index 0000000..19e3e5b
--- /dev/null
+++ b/ambari-server/src/test/resources/custom_actions/check_host_ip_addresses.json
@@ -0,0 +1,51 @@
+{
+ "roleCommand": "ACTIONEXECUTE",
+ "clusterName": "c1",
+ "hostname": "c6401.ambari.apache.org",
+ "passiveInfo": [],
+ "hostLevelParams": {
+ "jdk_location": "http://192.168.64.1:8080/resources/",
+ "ambari_db_rca_password": "mapred",
+ "java_home": "/usr/jdk64/jdk1.6.0_31",
+ "ambari_db_rca_url": "jdbc:postgresql://192.168.64.1/ambarirca",
+ "stack_name": "HDP",
+ "oracle_jdbc_url": "http://192.168.64.1:8080/resources//ojdbc6.jar",
+ "stack_version": "2.1",
+ "db_name": "ambari",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "jdk_name": "jdk-6u31-linux-x64.bin",
+ "ambari_db_rca_username": "mapred",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "mysql_jdbc_url": "http://192.168.64.1:8080/resources//mysql-connector-java.jar"
+ },
+ "commandType": "EXECUTION_COMMAND",
+ "roleParams": {
+ "script": "check_host.py",
+ "check_execute_list": "host_resolution_check",
+ "threshold": "20",
+ "hosts": "c6401.ambari.apache.org, c6402.ambari.apache.org, c6403.ambari.apache.org, foobar, !!!",
+ "command_timeout": "60",
+ "script_type": "PYTHON"
+ },
+ "serviceName": "null",
+ "role": "check_host",
+ "commandParams": {
+ "script": "check_host.py",
+ "check_execute_list": "host_resolution_check",
+ "threshold": "20",
+ "hosts": "c6401.ambari.apache.org, c6402.ambari.apache.org, c6403.ambari.apache.org, foobar, !!!",
+ "command_timeout": "60",
+ "script_type": "PYTHON"
+ },
+ "taskId": 158,
+ "public_hostname": "c6401.ambari.apache.org",
+ "configurations": {},
+ "commandId": "27-1",
+ "clusterHostInfo": {
+ "ambari_server_host": [
+ "192.168.64.1"
+ ],
+ "all_hosts": [],
+ "all_ping_ports": []
+ }
+}
\ No newline at end of file