You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/03 14:53:22 UTC
[18/50] [abbrv] ambari git commit: AMBARI-19816. Agent heartbeat lost
due to dead service check process. (Attila Doroszlai via stoader)
AMBARI-19816. Agent heartbeat lost due to dead service check process. (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0458adda
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0458adda
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0458adda
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 0458addaaa3b99943e3e3ecf93483f4009b2c120
Parents: a8a43bc
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Wed Feb 1 19:21:32 2017 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Wed Feb 1 19:21:54 2017 +0100
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostInfo.py | 19 ++++---
.../test/python/ambari_agent/TestHostInfo.py | 45 +++++++++--------
.../python/ambari_agent/TestRegistration.py | 2 +-
ambari-agent/src/test/python/unitTests.py | 3 ++
.../src/main/python/ambari_commons/firewall.py | 21 ++++----
.../src/test/python/TestAmbariServer.py | 52 +++++++-------------
6 files changed, 68 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index 3810b52..7f041eb 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -30,6 +30,7 @@ import time
from ambari_commons import OSCheck, OSConst
from ambari_commons.firewall import Firewall
from ambari_commons.os_family_impl import OsFamilyImpl
+from resource_management.core import shell
from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
@@ -169,6 +170,7 @@ class HostInfoLinux(HostInfo):
DEFAULT_SERVICE_NAME = "ntpd"
SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME)
+ SERVICE_STATUS_CMD_LIST = shlex.split(SERVICE_STATUS_CMD)
THP_FILE_REDHAT = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled"
@@ -333,13 +335,16 @@ class HostInfoLinux(HostInfo):
pass
- def getServiceStatus(self, serivce_name):
- service_check_live = shlex.split(self.SERVICE_STATUS_CMD)
- service_check_live[1] = serivce_name
- osStat = subprocess.Popen(service_check_live, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = osStat.communicate()
- return out, err, osStat.returncode
+ def getServiceStatus(self, service_name):
+ service_check_live = list(self.SERVICE_STATUS_CMD_LIST)
+ service_check_live[1] = service_name
+ try:
+ code, out, err = shell.call(service_check_live, stdout = subprocess.PIPE, stderr = subprocess.PIPE, timeout = 5, quiet = True)
+ return out, err, code
+ except Exception as ex:
+ logger.warn("Checking service {0} status failed".format(service_name))
+ return '', str(ex), 1
+
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index fcfbc85..67294e0 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -22,7 +22,6 @@ limitations under the License.
from unittest import TestCase
import logging
import unittest
-import subprocess
import socket
import platform
from mock.mock import patch
@@ -38,6 +37,7 @@ from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
from ambari_agent.HostInfo import HostInfo, HostInfoLinux
from ambari_agent.Hardware import Hardware
from ambari_agent.AmbariConfig import AmbariConfig
+from resource_management.core import shell
from resource_management.core.system import System
from resource_management.libraries.functions import packages_analyzer
@@ -360,25 +360,20 @@ class TestHostInfo(TestCase):
self.assertEquals(list[0]['user'], 'user')
@patch.object(OSCheck, "get_os_type")
- @patch("subprocess.Popen")
- def test_checkLiveServices(self, subproc_popen, get_os_type_method):
+ @patch("resource_management.core.shell.call")
+ def test_checkLiveServices(self, shell_call, get_os_type_method):
+ get_os_type_method.return_value = 'redhat'
hostInfo = HostInfoLinux()
- p = MagicMock()
- p.returncode = 0
- p.communicate.return_value = ('', 'err')
- subproc_popen.return_value = p
+
+ shell_call.return_value = (0, '', 'err')
result = []
- get_os_type_method.return_value = 'redhat'
hostInfo.checkLiveServices([('service1',)], result)
+ self.assertEquals(result[0]['desc'], '')
self.assertEquals(result[0]['status'], 'Healthy')
self.assertEquals(result[0]['name'], 'service1')
- self.assertEquals(result[0]['desc'], '')
- self.assertEquals(str(subproc_popen.call_args_list),
- "[call(['service', 'service1', 'status'], stderr=-1, stdout=-1)]")
- p.returncode = 1
- p.communicate.return_value = ('out', 'err')
+ shell_call.return_value = (1, 'out', 'err')
result = []
hostInfo.checkLiveServices([('service1',)], result)
@@ -386,7 +381,7 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['name'], 'service1')
self.assertEquals(result[0]['desc'], 'out')
- p.communicate.return_value = ('', 'err')
+ shell_call.return_value = (1, '', 'err')
result = []
hostInfo.checkLiveServices([('service1',)], result)
@@ -394,7 +389,7 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['name'], 'service1')
self.assertEquals(result[0]['desc'], 'err')
- p.communicate.return_value = ('', 'err', '')
+ shell_call.return_value = (1, '', 'err')
result = []
hostInfo.checkLiveServices([('service1',)], result)
@@ -402,8 +397,7 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['name'], 'service1')
self.assertTrue(len(result[0]['desc']) > 0)
- p.returncode = 0
- p.communicate.return_value = ('', 'err')
+ shell_call.return_value = (0, '', 'err')
result = []
hostInfo.checkLiveServices([('service1', 'service2',)], result)
@@ -411,8 +405,7 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['name'], 'service1 or service2')
self.assertEquals(result[0]['desc'], '')
- p.returncode = 1
- p.communicate.return_value = ('out', 'err')
+ shell_call.return_value = (1, 'out', 'err')
result = []
hostInfo.checkLiveServices([('service1', 'service2',)], result)
@@ -420,6 +413,16 @@ class TestHostInfo(TestCase):
self.assertEquals(result[0]['name'], 'service1 or service2')
self.assertEquals(result[0]['desc'], 'out\nout')
+ msg = 'thrown by shell call'
+ shell_call.side_effect = Exception(msg)
+ result = []
+ hostInfo.checkLiveServices([('service1',)], result)
+
+ self.assertEquals(result[0]['status'], 'Unhealthy')
+ self.assertEquals(result[0]['name'], 'service1')
+ self.assertEquals(result[0]['desc'], msg)
+
+
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('redhat','11','Final')))
@patch("os.path.exists")
@patch("os.listdir", create=True, autospec=True)
@@ -445,7 +448,7 @@ class TestHostInfo(TestCase):
@patch.object(OSCheck, "get_os_family")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_major_version")
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
def test_FirewallRunning(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
get_os_type_mock.return_value = ""
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
@@ -480,7 +483,7 @@ class TestHostInfo(TestCase):
@patch.object(OSCheck, "get_os_family")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_major_version")
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
def test_FirewallStopped(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
get_os_type_mock.return_value = ""
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
index f5e0288..a5c23fa 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
@@ -37,7 +37,7 @@ class TestRegistration(TestCase):
@patch.object(Hardware, "_chk_writable_mount", new = MagicMock(return_value=True))
@patch.object(FacterLinux, "facterInfo", new = MagicMock(return_value={}))
@patch.object(FacterLinux, "__init__", new = MagicMock(return_value = None))
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_registration_build(self, get_os_version_mock, get_os_type_mock, run_os_cmd_mock, Popen_mock):
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-agent/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/unitTests.py b/ambari-agent/src/test/python/unitTests.py
index b70d226..785699f 100644
--- a/ambari-agent/src/test/python/unitTests.py
+++ b/ambari-agent/src/test/python/unitTests.py
@@ -47,6 +47,7 @@ import fnmatch
from os.path import isdir
import logging
from only_for_platform import get_platform, PLATFORM_WINDOWS
+from resource_management.core.logger import Logger
#TODO Add an option to randomize the tests' execution
#from random import shuffle
@@ -182,4 +183,6 @@ if __name__ == '__main__':
consoleLog = logging.StreamHandler(file)
consoleLog.setFormatter(formatter)
logger.addHandler(consoleLog)
+ Logger.initialize_logger(logging_level = logging.WARNING)
+
main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-common/src/main/python/ambari_commons/firewall.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/firewall.py b/ambari-common/src/main/python/ambari_commons/firewall.py
index 6868d3f..43983ca 100644
--- a/ambari-common/src/main/python/ambari_commons/firewall.py
+++ b/ambari-common/src/main/python/ambari_commons/firewall.py
@@ -18,11 +18,11 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import subprocess
from ambari_commons import OSCheck, OSConst
from ambari_commons.logging_utils import print_warning_msg
from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons.os_utils import run_os_command
-
+from resource_management.core import shell
class Firewall(object):
def __init__(self):
@@ -81,10 +81,13 @@ class FirewallChecks(object):
return result
def run_command(self):
- retcode, out, err = run_os_command(self.get_command())
- self.returncode = retcode
- self.stdoutdata = out
- self.stderrdata = err
+ try:
+ retcode, out, err = shell.call(self.get_command(), stdout = subprocess.PIPE, stderr = subprocess.PIPE, timeout = 5, quiet = True)
+ self.returncode = retcode
+ self.stdoutdata = out
+ self.stderrdata = err
+ except Exception as ex:
+ print_warning_msg("Unable to check firewall status: {0}".format(ex))
def check_firewall(self):
try:
@@ -129,12 +132,6 @@ class RedHat7FirewallChecks(FirewallChecks):
return False
- def run_command(self):
- retcode, out, err = run_os_command(self.get_command())
- self.returncode = retcode
- self.stdoutdata = out
- self.stderrdata = err
-
class Fedora18FirewallChecks(FirewallChecks):
def __init__(self):
super(Fedora18FirewallChecks, self).__init__()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0458adda/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 81e1a22..7dae7c0 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -1920,25 +1920,21 @@ class TestAmbariServer(TestCase):
self.assertTrue(set_file_permissions_mock.called)
pass
- @patch("subprocess.Popen")
+ @patch("resource_management.core.shell.call")
@patch.object(OSCheck, "get_os_family")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_major_version")
- def test_check_firewall_is_running(self, get_os_major_version_mock, get_os_type_mock, get_os_family_mock, popen_mock):
+ def test_check_firewall_is_running(self, get_os_major_version_mock, get_os_type_mock, get_os_family_mock, shell_call_mock):
get_os_major_version_mock.return_value = 18
get_os_type_mock.return_value = OSConst.OS_FEDORA
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
firewall_obj = Firewall().getFirewallObject()
- p = MagicMock()
- p.communicate.return_value = ("active", "err")
- p.returncode = 0
- popen_mock.return_value = p
+ shell_call_mock.return_value = (0, "active", "err")
self.assertEqual("Fedora18FirewallChecks", firewall_obj.__class__.__name__)
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("", "err")
- p.returncode = 3
+ shell_call_mock.return_value = (3, "", "err")
self.assertFalse(firewall_obj.check_firewall())
self.assertEqual("err", firewall_obj.stderrdata)
@@ -1947,12 +1943,10 @@ class TestAmbariServer(TestCase):
get_os_family_mock.return_value = OSConst.UBUNTU_FAMILY
firewall_obj = Firewall().getFirewallObject()
- p.communicate.return_value = ("Status: active", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "Status: active", "err")
self.assertEqual("UbuntuFirewallChecks", firewall_obj.__class__.__name__)
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("Status: inactive", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "Status: inactive", "err")
self.assertFalse(firewall_obj.check_firewall())
self.assertEqual("err", firewall_obj.stderrdata)
@@ -1960,12 +1954,10 @@ class TestAmbariServer(TestCase):
get_os_family_mock.return_value = OSConst.SUSE_FAMILY
firewall_obj = Firewall().getFirewallObject()
- p.communicate.return_value = ("running", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "running", "err")
self.assertEqual("SuseFirewallChecks", firewall_obj.__class__.__name__)
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("unused", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "unused", "err")
self.assertFalse(firewall_obj.check_firewall())
self.assertEqual("err", firewall_obj.stderrdata)
@@ -1974,12 +1966,10 @@ class TestAmbariServer(TestCase):
get_os_major_version_mock.return_value = 6
firewall_obj = Firewall().getFirewallObject()
- p.communicate.return_value = ("Table: filter", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "Table: filter", "err")
self.assertEqual("FirewallChecks", firewall_obj.__class__.__name__)
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("", "err")
- p.returncode = 3
+ shell_call_mock.return_value = (3, "", "err")
self.assertFalse(firewall_obj.check_firewall())
self.assertEqual("err", firewall_obj.stderrdata)
@@ -1988,18 +1978,14 @@ class TestAmbariServer(TestCase):
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
firewall_obj = Firewall().getFirewallObject()
- p.communicate.return_value = ("active\nactive", "err")
- p.returncode = 0
+ shell_call_mock.return_value = (0, "active\nactive", "err")
self.assertEqual("RedHat7FirewallChecks", firewall_obj.__class__.__name__)
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("inactive\nactive", "err")
- p.returncode = 3
+ shell_call_mock.return_value = (3, "inactive\nactive", "err")
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("active\ninactive", "err")
- p.returncode = 3
+ shell_call_mock.return_value = (3, "active\ninactive", "err")
self.assertTrue(firewall_obj.check_firewall())
- p.communicate.return_value = ("inactive\ninactive", "err")
- p.returncode = 3
+ shell_call_mock.return_value = (3, "inactive\ninactive", "err")
self.assertFalse(firewall_obj.check_firewall())
self.assertEqual("err", firewall_obj.stderrdata)
@@ -3666,7 +3652,7 @@ class TestAmbariServer(TestCase):
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
@patch("pwd.getpwnam")
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch("os.path.exists")
@patch("os.path.isfile")
@patch("ambari_commons.os_utils.remove_file")
@@ -3901,7 +3887,7 @@ class TestAmbariServer(TestCase):
@only_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch("os.path.exists")
@patch("os.path.isfile")
@patch("ambari_commons.os_utils.remove_file")
@@ -6224,7 +6210,7 @@ class TestAmbariServer(TestCase):
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch("ambari_server.serverSetup.verify_setup_allowed")
@patch("sys.exit")
@patch("ambari_server.serverSetup.get_YN_input")
@@ -6293,7 +6279,7 @@ class TestAmbariServer(TestCase):
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch("sys.exit")
@patch("ambari_server.userInput.get_YN_input")
@patch("ambari_commons.os_utils.is_root")
@@ -8067,7 +8053,7 @@ class TestAmbariServer(TestCase):
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch("ambari_commons.firewall.run_os_command")
+ @patch("resource_management.core.shell.call")
@patch("ambari_server.dbConfiguration_linux.PGConfig._is_jdbc_user_changed")
@patch("ambari_server.serverSetup.verify_setup_allowed")
@patch("ambari_server.serverSetup.get_YN_input")