You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2018/04/16 09:59:58 UTC
[ambari] 01/06: AMBARI-23536. Fix TestCustomServiceOrchestrator.py
and TestRegistration.py on branch-3.0-perf (aonishuk)
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 8f889e9a0fdf5435408bc9ebea9a280bdd806f15
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Wed Apr 11 10:07:51 2018 +0300
AMBARI-23536. Fix TestCustomServiceOrchestrator.py and TestRegistration.py on branch-3.0-perf (aonishuk)
---
.../ambari_agent/CustomServiceOrchestrator.py | 43 -----
.../main/python/ambari_agent/InitializerModule.py | 3 +-
.../ambari_agent/TestCustomServiceOrchestrator.py | 184 +++++++++------------
.../test/python/ambari_agent/TestRegistration.py | 10 +-
4 files changed, 79 insertions(+), 161 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 604d01c..a806199 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -553,49 +553,6 @@ class CustomServiceOrchestrator():
f.write(content)
return file_path
- def decompressClusterHostInfo(self, clusterHostInfo):
- info = clusterHostInfo.copy()
- #Pop info not related to host roles
- hostsList = info.pop(self.HOSTS_LIST_KEY)
- pingPorts = info.pop(self.PING_PORTS_KEY)
- racks = info.pop(self.RACKS_KEY)
- ipv4_addresses = info.pop(self.IPV4_ADDRESSES_KEY)
-
- ambariServerHost = info.pop(self.AMBARI_SERVER_HOST)
- ambariServerPort = info.pop(self.AMBARI_SERVER_PORT)
- ambariServerUseSsl = info.pop(self.AMBARI_SERVER_USE_SSL)
-
- decompressedMap = {}
-
- for k,v in info.items():
- # Convert from 1-3,5,6-8 to [1,2,3,5,6,7,8]
- indexes = self.convertRangeToList(v)
- # Convert from [1,2,3,5,6,7,8] to [host1,host2,host3...]
- decompressedMap[k] = [hostsList[i] for i in indexes]
-
- #Convert from ['1:0-2,4', '42:3,5-7'] to [1,1,1,42,1,42,42,42]
- pingPorts = self.convertMappedRangeToList(pingPorts)
- racks = self.convertMappedRangeToList(racks)
- ipv4_addresses = self.convertMappedRangeToList(ipv4_addresses)
-
- #Convert all elements to str
- pingPorts = map(str, pingPorts)
-
- #Add ping ports to result
- decompressedMap[self.PING_PORTS_KEY] = pingPorts
- #Add hosts list to result
- decompressedMap[self.HOSTS_LIST_KEY] = hostsList
- #Add racks list to result
- decompressedMap[self.RACKS_KEY] = racks
- #Add ips list to result
- decompressedMap[self.IPV4_ADDRESSES_KEY] = ipv4_addresses
- #Add ambari-server properties to result
- decompressedMap[self.AMBARI_SERVER_HOST] = ambariServerHost
- decompressedMap[self.AMBARI_SERVER_PORT] = ambariServerPort
- decompressedMap[self.AMBARI_SERVER_USE_SSL] = ambariServerUseSsl
-
- return decompressedMap
-
# Converts from 1-3,5,6-8 to [1,2,3,5,6,7,8]
def convertRangeToList(self, list):
diff --git a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py
index ea2e043..fbfab47 100644
--- a/ambari-agent/src/main/python/ambari_agent/InitializerModule.py
+++ b/ambari-agent/src/main/python/ambari_agent/InitializerModule.py
@@ -48,14 +48,13 @@ class InitializerModule:
"""
def __init__(self):
self.stop_event = threading.Event()
+ self.config = AmbariConfig.get_resolved_config()
self.init()
def init(self):
"""
Initialize properties
"""
- self.config = AmbariConfig.get_resolved_config()
-
self.is_registered = False
self.metadata_cache = ClusterMetadataCache(self.config.cluster_cache_dir)
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 30c4075..badcb5f 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -44,8 +44,10 @@ from ambari_agent.FileCache import FileCache
from ambari_agent.PythonExecutor import PythonExecutor
from ambari_commons import OSCheck
from only_for_platform import get_platform, os_distro_value, PLATFORM_WINDOWS
+from ambari_agent.InitializerModule import InitializerModule
+from ambari_agent.ConfigurationBuilder import ConfigurationBuilder
-class TestCustomServiceOrchestrator:#(TestCase):
+class TestCustomServiceOrchestrator(TestCase):
def setUp(self):
@@ -65,83 +67,6 @@ class TestCustomServiceOrchestrator:#(TestCase):
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch.object(FileCache, "__init__")
- def test_add_reg_listener_to_controller(self, FileCache_mock):
- FileCache_mock.return_value = None
- dummy_controller = MagicMock()
- config = AmbariConfig()
- tempdir = tempfile.gettempdir()
- config.set('agent', 'prefix', tempdir)
- CustomServiceOrchestrator(config, dummy_controller)
- self.assertTrue(dummy_controller.registration_listeners.append.called)
-
-
- @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
- @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo')
- @patch("ambari_agent.hostname.public_hostname")
- @patch("os.path.isfile")
- @patch("os.unlink")
- @patch.object(FileCache, "__init__")
- def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
- isfile_mock, hostname_mock,
- decompress_cluster_host_info_mock):
- FileCache_mock.return_value = None
- hostname_mock.return_value = "test.hst"
- command = {
- 'commandType': 'EXECUTION_COMMAND',
- 'role': u'DATANODE',
- 'roleCommand': u'INSTALL',
- 'commandId': '1-1',
- 'taskId': 3,
- 'clusterName': u'cc',
- 'serviceName': u'HDFS',
- 'configurations':{'global' : {}},
- 'configurationTags':{'global' : { 'tag': 'v1' }},
- 'clusterHostInfo':{'namenode_host' : ['1'],
- 'slave_hosts' : ['0', '1'],
- 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
- 'all_ping_ports': ['8670:0,1']},
- 'hostLevelParams':{}
- }
-
- decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
- 'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
- 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
- 'all_ping_ports': ['8670', '8670']}
-
- config = AmbariConfig()
- tempdir = tempfile.gettempdir()
- config.set('agent', 'prefix', tempdir)
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
- isfile_mock.return_value = True
- # Test dumping EXECUTION_COMMAND
- json_file = orchestrator.dump_command_to_json(command)
- self.assertTrue(os.path.exists(json_file))
- self.assertTrue(os.path.getsize(json_file) > 0)
- if get_platform() != PLATFORM_WINDOWS:
- self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
- self.assertTrue(json_file.endswith("command-3.json"))
- self.assertTrue(decompress_cluster_host_info_mock.called)
- os.unlink(json_file)
- # Test dumping STATUS_COMMAND
- command['commandType']='STATUS_COMMAND'
- decompress_cluster_host_info_mock.reset_mock()
- json_file = orchestrator.dump_command_to_json(command)
- self.assertTrue(os.path.exists(json_file))
- self.assertTrue(os.path.getsize(json_file) > 0)
- if get_platform() != PLATFORM_WINDOWS:
- self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
- self.assertTrue(json_file.endswith("status_command.json"))
- self.assertFalse(decompress_cluster_host_info_mock.called)
- os.unlink(json_file)
- # Testing side effect of dump_command_to_json
- self.assertEquals(command['public_hostname'], "test.hst")
- self.assertEquals(command['agentConfigParams']['agent']['parallel_execution'], 0)
- self.assertTrue(unlink_mock.called)
-
-
- @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
@patch("ambari_agent.hostname.public_hostname")
@patch("os.path.isfile")
@patch("os.unlink")
@@ -172,11 +97,11 @@ class TestCustomServiceOrchestrator:#(TestCase):
'hostLevelParams':{}
}
- config = AmbariConfig()
tempdir = tempfile.gettempdir()
- config.set('agent', 'prefix', tempdir)
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ initializer_module.config.set('agent', 'prefix', tempdir)
+ orchestrator = CustomServiceOrchestrator(initializer_module)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
@@ -195,8 +120,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
- self.assertEquals(command['public_hostname'], "test.hst")
- self.assertEquals(command['agentConfigParams']['agent']['parallel_execution'], 0)
+ self.assertNotEquals(command['clusterHostInfo'], {})
self.assertTrue(unlink_mock.called)
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
@@ -206,7 +130,9 @@ class TestCustomServiceOrchestrator:#(TestCase):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
config = AmbariConfig()
- orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
# Testing existing path
exists_mock.return_value = True
path = orchestrator.\
@@ -221,6 +147,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
except AgentException:
pass # Expected
+ @patch.object(ConfigurationBuilder, "get_configuration")
@patch.object(FileCache, "get_custom_resources_subdir")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@@ -236,15 +163,17 @@ class TestCustomServiceOrchestrator:#(TestCase):
get_host_scripts_base_dir_mock,
resolve_hook_script_path_mock,
resolve_script_path_mock,
- get_custom_resources_subdir_mock):
+ get_custom_resources_subdir_mock, get_configuration_mock):
FileCache_mock.return_value = None
command = {
'commandType' : 'EXECUTION_COMMAND',
'role' : 'REGION_SERVER',
- 'hostLevelParams' : {
+ 'clusterLevelParams' : {
'stack_name' : 'HDP',
'stack_version' : '2.0.7',
+ },
+ 'ambariLevelParams': {
'jdk_location' : 'some_location'
},
'commandParams': {
@@ -254,8 +183,10 @@ class TestCustomServiceOrchestrator:#(TestCase):
'service_package_folder' : 'HBASE'
},
'taskId' : '3',
- 'roleCommand': 'INSTALL'
+ 'roleCommand': 'INSTALL',
+ 'clusterId': '-1',
}
+ get_configuration_mock.return_value = command
get_host_scripts_base_dir_mock.return_value = "/host_scripts"
get_service_base_dir_mock.return_value = "/basedir/"
@@ -263,8 +194,9 @@ class TestCustomServiceOrchestrator:#(TestCase):
resolve_hook_script_path_mock.return_value = \
('/hooks_dir/prefix-command/scripts/hook.py',
'/hooks_dir/prefix-command')
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
unix_process_id = 111
orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
get_hook_base_dir_mock.return_value = "/hooks/"
@@ -329,6 +261,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
pass
+ @patch.object(ConfigurationBuilder, "get_configuration")
@patch("ambari_commons.shell.kill_process_with_children")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@@ -343,13 +276,15 @@ class TestCustomServiceOrchestrator:#(TestCase):
get_hook_base_dir_mock, get_service_base_dir_mock,
get_host_scripts_base_dir_mock,
resolve_hook_script_path_mock, resolve_script_path_mock,
- kill_process_with_children_mock):
+ kill_process_with_children_mock, get_configuration_mock):
FileCache_mock.return_value = None
command = {
'role' : 'REGION_SERVER',
- 'hostLevelParams' : {
+ 'clusterLevelParams' : {
'stack_name' : 'HDP',
- 'stack_version' : '2.0.7',
+ 'stack_version' : '2.0.7'
+ },
+ 'ambariLevelParams': {
'jdk_location' : 'some_location'
},
'commandParams': {
@@ -359,17 +294,20 @@ class TestCustomServiceOrchestrator:#(TestCase):
'service_package_folder' : 'HBASE'
},
'taskId' : '3',
- 'roleCommand': 'INSTALL'
+ 'roleCommand': 'INSTALL',
+ 'clusterId': '-1'
}
-
+ get_configuration_mock.return_value = command
+
get_host_scripts_base_dir_mock.return_value = "/host_scripts"
get_service_base_dir_mock.return_value = "/basedir/"
resolve_script_path_mock.return_value = "/basedir/scriptpath"
resolve_hook_script_path_mock.return_value = \
('/hooks_dir/prefix-command/scripts/hook.py',
'/hooks_dir/prefix-command')
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
unix_process_id = 111
orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
get_hook_base_dir_mock.return_value = "/hooks/"
@@ -408,6 +346,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
pass
@patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+ @patch.object(ConfigurationBuilder, "get_configuration")
@patch.object(CustomServiceOrchestrator, "get_py_executor")
@patch("ambari_commons.shell.kill_process_with_children")
@patch.object(FileCache, "__init__")
@@ -415,7 +354,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
@patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
def test_cancel_backgound_command(self, resolve_hook_script_path_mock,
resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock,
- get_py_executor_mock):
+ get_py_executor_mock, get_configuration_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
@@ -425,10 +364,14 @@ class TestCustomServiceOrchestrator:#(TestCase):
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
- actionQueue = ActionQueue(cfg, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+
+ actionQueue = ActionQueue(initializer_module)
+ orchestrator = CustomServiceOrchestrator(initializer_module)
- dummy_controller.actionQueue = actionQueue
- orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
+ initializer_module.actionQueue = actionQueue
+
orchestrator.file_cache = MagicMock()
def f (a, b):
return ""
@@ -455,6 +398,8 @@ class TestCustomServiceOrchestrator:#(TestCase):
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
+ get_configuration_mock.return_value = execute_command
+
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
@@ -477,6 +422,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
+ @patch.object(ConfigurationBuilder, "get_configuration")
@patch.object(AmbariConfig, "get")
@patch.object(CustomServiceOrchestrator, "dump_command_to_json")
@patch.object(PythonExecutor, "run_file")
@@ -484,7 +430,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
@patch.object(FileCache, "get_custom_actions_base_dir")
def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
- run_file_mock, dump_command_to_json_mock, ambari_config_get):
+ run_file_mock, dump_command_to_json_mock, ambari_config_get, get_configuration_mock):
ambari_config_get.return_value = "0"
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
@@ -495,13 +441,22 @@ class TestCustomServiceOrchestrator:#(TestCase):
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
+ },
+ 'ambariLevelParams': {
'jdk_location' : 'some_location'
},
'taskId' : '3',
- 'roleCommand': 'ACTIONEXECUTE'
+ 'roleCommand': 'ACTIONEXECUTE',
+ 'clusterId': '-1',
}
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ get_configuration_mock.return_value = command
+
+ initializer_module = InitializerModule()
+ initializer_module.config = self.config
+ initializer_module.init()
+
+
+ orchestrator = CustomServiceOrchestrator(initializer_module)
unix_process_id = 111
orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
# normal run case
@@ -523,7 +478,9 @@ class TestCustomServiceOrchestrator:#(TestCase):
def test_resolve_hook_script_path(self, FileCache_mock, isfile_mock):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
# Testing None param
res1 = orchestrator.resolve_hook_script_path(None, "prefix", "command",
"script_type")
@@ -553,7 +510,9 @@ class TestCustomServiceOrchestrator:#(TestCase):
'configurations':{}
}
dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
# Test alive case
runCommand_mock.return_value = {
"exitcode" : 0
@@ -570,6 +529,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
self.assertEqual(runCommand_mock.return_value, status)
+ @patch.object(ConfigurationBuilder, "get_configuration")
@patch.object(CustomServiceOrchestrator, "get_py_executor")
@patch.object(CustomServiceOrchestrator, "dump_command_to_json")
@patch.object(FileCache, "__init__")
@@ -577,7 +537,7 @@ class TestCustomServiceOrchestrator:#(TestCase):
def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
dump_command_to_json_mock,
- get_py_executor_mock):
+ get_py_executor_mock, get_configuration_mock):
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
_, script = tempfile.mkstemp()
@@ -587,16 +547,20 @@ class TestCustomServiceOrchestrator:#(TestCase):
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
+ },
+ 'ambariLevelParams': {
'jdk_location' : 'some_location'
},
+ 'clusterId': '-1',
'taskId' : '13',
'roleCommand': 'ACTIONEXECUTE',
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
MagicMock(), MagicMock())
}
- dummy_controller = MagicMock()
- orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+ initializer_module = InitializerModule()
+ initializer_module.init()
+ orchestrator = CustomServiceOrchestrator(initializer_module)
import TestActionQueue
pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
diff --git a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
index db30be0..a0a4283 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
@@ -30,7 +30,7 @@ from ambari_agent.Hardware import Hardware
from ambari_agent.Facter import FacterLinux
@not_for_platform(PLATFORM_WINDOWS)
-class TestRegistration:#(TestCase):
+class TestRegistration(TestCase):
@patch("subprocess32.Popen")
@patch.object(Hardware, "_chk_writable_mount", new = MagicMock(return_value=True))
@@ -50,17 +50,15 @@ class TestRegistration:#(TestCase):
run_os_cmd_mock.return_value = (3, "", "")
register = Register(config)
reference_version = '2.1.0'
- data = register.build(reference_version, 1)
+ data = register.build()
self.assertEquals(len(data['hardwareProfile']) > 0, True, "hardwareProfile should contain content")
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
- self.assertEquals(data['responseId'], 1)
+ self.assertEquals(data['id'], -1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(len(data['agentEnv']) > 0, True, "agentEnv should not be empty")
- self.assertEquals(data['agentVersion'], reference_version, "agentVersion should not be empty")
self.assertEquals(not data['agentEnv']['umask']== "", True, "agents umask should not be empty")
- self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
self.assertEquals(data['prefix'], config.get('agent', 'prefix'), 'The prefix path does not match')
- self.assertEquals(len(data), 9)
+ self.assertEquals(len(data), 10)
--
To stop receiving notification emails like this one, please contact
aonishuk@apache.org.