You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2018/08/14 00:23:52 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: AMBARI-24427:
Fix broken Python UTs in ambari-server code -- Part 1 (#2042)
This is an automated email from the ASF dual-hosted git repository.
jluniya pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 89abbc6 AMBARI-24427: Fix broken Python UTs in ambari-server code -- Part 1 (#2042)
89abbc6 is described below
commit 89abbc62538260a5f6a958c3d5dc9fcf0c46b339
Author: sduan <sd...@hortonworks.com>
AuthorDate: Mon Aug 13 17:23:50 2018 -0700
AMBARI-24427: Fix broken Python UTs in ambari-server code -- Part 1 (#2042)
* AMBARI-24427: Fix broken Python UTs in ambari-server code -- Part 1
* AMBARI-24427: Fix broken Python UTs in ambari-server code -- Part 1
---
.../libraries/functions/stack_features.py | 29 ++--
.../libraries/functions/stack_select.py | 12 +-
.../src/main/python/ambari_server/setupMpacks.py | 2 +-
ambari-server/src/test/python/TestBootstrap.py | 2 +-
ambari-server/src/test/python/TestMpacks.py | 10 +-
.../src/test/python/TestServiceAdvisor.py | 1 +
ambari-server/src/test/python/TestStackFeature.py | 24 ++--
ambari-server/src/test/python/TestStackSelect.py | 21 +--
.../python/mpacks/mpack_advisor_data/mpack.json | 159 +++++++++++++++++++++
9 files changed, 214 insertions(+), 46 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index d754cd9..aa2f5c4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -41,21 +41,22 @@ def check_stack_feature(stack_feature, stack_version):
:param stack_version: Version of the stack
:return: Will return True if successful, otherwise, False.
"""
-
- from resource_management.libraries.functions.default import default
+ from resource_management.libraries.script.script import Script
+ from resource_management.libraries.execution_command.execution_command import ExecutionCommand
from resource_management.libraries.functions.version import compare_versions
- stack_name = default("/stackSettings/stack_name", None)
+ execution_command = Script.get_execution_command()
+ stack_name = execution_command.get_mpack_name()
if stack_name is None:
Logger.warning("Cannot find the stack name in the command. Stack features cannot be loaded")
return False
-
- stack_features_setting = stack_settings.get_stack_setting_value(stack_settings.STACK_FEATURES_SETTING)
+ # TODO: If stack_features is needed, this should be added in execution_command lib
+ stack_features_setting = execution_command.get_value("stackSettings/"+stack_settings.STACK_FEATURES_SETTING)
# TODO : Removed the below if of reading from cluster_env, once we have removed stack_features from there
# and have started using /stackSettings as source of truth.
if stack_features_setting is None:
Logger.debug("Couldn't retrieve 'stack_features' from /stackSettings. Retrieving from cluster_env now.")
- stack_features_setting = default("/configurations/cluster-env/"+stack_settings.STACK_FEATURES_SETTING, None)
+ stack_features_setting = execution_command.get_value("clusterSettings/"+stack_settings.STACK_FEATURES_SETTING)
if not stack_version:
@@ -102,24 +103,28 @@ def get_stack_feature_version(config):
and commandParams from.
:return: the version to use when checking stack features.
"""
- from resource_management.libraries.functions.default import default
+ from resource_management.libraries.script.script import Script
+ from resource_management.libraries.execution_command.execution_command import ExecutionCommand
+
+ execution_command = Script.get_execution_command()
- if "stackSettings" not in config or "commandParams" not in config:
+ if "stackSettings" not in config and "commandParams" not in config:
raise Fail("Unable to determine the correct version since stackSettings and commandParams were not present in the configuration dictionary")
# should always be there
# Actually not always, i.e if we restart zookeeper service and no stack_version is included in command.json
- stack_version = default("/stackSettings/stack_version", None)
+ stack_version = execution_command.get_mpack_version()
# something like 2.4.0.0-1234; represents the version for the command
# (or None if this is a cluster install and it hasn't been calculated yet)
# this is always guaranteed to be the correct version for the command, even in
# upgrade and downgrade scenarios
- command_version = default("/commandParams/version", None)
- command_stack = default("/commandParams/target_stack", None)
+ command_version = execution_command.get_new_mpack_version_for_upgrade()
+ # TODO we may need add this to execution_command lib
+ command_stack = execution_command.get_value("commandParams/target_stack", None)
# UPGRADE or DOWNGRADE (or None)
- upgrade_direction = default("/commandParams/upgrade_direction", None)
+ upgrade_direction = execution_command.check_upgrade_direction()
# start out with the value that's right 99% of the time
version_for_stack_feature_checks = command_version if command_version is not None else stack_version
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index a50e7df..c7fa910 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -171,31 +171,33 @@ def get_packages(scope, service_name = None, component_name = None):
:param component_name: the component name, such as ZOOKEEPER_SERVER
:return: the packages to use with stack-select or None
"""
+ from resource_management.libraries.execution_command.execution_command import ExecutionCommand
from resource_management.libraries.functions.default import default
if scope not in _PACKAGE_SCOPES:
raise Fail("The specified scope of {0} is not valid".format(scope))
config = Script.get_config()
+ execution_command = Script.get_execution_command()
if service_name is None or component_name is None:
if 'role' not in config or 'serviceName' not in config:
raise Fail("Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool")
- service_name = config['serviceName']
- component_name = config['role']
+ service_name = execution_command.get_component_instance_name()
+ component_name = execution_command.get_component_type()
- stack_name = default("/stackSettings/stack_name", None)
+ stack_name = execution_command.get_mpack_name()
if stack_name is None:
raise Fail("The stack name is not present in the command. Packages for stack-select tool cannot be loaded.")
- stack_packages_setting = stack_settings.get_stack_setting_value(stack_settings.STACK_PACKAGES_SETTING)
+ stack_packages_setting = execution_command.get_value("stackSettings/"+stack_settings.STACK_PACKAGES_SETTING)
# TODO : Removed the below if of reading from cluster_env, once we have removed stack_packages from there
# and have started using /stackSettings as source of truth.
if stack_packages_setting is None:
Logger.debug("Couldn't retrieve 'stack_packages' from /stackSettings. Retrieving from cluster_env now.")
- stack_packages_setting = default("/configurations/cluster-env/"+stack_settings.STACK_PACKAGES_SETTING, None)
+ stack_packages_setting = execution_command.get_value("clusterSettings/"+stack_settings.STACK_PACKAGES_SETTING)
if stack_packages_setting is None:
raise Fail("The stack packages are not defined on the command. Unable to load packages for the stack-select tool")
diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py b/ambari-server/src/main/python/ambari_server/setupMpacks.py
index b8c0974..c59371c 100755
--- a/ambari-server/src/main/python/ambari_server/setupMpacks.py
+++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py
@@ -617,7 +617,7 @@ def _uninstall_mpack(mpack_name, mpack_version):
staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir)
if not staged_mpack_metadata:
print_error_msg("Skipping malformed management pack {0}-{1}. Metadata file missing!".format(
- staged_mpack_name, staged_mpack_version))
+ mpack_name, mpack_version))
continue
staged_mpack_name = staged_mpack_metadata.name
staged_mpack_version = staged_mpack_metadata.version
diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py
index 3fc1873..9547b18 100644
--- a/ambari-server/src/test/python/TestBootstrap.py
+++ b/ambari-server/src/test/python/TestBootstrap.py
@@ -36,7 +36,7 @@ from mock.mock import create_autospec
from only_for_platform import not_for_platform, os_distro_value, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
-class TestBootstrap:#(TestCase):
+class TestBootstrap(TestCase):
def setUp(self):
logging.basicConfig(level=logging.ERROR)
diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py
index 1a45038..f32f06a 100644
--- a/ambari-server/src/test/python/TestMpacks.py
+++ b/ambari-server/src/test/python/TestMpacks.py
@@ -589,11 +589,11 @@ class TestMpacks(TestCase):
@patch("ambari_server.setupMpacks.download_mpack")
@patch("ambari_server.setupMpacks.run_os_command")
@patch("ambari_server.setupMpacks.set_file_permissions")
-
def test_upgrade_stack_mpack(self, set_file_permissions_mock, run_os_command_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock,
_uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock,
get_ambari_version_mock, create_symlink_mock, read_ambari_user_mock, os_mkdir_mock, shutil_move_mock,
os_path_exists_mock, create_symlink_using_path_mock):
+
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/mystack-1.0.0.1.tar.gz"
download_mpack_mock.side_effect = ["/tmp/mystack-1.0.0.1.tar.gz", "/tmp/mystack-1.0.0.1.tar.gz"]
@@ -603,11 +603,11 @@ class TestMpacks(TestCase):
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, True, True,
- True, True, True, True, True, True, True, False, False, True,
+ True, True, True, True, True, True, True, False, False, False, True,
True, False, False, True, False, False, False, False, False, True,
True, True, True, False, True, True, False, True, True, False,
False, False, False, False, True, True, True, True, True, True,
- True, False, True, False, True, True, True, True, True, True,
+ True, False, True, False, True, True, True, True, True, True, True,
True]
get_ambari_properties_mock.return_value = configs
@@ -765,10 +765,10 @@ class TestMpacks(TestCase):
os.path.join(dashboards_directory, "grafana-dashboards/SERVICEC"), True)
]
- os_path_exists_mock.assert_has_calls(os_path_exists_calls)
+ os_path_exists_mock.assert_has_calls(os_path_exists_calls, True)
self.assertFalse(purge_stacks_and_mpacks_mock.called)
run_os_command_mock.assert_has_calls(run_os_command_calls)
- os_mkdir_mock.assert_has_calls(os_mkdir_calls)
+ os_mkdir_mock.assert_has_calls(os_mkdir_calls, True)
create_symlink_mock.assert_has_calls(create_symlink_calls)
self.assertEqual(18, create_symlink_mock.call_count)
create_symlink_using_path_mock.assert_has_calls(create_symlink_using_path_calls)
diff --git a/ambari-server/src/test/python/TestServiceAdvisor.py b/ambari-server/src/test/python/TestServiceAdvisor.py
index 41e5b5c..87e8a6d 100644
--- a/ambari-server/src/test/python/TestServiceAdvisor.py
+++ b/ambari-server/src/test/python/TestServiceAdvisor.py
@@ -35,6 +35,7 @@ class TestServiceAdvisor(TestCase):
imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE))
serviceAdvisorPath = os.path.join(resources_path, 'stacks/service_advisor.py')
+ os.environ["advisor"] = "mpack"
with open(serviceAdvisorPath, 'rb') as fp:
service_advisor_impl = imp.load_module('service_advisor_impl', fp, serviceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py
index 8e584b1..889b9f4 100644
--- a/ambari-server/src/test/python/TestStackFeature.py
+++ b/ambari-server/src/test/python/TestStackFeature.py
@@ -23,6 +23,7 @@ from resource_management.core.logger import Logger
from resource_management.libraries.functions.stack_features import get_stack_feature_version
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
+from resource_management.libraries.execution_command.execution_command import ExecutionCommand
from resource_management.core.exceptions import Fail
from unittest import TestCase
@@ -68,7 +69,7 @@ class TestStackFeature(TestCase):
"""
command_json = TestStackFeature._get_cluster_install_command_json()
Script.config = command_json
-
+ Script.execution_command = ExecutionCommand(Script.config)
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.4", stack_feature_version)
@@ -80,7 +81,7 @@ class TestStackFeature(TestCase):
"""
command_json = TestStackFeature._get_cluster_upgrade_restart_json()
Script.config = command_json
-
+ Script.execution_command = ExecutionCommand(Script.config)
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.5.9.9-9999", stack_feature_version)
@@ -92,7 +93,7 @@ class TestStackFeature(TestCase):
"""
command_json = TestStackFeature._get_cluster_downgrade_restart_json()
Script.config = command_json
-
+ Script.execution_command = ExecutionCommand(Script.config)
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.4.0.0-1234", stack_feature_version)
@@ -104,7 +105,7 @@ class TestStackFeature(TestCase):
"""
command_json = TestStackFeature._get_cluster_downgrade_stop_json()
Script.config = command_json
-
+ Script.execution_command = ExecutionCommand(Script.config)
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.5.9.9-9999", stack_feature_version)
@@ -122,11 +123,10 @@ class TestStackFeature(TestCase):
"""
command_json = TestStackFeature._get_cluster_upgrade_restart_json()
Script.config = command_json
-
- Script.config["configurations"] = {}
- Script.config["configurations"]["cluster-env"] = {}
- Script.config["configurations"]["cluster-env"]["stack_features"] = {}
- Script.config["configurations"]["cluster-env"]["stack_features"] = json.dumps(TestStackFeature._get_stack_feature_json())
+ Script.execution_command = ExecutionCommand(Script.config)
+ Script.config["clusterSettings"] = {}
+ Script.config["clusterSettings"]["stack_features"] = {}
+ Script.config["clusterSettings"]["stack_features"] = json.dumps(TestStackFeature._get_stack_feature_json())
stack_feature_version = get_stack_feature_version(command_json)
self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version))
@@ -151,7 +151,7 @@ class TestStackFeature(TestCase):
return {
"serviceName":"HDFS",
"roleCommand": "ACTIONEXECUTE",
- "clusterLevelParams": {
+ "stackSettings": {
"stack_name": "HDP",
"stack_version": "2.4",
},
@@ -171,7 +171,7 @@ class TestStackFeature(TestCase):
return {
"serviceName":"HDFS",
"roleCommand":"ACTIONEXECUTE",
- "clusterLevelParams": {
+ "stackSettings": {
"stack_name": "HDP",
"stack_version": "2.4",
},
@@ -212,7 +212,7 @@ class TestStackFeature(TestCase):
return {
"serviceName":"HDFS",
"roleCommand":"ACTIONEXECUTE",
- "clusterLevelParams":{
+ "stackSettings":{
"stack_name":"HDP",
"stack_version":"2.4"
},
diff --git a/ambari-server/src/test/python/TestStackSelect.py b/ambari-server/src/test/python/TestStackSelect.py
index 462d3a0..18a90d1 100644
--- a/ambari-server/src/test/python/TestStackSelect.py
+++ b/ambari-server/src/test/python/TestStackSelect.py
@@ -25,6 +25,7 @@ from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions import stack_select
from resource_management.libraries.script import Script
+from resource_management.libraries.execution_command.execution_command import ExecutionCommand
from unittest import TestCase
@@ -59,9 +60,9 @@ class TestStackSelect(TestCase):
Script.config = dict()
Script.config.update(command_json)
- Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } )
- Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages()
- Script.config["clusterLevelParams"] = { "stack_name" : "HDP" }
+ Script.config.update( { "clusterSettings" : {}, "clusterLevelParams": {} } )
+ Script.config["clusterSettings"]["stack_packages"] = self._get_stack_packages()
+ Script.execution_command = ExecutionCommand(Script.config)
stack_select.select_packages(version)
@@ -85,9 +86,9 @@ class TestStackSelect(TestCase):
Script.config = dict()
Script.config.update(command_json)
- Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } )
- Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages()
- Script.config["clusterLevelParams"] = { "stack_name" : "HDP" }
+ Script.config.update( { "clusterSettings" : {}, "clusterLevelParams": {} } )
+ Script.config["clusterSettings"]["stack_packages"] = self._get_stack_packages()
+ Script.execution_command = ExecutionCommand(Script.config)
stack_select.select_packages(version)
@@ -119,9 +120,9 @@ class TestStackSelect(TestCase):
Script.config = dict()
Script.config.update(command_json)
- Script.config.update( { "configurations" : { "cluster-env" : {} }, "clusterLevelParams": {} } )
- Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages_with_legacy()
- Script.config["clusterLevelParams"] = { "stack_name" : "HDP" }
+ Script.config.update({"clusterSettings": {}, "clusterLevelParams": {}})
+ Script.config["clusterSettings"]["stack_packages"] = self._get_stack_packages_with_legacy()
+ Script.execution_command = ExecutionCommand(Script.config)
stack_select.select_packages(version)
@@ -178,7 +179,7 @@ class TestStackSelect(TestCase):
"roleCommand":"ACTIONEXECUTE",
"serviceName": "FOO_SERVICE",
"role": "FOO_MASTER",
- "clusterLevelParams": {
+ "stackSettings": {
"stack_name": "HDP",
"stack_version": "2.4",
},
diff --git a/ambari-server/src/test/python/mpacks/mpack_advisor_data/mpack.json b/ambari-server/src/test/python/mpacks/mpack_advisor_data/mpack.json
new file mode 100644
index 0000000..fb20d43
--- /dev/null
+++ b/ambari-server/src/test/python/mpacks/mpack_advisor_data/mpack.json
@@ -0,0 +1,159 @@
+{
+ "definition": "ods-1.0.0-b509-definition.tar.gz",
+ "description": "Hortonworks Operational Data Store for services like HBASE.",
+ "displayName": "Hortonworks Operational Data Store",
+ "id": "ods",
+ "modules": [
+ {
+ "category": "CLIENT",
+ "components": [
+ {
+ "category": "CLIENT",
+ "id": "hadoop_client",
+ "isExternal": false,
+ "name": "HADOOP_CLIENT",
+ "version": "3.0.0.0-b235"
+ }
+ ],
+ "definition": "hadoop_clients-3.0.0.0-b235-definition.tar.gz",
+ "dependencies": [
+ {
+ "id": "hdfs",
+ "name": "HDFS",
+ "type": "RUNTIME"
+ },
+ {
+ "id": "yarn",
+ "name": "YARN",
+ "type": "RUNTIME"
+ },
+ {
+ "id": "mapreduce2",
+ "name": "MAPREDUCE2",
+ "type": "RUNTIME"
+ }
+ ],
+ "description": "Clients for HDFS, YARN and MAPREDUCE services",
+ "displayName": "Hadoop Clients",
+ "hidden": true,
+ "id": "hadoop_clients",
+ "name": "HADOOP_CLIENTS",
+ "version": "3.0.0.0-b235"
+ },
+ {
+ "category": "CLIENT",
+ "components": [
+ {
+ "category": "CLIENT",
+ "id": "zookeeper_client",
+ "isExternal": false,
+ "name": "ZOOKEEPER_CLIENT",
+ "version": "3.4.0.0-b43"
+ }
+ ],
+ "definition": "zookeeper_clients-3.4.0.0-b43-definition.tar.gz",
+ "dependencies": [
+ {
+ "id": "zookeeper",
+ "name": "ZOOKEEPER",
+ "type": "RUNTIME"
+ }
+ ],
+ "description": "Clients for ZooKeeper",
+ "displayName": "ZooKeeper Clients",
+ "hidden": true,
+ "id": "zookeeper_clients",
+ "name": "ZOOKEEPER_CLIENTS",
+ "version": "3.4.0.0-b43"
+ },
+ {
+ "category": "SERVER",
+ "components": [
+ {
+ "category": "MASTER",
+ "id": "hbase_master",
+ "isExternal": false,
+ "name": "HBASE_MASTER",
+ "version": "2.0.0.0-b257"
+ },
+ {
+ "category": "SLAVE",
+ "id": "hbase_regionserver",
+ "isExternal": false,
+ "name": "HBASE_REGIONSERVER",
+ "version": "2.0.0.0-b257"
+ }
+ ],
+ "definition": "hbase-2.0.0.0-b257-definition.tar.gz",
+ "dependencies": [
+ {
+ "id": "zookeeper_clients",
+ "name": "ZOOKEEPER_CLIENTS",
+ "type": "INSTALL"
+ },
+ {
+ "id": "hbase_clients",
+ "name": "HBASE_CLIENTS",
+ "type": "INSTALL"
+ }
+ ],
+ "description": "Non-relational distributed database and centralized service for configuration management &\n synchronization\n ",
+ "displayName": "HBase",
+ "hidden": false,
+ "id": "hbase",
+ "name": "HBASE",
+ "version": "2.0.0.0-b257"
+ },
+ {
+ "category": "CLIENT",
+ "components": [
+ {
+ "category": "CLIENT",
+ "id": "hbase_client",
+ "isExternal": false,
+ "name": "HBASE_CLIENT",
+ "version": "2.0.0.0-b257"
+ }
+ ],
+ "definition": "hbase_clients-2.0.0.0-b257-definition.tar.gz",
+ "dependencies": [
+ {
+ "id": "zookeeper_clients",
+ "name": "ZOOKEEPER_CLIENTS",
+ "type": "INSTALL"
+ },
+ {
+ "id": "hbase",
+ "name": "HBASE",
+ "type": "RUNTIME"
+ }
+ ],
+ "description": "Non-relational distributed database and centralized service for configuration management &\n synchronization\n ",
+ "displayName": "HBase",
+ "hidden": false,
+ "id": "hbase_clients",
+ "name": "HBASE_CLIENTS",
+ "version": "2.0.0.0-b257"
+ }
+ ],
+ "name": "ODS",
+ "osSpecifics": [
+ {
+ "osFamily": "ubuntu12,ubuntu14",
+ "packages": [
+ "ods-1-0-0-b509"
+ ]
+ },
+ {
+ "osFamily": "redhat7",
+ "packages": [
+ "ods_1_0_0_b509"
+ ]
+ }
+ ],
+ "prerequisites": {
+ "max-ambari-version": "3.1.0.0",
+ "min-ambari-version": "3.0.0.0"
+ },
+ "version": "1.0.0-b509"
+}
\ No newline at end of file