You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2018/03/02 13:30:17 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-23121]
Fix few instance manager issues found during ZK deploment (dsen) (#518)
This is an automated email from the ASF dual-hosted git repository.
dsen pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new d7bcc61 [AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) (#518)
d7bcc61 is described below
commit d7bcc61b68539612c4936669b305223b91623ad4
Author: Dmitry Sen <ds...@apache.org>
AuthorDate: Fri Mar 2 15:30:14 2018 +0200
[AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) (#518)
* [AMBARI-23121] Fix few instance manager issues found during ZK deployment (dsen)
* [AMBARI-23121] Fix few instance manager issues found during ZK deployment (dsen) - changes according to review
---
.../python/instance_manager/instance_manager.py | 45 +++++++++++++++-------
.../instance_manager/mpack-instance-manager.py | 17 ++++++--
.../instance_manager/test_instance_manager.py | 43 ++++++++++++++-------
3 files changed, 74 insertions(+), 31 deletions(-)
diff --git a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
index 6682234..c9f610b 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
@@ -51,8 +51,8 @@ def create_mpack(mpack_name, mpack_version, mpack_instance, subgroup_name=DEFAUL
OR
list of 'components_instances_name' to be created. (default) OR '*' for all components
"""
- mpack_name = mpack_name.lower()
- module_name = module_name.lower()
+ mpack_name, module_name, components, components_map = normalize_parameters(
+ mpack_name, module_name, components, components_map)
validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map)
@@ -71,14 +71,17 @@ def set_mpack_instance(mpack, mpack_version, mpack_instance, subgroup_name=DEFAU
OR
list of 'components_instances_name' to be created. (default) OR '*' for all components
"""
- mpack = mpack.lower()
- module_name = module_name.lower()
+ mpack, module_name, components, components_map = normalize_parameters(
+ mpack, module_name, components, components_map)
instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME), mpack,
mpack_instance, subgroup_name, module_name,
components, components_map)
if not instances:
- raise ValueError("Found no instances for the given filters.")
+ raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
+ " subgroup_name:{2}, module_name:{3}, components:{4}, components_map:{5}".
+ format(mpack, mpack_instance, subgroup_name, module_name,
+ components, components_map))
validate_mpack_for_creation_or_changing(mpack, mpack_version, module_name, components, components_map)
@@ -142,11 +145,8 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
The output_conf_dir or output_path for each component instance will be included in json depending on given parameters.
"""
-
- if mpack_name_filter:
- mpack_name_filter = mpack_name_filter.lower()
- if module_name_filter:
- module_name_filter = module_name_filter.lower()
+ mpack_name_filter, module_name_filter, skipped, components_name_filter_map = normalize_parameters(
+ mpack_name_filter, module_name_filter, [], components_name_filter_map)
instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME),
mpack_name_filter,
@@ -155,7 +155,10 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
None,
components_name_filter_map)
if not instances:
- raise ValueError("Found no instances for the given filters.")
+ raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
+ " subgroup_name:{2}, module_name:{3}, components_name_map:{4}".
+ format(mpack_name_filter, instance_name_filter, subgroup_name_filter,
+ module_name_filter, components_name_filter_map))
full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path)
@@ -229,6 +232,22 @@ def build_json_output(instances, output_conf_dir=False, output_path=False):
return {'mpacks': result}
+def normalize_parameters(mpack_name, module_name, components, components_map):
+ """
+ normalize parameters to be lowercase, for components_map dictionary only keys are normalized
+ """
+ if mpack_name:
+ mpack_name = mpack_name.lower()
+ if module_name:
+ module_name = module_name.lower()
+ if components and components != "*":
+ components = [component.lower() for component in components]
+ if components_map:
+ components_map = dict((k.lower(), v) for k, v in components_map.iteritems())
+
+ return mpack_name, module_name, components, components_map
+
+
def validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map):
mpack_root_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name)
if not os.path.exists(mpack_root_path):
@@ -478,7 +497,7 @@ class ModuleInstance(Instance):
(components_name_filter_map and folder_name in components_name_filter_map)):
components_map = ComponentInstance(name=DEFAULT_COMPONENT_INSTANCE_NAME,
component_path=os.path.join(path, folder_name),
- path_exec=os.path.realpath(
+ path_exec=os.readlink(
os.path.join(path, folder_name, CURRENT_SOFTLINK_NAME)))
else:
components_map = ComponentInstance.parse_into_components_dict(os.path.join(path, folder_name),
@@ -566,7 +585,7 @@ class ComponentInstance(Instance):
if not component_names_filter or component_instance_name in component_names_filter:
result[component_instance_name] = ComponentInstance(name=component_instance_name,
component_path=os.path.join(path, component_instance_name),
- path_exec=os.path.realpath(
+ path_exec=os.readlink(
os.path.join(path, component_instance_name,
CURRENT_SOFTLINK_NAME)))
return result
diff --git a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
index 38c29f8..f9458b1 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
@@ -56,8 +56,14 @@ def check_required_options(options, action, parser):
if not options.module_name:
missing_options.append("module-name")
- if missing_options:
- parser.error("Missing following required command options: {0}".format(missing_options))
+ if missing_options:
+ parser.error("Missing following required command options: {0}".format(missing_options))
+
+ if not options.components_map and not options.components:
+ parser.error("Either components or components-map option must be specified.")
+
+ if options.components_map and options.components:
+ parser.error("Only components or components-map option could be specified. Can't use both.")
def init_create_parser_options(parser):
@@ -117,8 +123,11 @@ def main(options, args):
parsed_components = None
parsed_components_map = None
try:
- if hasattr(options, 'components') and options.components and options.components != '*':
- parsed_components = ast.literal_eval(options.components)
+ if hasattr(options, 'components') and options.components:
+ if options.components == '*':
+ parsed_components = '*'
+ else:
+ parsed_components = ast.literal_eval(options.components)
if options.components_map:
parsed_components_map = ast.literal_eval(options.components_map)
except ValueError:
diff --git a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
index 5315094..0d834cc 100644
--- a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
+++ b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
@@ -84,7 +84,7 @@ class TestInstanceManager(TestCase):
instance_manager.DEFAULT_MPACK_INSTANCE_NAME)))
def test_create_mpack_server_module_with_multiple_component_instances(self):
- create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME: ['server1', 'server2']})
+ create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME.upper(): ['server1', 'server2']})
current_link_1 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME,
INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME,
@@ -121,12 +121,12 @@ class TestInstanceManager(TestCase):
MPACK_VERSION_2, SERVER_COMPONENT_NAME))
def test_set_version_client_module_asterisk(self):
- create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
+ create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False)
- instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME,
- CLIENT_MODULE_NAME, '*')
+ instance_manager.set_mpack_instance(MPACK_NAME.upper(), MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME,
+ CLIENT_MODULE_NAME.upper(), '*')
current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME,
INSTANCE_NAME_1, SUBGROUP_NAME, CLIENT_COMPONENT_NAME,
@@ -160,9 +160,9 @@ class TestInstanceManager(TestCase):
MPACK_VERSION_2, SERVER_COMPONENT_NAME))
def test_get_conf_dir_all(self):
- create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
- create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
- components_map={SERVER_COMPONENT_NAME: ['server1']})
+ create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+ create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+ components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
conf_dir_json = instance_manager.get_conf_dir()
@@ -210,9 +210,9 @@ class TestInstanceManager(TestCase):
self.assertEqual(conf_dir_json, expected_json)
def test_list_instances_all(self):
- create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
- create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
- components_map={SERVER_COMPONENT_NAME: ['server1']})
+ create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+ create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+ components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
conf_dir_json = instance_manager.list_instances()
@@ -232,7 +232,7 @@ class TestInstanceManager(TestCase):
"hdfs_server": {
"component-instances": {
"server1": {
- "path": "/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1",
+ "path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_server",
"name": "server1"
}
}
@@ -243,7 +243,7 @@ class TestInstanceManager(TestCase):
"category": "CLIENT",
"component_instances": {
"default": {
- "path": "/tmp/instance_manager_test/modules/hdfs-clients/3.1.0.0-b1",
+ "path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_client",
"name": "default"
}
},
@@ -317,7 +317,7 @@ class TestInstanceManager(TestCase):
expected_filter_result = {'mpacks': {'edw': {'mpack-instances': {'eCommerce': {'name': 'eCommerce', 'subgroups': {
'default': {'modules': {'hdfs': {'category': 'SERVER', 'name': 'hdfs', 'components': {'hdfs_server': {
'component-instances': {
- 'server2': {'path': '/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1', 'name': 'server2'}}}}}}}}}}}}}
+ 'server2': {'path': '/tmp/instance_manager_test/mpacks/edw/1.0.0-b1/hdfs_server', 'name': 'server2'}}}}}}}}}}}}}
self.assertEquals(expected_filter_result, filter_by_component_instance_name_json)
def test_validation(self):
@@ -363,6 +363,19 @@ class TestInstanceManager(TestCase):
"The instance /tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/"
"hdfs_server/default already exist. To change the version use set-mpack-instance command")
+ def test_normalize_parameters(self):
+ mpack_name = MPACK_NAME.upper()
+ module_name = SERVER_MODULE_NAME.upper()
+ components = [SERVER_COMPONENT_NAME.upper()]
+ components_map = {SERVER_COMPONENT_NAME.upper(): ["DEFAULT"]}
+ mpack_name, module_name, components, components_map = instance_manager.normalize_parameters(
+ mpack_name, module_name, components, components_map)
+
+ self.assertEquals(mpack_name, MPACK_NAME.lower())
+ self.assertEquals(module_name, SERVER_MODULE_NAME.lower())
+ self.assertEquals(components, [SERVER_COMPONENT_NAME.lower()])
+ self.assertEquals(components_map, {SERVER_COMPONENT_NAME.lower(): ["DEFAULT"]})
+
def test_set_non_existing_instance(self):
try:
instance_manager.set_mpack_instance(mpack=MPACK_NAME, mpack_version=MPACK_VERSION_1,
@@ -383,7 +396,9 @@ class TestInstanceManager(TestCase):
raise AssertionError("The previous call should have thrown exception")
except ValueError as e:
self.assertEquals(e.message,
- "Found no instances for the given filters.")
+ "Found no instances for the given filters: mpack_name:hdpcore, instance_name:Production,"
+ " subgroup_name:default, module_name:hdfs, components:None,"
+ " components_map:{'hdfs_server': ['non-existing-instance']}")
def create_mpack_with_defaults(mpack_name=MPACK_NAME, mpack_version=MPACK_VERSION_1, mpack_instance=INSTANCE_NAME_1,
--
To stop receiving notification emails like this one, please contact
dsen@apache.org.