You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2018/05/31 19:03:03 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-23993]
Mpack Instance Manager should create pid dir and log dir for each instance
(#1424)
This is an automated email from the ASF dual-hosted git repository.
jluniya pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 7548975 [AMBARI-23993] Mpack Instance Manager should create pid dir and log dir for each instance (#1424)
7548975 is described below
commit 75489756dd893caf473a5825550c87e3240a4634
Author: sduan <sd...@hortonworks.com>
AuthorDate: Thu May 31 12:02:59 2018 -0700
[AMBARI-23993] Mpack Instance Manager should create pid dir and log dir for each instance (#1424)
* AMBARI-23993: Mpack Instance Manager should create pid dir and log dir for each instance
* AMBARI-23993: Mpack Instance Manager should create pid dir and log dir for each instance
---
.../libraries/functions/mpack_manager_helper.py | 27 ++++++
.../python/instance_manager/instance_manager.py | 74 +++++++++++---
.../instance_manager/mpack-instance-manager.py | 18 +++-
.../instance_manager/test_instance_manager.py | 108 +++++++++++++++++++++
4 files changed, 212 insertions(+), 15 deletions(-)
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py
index e99ff27..1bae115 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py
@@ -23,6 +23,8 @@ import os
from instance_manager import create_mpack, set_mpack_instance, get_conf_dir, list_instances
CONFIG_DIR_KEY_NAME = 'config_dir'
+LOG_DIR_KEY_NAME = 'log_dir'
+RUN_DIR_KEY_NAME = 'run_dir'
PATH_KEY_NAME = 'path'
COMPONENTS_PLURAL_KEY_NAME = 'components'
COMPONENT_INSTANCES_PLURAL_KEY_NAME = 'component-instances'
@@ -41,6 +43,31 @@ def get_component_conf_path(mpack_name, instance_name, module_name, components_i
return conf_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][
component_instance_name][CONFIG_DIR_KEY_NAME]
+def get_component_log_path(mpack_name, instance_name, module_name, components_instance_type,
+ subgroup_name='default', component_instance_name='default'):
+ """
+ :returns the single string that contains the path to the log folder of given component instance
+ :raises ValueError if the parameters doesn't match the mpack or instances structure
+ """
+
+ log_json = get_log_dir(mpack_name, instance_name, subgroup_name, module_name,
+ {components_instance_type: [component_instance_name]})
+
+ return log_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][
+ component_instance_name][LOG_DIR_KEY_NAME]
+
+def get_component_rundir_path(mpack_name, instance_name, module_name, components_instance_type,
+ subgroup_name='default', component_instance_name='default'):
+ """
+ :returns the single string that contains the path to the rundir folder of given component instance
+ :raises ValueError if the parameters doesn't match the mpack or instances structure
+ """
+
+ run_json = get_run_dir(mpack_name, instance_name, subgroup_name, module_name,
+ {components_instance_type: [component_instance_name]})
+
+ return run_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][
+ component_instance_name][RUN_DIR_KEY_NAME]
def get_component_target_path(mpack_name, instance_name, module_name, components_instance_type,
subgroup_name='default', component_instance_name='default'):
diff --git a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
index 39e5ea6..0a9ed83 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
@@ -17,7 +17,7 @@ limitations under the License.
"""
-__all__ = ["create_mpack", "set_mpack_instance", "get_conf_dir", "list_instances"]
+__all__ = ["create_mpack", "set_mpack_instance", "get_conf_dir", "get_log_dir", "get_run_dir", "list_instances"]
import sys
import os
@@ -26,6 +26,8 @@ import json
MPACK_JSON_FILE_NAME = 'mpack.json'
CURRENT_SOFTLINK_NAME = 'current'
CONFIGS_DIRECTORY_NAME = 'conf'
+LOG_DIRECTORY_NAME = 'log'
+RUN_DIRECTORY_NAME = 'run'
ROOT_FOLDER_PATH = "/usr/hwx/"
@@ -110,6 +112,46 @@ def get_conf_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP
return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map,
output_conf_dir=True)
+def get_log_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None,
+ components_map=None):
+ """
+ Use case: retrieve log directory paths for a given component instances based on the granularity specified
+ ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance
+ AND with a filtering on each level
+
+ Granularity works only while names for all consecutive levels are specified.
+ Note that subgroup has default value of 'default'
+ Levels: mpack/instance/subgroup/module
+ E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level,
+ though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output
+ of subgroup will be returned.
+
+ Components are provided as map with key as 'component type' and value as 'list of individual component instances
+ names' OR empty map for all component instances present
+ """
+ return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map,
+ output_log_dir=True)
+
+def get_run_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None,
+ components_map=None):
+ """
+ Use case: retrieve run directory paths for a given component instances based on the granularity specified
+ ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance
+ AND with a filtering on each level
+
+ Granularity works only while names for all consecutive levels are specified.
+ Note that subgroup has default value of 'default'
+ Levels: mpack/instance/subgroup/module
+ E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level,
+ though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output
+ of subgroup will be returned.
+
+ Components are provided as map with key as 'component type' and value as 'list of individual component instances
+ names' OR empty map for all component instances present
+ """
+ return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map,
+ output_run_dir=True)
+
def list_instances(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None,
components_map=None):
@@ -134,7 +176,7 @@ def list_instances(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGRO
def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter, subgroup_name_filter,
module_name_filter, components_name_filter_map, output_conf_dir=False,
- output_path=False):
+ output_log_dir=False, output_run_dir=False, output_path=False):
"""
Builds the json that contains all instances specified in filters or all instances if filters are not specified.
The level of granularity depends on the consecutive levels of specified filters
@@ -160,7 +202,7 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
format(mpack_name_filter, instance_name_filter, subgroup_name_filter,
module_name_filter, components_name_filter_map))
- full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path)
+ full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_log_dir=output_log_dir, output_run_dir=output_run_dir, output_path=output_path)
granular_json_output = build_granular_output(full_json_output, mpack_name_filter, instance_name_filter,
subgroup_name_filter,
@@ -190,14 +232,14 @@ def build_granular_output(json_output, mpack_name_filter, instance_name_filter,
return json_output
-def build_json_output_from_instances_dict(instances_dict, plural_name, output_conf_dir, output_path):
+def build_json_output_from_instances_dict(instances_dict, plural_name, output_conf_dir, output_log_dir, output_run_dir, output_path):
"""
Build the json from the dictionary of Instance objects.
The plural_name is used to form the upper level of the json output.
"""
result = {}
for instance_name in instances_dict:
- result[instance_name] = instances_dict[instance_name].build_json_output(output_conf_dir, output_path)
+ result[instance_name] = instances_dict[instance_name].build_json_output(output_conf_dir, output_log_dir, output_run_dir, output_path)
return {plural_name: result}
@@ -224,11 +266,11 @@ def get_module_meta_mpack(path, module_name):
return MetaMpack.parse_mpack(os.path.dirname(current_target))
-def build_json_output(instances, output_conf_dir=False, output_path=False):
+def build_json_output(instances, output_conf_dir=False, output_log_dir=False, output_run_dir=False, output_path=False):
result = {}
for mpack_name in instances.keys():
result[mpack_name] = build_json_output_from_instances_dict(instances[mpack_name], MpackInstance.plural_name,
- output_conf_dir, output_path)
+ output_conf_dir, output_log_dir, output_run_dir, output_path)
return {'mpacks': result}
@@ -365,7 +407,7 @@ class MetaMpack:
class Instance:
- def build_json_output(self, output_conf_dir, output_path):
+ def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path):
raise NotImplementedError("Should have implemented this")
@@ -377,11 +419,11 @@ class MpackInstance(Instance):
self.instance_name = instance_name
self.groups_dict = groups_dict
- def build_json_output(self, output_conf_dir, output_path):
+ def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path):
result = {}
for group in self.groups_dict.keys():
result[group] = build_json_output_from_instances_dict(self.groups_dict[group], ModuleInstance.plural_name,
- output_conf_dir, output_path)
+ output_conf_dir, output_log_dir, output_run_dir, output_path)
return {"subgroups": result, 'name': self.instance_name}
def set_new_version(self, mpack_name, mpack_version):
@@ -536,12 +578,12 @@ class ModuleInstance(Instance):
print("\nSetting new version for component : " + component_instance.component_path)
component_instance.set_new_version(mpack_name, mpack_version, component_type)
- def build_json_output(self, output_conf_dir, output_path):
+ def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path):
result = {}
for component_type in self.components_map.keys():
result[component_type] = build_json_output_from_instances_dict(self.components_map[component_type],
ComponentInstance.plural_name,
- output_conf_dir, output_path)
+ output_conf_dir, output_log_dir, output_run_dir, output_path)
result = {'components': result}
result['category'] = self.category
result['name'] = self.module_name
@@ -631,12 +673,18 @@ class ComponentInstance(Instance):
os.symlink(mpack_path, os.path.join(component_path, CURRENT_SOFTLINK_NAME))
os.makedirs(os.path.join(component_path, CONFIGS_DIRECTORY_NAME))
+ os.makedirs(os.path.join(component_path, LOG_DIRECTORY_NAME))
+ os.makedirs(os.path.join(component_path, RUN_DIRECTORY_NAME))
print "\n Created " + component_path
- def build_json_output(self, output_conf_dir, output_path):
+ def build_json_output(self, output_conf_dir, output_log_dir, output_run_dir, output_path):
result = {'name': self.name}
if output_conf_dir:
result['config_dir'] = os.path.join(self.component_path, CONFIGS_DIRECTORY_NAME)
+ if output_log_dir:
+ result['log_dir'] = os.path.join(self.component_path, LOG_DIRECTORY_NAME)
+ if output_run_dir:
+ result['run_dir'] = os.path.join(self.component_path, RUN_DIRECTORY_NAME)
if output_path:
result['path'] = self.path_exec
return result
diff --git a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
index f9458b1..4332d25 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
@@ -26,6 +26,8 @@ import ast
CREATE_MPACK_INSTANCE_ACTION = 'create-mpack-instance'
SET_MPACK_INSTANCE_ACTION = 'set-mpack-instance'
GET_CONF_DIR_ACTION = 'get-conf-dir'
+GET_LOG_DIR_ACTION = 'get-log-dir'
+GET_RUN_DIR_ACTION = 'get-run-dir'
LIST_INSTANCES_ACTION = 'list-instances'
@@ -34,6 +36,8 @@ def init_action_parser(action, parser):
CREATE_MPACK_INSTANCE_ACTION: init_create_parser_options,
SET_MPACK_INSTANCE_ACTION: init_set_parser_options,
GET_CONF_DIR_ACTION: init_get_parser_options,
+ GET_LOG_DIR_ACTION: init_get_parser_options,
+ GET_RUN_DIR_ACTION: init_get_parser_options,
LIST_INSTANCES_ACTION: init_get_parser_options
}
try:
@@ -152,6 +156,16 @@ def main(options, args):
subgroup_name=options.subgroup_name, module_name=options.module_name,
components_map=parsed_components_map)
+ elif action == GET_LOG_DIR_ACTION:
+ print get_log_dir(mpack=options.mpack, mpack_instance=options.mpack_instance,
+ subgroup_name=options.subgroup_name, module_name=options.module_name,
+ components_map=parsed_components_map)
+
+ elif action == GET_RUN_DIR_ACTION:
+ print get_run_dir(mpack=options.mpack, mpack_instance=options.mpack_instance,
+ subgroup_name=options.subgroup_name, module_name=options.module_name,
+ components_map=parsed_components_map)
+
elif action == LIST_INSTANCES_ACTION:
print list_instances(mpack=options.mpack, mpack_instance=options.mpack_instance,
subgroup_name=options.subgroup_name, module_name=options.module_name,
@@ -161,10 +175,10 @@ def main(options, args):
if __name__ == "__main__":
if len(sys.argv) < 2:
print(
- "Missing the command. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances")
+ "Missing the command. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|get-log-dir|get-run-dir|list-instances")
sys.exit(1)
- usage = "Usage: %prog create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances [options]"
+ usage = "Usage: %prog create-mpack-instance|set-mpack-instance|get-conf-dir|get-log-dir|get-run-dir|list-instances [options]"
parser = optparse.OptionParser(usage=usage)
action = sys.argv[1]
init_action_parser(action, parser)
diff --git a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
index 5d359a4..db5c521 100644
--- a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
+++ b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
@@ -213,6 +213,114 @@ class TestInstanceManager(TestCase):
}
self.assertEqual(conf_dir_json, expected_json)
+ def test_get_log_dir_all(self):
+ create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+ create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+ components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
+
+ log_dir_json = instance_manager.get_log_dir()
+
+ expected_json = {
+ "mpacks": {
+ "hdpcore": {
+ "mpack-instances": {
+ "Production": {
+ "name": "Production",
+ "subgroups": {
+ "default": {
+ "modules": {
+ "hdfs": {
+ "category": "SERVER",
+ "name": "hdfs",
+ "components": {
+ "hdfs_server": {
+ "component-instances": {
+ "server1": {
+ "log_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/hdfs_server/server1/log",
+ "name": "server1"
+ }
+ }
+ }
+ }
+ },
+ "hdfs-clients": {
+ "category": "CLIENT",
+ "components": {
+ "hdfs_client": {
+ "component-instances": {
+ "default": {
+ "log_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs_client/log",
+ "name": "default"
+ }
+ }
+ }
+ },
+ "name": "hdfs-clients"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ self.assertEqual(log_dir_json, expected_json)
+
+ def test_get_run_dir_all(self):
+ create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+ create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+ components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
+
+ run_dir_json = instance_manager.get_run_dir()
+
+ expected_json = {
+ "mpacks": {
+ "hdpcore": {
+ "mpack-instances": {
+ "Production": {
+ "name": "Production",
+ "subgroups": {
+ "default": {
+ "modules": {
+ "hdfs": {
+ "category": "SERVER",
+ "name": "hdfs",
+ "components": {
+ "hdfs_server": {
+ "component-instances": {
+ "server1": {
+ "name": "server1",
+ "run_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/hdfs_server/server1/run"
+ }
+ }
+ }
+ }
+ },
+ "hdfs-clients": {
+ "category": "CLIENT",
+ "components": {
+ "hdfs_client": {
+ "component-instances": {
+ "default": {
+ "name": "default",
+ "run_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs_client/run"
+ }
+ }
+ }
+ },
+ "name": "hdfs-clients"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ self.assertEqual(run_dir_json, expected_json)
+
def test_list_instances_all(self):
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
--
To stop receiving notification emails like this one, please contact
jluniya@apache.org.