You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by lp...@apache.org on 2017/09/14 14:38:38 UTC
[02/38] ambari git commit: AMBARI-21928 In case if component package
doesn't advertise his version,
Ambari silently failing on deploy stack steps (dgrinenko)
AMBARI-21928 In case if component package doesn't advertise his version, Ambari silently failing on deploy stack steps (dgrinenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/001f290b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/001f290b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/001f290b
Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 001f290be2622375786f27cc054727adb7be8905
Parents: 252e989
Author: Dmytro Grinenko <ha...@apache.org>
Authored: Tue Sep 12 15:12:44 2017 +0300
Committer: Dmytro Grinenko <ha...@apache.org>
Committed: Tue Sep 12 15:12:44 2017 +0300
----------------------------------------------------------------------
.../libraries/functions/stack_tools.py | 12 ++---
.../libraries/functions/version_select_util.py | 57 +++++++++-----------
.../libraries/script/script.py | 2 +
.../scripts/shared_initialization.py | 19 +++----
.../hooks/after-INSTALL/test_after_install.py | 2 +-
5 files changed, 42 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
index fa97fd1..d9233a3 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
@@ -32,6 +32,7 @@ from resource_management.core.utils import pad
STACK_SELECTOR_NAME = "stack_selector"
CONF_SELECTOR_NAME = "conf_selector"
+
def get_stack_tool(name):
"""
Give a tool selector name get the stack-specific tool name, tool path, tool package
@@ -43,7 +44,7 @@ def get_stack_tool(name):
stack_name = default("/hostLevelParams/stack_name", None)
if stack_name is None:
Logger.warning("Cannot find the stack name in the command. Stack tools cannot be loaded")
- return (None, None, None)
+ return None, None, None
stack_tools = None
stack_tools_config = default("/configurations/cluster-env/stack_tools", None)
@@ -52,23 +53,22 @@ def get_stack_tool(name):
if stack_tools is None:
Logger.warning("The stack tools could not be found in cluster-env")
- return (None, None, None)
+ return None, None, None
if stack_name not in stack_tools:
Logger.warning("Cannot find stack tools for the stack named {0}".format(stack_name))
- return (None, None, None)
+ return None, None, None
# load the stack tooks keyed by the stack name
stack_tools = stack_tools[stack_name]
if not stack_tools or not name or name.lower() not in stack_tools:
Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools)))
- return (None, None, None)
-
+ return None, None, None
tool_config = stack_tools[name.lower()]
- # Return fixed length (tool_name, tool_path tool_package) tuple
+ # Return fixed length (tool_name, tool_path, tool_package) tuple
return tuple(pad(tool_config[:3], 3))
def get_stack_tool_name(name):
http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index 79dc874..9fbb42b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -26,7 +26,6 @@ import tempfile
from resource_management.core.logger import Logger
from resource_management.core import shell
from resource_management.libraries.functions import stack_tools
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
def get_component_version(stack_name, component_name):
@@ -39,42 +38,36 @@ def get_component_version(stack_name, component_name):
:return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
"""
version = None
- if stack_name is None or component_name is None:
+ if not stack_name or not component_name:
Logger.error("Could not determine component version because of the parameters is empty. " \
"stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
return version
- out = None
- code = -1
- if not stack_name:
- Logger.error("Stack name not provided")
- elif not component_name:
- Logger.error("Component name not provided")
+ stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
+ if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path):
+ tmpfile = tempfile.NamedTemporaryFile(delete=True)
+ out, code = None, -1
+ get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name)
+
+ try:
+ # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
+ # ToDo: original problem looks strange
+ with open(tmpfile.name, 'r') as f:
+ code = shell.call(get_stack_comp_version_cmd, quiet=True)[0]
+ out = f.read()
+
+ if code != 0 or out is None:
+ raise ValueError("Code is nonzero or output is empty")
+
+ Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
+ matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
+ version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0]) > 0 else None
+ Logger.debug("Version for component %s: %s" % (component_name, str(version)))
+ except Exception, e:
+ Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
+ (component_name, get_stack_comp_version_cmd, str(code), str(out)))
else:
- (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
- if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path):
- tmpfile = tempfile.NamedTemporaryFile()
-
- get_stack_comp_version_cmd = ""
- try:
- # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
- with open(tmpfile.name, 'r') as file:
- get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name)
- code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
- out = file.read()
-
- if code != 0 or out is None:
- raise Exception("Code is nonzero or output is empty")
-
- Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
- matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
- version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0]) > 0 else None
- Logger.debug("Version for component %s: %s" % (component_name, str(version)))
- except Exception, e:
- Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
- (component_name, get_stack_comp_version_cmd, str(code), str(out)))
- else:
- Logger.error("Could not find stack selector for stack: %s" % str(stack_name))
+ Logger.error("Could not find stack selector for stack: %s" % str(stack_name))
return version
http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index fcfac2b..0e352d1 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -228,6 +228,8 @@ class Script(object):
repo_version_id = default("/hostLevelParams/repository_version_id", None)
if repo_version_id:
self.put_structured_out({"repository_version_id": repo_version_id})
+ else:
+ Logger.error("Component '{0}' did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name))
def should_expose_component_version(self, command_name):
http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index daaffd8..67c3ba8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -46,7 +46,7 @@ def setup_stack_symlinks(struct_out_file):
return
if params.host_sys_prepped:
- Logger.warning("Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.")
+ Logger.warning("Skipping running stack-selector-tool because this is a sys_prepped host. This may cause symlink pointers not to be created for HDP components installed later on top of an already sys_prepped host")
return
# get the packages which the stack-select tool should be used on
@@ -105,17 +105,14 @@ def load_version(struct_out_file):
"""
Load version from file. Made a separate method for testing
"""
- json_version = None
try:
- if os.path.exists(struct_out_file):
- with open(struct_out_file, 'r') as fp:
- json_info = json.load(fp)
- json_version = json_info['version']
- except:
- pass
-
- return json_version
-
+ with open(struct_out_file, 'r') as fp:
+ json_info = json.load(fp)
+
+ return json_info['version']
+ except (IOError, KeyError, TypeError):
+ return None
+
def link_configs(struct_out_file):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index d6740cb..f905cf9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -344,4 +344,4 @@ class TestHookAfterInstall(RMFTestCase):
config_dict = json_content,
config_overrides = self.CONFIG_OVERRIDES)
- logger_warning_mock.assert_any_call('Skipping running stack-selector-tool becase this is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets installed later on top of an already sys_prepped host.')
+ logger_warning_mock.assert_any_call('Skipping running stack-selector-tool because this is a sys_prepped host. This may cause symlink pointers not to be created for HDP components installed later on top of an already sys_prepped host')