You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/02/24 13:52:39 UTC
ambari git commit: AMBARI-20148 - "Set Version on All Hosts" Fails
For Hosts Without Any Stack Components (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk b511b5b22 -> 1d3dce261
AMBARI-20148 - "Set Version on All Hosts" Fails For Hosts Without Any Stack Components (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d3dce26
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d3dce26
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d3dce26
Branch: refs/heads/trunk
Commit: 1d3dce26190cd9105e47d1c18472075fdda75ef6
Parents: b511b5b
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Feb 23 14:59:01 2017 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Feb 24 08:51:54 2017 -0500
----------------------------------------------------------------------
.../custom_actions/scripts/ru_set_all.py | 65 ++++++++++++++++----
.../python/custom_actions/test_ru_set_all.py | 61 +++++++++++++++++-
2 files changed, 110 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d3dce26/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
index b7fcc51..a7732d9 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
@@ -22,6 +22,7 @@ Ambari Agent
import os
import shutil
+import socket
from ambari_commons.os_check import OSCheck
from resource_management.libraries.script import Script
from resource_management.libraries.functions import conf_select
@@ -38,15 +39,12 @@ from resource_management.libraries.functions import StackFeature
class UpgradeSetAll(Script):
"""
- This script is a part of Rolling Upgrade workflow and is used to set the
- component versions as a final step in the upgrade process
+ This script is a part of stack upgrade workflow and is used to set the
+ all of the component versions as a final step in the upgrade process
"""
def actionexecute(self, env):
- config = Script.get_config()
-
version = default('/commandParams/version', None)
- stack_name = default('/hostLevelParams/stack_name', "")
if not version:
raise Fail("Value is required for '/commandParams/version'")
@@ -56,15 +54,25 @@ class UpgradeSetAll(Script):
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
- real_ver = format_stack_version(version)
- if real_ver and check_stack_feature(StackFeature.ROLLING_UPGRADE, real_ver):
- stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
- cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version)
- code, out = shell.call(cmd, sudo=True)
- if code != 0:
- raise Exception("Command '{0}' exit code is nonzero".format(cmd))
+ formatted_version = format_stack_version(version)
+ if not formatted_version:
+ raise Fail("Unable to determine a properly formatted stack version from {0}".format(version))
+
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+
+ # this script runs on all hosts; if this host doesn't have stack components,
+ # then don't invoke the stack tool
+ # (no need to log that it's skipped - the function will do that)
+ if is_host_skippable(stack_selector_path, formatted_version):
+ return
+
+ # invoke "set all"
+ cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version)
+ code, out = shell.call(cmd, sudo=True)
+ if code != 0:
+ raise Exception("Command '{0}' exit code is nonzero".format(cmd))
- if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver):
+ if check_stack_feature(StackFeature.CONFIG_VERSIONING, formatted_version):
# backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component]
for k, v in conf_select.get_package_dirs().iteritems():
for dir_def in v:
@@ -148,6 +156,37 @@ class UpgradeSetAll(Script):
Logger.info(" Skipping restoring config from backup {0} since it does not exist".format(backup_conf_directory))
+def is_host_skippable(stack_selector_path, formatted_version):
+ """
+ Gets whether this host should not have the stack select tool called.
+ :param stack_selector_path the path to the stack selector tool.
+ :param formatted_version: the version to use with the stack selector tool.
+ :return: True if this host should be skipped, False otherwise.
+ """
+ if not os.path.exists(stack_selector_path):
+ Logger.info("{0} does not have any stack components installed and will not invoke {1}".format(
+ socket.gethostname(), stack_selector_path))
+
+ return True
+
+ # invoke the tool, checking its output
+ cmd = ('ambari-python-wrap', stack_selector_path, "versions")
+ code, out = shell.call(cmd, sudo=True)
+
+ if code != 0:
+ Logger.info("{0} is unable to determine which stack versions are available using {1}".format(
+ socket.gethostname(), stack_selector_path))
+
+ return True
+
+ # check to see if the output is empty, indicating no versions installed
+ if not out.strip():
+ Logger.info("{0} has no stack versions installed".format(socket.gethostname()))
+ return True
+
+ return False
+
+
def link_config(old_conf, link_conf):
"""
Creates a config link following:
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d3dce26/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
index b018605..e1a89a8 100644
--- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
+++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
@@ -46,7 +46,7 @@ def fake_call(command, **kwargs):
:param command: Command that will be echoed.
:return: Returns a tuple of (process output code, output)
"""
- return (0, command)
+ return (0, str(command))
class TestRUSetAll(RMFTestCase):
@@ -69,10 +69,11 @@ class TestRUSetAll(RMFTestCase):
def tearDown(self):
Logger.logger = None
+ @patch("os.path.exists")
@patch("resource_management.core.shell.call")
@patch.object(Script, 'get_config')
@patch.object(OSCheck, 'is_redhat_family')
- def test_execution(self, family_mock, get_config_mock, call_mock):
+ def test_execution(self, family_mock, get_config_mock, call_mock, exists_mock):
# Mock the config objects
json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
@@ -88,6 +89,7 @@ class TestRUSetAll(RMFTestCase):
family_mock.return_value = True
get_config_mock.return_value = config_dict
call_mock.side_effect = fake_call # echo the command
+ exists_mock.return_value = True
# Ensure that the json file was actually read.
stack_name = default("/hostLevelParams/stack_name", None)
@@ -104,11 +106,12 @@ class TestRUSetAll(RMFTestCase):
call_mock.assert_called_with(('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'all', u'2.2.1.0-2260'), sudo=True)
+ @patch("os.path.exists")
@patch("resource_management.core.shell.call")
@patch.object(Script, 'get_config')
@patch.object(OSCheck, 'is_redhat_family')
@patch("ru_set_all.link_config")
- def test_execution_23(self, link_mock, family_mock, get_config_mock, call_mock):
+ def test_execution_23(self, link_mock, family_mock, get_config_mock, call_mock, exists_mock):
# Mock the config objects
json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
@@ -125,6 +128,7 @@ class TestRUSetAll(RMFTestCase):
family_mock.return_value = True
get_config_mock.return_value = config_dict
call_mock.side_effect = fake_call # echo the command
+ exists_mock.return_value = True
# Ensure that the json file was actually read.
stack_name = default("/hostLevelParams/stack_name", None)
@@ -142,6 +146,57 @@ class TestRUSetAll(RMFTestCase):
self.assertTrue(link_mock.called)
call_mock.assert_called_with(('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'all', '2.3.0.0-1234'), sudo=True)
+ @patch("os.path.exists")
+ @patch("resource_management.core.shell.call")
+ @patch.object(Script, 'get_config')
+ @patch.object(OSCheck, 'is_redhat_family')
+ def test_skippable_hosts(self, family_mock, get_config_mock, call_mock, exists_mock):
+ """
+ Tests that hosts are skippable if they don't have stack components installed
+ :return:
+ """
+ # Mock the config objects
+ json_file_path = os.path.join(self.get_custom_actions_dir(),
+ "ru_execute_tasks_namenode_prepare.json")
+ self.assertTrue(os.path.isfile(json_file_path))
+
+ with open(json_file_path, "r") as json_file:
+ json_payload = json.load(json_file)
+
+ json_payload["configurations"]["cluster-env"]["stack_tools"] = self.get_stack_tools()
+ json_payload["configurations"]["cluster-env"]["stack_features"] = self.get_stack_features()
+
+ config_dict = ConfigDictionary(json_payload)
+
+ family_mock.return_value = False
+ get_config_mock.return_value = config_dict
+ exists_mock.return_value = True
+
+ def hdp_select_call(command, **kwargs):
+ # return no versions
+ if "versions" in command:
+ return (0,"")
+
+ return (0,command)
+
+ call_mock.side_effect = hdp_select_call
+
+ # Ensure that the json file was actually read.
+ stack_name = default("/hostLevelParams/stack_name", None)
+ stack_version = default("/hostLevelParams/stack_version", None)
+ service_package_folder = default('/roleParams/service_package_folder', None)
+
+ self.assertEqual(stack_name, "HDP")
+ self.assertEqual(stack_version, '2.2')
+ self.assertEqual(service_package_folder, "common-services/HDFS/2.1.0.2.0/package")
+
+ # Begin the test
+ ru_execute = UpgradeSetAll()
+ ru_execute.actionexecute(None)
+
+ call_mock.assert_called_with(('ambari-python-wrap', u'/usr/bin/hdp-select', 'versions'), sudo = True)
+ self.assertEqual(call_mock.call_count, 1)
+
@patch("os.path.islink")
@patch("os.path.isdir")