You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/09/12 16:13:01 UTC
ambari git commit: AMBARI-21931 - Use Correct Packages For Clients
Where Stack Tools Support It (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 379a56a24 -> 4cc2ecca8
AMBARI-21931 - Use Correct Packages For Clients Where Stack Tools Support It (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4cc2ecca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4cc2ecca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4cc2ecca
Branch: refs/heads/branch-2.6
Commit: 4cc2ecca8d56c3b778929ca524e3b4e8258c657b
Parents: 379a56a
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Sep 11 15:41:22 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Tue Sep 12 12:12:46 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/stack_select.py | 65 ++++++++++++++++++-
.../ambari/server/topology/AmbariContext.java | 2 +-
.../HDP/2.0.6/properties/stack_packages.json | 66 +++++++++++++------
.../src/test/python/TestStackSelect.py | 67 ++++++++++++++++++--
.../src/test/python/stacks/utils/RMFTestCase.py | 28 ++++----
5 files changed, 191 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index eac1bef..f5068e4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -33,6 +33,7 @@ from resource_management.libraries.functions.get_stack_version import get_stack_
from resource_management.libraries.functions.format import format
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions import stack_tools
+from resource_management.core import shell
from resource_management.core.shell import call
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
@@ -75,6 +76,11 @@ PACKAGE_SCOPE_INSTALL = "INSTALL"
PACKAGE_SCOPE_STANDARD = "STANDARD"
PACKAGE_SCOPE_PATCH = "PATCH"
PACKAGE_SCOPE_STACK_SELECT = "STACK-SELECT-PACKAGE"
+
+# the legacy key is used when a package has changed from one version of the stack select tool to another.
+_PACKAGE_SCOPE_LEGACY = "LEGACY"
+
+# the valid scopes which can be requested
_PACKAGE_SCOPES = (PACKAGE_SCOPE_INSTALL, PACKAGE_SCOPE_STANDARD, PACKAGE_SCOPE_PATCH, PACKAGE_SCOPE_STACK_SELECT)
# the orchestration types which equal to a partial (non-STANDARD) upgrade
@@ -110,6 +116,37 @@ def get_package_name(default_package = None):
raise
+def is_package_supported(package, supported_packages = None):
+ """
+ Gets whether the specified package is supported by the <stack_select> tool.
+ :param package: the package to check
+ :param supported_packages: the list of supported packages pre-fetched
+ :return: True if the package is support, False otherwise
+ """
+ if supported_packages is None:
+ supported_packages = get_supported_packages()
+
+ if package in supported_packages:
+ return True
+
+ return False
+
+
+def get_supported_packages():
+ """
+ Parses the output from <stack-select> packages and returns an array of the various packages.
+ :return: and array of packages support by <stack-select>
+ """
+ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
+ command = (STACK_SELECT_PREFIX, stack_selector_path, "packages")
+ code, stdout = shell.call(command, sudo = True, quiet = True)
+
+ if code != 0 or stdout is None:
+ raise Fail("Unable to query for supported packages using {0}".format(stack_selector_path))
+
+ # turn the output into lines, stripping each line
+ return [line.strip() for line in stdout.splitlines()]
+
def get_packages(scope, service_name = None, component_name = None):
"""
@@ -172,7 +209,33 @@ def get_packages(scope, service_name = None, component_name = None):
Logger.info("Skipping stack-select on {0} because it does not exist in the stack-select package structure.".format(component_name))
return None
- return data[component_name][scope]
+ # this one scope is not an array, so transform it into one for now so we can
+ # use the same code below
+ packages = data[component_name][scope]
+ if scope == PACKAGE_SCOPE_STACK_SELECT:
+ packages = [packages]
+
+ # grab the package name from the JSON and validate it against the packages
+ # that the stack-select tool supports - if it doesn't support it, then try to find the legacy
+ # package name if it exists
+ supported_packages = get_supported_packages()
+ for index, package in enumerate(packages):
+ if not is_package_supported(package, supported_packages=supported_packages):
+ if _PACKAGE_SCOPE_LEGACY in data[component_name]:
+ legacy_package = data[component_name][_PACKAGE_SCOPE_LEGACY]
+ Logger.info(
+ "The package {0} is not supported by this version of the stack-select tool, defaulting to the legacy package of {1}".format(package, legacy_package))
+
+ # use the legacy package
+ packages[index] = legacy_package
+ else:
+ raise Fail("The package {0} is not supported by this version of the stack-select tool.".format(package))
+
+ # transform the array bcak to a single element
+ if scope == PACKAGE_SCOPE_STACK_SELECT:
+ packages = packages[0]
+
+ return packages
def select_all(version_to_select):
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 6a2d58d..134fffc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -206,7 +206,7 @@ public class AmbariContext {
StackId stackId = new StackId(stack.getName(), stack.getVersion());
RepositoryVersionEntity repoVersion = null;
- if (null == repoVersionString && null == repoVersionId) {
+ if (StringUtils.isEmpty(repoVersionString) && null == repoVersionId) {
List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId);
if (stackRepoVersions.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
index 704fb54..ec35f2b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json
@@ -298,12 +298,13 @@
]
},
"HDFS_CLIENT": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "hadoop-hdfs-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "hadoop-hdfs-client"
],
"PATCH": [
- "INVALID"
+ "hadoop-hdfs-client"
],
"STANDARD": [
"hadoop-client"
@@ -420,12 +421,13 @@
]
},
"HIVE_CLIENT": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "hive-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "hive-client"
],
"PATCH": [
- "INVALID"
+ "hive-client"
],
"STANDARD": [
"hadoop-client"
@@ -500,12 +502,13 @@
]
},
"MAPREDUCE2_CLIENT": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "hadoop-mapreduce-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "hadoop-mapreduce-client"
],
"PATCH": [
- "hadoop-mapreduce-INVALID"
+ "hadoop-mapreduce-client"
],
"STANDARD": [
"hadoop-client"
@@ -542,12 +545,13 @@
},
"PIG": {
"PIG": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "pig-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "pig-client"
],
"PATCH": [
- "INVALID"
+ "pig-client"
],
"STANDARD": [
"hadoop-client"
@@ -636,6 +640,18 @@
}
},
"SPARK": {
+ "LIVY_CLIENT": {
+ "STACK-SELECT-PACKAGE": "livy-client",
+ "INSTALL": [
+ "livy-client"
+ ],
+ "PATCH": [
+ "livy-client"
+ ],
+ "STANDARD": [
+ "livy-client"
+ ]
+ },
"LIVY_SERVER": {
"STACK-SELECT-PACKAGE": "livy-server",
"INSTALL": [
@@ -686,6 +702,18 @@
}
},
"SPARK2": {
+ "LIVY2_CLIENT": {
+ "STACK-SELECT-PACKAGE": "livy2-client",
+ "INSTALL": [
+ "livy2-client"
+ ],
+ "PATCH": [
+ "livy2-client"
+ ],
+ "STANDARD": [
+ "livy2-client"
+ ]
+ },
"LIVY2_SERVER": {
"STACK-SELECT-PACKAGE": "livy2-server",
"INSTALL": [
@@ -819,12 +847,13 @@
},
"TEZ": {
"TEZ_CLIENT": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "tez-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "tez-client"
],
"PATCH": [
- "INVALID"
+ "tez-client"
],
"STANDARD": [
"hadoop-client"
@@ -895,12 +924,13 @@
]
},
"YARN_CLIENT": {
- "STACK-SELECT-PACKAGE": "hadoop-client",
+ "STACK-SELECT-PACKAGE": "hadoop-yarn-client",
+ "LEGACY": "hadoop-client",
"INSTALL": [
- "hadoop-client"
+ "hadoop-yarn-client"
],
"PATCH": [
- "INVALID"
+ "hadoop-yarn-client"
],
"STANDARD": [
"hadoop-client"
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/test/python/TestStackSelect.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackSelect.py b/ambari-server/src/test/python/TestStackSelect.py
index 3d4e5b6..8cd8f2f 100644
--- a/ambari-server/src/test/python/TestStackSelect.py
+++ b/ambari-server/src/test/python/TestStackSelect.py
@@ -19,6 +19,7 @@ limitations under the License.
'''
from mock.mock import patch
+from mock.mock import MagicMock
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
@@ -43,13 +44,15 @@ class TestStackSelect(TestCase):
self.assertRaises(Fail, stack_select.select_packages, version)
-
+ @patch.object(stack_select, "get_supported_packages")
@patch("resource_management.libraries.functions.stack_select.select")
- def test_select_package_for_standard_orchestration(self, stack_select_select_mock):
+ def test_select_package_for_standard_orchestration(self, stack_select_select_mock, get_supported_packages_mock):
"""
Tests that missing the service & role throws an excpetion
:return:
"""
+ get_supported_packages_mock.return_value = TestStackSelect._get_supported_packages()
+
version = "2.5.9.9-9999"
command_json = TestStackSelect._get_cluster_simple_upgrade_json()
@@ -66,13 +69,15 @@ class TestStackSelect(TestCase):
self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version))
self.assertEqual(stack_select_select_mock.call_args_list[1][0], ("foo-client", version))
-
+ @patch.object(stack_select, "get_supported_packages")
@patch("resource_management.libraries.functions.stack_select.select")
- def test_select_package_for_patch_orchestration(self, stack_select_select_mock):
+ def test_select_package_for_patch_orchestration(self, stack_select_select_mock, get_supported_packages_mock):
"""
Tests that missing the service & role throws an excpetion
:return:
"""
+ get_supported_packages_mock.return_value = TestStackSelect._get_supported_packages()
+
version = "2.5.9.9-9999"
command_json = TestStackSelect._get_cluster_simple_upgrade_json()
@@ -98,6 +103,31 @@ class TestStackSelect(TestCase):
self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-master", version))
+ @patch.object(stack_select, "get_supported_packages")
+ @patch("resource_management.libraries.functions.stack_select.select")
+ def test_legacy_package_fallback(self, stack_select_select_mock, get_supported_packages_mock):
+ """
+ Tests that if the package specified by the JSON isn't support by the stack-select tool,
+ the the fallback legacy value is used.
+ :return:
+ """
+ get_supported_packages_mock.return_value = ["foo-legacy"]
+
+ version = "2.5.9.9-9999"
+
+ command_json = TestStackSelect._get_cluster_simple_upgrade_json()
+
+ Script.config = dict()
+ Script.config.update(command_json)
+ Script.config.update( { "configurations" : { "cluster-env" : {} }, "hostLevelParams": {} } )
+ Script.config["configurations"]["cluster-env"]["stack_packages"] = self._get_stack_packages_with_legacy()
+ Script.config["hostLevelParams"] = { "stack_name" : "HDP" }
+
+ stack_select.select_packages(version)
+
+ self.assertEqual(len(stack_select_select_mock.call_args_list), 1)
+ self.assertEqual(stack_select_select_mock.call_args_list[0][0], ("foo-legacy", version))
+
@staticmethod
def _get_incomplete_cluster_simple_upgrade_json():
"""
@@ -197,3 +227,32 @@ class TestStackSelect(TestCase):
}
}
} )
+
+ @staticmethod
+ def _get_stack_packages_with_legacy():
+ import json
+ return json.dumps( {
+ "HDP": {
+ "stack-select": {
+ "FOO_SERVICE": {
+ "FOO_MASTER": {
+ "LEGACY":"foo-legacy",
+ "STACK-SELECT-PACKAGE": "foo-master",
+ "INSTALL": [
+ "foo-master"
+ ],
+ "PATCH": [
+ "foo-master"
+ ],
+ "STANDARD": [
+ "foo-master"
+ ]
+ }
+ }
+ }
+ }
+ } )
+
+ @staticmethod
+ def _get_supported_packages():
+ return ["foo-master", "foo-client"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/4cc2ecca/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index 0341092..c10ff64 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -139,19 +139,21 @@ class RMFTestCase(TestCase):
if 'status_params' in sys.modules:
del(sys.modules["status_params"])
- with Environment(basedir, test_mode=True) as RMFTestCase.env:
- with patch('resource_management.core.shell.checked_call', side_effect=checked_call_mocks) as mocks_dict['checked_call']:
- with patch('resource_management.core.shell.call', side_effect=call_mocks) as mocks_dict['call']:
- with patch.object(Script, 'get_config', return_value=self.config_dict) as mocks_dict['get_config']: # mocking configurations
- with patch.object(Script, 'get_tmp_dir', return_value="/tmp") as mocks_dict['get_tmp_dir']:
- with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path']:
- with patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution']:
- with patch.object(os, "environ", new=os_env) as mocks_dict['environ']:
- if not try_install:
- with patch.object(Script, 'install_packages') as install_mock_value:
- method(RMFTestCase.env, *command_args)
- else:
- method(RMFTestCase.env, *command_args)
+ with Environment(basedir, test_mode=True) as RMFTestCase.env,\
+ patch('resource_management.core.shell.checked_call', side_effect=checked_call_mocks) as mocks_dict['checked_call'],\
+ patch('resource_management.core.shell.call', side_effect=call_mocks) as mocks_dict['call'],\
+ patch.object(Script, 'get_config', return_value=self.config_dict) as mocks_dict['get_config'],\
+ patch.object(Script, 'get_tmp_dir', return_value="/tmp") as mocks_dict['get_tmp_dir'],\
+ patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path'],\
+ patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution'],\
+ patch('resource_management.libraries.functions.stack_select.is_package_supported', return_value=True),\
+ patch('resource_management.libraries.functions.stack_select.get_supported_packages', return_value=MagicMock()),\
+ patch.object(os, "environ", new=os_env) as mocks_dict['environ']:
+ if not try_install:
+ with patch.object(Script, 'install_packages') as install_mock_value:
+ method(RMFTestCase.env, *command_args)
+ else:
+ method(RMFTestCase.env, *command_args)
sys.path.remove(scriptsdir)