You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/20 19:44:08 UTC
[2/2] ambari git commit: AMBARI-21450 - Cherry picking AMBARI-20871,
AMBARI-20923, and AMBARI-21091 (jonathanhurley)
AMBARI-21450 - Cherry picking AMBARI-20871, AMBARI-20923, and AMBARI-21091 (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/81035d86
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/81035d86
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/81035d86
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 81035d86c1b90b0a54896c67a124d7748dfb6aa3
Parents: 56362fd
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Jul 20 15:43:49 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Jul 20 15:43:49 2017 -0400
----------------------------------------------------------------------
.../test/python/ambari_agent/TestHostInfo.py | 109 ++++-
.../libraries/functions/packages_analyzer.py | 66 ++-
.../libraries/functions/repository_util.py | 121 +++++
.../libraries/script/script.py | 57 ++-
.../DistributeRepositoriesStructuredOutput.java | 16 +-
.../controller/AmbariActionExecutionHelper.java | 3 +-
.../stack/upgrade/RepositoryVersionHelper.java | 2 +-
.../custom_actions/scripts/install_packages.py | 107 ++---
.../hooks/before-INSTALL/scripts/params.py | 3 +
.../scripts/repo_initialization.py | 24 +-
.../custom_actions/TestInstallPackages.py | 254 +++++++----
.../configs/install_packages_config.json | 97 ++--
.../install_packages_repository_file.json | 112 +++++
.../stacks/2.0.6/HBASE/test_hbase_master.py | 52 ++-
.../stacks/2.0.6/configs/hbase_with_phx.json | 446 ++++++++++---------
utility/pom.xml | 21 +
16 files changed, 998 insertions(+), 492 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index a00771d..28e7d2b 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -538,5 +538,112 @@ class TestHostInfo(TestCase):
os_path_isfile_mock.return_value = False
self.assertEqual("", hostInfo.getTransparentHugePage())
+ @staticmethod
+ def _add_packages_available(command, arg):
+ arg.append(["hadoop_2_2_0_1_885", "1.0", "HDP-2.2"])
+ arg.append(["hadooplzo_2_2_0_1_885", "1.0", "HDP-2.2"])
+ arg.append(["hadoop_2_2_0_1_885-libhdfs", "1.0", "HDP-2.2"])
+
+ @staticmethod
+ def _add_packages_lookUpYum(command, key, arg):
+ TestHostInfo._add_packages_available(command, arg)
+
+ @patch("ambari_commons.os_check.OSCheck.is_suse_family")
+ @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family")
+ @patch("ambari_commons.os_check.OSCheck.is_redhat_family")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages")
+ def test_get_available_packages_in_repos_suse(self, lookUpZypperPackages, is_redhat_family, is_ubuntu_family,
+ is_suse_family_mock):
+ is_suse_family_mock.return_value = True
+ is_redhat_family.return_value = False
+ is_ubuntu_family.return_value = False
+ lookUpZypperPackages.side_effect = TestHostInfo._add_packages_available
+
+ command_json = {
+ "repositoryFile": {
+ "stackName": "HDP",
+ "repoVersionId": 1,
+ "repoVersion": "2",
+ "repositories": [
+ {
+ "repoName": "HDP",
+ "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0",
+ "repoId": "HDP-2.2"
+ }
+ ]
+ }
+ }
+
+ available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(
+ command_json['repositoryFile']['repositories'])
+
+ self.assertEqual(available_packages_in_repos,
+ ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"])
+
+ @patch("ambari_commons.os_check.OSCheck.is_suse_family")
+ @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family")
+ @patch("ambari_commons.os_check.OSCheck.is_redhat_family")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages")
+ def test_get_available_packages_in_repos_rhel(self, lookUpYumPackages, is_redhat_family, is_ubuntu_family,
+ is_suse_family_mock):
+ is_suse_family_mock.return_value = False
+ is_redhat_family.return_value = True
+ is_ubuntu_family.return_value = False
+ lookUpYumPackages.side_effect = TestHostInfo._add_packages_lookUpYum
+
+ command_json = {
+ "repositoryFile": {
+ "stackName": "HDP",
+ "repoVersionId": 1,
+ "repoVersion": "2",
+ "repositories": [
+ {
+ "repoName": "HDP",
+ "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0",
+ "repoId": "HDP-2.2"
+ }
+ ]
+ }
+ }
+
+ available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(
+ command_json['repositoryFile']['repositories'])
+
+ self.assertEqual(available_packages_in_repos,
+ ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs", "hadoop_2_2_0_1_885",
+ "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"])
+
+ @patch("ambari_commons.os_check.OSCheck.is_suse_family")
+ @patch("ambari_commons.os_check.OSCheck.is_ubuntu_family")
+ @patch("ambari_commons.os_check.OSCheck.is_redhat_family")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpAptPackages")
+ def test_get_available_packages_in_repos_ubuntu(self, lookUpAptPackages, is_redhat_family, is_ubuntu_family,
+ is_suse_family_mock):
+ is_suse_family_mock.return_value = False
+ is_redhat_family.return_value = False
+ is_ubuntu_family.return_value = True
+ lookUpAptPackages.side_effect = TestHostInfo._add_packages_available
+
+ command_json = {
+ "repositoryFile": {
+ "stackName": "HDP",
+ "repoVersionId": 1,
+ "repoVersion": "2",
+ "repositories": [
+ {
+ "repoName": "HDP",
+ "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0",
+ "repoId": "HDP-2.2"
+ }
+ ]
+ }
+ }
+
+ available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(
+ command_json['repositoryFile']['repositories'])
+
+ self.assertEqual(available_packages_in_repos,
+ ["hadoop_2_2_0_1_885", "hadooplzo_2_2_0_1_885", "hadoop_2_2_0_1_885-libhdfs"])
+
if __name__ == "__main__":
- unittest.main()
+ unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
index 9df2b64..1cf7b5c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
@@ -26,6 +26,7 @@ from threading import Thread
import threading
from ambari_commons import OSCheck, OSConst
from ambari_commons import shell
+from ambari_commons.constants import AMBARI_SUDO_BINARY
from resource_management.core.logger import Logger
from resource_management.core import shell as rmf_shell
from resource_management.core.exceptions import Fail
@@ -34,8 +35,8 @@ __all__ = ["installedPkgsByName", "allInstalledPackages", "allAvailablePackages"
"getInstalledRepos", "getInstalledPkgsByRepo", "getInstalledPkgsByNames", "getPackageDetails"]
LIST_INSTALLED_PACKAGES_UBUNTU = "COLUMNS=9999 ; for i in $(dpkg -l |grep ^ii |awk -F' ' '{print $2}'); do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done"
-LIST_AVAILABLE_PACKAGES_UBUNTU = "packages=`for i in $(ls -1 /var/lib/apt/lists | grep -v \"ubuntu.com\") ; do grep ^Package: /var/lib/apt/lists/$i | awk '{print $2}' ; done` ; for i in $packages; do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done"
-
+LIST_AVAILABLE_PACKAGES_UBUNTU = "packages=`for i in $(ls -1 /var/lib/apt/lists | grep %s ) ; do grep ^Package: /var/lib/apt/lists/$i | awk '{print $2}' ; done` ; for i in $packages; do apt-cache showpkg \"$i\"|head -3|grep -v '^Versions'| tr -d '()' | awk '{ print $1\" \"$2 }'|sed -e 's/^Package: //;' | paste -d ' ' - -; done"
+GREP_REPO_EXCLUDE_SYSTEM = "-v \"ubuntu.com\""
logger = logging.getLogger()
# default timeout for async invoked processes
@@ -91,15 +92,13 @@ def allInstalledPackages(allInstalledPackages):
"""
All installed packages in system
"""
- osType = OSCheck.get_os_family()
-
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
- ["sudo", "zypper", "--no-gpg-checks", "search", "--installed-only", "--details"],
+ [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--installed-only", "--details"],
allInstalledPackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
- ["sudo", "yum", "list", "installed"],
+ [AMBARI_SUDO_BINARY, "yum", "list", "installed"],
'Installed Packages',
allInstalledPackages)
elif OSCheck.is_ubuntu_family():
@@ -108,21 +107,59 @@ def allInstalledPackages(allInstalledPackages):
allInstalledPackages)
-def allAvailablePackages(allAvailablePackages):
- osType = OSCheck.get_os_family()
+def get_available_packages_in_repos(repositories):
+ """
+ Gets all (both installed and available) packages that are available at given repositories.
+ :param repositories: from command configs like config['repositoryFile']['repositories']
+ :return: installed and available packages from these repositories
+ """
+ available_packages = []
+ installed_packages = []
+ available_packages_in_repos = []
+ repo_ids = [repository['repoId'] for repository in repositories]
+ if OSCheck.is_ubuntu_family():
+ allInstalledPackages(installed_packages)
+ repo_urls = [repository['baseUrl'] for repository in repositories]
+ repo_urls = [repo_url.replace("http://","") for repo_url in repo_urls]
+ repo_urls = [repo_url.replace("/","_") for repo_url in repo_urls]
+ for url in repo_urls:
+ _lookUpAptPackages(
+ LIST_AVAILABLE_PACKAGES_UBUNTU % url,
+ available_packages)
+ for package in installed_packages:
+ if url in package[2]:
+ available_packages_in_repos.append(package[0])
+ for package in available_packages:
+ available_packages_in_repos.append(package[0])
+ elif OSCheck.is_suse_family():
+ for repo in repo_ids:
+ _lookUpZypperPackages([AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--details", "--repo", repo],
+ available_packages)
+ available_packages_in_repos += [package[0] for package in available_packages]
+ elif OSCheck.is_redhat_family():
+ for repo in repo_ids:
+ _lookUpYumPackages([AMBARI_SUDO_BINARY, "yum", "list", "available", "--disablerepo=*", "--enablerepo=" + repo],
+ 'Available Packages', available_packages)
+ _lookUpYumPackages([AMBARI_SUDO_BINARY, "yum", "list", "installed", "--disablerepo=*", "--enablerepo=" + repo],
+ 'Installed Packages', installed_packages)
+ available_packages_in_repos += [package[0] for package in available_packages + installed_packages]
+ return available_packages_in_repos
+
+
+def allAvailablePackages(allAvailablePackages):
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
- ["sudo", "zypper", "--no-gpg-checks", "search", "--uninstalled-only", "--details"],
+ [AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search", "--uninstalled-only", "--details"],
allAvailablePackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
- ["sudo", "yum", "list", "available"],
+ [AMBARI_SUDO_BINARY, "yum", "list", "available"],
'Available Packages',
allAvailablePackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(
- LIST_AVAILABLE_PACKAGES_UBUNTU,
+ LIST_AVAILABLE_PACKAGES_UBUNTU % GREP_REPO_EXCLUDE_SYSTEM,
allAvailablePackages)
# ToDo: add execution via sudo for ubuntu (currently Ubuntu is not supported)
@@ -170,7 +207,6 @@ def _lookUpZypperPackages(command, allPackages):
if 0 == result['retCode']:
lines = result['out'].split('\n')
lines = [line.strip() for line in lines]
- items = []
for index in range(len(lines)):
if "--+--" in lines[index]:
skipIndex = index + 1
@@ -186,7 +222,7 @@ def _lookUpZypperPackages(command, allPackages):
def nameMatch(lookupName, actualName):
tokens = actualName.strip().split()
for token in tokens:
- if token.lower().find(lookupName.lower()) == 0:
+ if lookupName.lower() in token.lower():
return True
return False
@@ -281,7 +317,7 @@ def getInstalledPackageVersion(package_name):
code, out, err = rmf_shell.checked_call("dpkg -s {0} | grep Version | awk '{{print $2}}'".format(package_name), stderr=subprocess.PIPE)
else:
code, out, err = rmf_shell.checked_call("rpm -q --queryformat '%{{version}}-%{{release}}' {0} | sed -e 's/\.el[0-9]//g'".format(package_name), stderr=subprocess.PIPE)
-
+
return out
@@ -317,4 +353,4 @@ def verifyDependencies():
Logger.error(err_msg)
return False
- return True
+ return True
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
new file mode 100644
index 0000000..f1edbec
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -0,0 +1,121 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.libraries.resources.repository import Repository
+import ambari_simplejson as json
+
+
+__all__ = ["create_repo_files", "CommandRepository"]
+
+# components_lits = repoName + postfix
+UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+
+
+def create_repo_files(template, command_repository):
+ """
+ Creates repositories in a consistent manner for all types
+ :param command_repository: a CommandRepository instance
+ :return:
+ """
+
+ if command_repository.version_id is None:
+ raise Fail("The command repository was not parsed correctly")
+
+ if 0 == len(command_repository.repositories):
+ raise Fail("Cannot create repository files when no repositories are defined")
+
+ # add the stack name to the file name just to make it a little easier to debug
+ # version_id is the primary id of the repo_version table in the database
+ file_name = "ambari-{0}-{1}".format(command_repository.stack_name.lower(),
+ command_repository.version_id)
+
+ append_to_file = False # initialize to False to create the file anew.
+
+ for repository in command_repository.repositories:
+
+ if repository.repo_id is None:
+ raise Fail("Repository with url {0} has no id".format(repository.base_url))
+
+ Repository(repository.repo_id,
+ action = "create",
+ base_url = repository.base_url,
+ mirror_list = repository.mirrors_list,
+ repo_file_name = file_name,
+ repo_template = template,
+ components = repository.ubuntu_components,
+ append_to_file = append_to_file)
+ append_to_file = True
+
+
+def _find_value(dictionary, key):
+ """
+ Helper to find a value in a dictionary
+ """
+ if key not in dictionary:
+ return None
+
+ return dictionary[key]
+
+
+class CommandRepository(object):
+ """
+ Class that encapsulates the representation of repositories passed in a command. This class
+ should match the CommandRepository class.
+ """
+
+ def __init__(self, jsonvalue):
+
+ if isinstance(jsonvalue, dict):
+ json_dict = jsonvalue
+ elif isinstance(jsonvalue, basestring):
+ json_dict = json.loads(jsonvalue)
+
+ if json_dict is None:
+ raise Fail("Cannot deserialize command repository {0}".format(str(jsonvalue)))
+
+ # version_id is the primary id of the repo_version table in the database
+ self.version_id = _find_value(json_dict, 'repoVersionId')
+ self.stack_name = _find_value(json_dict, 'stackName')
+ self.repositories = []
+
+ repos_def = _find_value(json_dict, 'repositories')
+ if repos_def is not None:
+ if not isinstance(repos_def, list):
+ repos_def = [repos_def]
+
+ for repo_def in repos_def:
+ self.repositories.append(_CommandRepositoryEntry(repo_def))
+
+
+class _CommandRepositoryEntry(object):
+ """
+ Class that represents the entries of a CommandRepository. This isn't meant to be instantiated
+ outside a CommandRepository
+ """
+ def __init__(self, json_dict):
+ self.repo_id = _find_value(json_dict, 'repoId') # this is the id within the repo file, not an Ambari artifact
+ self.repo_name = _find_value(json_dict, 'repoName')
+ self.base_url = _find_value(json_dict, 'baseUrl')
+ self.mirrors_list = _find_value(json_dict, 'mirrorsList')
+
+ # if repoName is changed on the java side, this will fail for ubuntu since we rely on the
+ # name being the same as how the repository was built
+ self.ubuntu_components = [self.repo_name] + UBUNTU_REPO_COMPONENTS_POSTFIX
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 0df6900..cce3ac4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -417,11 +417,11 @@ class Script(object):
def get_stack_version_before_packages_installed(self):
"""
- This works in a lazy way (calculates the version first time and stores it).
+ This works in a lazy way (calculates the version first time and stores it).
If you need to recalculate the version explicitly set:
-
+
Script.stack_version_from_distro_select = None
-
+
before the call. However takes a bit of time, so better to avoid.
:return: stack version including the build number. e.g.: 2.3.4.0-1234.
@@ -443,21 +443,45 @@ class Script(object):
return Script.stack_version_from_distro_select
+
+ def get_package_from_available(self, name, available_packages_in_repos):
+ """
+ This function matches package names with ${stack_version} placeholder to actual package names from
+ Ambari-managed repository.
+ Package names without ${stack_version} placeholder are returned as is.
+ """
+ if STACK_VERSION_PLACEHOLDER not in name:
+ return name
+ package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
+ package_regex = name.replace(STACK_VERSION_PLACEHOLDER, '(\d|{0})+'.format(package_delimiter))
+ for package in available_packages_in_repos:
+ if re.match(package_regex, package):
+ return package
+
+
def format_package_name(self, name):
from resource_management.libraries.functions.default import default
"""
- This function replaces ${stack_version} placeholder into actual version. If the package
+ This function replaces ${stack_version} placeholder with actual version. If the package
version is passed from the server, use that as an absolute truth.
"""
- # two different command types put things in different objects. WHY.
- # package_version is the form W_X_Y_Z_nnnn
- package_version = default("roleParams/package_version", None)
- if not package_version:
- package_version = default("hostLevelParams/package_version", None)
-
package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
+ # repositoryFile is the truth
+ # package_version should be made to the form W_X_Y_Z_nnnn
+ package_version = default("repositoryFile/repoVersion", None)
+ if package_version is not None:
+ package_version = package_version.replace('.', package_delimiter).replace('-', package_delimiter)
+
+ # TODO remove legacy checks
+ if package_version is None:
+ package_version = default("roleParams/package_version", None)
+
+ # TODO remove legacy checks
+ if package_version is None:
+ package_version = default("hostLevelParams/package_version", None)
+
# The cluster effective version comes down when the version is known after the initial
# install. In that case we should not be guessing which version when invoking INSTALL, but
# use the supplied version to build the package_version
@@ -673,7 +697,7 @@ class Script(object):
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
-
+
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
@@ -690,12 +714,15 @@ class Script(object):
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count'])
-
+ try:
+ available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories'])
+ except Exception as err:
+ available_packages_in_repos = []
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
- name = self.format_package_name(package['name'])
+ name = self.get_package_from_available(package['name'], available_packages_in_repos)
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
@@ -740,7 +767,7 @@ class Script(object):
@staticmethod
def matches_any_regexp(string, regexp_list):
for regex in regexp_list:
- # we cannot use here Python regex, since * will create some troubles matching plaintext names.
+ # we cannot use here Python regex, since * will create some troubles matching plaintext names.
package_regex = '^' + re.escape(regex).replace('\\.\\*','.*').replace("\\?", ".").replace("\\*", ".*") + '$'
if re.match(package_regex, string):
return True
@@ -995,4 +1022,4 @@ class Script(object):
def __init__(self):
if Script.instance is not None:
- raise Fail("An instantiation already exists! Use, get_instance() method.")
+ raise Fail("An instantiation already exists! Use, get_instance() method.")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java
index 662b856..0cf2d78 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,8 +19,6 @@
package org.apache.ambari.server.bootstrap;
-import java.util.List;
-
import com.google.gson.annotations.SerializedName;
/**
@@ -36,12 +34,6 @@ public class DistributeRepositoriesStructuredOutput {
private String installedRepositoryVersion;
/**
- * All Ambari-managed repositories that are installed side by side on host
- */
- @SerializedName("ambari_repositories")
- private List<String> ambariRepositories;
-
- /**
* Either SUCCESS or FAIL
*/
@SerializedName("package_installation_result")
@@ -64,10 +56,6 @@ public class DistributeRepositoriesStructuredOutput {
return installedRepositoryVersion;
}
- public List<String> getAmbariRepositories() {
- return ambariRepositories;
- }
-
public String getPackageInstallationResult() {
return packageInstallationResult;
}
@@ -79,4 +67,4 @@ public class DistributeRepositoriesStructuredOutput {
public String getStackId() {
return stackId;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 04f1cb3..a644eda 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -466,9 +466,10 @@ public class AmbariActionExecutionHelper {
Service service = cluster.getService(serviceName);
addRepoInfoToHostLevelParams(actionContext, service.getDesiredRepositoryVersion(),
hostLevelParams, hostName);
+ } else {
+ addRepoInfoToHostLevelParams(actionContext, null, hostLevelParams, hostName);
}
-
Map<String, String> roleParams = execCmd.getRoleParams();
if (roleParams == null) {
roleParams = new TreeMap<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index 471e30f..8bf2a24 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@ -340,4 +340,4 @@ public class RepositoryVersionHelper {
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index a6645be..9a132df 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -16,36 +16,33 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Ambari Agent
-
"""
-import os
import signal
import re
-import os.path
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import ambari_simplejson as json
+import sys, traceback
-from resource_management import *
-import resource_management
-from resource_management.libraries.functions.list_ambari_managed_repos import list_ambari_managed_repos
-from ambari_commons.os_check import OSCheck, OSConst
+from ambari_commons.os_check import OSCheck
from ambari_commons.str_utils import cbool, cint
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.core.resources import Package
from resource_management.libraries.functions.packages_analyzer import allInstalledPackages, verifyDependencies
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.stack_select import get_stack_versions
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions.repo_version_history \
- import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
-from resource_management.libraries.script.script import Script
+ import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
from resource_management.core.resources.system import Link
-from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions import StackFeature
-
-from resource_management.core.logger import Logger
-
+from resource_management.libraries.functions import packages_analyzer
+from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.repository import Repository
+from resource_management.libraries.script.script import Script
class InstallPackages(Script):
"""
@@ -72,25 +69,21 @@ class InstallPackages(Script):
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version_id = None
- self.ignore_package_dependencies = False
+ base_urls = []
# Select dict that contains parameters
try:
- self.ignore_package_dependencies = 'ignore_package_dependencies' in config['roleParams'] and config['roleParams']['ignore_package_dependencies']
+ if 'base_urls' in config['roleParams']:
+ base_urls = json.loads(config['roleParams']['base_urls'])
+
self.repository_version = config['roleParams']['repository_version']
- base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
+
if 'repository_version_id' in config['roleParams']:
self.repository_version_id = config['roleParams']['repository_version_id']
except KeyError:
- # Last try
- self.repository_version = config['commandParams']['repository_version']
- base_urls = json.loads(config['commandParams']['base_urls'])
- package_list = json.loads(config['commandParams']['package_list'])
- stack_id = config['commandParams']['stack_id']
- if 'repository_version_id' in config['commandParams']:
- self.repository_version_id = config['commandParams']['repository_version_id']
+ pass
# current stack information
self.current_stack_version_formatted = None
@@ -112,9 +105,7 @@ class InstallPackages(Script):
self.repository_version = self.repository_version.strip()
-
# Install/update repositories
- installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
@@ -129,24 +120,25 @@ class InstallPackages(Script):
Logger.info("Will install packages for repository version {0}".format(self.repository_version))
if 0 == len(base_urls):
- Logger.info("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))
+ Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))
try:
- append_to_file = False
- for url_info in base_urls:
- repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
- self.current_repositories.append(repo_name)
- self.current_repo_files.add(repo_file)
- append_to_file = True
-
- installed_repositories = list_ambari_managed_repos(self.stack_name)
+ if 'repositoryFile' in config:
+ create_repo_files(template, CommandRepository(config['repositoryFile']))
+ else:
+ append_to_file = False
+ for url_info in base_urls:
+ repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
+ self.current_repositories.append(repo_name)
+ self.current_repo_files.add(repo_file)
+ append_to_file = True
+
except Exception, err:
- Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err)))
+ Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
- 'ambari_repositories': installed_repositories,
'installed_repository_version': self.repository_version,
'stack_id': stack_id,
'package_installation_result': 'FAIL'
@@ -213,18 +205,11 @@ class InstallPackages(Script):
Link("/usr/bin/conf-select", to = "/usr/bin/hdfconf-select")
for package_name, directories in conf_select.get_package_dirs().iteritems():
- # if already on HDP 2.3, then we should skip making conf.backup folders
- if self.current_stack_version_formatted and check_stack_feature(StackFeature.CONFIG_VERSIONING, self.current_stack_version_formatted):
- conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
- Logger.info("The current cluster stack of {0} does not require backing up configurations; "
- "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name))
- # only link configs for all known packages
- conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
- else:
- # link configs and create conf.backup folders for all known packages
- # this will also call conf-select select
- conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories,
- skip_existing_links = False, link_to = "backup")
+ conf_selector_name = stack_tools.get_stack_tool_name(stack_tools.CONF_SELECTOR_NAME)
+ Logger.info("The current cluster stack of {0} does not require backing up configurations; "
+ "only {1} versioned config directories will be created.".format(stack_version, conf_selector_name))
+ # only link configs for all known packages
+ conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
def compute_actual_version(self):
@@ -355,7 +340,7 @@ class InstallPackages(Script):
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
-
+
config = self.get_config()
agent_stack_retry_on_unavailability = cbool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = cint(config['hostLevelParams']['agent_stack_retry_count'])
@@ -369,16 +354,20 @@ class InstallPackages(Script):
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
)
-
+
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [package[0] for package in packages_installed_before]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
+ try:
+ available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories'])
+ except Exception:
+ available_packages_in_repos = []
for package in filtered_package_list:
- name = self.format_package_name(package['name'])
+ name = self.get_package_from_available(package['name'], available_packages_in_repos)
Package(name,
- action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated
+ action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
)
@@ -404,9 +393,7 @@ class InstallPackages(Script):
if package_version_string and (package_version_string in package):
Package(package, action="remove")
- if self.ignore_package_dependencies:
- Logger.info("Ignoring package dependencies")
- elif not verifyDependencies():
+ if not verifyDependencies():
ret_code = 1
Logger.logger.error("Failure while verifying dependencies")
Logger.logger.error("*******************************************************************************")
@@ -457,13 +444,13 @@ class InstallPackages(Script):
def abort_handler(self, signum, frame):
Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version if possible before exiting.".format(signum))
self.check_partial_install()
-
+
def filter_package_list(self, package_list):
"""
Note: that we have skipUpgrade option in metainfo.xml to filter packages,
as well as condition option to filter them conditionally,
so use this method only if, for some reason the metainfo option cannot be used.
-
+
:param package_list: original list
:return: filtered package_list
"""
@@ -475,4 +462,4 @@ class InstallPackages(Script):
if __name__ == "__main__":
- InstallPackages().execute()
+ InstallPackages().execute()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
index 6193c11..da75a8e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
@@ -108,6 +108,9 @@ host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
if has_hbase_masters:
hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+
#repo params
repo_info = config['hostLevelParams']['repo_info']
service_repo_info = default("/hostLevelParams/service_repo_info",None)
+
+repo_file = default("/repositoryFile", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
index a35dce7..058c45b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
@@ -19,11 +19,10 @@ limitations under the License.
from ambari_commons.os_check import OSCheck
from resource_management.libraries.resources.repository import Repository
+from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX
from resource_management.core.logger import Logger
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import ambari_simplejson as json
-# components_lits = repoName + postfix
-_UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
def _alter_repo(action, repo_string, repo_template):
"""
@@ -45,17 +44,17 @@ def _alter_repo(action, repo_string, repo_template):
repo['baseUrl'] = None
if not 'mirrorsList' in repo:
repo['mirrorsList'] = None
-
- ubuntu_components = [ repo['repoName'] ] + _UBUNTU_REPO_COMPONENTS_POSTFIX
-
+
+ ubuntu_components = [ repo['repoName'] ] + UBUNTU_REPO_COMPONENTS_POSTFIX
+
Repository(repo['repoId'],
action = action,
base_url = repo['baseUrl'],
mirror_list = repo['mirrorsList'],
repo_file_name = repo['repoName'],
repo_template = repo_template,
- components = ubuntu_components, # ubuntu specific
- )
+ components = ubuntu_components) # ubuntu specific
+
def install_repos():
import params
@@ -63,6 +62,13 @@ def install_repos():
return
template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu
+
+ # use this newer way of specifying repositories, if available
+ if params.repo_file is not None:
+ create_repo_files(template, CommandRepository(params.repo_file))
+ return
+
_alter_repo("create", params.repo_info, template)
+
if params.service_repo_info:
- _alter_repo("create", params.service_repo_info, template)
+ _alter_repo("create", params.service_repo_info, template)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
index 77e3bee..134d9aa 100644
--- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
+++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
@@ -61,8 +61,18 @@ class TestInstallPackages(RMFTestCase):
arg.append(["pkg1", "1.0", "repo"])
arg.append(["pkg2", "2.0", "repo2"])
+ @staticmethod
+ def _add_packages_available(command, arg):
+ arg.append(["hadoop_2_2_0_1_885", "1.0", "HDP-2.2"])
+ arg.append(["hadooplzo_2_2_0_1_885", "1.0", "HDP-2.2"])
+ arg.append(["hadoop_2_2_0_1_885-libhdfs", "1.0", "HDP-2.2"])
+
+ @staticmethod
+ def _add_packages_lookUpYum(command, key, arg):
+ TestInstallPackages._add_packages_available(command, arg)
+
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
- @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages")
@patch("resource_management.libraries.script.Script.put_structured_out")
@patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
@@ -71,12 +81,13 @@ class TestInstallPackages(RMFTestCase):
write_actual_version_to_history_file_mock,
read_actual_version_from_history_file_mock,
stack_versions_mock,
- put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock):
+ put_structured_out_mock,
+ lookUpYumPackages, list_ambari_managed_repos_mock):
stack_versions_mock.side_effect = [
[], # before installation attempt
[VERSION_STUB]
]
- allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
+ lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum
list_ambari_managed_repos_mock.return_value=[]
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -90,23 +101,22 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ 'actual_version': VERSION_STUB})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=True,
)
@@ -141,6 +151,7 @@ class TestInstallPackages(RMFTestCase):
command_json = json.load(f)
command_json['roleParams']['base_urls'] = "[]"
+ del command_json['repositoryFile']
allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
@@ -157,36 +168,36 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
-
+ 'actual_version': VERSION_STUB})
+
self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', None, action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertNoMoreResources()
@patch("ambari_commons.os_check.OSCheck.is_suse_family")
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
- @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages")
@patch("resource_management.libraries.script.Script.put_structured_out")
@patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
@patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
def test_normal_flow_sles(self, write_actual_version_to_history_file_mock,
read_actual_version_from_history_file_mock,
- stack_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, is_suse_family_mock):
+ stack_versions_mock, put_structured_out_mock,
+ lookUpZypperPackages, list_ambari_managed_repos_mock, is_suse_family_mock):
is_suse_family_mock = True
Script.stack_version_from_distro_select = VERSION_STUB
stack_versions_mock.side_effect = [
[], # before installation attempt
[VERSION_STUB]
]
- allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
+ lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available
list_ambari_managed_repos_mock.return_value=[]
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -200,23 +211,22 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ 'actual_version': VERSION_STUB})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=True,
)
@@ -234,21 +244,21 @@ class TestInstallPackages(RMFTestCase):
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
@patch("ambari_commons.os_check.OSCheck.is_redhat_family")
@patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages")
@patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
@patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- def test_exclude_existing_repo(self, write_actual_version_to_history_file_mock,
+ def test_exclude_existing_repo(self, write_actual_version_to_history_file_mock,
read_actual_version_from_history_file_mock,
stack_versions_mock,
- allInstalledPackages_mock, put_structured_out_mock,
+ lookUpYumPackages, put_structured_out_mock,
is_redhat_family_mock, list_ambari_managed_repos_mock):
stack_versions_mock.side_effect = [
[], # before installation attempt
[VERSION_STUB]
]
Script.stack_version_from_distro_select = VERSION_STUB
- allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
+ lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum
list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"]
is_redhat_family_mock.return_value = True
self.executeScript("scripts/install_packages.py",
@@ -263,23 +273,22 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': ["HDP-UTILS-2.2.0.1-885"]})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ 'actual_version': VERSION_STUB})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name='HDP-2.2.0.1-885',
+ repo_file_name='ambari-hdp-1',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2',
base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=True,
)
@@ -340,23 +349,22 @@ class TestInstallPackages(RMFTestCase):
self.assertEquals(put_structured_out_mock.call_args[0][0],
{'stack_id': 'HDP-2.2',
'installed_repository_version': VERSION_STUB,
- 'ambari_repositories': [],
'package_installation_result': 'FAIL'})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=True,
)
@@ -368,14 +376,14 @@ class TestInstallPackages(RMFTestCase):
@patch("ambari_commons.os_check.OSCheck.is_suse_family")
@patch("resource_management.core.resources.packaging.Package")
@patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages")
@patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
@patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- def test_format_package_name(self, write_actual_version_to_history_file_mock,
+ def test_format_package_name(self, write_actual_version_to_history_file_mock,
read_actual_version_from_history_file_mock,
stack_versions_mock,
- allInstalledPackages_mock, put_structured_out_mock,
+ lookUpZypperPackages, put_structured_out_mock,
package_mock, is_suse_family_mock):
Script.stack_version_from_distro_select = VERSION_STUB
stack_versions_mock.side_effect = [
@@ -383,7 +391,7 @@ class TestInstallPackages(RMFTestCase):
[VERSION_STUB]
]
read_actual_version_from_history_file_mock.return_value = VERSION_STUB
- allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
+ lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available
is_suse_family_mock.return_value = True
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -397,23 +405,22 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ 'actual_version': VERSION_STUB})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-1',
mirror_list=None,
append_to_file=True,
)
@@ -427,34 +434,35 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertNoMoreResources()
+
+
@patch("ambari_commons.os_check.OSCheck.is_suse_family")
@patch("resource_management.core.resources.packaging.Package")
@patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpZypperPackages")
@patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
@patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- def test_format_package_name_specific(self, write_actual_version_to_history_file_mock,
- read_actual_version_from_history_file_mock,
- stack_versions_mock,
- allInstalledPackages_mock, put_structured_out_mock,
- package_mock, is_suse_family_mock):
+ def test_format_package_name_via_repositoryFile(self, write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ lookUpZypperPackages, put_structured_out_mock,
+ package_mock, is_suse_family_mock):
Script.stack_version_from_distro_select = VERSION_STUB
stack_versions_mock.side_effect = [
[], # before installation attempt
[VERSION_STUB]
]
read_actual_version_from_history_file_mock.return_value = VERSION_STUB
- allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
+ lookUpZypperPackages.side_effect = TestInstallPackages._add_packages_available
is_suse_family_mock.return_value = True
-
- config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_config.json"
+
+ config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_repository_file.json"
with open(config_file, "r") as f:
command_json = json.load(f)
- command_json['roleParams']['package_version'] = "2_2_0_1_889"
-
+ command_json['repositoryFile']['repoVersion'] = '2.2.0.1-990'
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -468,33 +476,35 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
- self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ 'actual_version': VERSION_STUB})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4',
+ base_url=u'http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-4',
mirror_list=None,
append_to_file=False,
)
- self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ self.assertResourceCalled('Repository', 'HDP-2.2-repo-4',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- repo_file_name=u'HDP-2.2.0.1-885',
+ repo_file_name=u'ambari-hdp-4',
mirror_list=None,
append_to_file=True,
)
self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
- self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
self.assertNoMoreResources()
@@ -535,8 +545,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertTrue(write_actual_version_to_history_file_mock.called)
self.assertEquals(write_actual_version_to_history_file_mock.call_args[0], (VERSION_STUB_WITHOUT_BUILD_NUMBER, VERSION_STUB))
@@ -571,8 +580,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -622,7 +630,7 @@ class TestInstallPackages(RMFTestCase):
self.assertTrue(put_structured_out_mock.called)
self.assertEquals(put_structured_out_mock.call_args_list[-1][0][0],
- { 'ambari_repositories': [],
+ {
'package_installation_result': 'FAIL',
'installed_repository_version': '2.2.0.1',
'stack_id': u'HDP-2.2'})
@@ -678,8 +686,7 @@ class TestInstallPackages(RMFTestCase):
self.assertEquals(put_structured_out_mock.call_args_list[-1][0][0],
{'package_installation_result': 'FAIL',
'stack_id': u'HDP-2.2',
- 'installed_repository_version': '2.2.0.1',
- 'ambari_repositories': []})
+ 'installed_repository_version': '2.2.0.1'})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -720,8 +727,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'FAIL',
'stack_id': u'HDP-2.2',
'installed_repository_version': VERSION_STUB,
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -762,8 +768,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB_WITHOUT_BUILD_NUMBER,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertTrue(write_actual_version_to_history_file_mock.called)
self.assertEquals(write_actual_version_to_history_file_mock.call_args[0], (VERSION_STUB_WITHOUT_BUILD_NUMBER, VERSION_STUB))
@@ -798,8 +803,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': VERSION_STUB_WITHOUT_BUILD_NUMBER,
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -840,8 +844,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': '2.2.0.1-500',
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertTrue(write_actual_version_to_history_file_mock.called)
self.assertEquals(write_actual_version_to_history_file_mock.call_args[0], ('2.2.0.1', VERSION_STUB))
@@ -876,8 +879,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'SUCCESS',
'installed_repository_version': '2.2.0.1-500',
'stack_id': 'HDP-2.2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -926,8 +928,7 @@ class TestInstallPackages(RMFTestCase):
self.assertEquals(put_structured_out_mock.call_args_list[-1][0][0],
{'package_installation_result': 'FAIL',
'stack_id': u'HDP-2.2',
- 'installed_repository_version': '2.2.0.1',
- 'ambari_repositories': []})
+ 'installed_repository_version': '2.2.0.1'})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -968,8 +969,7 @@ class TestInstallPackages(RMFTestCase):
{'package_installation_result': 'FAIL',
'stack_id': u'HDP-2.2',
'installed_repository_version': VERSION_STUB,
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertFalse(write_actual_version_to_history_file_mock.called)
@@ -1011,8 +1011,7 @@ class TestInstallPackages(RMFTestCase):
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
'repository_version_id': '2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
self.assertTrue(write_actual_version_to_history_file_mock.called)
self.assertEquals(write_actual_version_to_history_file_mock.call_args[0], (VERSION_STUB_WITHOUT_BUILD_NUMBER, VERSION_STUB))
@@ -1049,7 +1048,70 @@ class TestInstallPackages(RMFTestCase):
'installed_repository_version': VERSION_STUB,
'stack_id': 'HDP-2.2',
'repository_version_id': '2',
- 'actual_version': VERSION_STUB,
- 'ambari_repositories': []})
+ 'actual_version': VERSION_STUB})
+
+ self.assertFalse(write_actual_version_to_history_file_mock.called)
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer._lookUpYumPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ def test_normal_flow_rhel_with_command_repo(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock,
+ lookUpYumPackages,
+ list_ambari_managed_repos_mock):
+ stack_versions_mock.side_effect = [
+ [], # before installation attempt
+ [VERSION_STUB]
+ ]
+ lookUpYumPackages.side_effect = TestInstallPackages._add_packages_lookUpYum
+ list_ambari_managed_repos_mock.return_value=[]
+ self.executeScript("scripts/install_packages.py",
+ classname="InstallPackages",
+ command="actionexecute",
+ config_file="install_packages_repository_file.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final'),
+ )
+ self.assertTrue(put_structured_out_mock.called)
+ self.assertEquals(put_structured_out_mock.call_args[0][0],
+ {'package_installation_result': 'SUCCESS',
+ 'installed_repository_version': VERSION_STUB,
+ 'stack_id': 'HDP-2.2',
+ 'actual_version': VERSION_STUB})
- self.assertFalse(write_actual_version_to_history_file_mock.called)
\ No newline at end of file
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4',
+ base_url=u'http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP-UTILS', 'main'],
+ repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
+ repo_file_name=u'ambari-hdp-4',
+ mirror_list=None,
+ append_to_file=False,
+ )
+ self.assertResourceCalled('Repository', 'HDP-2.2-repo-4',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP', 'main'],
+ repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
+ repo_file_name=u'ambari-hdp-4',
+ mirror_list=None,
+ append_to_file=True,
+ )
+ self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885-libhdfs', action=["upgrade"], retry_count=5,
+ retry_on_repo_unavailability=False)
+ self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False)
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/81035d86/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
index c6575b5..c6b1c0c 100644
--- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
+++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
@@ -1,40 +1,40 @@
{
- "configuration_attributes": {},
- "roleCommand": "ACTIONEXECUTE",
- "clusterName": "cc",
- "hostname": "0b3.vm",
- "passiveInfo": [],
+ "configuration_attributes": {},
+ "roleCommand": "ACTIONEXECUTE",
+ "clusterName": "cc",
+ "hostname": "0b3.vm",
+ "passiveInfo": [],
"hostLevelParams": {
"agent_stack_retry_count": "5",
"agent_stack_retry_on_unavailability": "false",
- "jdk_location": "http://0b3.vm:8080/resources/",
- "ambari_db_rca_password": "mapred",
+ "jdk_location": "http://0b3.vm:8080/resources/",
+ "ambari_db_rca_password": "mapred",
"java_home": "/usr/jdk64/jdk1.7.0_67",
"java_version": "8",
- "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
- "jce_name": "UnlimitedJCEPolicyJDK7.zip",
- "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar",
- "stack_version": "2.1",
- "stack_name": "HDP",
- "db_name": "ambari",
- "ambari_db_rca_driver": "org.postgresql.Driver",
- "jdk_name": "jdk-7u67-linux-x64.tar.gz",
- "ambari_db_rca_username": "mapred",
- "db_driver_filename": "mysql-connector-java.jar",
+ "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
+ "jce_name": "UnlimitedJCEPolicyJDK7.zip",
+ "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar",
+ "stack_version": "2.1",
+ "stack_name": "HDP",
+ "db_name": "ambari",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "jdk_name": "jdk-7u67-linux-x64.tar.gz",
+ "ambari_db_rca_username": "mapred",
+ "db_driver_filename": "mysql-connector-java.jar",
"mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"
- },
- "commandType": "EXECUTION_COMMAND",
+ },
+ "commandType": "EXECUTION_COMMAND",
"roleParams": {
"stack_id": "HDP-2.2",
"repository_version": "2.2.0.1-885",
"base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_${stack_version}\", \"condition\": \"\"},{\"name\":\"snappy\", \"condition\": \"\"},{\"name\":\"snappy-devel\", \"condition\": \"\"},{\"name\":\"lzo\", \"condition\": \"\"},{\"name\":\"hadooplzo_${stack_version}\", \"condition\": \"\"},{\"name\":\"hadoop_${stack_version}-libhdfs\", \"condition\": \"\"},{\"name\":\"ambari-log4j\", \"condition\": \"\"}]"
- },
- "serviceName": "null",
- "role": "install_packages",
- "forceRefreshConfigTags": [],
- "taskId": 61,
- "public_hostname": "0b3.vm",
+ },
+ "serviceName": "null",
+ "role": "install_packages",
+ "forceRefreshConfigTags": [],
+ "taskId": 61,
+ "public_hostname": "0b3.vm",
"configurations": {
"cluster-env": {
"repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
@@ -42,56 +42,73 @@
}
},
"commandParams": {
- "command_timeout": "60",
+ "command_timeout": "60",
"script_type": "PYTHON",
"repository_version": "2.2.0.1-885",
"base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_${stack_version}\", \"condition\": \"\"},{\"name\":\"snappy\", \"condition\": \"\"},{\"name\":\"snappy-devel\", \"condition\": \"\"},{\"name\":\"lzo\", \"condition\": \"\"},{\"name\":\"hadooplzo_${stack_version}\", \"condition\": \"\"},{\"name\":\"hadoop_${stack_version}-libhdfs\", \"condition\": \"\"},{\"name\":\"ambari-log4j\", \"condition\": \"\"}]",
"script": "install_packages.py"
- },
- "commandId": "14-1",
+ },
+ "commandId": "14-1",
+ "repositoryFile": {
+ "stackName": "HDP",
+ "repoVersionId": 1,
+ "repoVersion": "2",
+ "repositories": [
+ {
+ "repoName": "HDP-UTILS",
+ "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0",
+ "repoId": "HDP-UTILS-1.1.0.20"
+ },
+ {
+ "repoName": "HDP",
+ "baseUrl": "http://repo1/HDP/centos5/2.x/updates/2.2.0.0",
+ "repoId": "HDP-2.2"
+ }
+ ]
+ },
"clusterHostInfo": {
"snamenode_host": [
"0b3.vm"
- ],
+ ],
"nm_hosts": [
"0b3.vm"
- ],
+ ],
"app_timeline_server_hosts": [
"0b3.vm"
- ],
+ ],
"all_ping_ports": [
"8670"
- ],
+ ],
"rm_host": [
"0b3.vm"
- ],
+ ],
"all_hosts": [
"0b3.vm"
- ],
+ ],
"slave_hosts": [
"0b3.vm"
- ],
+ ],
"namenode_host": [
"0b3.vm"
- ],
+ ],
"ambari_server_host": [
"0b3.vm"
- ],
+ ],
"zookeeper_hosts": [
"0b3.vm"
- ],
+ ],
"hs_host": [
"0b3.vm"
]
},
"configurations": {
"cluster-env": {
- "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
"repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
},
"core-site": {
"io.compression.codecs": "com.hadoop.compression.lzo"
}
}
-}
+}
\ No newline at end of file