You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2018/08/29 11:02:55 UTC
[ambari] branch trunk updated (1edde0c -> 78f54fe)
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git.
from 1edde0c AMBARI-24516 - Default value for LDAP type (#2181)
new e7c6923 AMBARI-24543. Client installs failing with Ambari-2.7.0.0 on Ubuntu14 (aonishuk)
new 78f54fe AMBARI-24543. Client installs failing with Ambari-2.7.0.0 on Ubuntu14 (aonishuk)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/main/python/ambari_agent/PythonExecutor.py | 6 +-
.../resource_management/TestRepositoryResource.py | 45 +++----
.../libraries/functions/repository_util.py | 10 +-
.../libraries/providers/__init__.py | 6 +-
.../libraries/providers/repository.py | 139 +++++++++++----------
.../libraries/resources/repository.py | 9 +-
.../custom_actions/scripts/update_repo.py | 3 +-
.../before-INSTALL/scripts/repo_initialization.py | 4 +-
.../python/custom_actions/TestInstallPackages.py | 66 +++++-----
.../test/python/custom_actions/TestUpdateRepo.py | 4 +-
.../before-INSTALL/test_before_install.py | 23 ++--
11 files changed, 167 insertions(+), 148 deletions(-)
[ambari] 02/02: AMBARI-24543. Client installs failing with
Ambari-2.7.0.0 on Ubuntu14 (aonishuk)
Posted by ao...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 78f54fe3dbc0f59dbdf514fa9174549c89195be5
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Tue Aug 28 14:54:24 2018 +0300
AMBARI-24543. Client installs failing with Ambari-2.7.0.0 on Ubuntu14
(aonishuk)
---
.../src/main/python/ambari_agent/PythonExecutor.py | 1 +
.../before-INSTALL/test_before_install.py | 23 ++++++++++++----------
2 files changed, 14 insertions(+), 10 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
index 8346f7c..73088ce 100644
--- a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
@@ -31,6 +31,7 @@ from ambari_commons import shell
from Grep import Grep
from BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle
+from resource_management.libraries.functions.log_process_information import log_process_information
class PythonExecutor(object):
diff --git a/ambari-server/src/test/python/stacks/stack-hooks/before-INSTALL/test_before_install.py b/ambari-server/src/test/python/stacks/stack-hooks/before-INSTALL/test_before_install.py
index 37ac094..b3d9f8f 100644
--- a/ambari-server/src/test/python/stacks/stack-hooks/before-INSTALL/test_before_install.py
+++ b/ambari-server/src/test/python/stacks/stack-hooks/before-INSTALL/test_before_install.py
@@ -36,32 +36,32 @@ class TestHookBeforeInstall(RMFTestCase):
config_file="default.json"
)
self.assertResourceCalled('Repository', 'HDP-2.6-repo-1',
- append_to_file = False,
base_url = 'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.6.4.0-60',
- action = ['create'],
+ action = ['prepare'],
components = [u'HDP', 'main'],
repo_template = '[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name = None,
mirror_list = None,
)
self.assertResourceCalled('Repository', 'HDP-2.6-GPL-repo-1',
- append_to_file = True,
base_url = 'http://s3.amazonaws.com/dev.hortonworks.com/HDP-GPL/centos6/2.x/BUILDS/2.6.4.0-60',
- action = ['create'],
+ action = ['prepare'],
components = [u'HDP-GPL', 'main'],
repo_template = '[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name = None,
mirror_list = None,
)
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.22-repo-1',
- append_to_file = True,
base_url = 'http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6',
- action = ['create'],
+ action = ['prepare'],
components = [u'HDP-UTILS', 'main'],
repo_template = '[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name = None,
mirror_list = None,
)
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertResourceCalled('Package', 'unzip', retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'curl', retry_count=5, retry_on_repo_unavailability=False)
@@ -95,22 +95,25 @@ class TestHookBeforeInstall(RMFTestCase):
config_file="repository_file.json"
)
self.assertResourceCalled('Repository', 'HDP-2.2-repo-4',
- action=['create'],
+ action=['prepare'],
base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
components=['HDP', 'main'],
mirror_list=None,
repo_file_name='ambari-hdp-4',
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- append_to_file=False)
+ )
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4',
- action=['create'],
+ action=['prepare'],
base_url='http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0',
components=['HDP-UTILS', 'main'],
mirror_list=None,
repo_file_name='ambari-hdp-4',
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
- append_to_file=True)
+ )
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertResourceCalled('Package', 'unzip', retry_count=5, retry_on_repo_unavailability=False)
self.assertResourceCalled('Package', 'curl', retry_count=5, retry_on_repo_unavailability=False)
[ambari] 01/02: AMBARI-24543. Client installs failing with
Ambari-2.7.0.0 on Ubuntu14 (aonishuk)
Posted by ao...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit e7c69231d682fcc99d85239e2a2e037ec300fecd
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Mon Aug 27 10:15:45 2018 +0300
AMBARI-24543. Client installs failing with Ambari-2.7.0.0 on Ubuntu14 (aonishuk)
---
.../src/main/python/ambari_agent/PythonExecutor.py | 5 +-
.../resource_management/TestRepositoryResource.py | 45 +++----
.../libraries/functions/repository_util.py | 10 +-
.../libraries/providers/__init__.py | 6 +-
.../libraries/providers/repository.py | 139 +++++++++++----------
.../libraries/resources/repository.py | 9 +-
.../custom_actions/scripts/update_repo.py | 3 +-
.../before-INSTALL/scripts/repo_initialization.py | 4 +-
.../python/custom_actions/TestInstallPackages.py | 66 +++++-----
.../test/python/custom_actions/TestUpdateRepo.py | 4 +-
10 files changed, 153 insertions(+), 138 deletions(-)
diff --git a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
index 977562b..8346f7c 100644
--- a/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/PythonExecutor.py
@@ -139,9 +139,8 @@ class PythonExecutor(object):
"""
Log some useful information after task failure.
"""
- pass
- #logger.info("Command %s failed with exitcode=%s", pprint.pformat(pythonCommand), result['exitcode'])
- #log_process_information(logger)
+ self.logger.info("Command %s failed with exitcode=%s", pprint.pformat(python_command), result['exitcode'])
+ log_process_information(self.logger)
def prepare_process_result(self, returncode, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=None):
out, error, structured_out = self.read_result_from_files(tmpoutfile, tmperrfile, tmpstructedoutfile)
diff --git a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
index a69b57b..91d4939 100644
--- a/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestRepositoryResource.py
@@ -64,6 +64,8 @@ class TestRepositoryResource(TestCase):
@patch.object(OSCheck, "is_ubuntu_family")
@patch.object(OSCheck, "is_redhat_family")
@patch("resource_management.libraries.providers.repository.File")
+ @patch("filecmp.cmp", new=MagicMock(return_value=False))
+ @patch("os.path.isfile", new=MagicMock(return_value=True))
@patch.object(System, "os_family", new='redhat')
def test_create_repo_redhat(self, file_mock,
is_redhat_family, is_ubuntu_family, is_suse_family):
@@ -77,6 +79,8 @@ class TestRepositoryResource(TestCase):
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository',
repo_template=RHEL_SUSE_DEFAULT_TEMPLATE)
+
+ Repository(None, action="create")
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
@@ -91,20 +95,16 @@ class TestRepositoryResource(TestCase):
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0], '/etc/yum.repos.d/Repository.repo')
- template_item = file_mock.call_args[1]['content']
- template = str(template_item.name)
- expected_template_arguments.update({'repo_id': 'hadoop'})
-
- self.assertEqual(expected_template_arguments, template_item.context._dict)
- self.assertEqual(RHEL_SUSE_DEFAULT_TEMPLATE, template)
-
@patch.object(OSCheck, "is_suse_family")
@patch.object(OSCheck, "is_ubuntu_family")
@patch.object(OSCheck, "is_redhat_family")
@patch.object(System, "os_family", new='suse')
+ @patch("resource_management.libraries.providers.repository.checked_call")
+ @patch("os.path.isfile", new=MagicMock(return_value=True))
+ @patch("filecmp.cmp", new=MagicMock(return_value=False))
@patch("resource_management.libraries.providers.repository.File")
- def test_create_repo_suse(self, file_mock,
+ def test_create_repo_suse(self, file_mock, checked_call,
is_redhat_family, is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = False
is_ubuntu_family.return_value = False
@@ -116,6 +116,8 @@ class TestRepositoryResource(TestCase):
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template = RHEL_SUSE_DEFAULT_TEMPLATE,
repo_file_name='Repository')
+
+ Repository(None, action="create")
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
@@ -130,13 +132,6 @@ class TestRepositoryResource(TestCase):
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0], '/etc/zypp/repos.d/Repository.repo')
- template_item = file_mock.call_args[1]['content']
- template = str(template_item.name)
- expected_template_arguments.update({'repo_id': 'hadoop'})
-
- self.assertEqual(expected_template_arguments, template_item.context._dict)
- self.assertEqual(RHEL_SUSE_DEFAULT_TEMPLATE, template)
-
@patch.object(OSCheck, "is_suse_family")
@patch.object(OSCheck, "is_ubuntu_family")
@@ -146,8 +141,10 @@ class TestRepositoryResource(TestCase):
@patch("resource_management.libraries.providers.repository.checked_call")
@patch("resource_management.core.sudo.read_file")
@patch("os.path.isfile", new=MagicMock(return_value=True))
- def test_recreate_repo_suse(self, read_file_mock, checked_call_mock, file_mock,
+ @patch("filecmp.cmp")
+ def test_recreate_repo_suse(self, filecmp_mock, read_file_mock, checked_call_mock, file_mock,
is_redhat_family, is_ubuntu_family, is_suse_family):
+ filecmp_mock.return_value = False
is_redhat_family.return_value = False
is_ubuntu_family.return_value = False
is_suse_family.return_value = True
@@ -161,27 +158,30 @@ class TestRepositoryResource(TestCase):
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template = RHEL_SUSE_DEFAULT_TEMPLATE,
repo_file_name='Repository')
-
+
+ Repository(None, action="create")
+
self.assertTrue(checked_call_mock.called)
expected_repo_file_content = "[hadoop]\nname=hadoop\nmirrorlist=https://mirrors.base_url.org/?repo=Repository&arch=$basearch\n\npath=/\nenabled=1\ngpgcheck=0"
- template = file_mock.call_args[1]['content']
+ template = file_mock.call_args_list[0][1]['content']
self.assertEqual(expected_repo_file_content, template)
# Check that if content is equal, zypper cache is not flushed
checked_call_mock.reset_mock()
- read_file_mock.return_value = expected_repo_file_content
+ filecmp_mock.return_value = True
Repository('hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template = RHEL_SUSE_DEFAULT_TEMPLATE,
repo_file_name='Repository')
+ Repository(None, action="create")
self.assertFalse(checked_call_mock.called)
expected_repo_file_content = "[hadoop]\nname=hadoop\nmirrorlist=https://mirrors.base_url.org/?repo=Repository&arch=$basearch\n\npath=/\nenabled=1\ngpgcheck=0"
- template = file_mock.call_args[1]['content']
+ template = file_mock.call_args_list[0][1]['content']
self.assertEqual(expected_repo_file_content, template)
@@ -213,6 +213,7 @@ class TestRepositoryResource(TestCase):
repo_template = DEBIAN_DEFAUTL_TEMPLATE,
components = ['a','b','c']
)
+ Repository(None, action="create")
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
@@ -256,6 +257,7 @@ class TestRepositoryResource(TestCase):
repo_template = DEBIAN_DEFAUTL_TEMPLATE,
components = ['a','b','c']
)
+ Repository(None, action="create")
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
@@ -292,6 +294,7 @@ class TestRepositoryResource(TestCase):
repo_template = DEBIAN_DEFAUTL_TEMPLATE,
components = ['a','b','c']
)
+ Repository(None, action="create")
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
@@ -347,7 +350,7 @@ class TestRepositoryResource(TestCase):
action='remove',
base_url='http://download.base_url.org/rpm/',
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
- repo_file_name='Repository')
+ repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
index 684e3dc..c2123da 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -59,7 +59,6 @@ class RepositoryUtil:
self.command_repository.stack_name, self.command_repository.version_string))
return {}
- append_to_file = False # initialize to False to create the file anew.
repo_files = {}
for repository in self.command_repository.items:
if repository.repo_id is None:
@@ -75,16 +74,17 @@ class RepositoryUtil:
self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
else:
Repository(repository.repo_id,
- action="create",
+ action="prepare",
base_url=repository.base_url,
mirror_list=repository.mirrors_list,
repo_file_name=self.command_repository.repo_filename,
repo_template=self.template,
- components=repository.ubuntu_components,
- append_to_file=append_to_file)
- append_to_file = True
+ components=repository.ubuntu_components
+ )
repo_files[repository.repo_id] = self.command_repository.repo_filename
+ Repository(None, action="create")
+
return repo_files
def create_repo_files(template, command_repository):
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
index 770f9b5..aed6c5f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py
@@ -22,13 +22,13 @@ Ambari Agent
PROVIDERS = dict(
amazon=dict(
- Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider",
+ Repository="resource_management.libraries.providers.repository.RhelRepositoryProvider",
),
redhat=dict(
- Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider",
+ Repository="resource_management.libraries.providers.repository.RhelRepositoryProvider",
),
suse=dict(
- Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider",
+ Repository="resource_management.libraries.providers.repository.SuseRepositoryProvider",
),
ubuntu=dict(
Repository="resource_management.libraries.providers.repository.UbuntuRepositoryProvider",
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py
index f476e29..8b3fe38 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py
@@ -35,102 +35,90 @@ from resource_management.core.shell import checked_call
from resource_management.core import sudo
from resource_management.core.logger import Logger
import re
+from collections import defaultdict
REPO_TEMPLATE_FOLDER = 'data'
+class RepositoryProvider(Provider):
+ repo_files_content = defaultdict(lambda:'')
-class RhelSuseRepositoryProvider(Provider):
+ def action_create(self):
+ with tempfile.NamedTemporaryFile() as tmpf:
+ with tempfile.NamedTemporaryFile() as old_repo_tmpf:
+ for repo_file_path, repo_file_content in RepositoryProvider.repo_files_content.iteritems():
+ repo_file_content = repo_file_content.strip()
- update_cmd = ['zypper', 'clean', '--all']
+ File(tmpf.name,
+ content=repo_file_content
+ )
- def action_create(self):
+ if os.path.isfile(repo_file_path):
+ # a copy of old repo file, which will be readable by current user
+ File(old_repo_tmpf.name,
+ content=StaticFile(repo_file_path),
+ )
+
+ if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, old_repo_tmpf.name):
+ Logger.info(format("Rewriting {repo_file_path} since it has changed."))
+ File(repo_file_path,
+ content = StaticFile(tmpf.name)
+ )
+
+ self.update(repo_file_path)
+
+ RepositoryProvider.repo_files_content.clear()
+
+class RhelRepositoryProvider(RepositoryProvider):
+ def action_prepare(self):
repo_file_name = self.resource.repo_file_name
- repo_dir = get_repo_dir()
+ repo_dir = self.get_repo_dir()
new_content = InlineTemplate(self.resource.repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name,
- base_url=self.resource.base_url, mirror_list=self.resource.mirror_list)
+ base_url=self.resource.base_url, mirror_list=self.resource.mirror_list).get_content() + '\n'
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
- if os.path.isfile(repo_file_path):
- existing_content_str = sudo.read_file(repo_file_path)
- new_content_str = new_content.get_content()
- if existing_content_str != new_content_str and OSCheck.is_suse_family():
- # We need to reset package manager's cache when we replace base urls
- # at existing repo. That is a case at least under SLES
- Logger.info("Flushing package manager cache since repo file content is about to change")
- checked_call(self.update_cmd, sudo=True)
- if self.resource.append_to_file:
- content = existing_content_str + '\n' + new_content_str
- else:
- content = new_content_str
- else: # If repo file does not exist yet
- content = new_content
-
- File(repo_file_path,
- content=content
- )
+ RepositoryProvider.repo_files_content[repo_file_path] += new_content
def action_remove(self):
repo_file_name = self.resource.repo_file_name
- repo_dir = get_repo_dir()
+ repo_dir = self.get_repo_dir()
File(format("{repo_dir}/{repo_file_name}.repo"),
action="delete")
-
-def get_repo_dir():
- if OSCheck.is_redhat_family():
+ def get_repo_dir(self):
return '/etc/yum.repos.d'
- elif OSCheck.is_suse_family():
+
+ def update(self, repo_file_path):
+ # Centos will usually update automatically. Don't need to waste deploy time.
+ # Also in cases of failure of package install 'yum clean metadata' and retry is ran anyway.
+ pass
+
+class SuseRepositoryProvider(RhelRepositoryProvider):
+ update_cmd = ['zypper', 'clean', '--all']
+
+ def get_repo_dir(self):
return '/etc/zypp/repos.d'
+ def update(self, repo_file_path):
+ Logger.info("Flushing package manager cache since repo file content is about to change")
+ checked_call(self.update_cmd, sudo=True)
-class UbuntuRepositoryProvider(Provider):
+class UbuntuRepositoryProvider(RepositoryProvider):
package_type = "deb"
repo_dir = "/etc/apt/sources.list.d"
update_cmd = ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/{repo_file_name}', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0']
missing_pkey_regex = "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY ([A-Z0-9]+)"
app_pkey_cmd_prefix = ('apt-key', 'adv', '--recv-keys', '--keyserver', 'keyserver.ubuntu.com')
- def action_create(self):
- with tempfile.NamedTemporaryFile() as tmpf:
- with tempfile.NamedTemporaryFile() as old_repo_tmpf:
- repo_file_name = format("{repo_file_name}.list",repo_file_name=self.resource.repo_file_name)
- repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir=self.repo_dir)
-
- new_content = InlineTemplate(self.resource.repo_template, package_type=self.package_type,
- base_url=self.resource.base_url,
- components=' '.join(self.resource.components)).get_content()
- old_content = ''
- if self.resource.append_to_file and os.path.isfile(repo_file_path):
- old_content = sudo.read_file(repo_file_path) + '\n'
-
- File(tmpf.name,
- content=old_content+new_content
- )
-
- if os.path.isfile(repo_file_path):
- # a copy of old repo file, which will be readable by current user
- File(old_repo_tmpf.name,
- content=StaticFile(repo_file_path),
- )
+ def action_prepare(self):
+ repo_file_name = format("{repo_file_name}.list",repo_file_name=self.resource.repo_file_name)
+ repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir=self.repo_dir)
- if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, old_repo_tmpf.name):
- File(repo_file_path,
- content = StaticFile(tmpf.name)
- )
-
- update_cmd_formatted = [format(x) for x in self.update_cmd]
- # this is time expensive
- retcode, out = checked_call(update_cmd_formatted, sudo=True, quiet=False)
-
- # add public keys for new repos
- missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
- for pkey in missing_pkeys:
- Execute(self.app_pkey_cmd_prefix + (pkey,),
- timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
- ignore_failures = True,
- sudo = True,
- )
+ new_content = InlineTemplate(self.resource.repo_template, package_type=self.package_type,
+ base_url=self.resource.base_url,
+ components=' '.join(self.resource.components)).get_content() + '\n'
+
+ RepositoryProvider.repo_files_content[repo_file_path] += new_content
def action_remove(self):
repo_file_name = format("{repo_file_name}.list", repo_file_name=self.resource.repo_file_name)
@@ -143,3 +131,18 @@ class UbuntuRepositoryProvider(Provider):
# this is time expensive
update_cmd_formatted = [format(x) for x in self.update_cmd]
Execute(update_cmd_formatted)
+
+ def update(self, repo_file_path):
+ repo_file_name = os.path.basename(repo_file_path)
+ update_cmd_formatted = [format(x) for x in self.update_cmd]
+ # this is time expensive
+ retcode, out = checked_call(update_cmd_formatted, sudo=True, quiet=False)
+
+ # add public keys for new repos
+ missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
+ for pkey in missing_pkeys:
+ Execute(self.app_pkey_cmd_prefix + (pkey,),
+ timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
+ ignore_failures = True,
+ sudo = True,
+ )
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/repository.py b/ambari-common/src/main/python/resource_management/libraries/resources/repository.py
index 2484d67..99bdc40 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/repository.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/repository.py
@@ -25,13 +25,16 @@ _all__ = ["Repository"]
from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
class Repository(Resource):
- action = ForcedListArgument(default="create")
+ action = ForcedListArgument(default="prepare")
repo_id = ResourceArgument(default=lambda obj: obj.name)
base_url = ResourceArgument()
mirror_list = ResourceArgument()
repo_file_name = ResourceArgument()
repo_template = ResourceArgument()
- append_to_file = ResourceArgument(default=False)
components = ForcedListArgument(default=[]) # ubuntu specific
- actions = Resource.actions + ["create","remove"]
\ No newline at end of file
+ """
+ 'prepare' action only adds repositories information into memory.
+ 'create' action writes prepared repositories into file (possibly multiple repos into single file)
+ """
+ actions = Resource.actions + ["prepare", "create","remove"]
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py
index 6f8a939..ece27f5 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py
@@ -54,7 +54,7 @@ class UpdateRepo(Script):
[components.replace(",", " ") if components else self.UBUNTU_REPO_COMPONENTS_POSTFIX]
Repository(repo_id,
- action = "create",
+ action = "prepare",
base_url = base_url,
mirror_list = None,
repo_file_name = repo_name,
@@ -62,6 +62,7 @@ class UpdateRepo(Script):
components = ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")}
+ Repository(None, action="create")
except Exception, exception:
Logger.logger.exception("ERROR: There was an unexpected error while updating repositories")
raise Fail("Failed to update repo files!")
diff --git a/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/repo_initialization.py
index 382b506..f6f2a12 100644
--- a/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/repo_initialization.py
+++ b/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/repo_initialization.py
@@ -48,13 +48,15 @@ def _alter_repo(action, repo_dicts, repo_template):
+ [repo['components'].replace(",", " ") if 'components' in repo and repo['components'] else UBUNTU_REPO_COMPONENTS_POSTFIX]
Repository(repo['repoId'],
- action = action,
+ action = "prepare",
base_url = repo['baseUrl'],
mirror_list = repo['mirrorsList'],
repo_file_name = repo['repoName'],
repo_template = repo_template,
components = ubuntu_components) # ubuntu specific
+ Repository(None, action = "create")
+
def install_repos():
import params
diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
index 6e04938..a8840f0 100644
--- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
+++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
@@ -130,21 +130,22 @@ class TestInstallPackages(RMFTestCase):
'actual_version': VERSION_STUB})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
+ )
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
)
self.assertNoMoreResources()
@@ -249,23 +250,23 @@ class TestInstallPackages(RMFTestCase):
'actual_version': VERSION_STUB})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
)
-
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertNoMoreResources()
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
@@ -317,21 +318,22 @@ class TestInstallPackages(RMFTestCase):
'actual_version': VERSION_STUB})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2',
base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
+ )
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
)
self.assertNoMoreResources()
@@ -403,22 +405,23 @@ class TestInstallPackages(RMFTestCase):
'package_installation_result': 'FAIL'})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=u'ambari-hdp-1',
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=u'ambari-hdp-1',
mirror_list=None,
- append_to_file=True,
)
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertNoMoreResources()
TestInstallPackages._install_failed = False
@@ -473,23 +476,23 @@ class TestInstallPackages(RMFTestCase):
'actual_version': VERSION_STUB})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
)
-
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertNoMoreResources()
@@ -551,22 +554,23 @@ class TestInstallPackages(RMFTestCase):
'actual_version': VERSION_STUB})
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4',
base_url=u'http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2-repo-4',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
)
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertNoMoreResources()
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
@@ -1257,23 +1261,23 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4',
base_url=u'http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP-UTILS', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2-repo-4',
base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
- action=['create'],
+ action=['prepare'],
components=[u'HDP', 'main'],
repo_template='[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0',
repo_file_name=repo_file_name,
mirror_list=None,
- append_to_file=True,
)
-
+ self.assertResourceCalled('Repository', None,
+ action=['create'],
+ )
self.assertNoMoreResources()
def test_os_family_check_with_inheritance(self):
diff --git a/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py b/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py
index f984da9..43114ac 100644
--- a/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py
+++ b/ambari-server/src/test/python/custom_actions/TestUpdateRepo.py
@@ -92,14 +92,14 @@ class TestUpdateRepo(TestCase):
updateRepo.actionexecute(None)
self.assertTrue(file_mock.called)
- self.assertEquals(file_mock.call_args[0][0], "/etc/yum.repos.d/HDP-UTILS.repo")
+ self.assertEquals(file_mock.call_args[0][0], "/etc/yum.repos.d/HDP.repo")
self.assertEquals(structured_out_mock.call_args[0][0], {'repo_update': {'message': 'Repository files successfully updated!', 'exit_code': 0}})
###### invalid repo info
file_mock.reset_mock()
failed = False
mock_config.return_value = { "configurations": {
- "cluster-env": {
+ "clugit ster-env": {
"repo_suse_rhel_template": "REPO_SUSE_RHEL_TEST_TEMPLATE",
"repo_ubuntu_template": "REPO_UBUNTU_TEMPLATE"
}