You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2015/03/06 11:06:26 UTC
[1/2] ambari git commit: AMBARI-9951. hadooplzo_2_2_2_0_2538-native
can not be upgraded during RU (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.0.0 1956e0cd6 -> 8ce17b991
refs/heads/trunk 9c14b8f66 -> b5ef4a1bc
AMBARI-9951. hadooplzo_2_2_2_0_2538-native can not be upgraded during RU (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b5ef4a1b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b5ef4a1b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b5ef4a1b
Branch: refs/heads/trunk
Commit: b5ef4a1bc2052b628b20d4ac85976e6e12517663
Parents: 9c14b8f
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Mar 6 11:57:59 2015 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Mar 6 11:57:59 2015 +0200
----------------------------------------------------------------------
.../custom_actions/scripts/install_packages.py | 18 +++-
.../custom_actions/TestInstallPackages.py | 91 ++++++++++++++++----
.../configs/install_packages_config.json | 4 +-
3 files changed, 91 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5ef4a1b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index 5819390..eb93e18 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -29,6 +29,7 @@ from resource_management.libraries.functions.list_ambari_managed_repos import li
from ambari_commons.os_check import OSCheck, OSConst
from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
from resource_management.core.shell import call
+from resource_management.libraries.functions.default import default
class InstallPackages(Script):
@@ -204,13 +205,24 @@ class InstallPackages(Script):
"""
filtered_package_list = []
for package in package_list:
+ skip_package = False
# mysql* package logic is managed at HIVE scripts
if package['name'].startswith('mysql'):
- continue
+ skip_package = True
# Ambari metrics packages should not be upgraded during RU
if package['name'].startswith('ambari-metrics'):
- continue
- filtered_package_list.append(package)
+ skip_package = True
+ # hadooplzo package is installed only if LZO comperession is enabled
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ if not io_compression_codecs or "com.hadoop.compression.lzo" not in io_compression_codecs:
+ lzo_packages = ['hadoop-lzo', 'lzo', 'hadoop-lzo-native', 'liblzo2-2', 'hadooplzo']
+ for lzo_package in lzo_packages:
+ if package['name'].startswith(lzo_package):
+ skip_package = True
+ break
+
+ if not skip_package:
+ filtered_package_list.append(package)
return filtered_package_list
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5ef4a1b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
index 4975757..d2c340b 100644
--- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
+++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
@@ -63,7 +63,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -72,7 +72,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -83,8 +83,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -113,7 +111,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': ["HDP-UTILS-2.2.0.1-885"]})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -122,7 +120,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url='http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -133,8 +131,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -186,7 +182,7 @@ class TestInstallPackages(RMFTestCase):
self.assertEquals(put_structured_out_mock.call_args[0][0],
{'stack_id': 'HDP-2.2', 'installed_repository_version': u'2.2.0.1-885', 'ambari_repositories': [], 'package_installation_result': 'FAIL'})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -195,7 +191,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -230,7 +226,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -239,7 +235,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -250,8 +246,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -272,7 +266,8 @@ class TestInstallPackages(RMFTestCase):
config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \
'{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \
- '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"}]'
+ '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \
+ ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]'
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -288,7 +283,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -297,7 +292,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -319,3 +314,65 @@ class TestInstallPackages(RMFTestCase):
)
self.assertNoMoreResources()
+ # Check case when LZO is enabled
+ config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+ with open(config_file, "r") as f:
+ config_dict = json.load(f)
+ config_dict['configurations']['core-site'] = {
+ 'io.compression.codecs': "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,"
+ "org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo"
+ }
+ config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \
+ '{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \
+ '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \
+ ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]'
+
+ self.executeScript("scripts/install_packages.py",
+ classname="InstallPackages",
+ command="actionexecute",
+ config_dict=config_dict,
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Suse', '11', 'Final'),
+ )
+ self.assertTrue(put_structured_out_mock.called)
+ self.assertEquals(put_structured_out_mock.call_args[0][0],
+ {'package_installation_result': 'SUCCESS',
+ 'installed_repository_version': u'2.2.0.1-885',
+ 'stack_id': 'HDP-2.2',
+ 'ambari_repositories': []})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP-UTILS', 'main'],
+ repo_template='repo_suse_rhel.j2',
+ repo_file_name=u'HDP-2.2.0.1-885',
+ mirror_list=None,
+ append_to_file=False,
+ )
+ self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP', 'main'],
+ repo_template='repo_suse_rhel.j2',
+ repo_file_name=u'HDP-2.2.0.1-885',
+ mirror_list=None,
+ append_to_file=True,
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-hcatalog',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-webhcat',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hadooplzo_2_2_2_0_2538-native',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertNoMoreResources()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b5ef4a1b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
index 4f262ea..f9476a4 100644
--- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
+++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
@@ -24,7 +24,7 @@
"roleParams": {
"stack_id": "HDP-2.2",
"repository_version": "2.2.0.1-885",
- "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
+ "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]"
},
"serviceName": "null",
@@ -37,7 +37,7 @@
"command_timeout": "60",
"script_type": "PYTHON",
"repository_version": "2.2.0.1-885",
- "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
+ "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]",
"script": "install_packages.py"
},
[2/2] ambari git commit: AMBARI-9951. hadooplzo_2_2_2_0_2538-native
can not be upgraded during RU (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-9951. hadooplzo_2_2_2_0_2538-native can not be upgraded during RU (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ce17b99
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ce17b99
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ce17b99
Branch: refs/heads/branch-2.0.0
Commit: 8ce17b991e7bdaba82622efbfb2b0fc888618707
Parents: 1956e0c
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Mar 6 11:57:59 2015 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Mar 6 12:03:20 2015 +0200
----------------------------------------------------------------------
.../custom_actions/scripts/install_packages.py | 18 +++-
.../custom_actions/TestInstallPackages.py | 91 ++++++++++++++++----
.../configs/install_packages_config.json | 4 +-
3 files changed, 91 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index 5819390..eb93e18 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -29,6 +29,7 @@ from resource_management.libraries.functions.list_ambari_managed_repos import li
from ambari_commons.os_check import OSCheck, OSConst
from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
from resource_management.core.shell import call
+from resource_management.libraries.functions.default import default
class InstallPackages(Script):
@@ -204,13 +205,24 @@ class InstallPackages(Script):
"""
filtered_package_list = []
for package in package_list:
+ skip_package = False
# mysql* package logic is managed at HIVE scripts
if package['name'].startswith('mysql'):
- continue
+ skip_package = True
# Ambari metrics packages should not be upgraded during RU
if package['name'].startswith('ambari-metrics'):
- continue
- filtered_package_list.append(package)
+ skip_package = True
+ # hadooplzo package is installed only if LZO comperession is enabled
+ io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+ if not io_compression_codecs or "com.hadoop.compression.lzo" not in io_compression_codecs:
+ lzo_packages = ['hadoop-lzo', 'lzo', 'hadoop-lzo-native', 'liblzo2-2', 'hadooplzo']
+ for lzo_package in lzo_packages:
+ if package['name'].startswith(lzo_package):
+ skip_package = True
+ break
+
+ if not skip_package:
+ filtered_package_list.append(package)
return filtered_package_list
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
index 4975757..d2c340b 100644
--- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
+++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
@@ -63,7 +63,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -72,7 +72,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -83,8 +83,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -113,7 +111,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': ["HDP-UTILS-2.2.0.1-885"]})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -122,7 +120,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url='http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url='http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -133,8 +131,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -186,7 +182,7 @@ class TestInstallPackages(RMFTestCase):
self.assertEquals(put_structured_out_mock.call_args[0][0],
{'stack_id': 'HDP-2.2', 'installed_repository_version': u'2.2.0.1-885', 'ambari_repositories': [], 'package_installation_result': 'FAIL'})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -195,7 +191,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template=u'repo_suse_rhel.j2',
@@ -230,7 +226,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -239,7 +235,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -250,8 +246,6 @@ class TestInstallPackages(RMFTestCase):
self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'snappy-devel', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'lzo', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
- self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_885*', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'hadoop_2_2_0_1_885*-libhdfs', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertResourceCalled('Package', 'ambari-log4j', use_repos=['base', 'HDP-UTILS-2.2.0.1-885', 'HDP-2.2.0.1-885'])
self.assertNoMoreResources()
@@ -272,7 +266,8 @@ class TestInstallPackages(RMFTestCase):
config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \
'{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \
- '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"}]'
+ '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \
+ ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]'
self.executeScript("scripts/install_packages.py",
classname="InstallPackages",
@@ -288,7 +283,7 @@ class TestInstallPackages(RMFTestCase):
'stack_id': 'HDP-2.2',
'ambari_repositories': []})
self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP-UTILS', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -297,7 +292,7 @@ class TestInstallPackages(RMFTestCase):
append_to_file=False,
)
self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
- base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
action=['create'],
components=[u'HDP', 'main'],
repo_template='repo_suse_rhel.j2',
@@ -319,3 +314,65 @@ class TestInstallPackages(RMFTestCase):
)
self.assertNoMoreResources()
+ # Check case when LZO is enabled
+ config_file = self._getSrcFolder()+"/test/python/custom_actions/configs/install_packages_config.json"
+ with open(config_file, "r") as f:
+ config_dict = json.load(f)
+ config_dict['configurations']['core-site'] = {
+ 'io.compression.codecs': "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,"
+ "org.apache.hadoop.io.compress.SnappyCodec,com.hadoop.compression.lzo"
+ }
+ config_dict['roleParams']['package_list'] = '[{\"name\":\"mysql-connector-java\"},{\"name\":\"hive_2_2_*\"},' \
+ '{\"name\":\"hive_2_2_*-hcatalog\"},{\"name\":\"hive_2_2_*-webhcat\"},' \
+ '{\"name\":\"mysql\"},{\"name\":\"mysql-client\"}, {\"name\":\"ambari-metrics-collector\"},' \
+ ' {\"name\":\"hadooplzo_2_2_2_0_2538-native\"}]'
+
+ self.executeScript("scripts/install_packages.py",
+ classname="InstallPackages",
+ command="actionexecute",
+ config_dict=config_dict,
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Suse', '11', 'Final'),
+ )
+ self.assertTrue(put_structured_out_mock.called)
+ self.assertEquals(put_structured_out_mock.call_args[0][0],
+ {'package_installation_result': 'SUCCESS',
+ 'installed_repository_version': u'2.2.0.1-885',
+ 'stack_id': 'HDP-2.2',
+ 'ambari_repositories': []})
+ self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP-UTILS', 'main'],
+ repo_template='repo_suse_rhel.j2',
+ repo_file_name=u'HDP-2.2.0.1-885',
+ mirror_list=None,
+ append_to_file=False,
+ )
+ self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885',
+ base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0',
+ action=['create'],
+ components=[u'HDP', 'main'],
+ repo_template='repo_suse_rhel.j2',
+ repo_file_name=u'HDP-2.2.0.1-885',
+ mirror_list=None,
+ append_to_file=True,
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-hcatalog',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hive_2_2_0_1_885*-webhcat',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertResourceCalled('Package', 'hadooplzo_2_2_2_0_2538-native',
+ use_repos=['base', 'HDP-UTILS-2.2.0.1-885',
+ 'HDP-2.2.0.1-885'],
+ )
+ self.assertNoMoreResources()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/8ce17b99/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
index 4f262ea..f9476a4 100644
--- a/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
+++ b/ambari-server/src/test/python/custom_actions/configs/install_packages_config.json
@@ -24,7 +24,7 @@
"roleParams": {
"stack_id": "HDP-2.2",
"repository_version": "2.2.0.1-885",
- "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
+ "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]"
},
"serviceName": "null",
@@ -37,7 +37,7 @@
"command_timeout": "60",
"script_type": "PYTHON",
"repository_version": "2.2.0.1-885",
- "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
+ "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
"package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]",
"script": "install_packages.py"
},