You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/09/26 11:09:55 UTC
[1/2] ambari git commit: AMBARI-18435. Provide script to delete an
old HDP stack version (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 f5f1272b8 -> a6c12bfc3
refs/heads/trunk a5fa754e1 -> 18ad001b3
AMBARI-18435. Provide script to delete an old HDP stack version (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18ad001b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18ad001b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18ad001b
Branch: refs/heads/trunk
Commit: 18ad001b374f627139e4e7b4e46c056a1c94c2fc
Parents: a5fa754
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Sep 26 14:05:11 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Sep 26 14:07:31 2016 +0300
----------------------------------------------------------------------
.../test/python/ambari_agent/TestHostInfo.py | 10 +-
.../libraries/functions/packages_analyzer.py | 2 +
.../system_action_definitions.xml | 10 ++
.../scripts/remove_previous_stacks.py | 118 ++++++++++++++++
.../custom_actions/TestRemoveStackVersion.py | 140 +++++++++++++++++++
.../configs/remove_previous_stacks.json | 90 ++++++++++++
6 files changed, 365 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index 616fd5b..0c0a3b3 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -105,9 +105,9 @@ class TestHostInfo(TestCase):
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(9, len(installedPackages))
for package in installedPackages:
- self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86",
- "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
- "hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch", "libconfuse.x86_64"])
+ self.assertTrue(package[0] in ["AMBARI.dev", "PyXML", "oracle-server-db",
+ "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+ "hcatalog", "hesiod", "hive", "ambari-log4j", "libconfuse"])
self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"])
self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0",
@@ -116,8 +116,8 @@ class TestHostInfo(TestCase):
packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"],
installedPackages)
self.assertEqual(4, len(packages))
- expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
- "hesiod.x86_64", "hive.noarch"]
+ expected = ["AMBARI.dev", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+ "hesiod", "hive"]
for package in expected:
self.assertTrue(package in packages)
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
index 7bbbd3d..52c46ed 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
@@ -155,6 +155,8 @@ def _lookUpYumPackages(command, skipTill, allPackages):
items = items + line.strip(' \t\n\r').split()
for i in range(0, len(items), 3):
+ if '.' in items[i]:
+ items[i] = items[i][:items[i].rindex('.')]
if items[i + 2].find('@') == 0:
items[i + 2] = items[i + 2][1:]
allPackages.append(items[i:i + 3])
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
index fc17584..0f50256 100644
--- a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
+++ b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
@@ -84,4 +84,14 @@
<targetType>ANY</targetType>
<permissions>CLUSTER.UPGRADE_DOWNGRADE_STACK</permissions>
</actionDefinition>
+ <actionDefinition>
+ <actionName>remove_previous_stacks</actionName>
+ <actionType>SYSTEM</actionType>
+ <inputs>version</inputs>
+ <targetService/>
+ <targetComponent/>
+ <defaultTimeout>600</defaultTimeout>
+ <description>Perform remove old stack version action</description>
+ <targetType>ANY</targetType>
+ </actionDefinition>
</actionDefinitions>
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
new file mode 100644
index 0000000..958b800
--- /dev/null
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+
+import os
+from resource_management import Script, format, Package, Execute, Fail
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
+from resource_management.libraries.functions.stack_select import get_stack_versions
+
+CURRENT_ = "/current/"
+stack_root = Script.get_stack_root()
+stack_root_current = stack_root + CURRENT_
+
+
+class RemovePreviousStacks(Script):
+
+
+ def actionexecute(self, env):
+ config = Script.get_config()
+ structured_output = {}
+ version = config['commandParams']['version']
+ self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+
+ versions_to_remove = self.get_lower_versions(version)
+
+ for low_version in versions_to_remove:
+ self.remove_stack_version(structured_output, low_version)
+
+ def remove_stack_version(self, structured_output, version):
+ # check simlinks not refer to version for remove
+ self.check_no_symlink_to_version(structured_output, version)
+ packages_to_remove = self.get_packages_to_remove(version)
+ for package in packages_to_remove:
+ Package(package, action="remove")
+ self.remove_stack_folder(structured_output, version)
+ structured_output["remove_previous_stacks"] = {"exit_code": 0,
+ "message": format("Stack version {0} successfully removed!".format(version))}
+ self.put_structured_out(structured_output)
+
+ def remove_stack_folder(self, structured_output, version):
+ if version and version != '' and stack_root and stack_root != '':
+
+ Logger.info("Removing {0}/{1}".format(stack_root, version))
+ try:
+ Execute(('rm', '-f', stack_root + version),
+ sudo=True)
+ finally:
+ structured_output["remove_previous_stacks"] = {"exit_code": -1,
+ "message": "Failed to remove version {0}{1}".format(stack_root, version)}
+ self.put_structured_out(structured_output)
+
+ def get_packages_to_remove(self, version):
+ packages = []
+ formated_version = version.replace('.', '_').replace('-', '_')
+ all_installed_packages = []
+ allInstalledPackages(all_installed_packages)
+ all_installed_packages = [package[0] for package in all_installed_packages]
+ for package in all_installed_packages:
+ if formated_version in package and self.stack_tool_package not in package:
+ packages.append(package)
+ Logger.info("%s added to remove" % (package))
+ return packages
+
+ def check_no_symlink_to_version(self, structured_output, version):
+ files = os.listdir(stack_root_current)
+ for file in files:
+ if version in os.path.realpath(stack_root_current + file):
+ structured_output["remove_previous_stacks"] = {"exit_code": -1,
+ "message": "{0} contains symlink to version for remove! {1}".format(
+ stack_root_current, version)}
+ self.put_structured_out(structured_output)
+ raise Fail("{0} contains symlink to version for remove! {1}".format(stack_root_current, version))
+
+ def get_lower_versions(self, current_version):
+ versions = get_stack_versions(stack_root)
+ Logger.info("available versions: {0}".format(str(versions)))
+
+ lover_versions = []
+ for version in versions:
+ if self.compare(version, current_version) < 0 :
+ lover_versions.append(version)
+ Logger.info("version %s added to remove" % (version))
+ return lover_versions
+
+ def compare(self, version1, version2):
+ """
+ Compare version1 and version2
+ :param version1:
+ :param version2:
+ :return: Return negative if version1<version2, zero if version1==version2, positive if version1>version2
+ """
+ vesion1_sections = re.findall(r"[\w']+", version1)
+ vesion2_sections = re.findall(r"[\w']+", version2)
+ return cmp(vesion1_sections, vesion2_sections)
+
+if __name__ == "__main__":
+ RemovePreviousStacks().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
new file mode 100644
index 0000000..5a05cc2
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
@@ -0,0 +1,140 @@
+# !/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from mock.mock import patch
+from mock.mock import MagicMock
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from stacks.utils.RMFTestCase import *
+
+OLD_VERSION_STUB = '2.1.0.0-400'
+VERSION_STUB = '2.2.0.1-885'
+
+@patch.object(Logger, 'logger', new=MagicMock())
+class TestRemoveStackVersion(RMFTestCase):
+
+ @staticmethod
+ def _add_packages(arg):
+ arg.append(["pkg12_1_0_0_400", "1.0", "repo"])
+ arg.append(["pkg22_1_0_1_885", "2.0", "repo2"])
+ arg.append(["hdp-select2_1_0_1_885", "2.0", "repo2"])
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+ def test_normal_flow(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, ):
+
+ stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+
+ self.assertResourceCalled('Package', "pkg12_1_0_0_400", action=["remove"])
+ self.assertTrue(put_structured_out_mock.called)
+ self.assertEquals(put_structured_out_mock.call_args[0][0],
+ {'remove_previous_stacks': {'exit_code': 0,
+ 'message': 'Stack version 2.1.0.0-400 successfully removed!'}})
+ self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp2.1.0.0-400'),
+ sudo = True,
+ )
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+ def test_without_versions(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock ):
+
+ stack_versions_mock.return_value = [VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile" + OLD_VERSION_STUB]))
+ def test_symlink_exist(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, ):
+
+ stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ try:
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.fail("Should throw exception")
+ except Fail, e:
+ self.assertEquals(str(e), '/usr/hdp/current/ contains symlink to version for remove! 2.1.0.0-400')
+ pass # Expected
+
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/18ad001b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
new file mode 100644
index 0000000..cc4a626
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
@@ -0,0 +1,90 @@
+{
+ "configuration_attributes": {},
+ "roleCommand": "ACTIONEXECUTE",
+ "clusterName": "cc",
+ "hostname": "0b3.vm",
+ "passiveInfo": [],
+ "hostLevelParams": {
+ "agent_stack_retry_count": "5",
+ "agent_stack_retry_on_unavailability": "false",
+ "jdk_location": "http://0b3.vm:8080/resources/",
+ "ambari_db_rca_password": "mapred",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "java_version": "8",
+ "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
+ "jce_name": "UnlimitedJCEPolicyJDK7.zip",
+ "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar",
+ "stack_version": "2.1",
+ "stack_name": "HDP",
+ "db_name": "ambari",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "jdk_name": "jdk-7u67-linux-x64.tar.gz",
+ "ambari_db_rca_username": "mapred",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"
+ },
+ "commandType": "SYSTEM",
+ "serviceName": "null",
+ "role": "remove_previous_stacks",
+ "forceRefreshConfigTags": [],
+ "taskId": 61,
+ "public_hostname": "0b3.vm",
+ "configurations": {
+ "cluster-env": {
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+ }
+ },
+ "commandParams": {
+ "command_timeout": "60",
+ "script_type": "PYTHON",
+ "repository_version": "2.2.0.1-885",
+ "script": "remove_previous_stacks.py",
+ "version": "2.2.0.1-885"
+ },
+ "commandId": "14-1",
+ "clusterHostInfo": {
+ "snamenode_host": [
+ "0b3.vm"
+ ],
+ "nm_hosts": [
+ "0b3.vm"
+ ],
+ "app_timeline_server_hosts": [
+ "0b3.vm"
+ ],
+ "all_ping_ports": [
+ "8670"
+ ],
+ "rm_host": [
+ "0b3.vm"
+ ],
+ "all_hosts": [
+ "0b3.vm"
+ ],
+ "slave_hosts": [
+ "0b3.vm"
+ ],
+ "namenode_host": [
+ "0b3.vm"
+ ],
+ "ambari_server_host": [
+ "0b3.vm"
+ ],
+ "zookeeper_hosts": [
+ "0b3.vm"
+ ],
+ "hs_host": [
+ "0b3.vm"
+ ]
+ },
+ "configurations": {
+ "cluster-env": {
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+ },
+ "core-site": {
+ "io.compression.codecs": "com.hadoop.compression.lzo"
+ }
+ }
+}
[2/2] ambari git commit: AMBARI-18435. Provide script to delete an
old HDP stack version (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-18435. Provide script to delete an old HDP stack version (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a6c12bfc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a6c12bfc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a6c12bfc
Branch: refs/heads/branch-2.5
Commit: a6c12bfc3df56d4b8a4f178edfc60bed48a449d4
Parents: f5f1272
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Sep 26 14:05:11 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Sep 26 14:09:09 2016 +0300
----------------------------------------------------------------------
.../test/python/ambari_agent/TestHostInfo.py | 10 +-
.../libraries/functions/packages_analyzer.py | 2 +
.../system_action_definitions.xml | 10 ++
.../scripts/remove_previous_stacks.py | 118 ++++++++++++++++
.../custom_actions/TestRemoveStackVersion.py | 140 +++++++++++++++++++
.../configs/remove_previous_stacks.json | 90 ++++++++++++
6 files changed, 365 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index 616fd5b..0c0a3b3 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -105,9 +105,9 @@ class TestHostInfo(TestCase):
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(9, len(installedPackages))
for package in installedPackages:
- self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86",
- "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
- "hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch", "libconfuse.x86_64"])
+ self.assertTrue(package[0] in ["AMBARI.dev", "PyXML", "oracle-server-db",
+ "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+ "hcatalog", "hesiod", "hive", "ambari-log4j", "libconfuse"])
self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"])
self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0",
@@ -116,8 +116,8 @@ class TestHostInfo(TestCase):
packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"],
installedPackages)
self.assertEqual(4, len(packages))
- expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
- "hesiod.x86_64", "hive.noarch"]
+ expected = ["AMBARI.dev", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+ "hesiod", "hive"]
for package in expected:
self.assertTrue(package in packages)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
index 7bbbd3d..52c46ed 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
@@ -155,6 +155,8 @@ def _lookUpYumPackages(command, skipTill, allPackages):
items = items + line.strip(' \t\n\r').split()
for i in range(0, len(items), 3):
+ if '.' in items[i]:
+ items[i] = items[i][:items[i].rindex('.')]
if items[i + 2].find('@') == 0:
items[i + 2] = items[i + 2][1:]
allPackages.append(items[i:i + 3])
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
index fc17584..0f50256 100644
--- a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
+++ b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
@@ -84,4 +84,14 @@
<targetType>ANY</targetType>
<permissions>CLUSTER.UPGRADE_DOWNGRADE_STACK</permissions>
</actionDefinition>
+ <actionDefinition>
+ <actionName>remove_previous_stacks</actionName>
+ <actionType>SYSTEM</actionType>
+ <inputs>version</inputs>
+ <targetService/>
+ <targetComponent/>
+ <defaultTimeout>600</defaultTimeout>
+ <description>Perform remove old stack version action</description>
+ <targetType>ANY</targetType>
+ </actionDefinition>
</actionDefinitions>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
new file mode 100644
index 0000000..958b800
--- /dev/null
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+
+import os
+from resource_management import Script, format, Package, Execute, Fail
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
+from resource_management.libraries.functions.stack_select import get_stack_versions
+
+CURRENT_ = "/current/"
+stack_root = Script.get_stack_root()
+stack_root_current = stack_root + CURRENT_
+
+
+class RemovePreviousStacks(Script):
+
+
+ def actionexecute(self, env):
+ config = Script.get_config()
+ structured_output = {}
+ version = config['commandParams']['version']
+ self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+
+ versions_to_remove = self.get_lower_versions(version)
+
+ for low_version in versions_to_remove:
+ self.remove_stack_version(structured_output, low_version)
+
+ def remove_stack_version(self, structured_output, version):
+ # check simlinks not refer to version for remove
+ self.check_no_symlink_to_version(structured_output, version)
+ packages_to_remove = self.get_packages_to_remove(version)
+ for package in packages_to_remove:
+ Package(package, action="remove")
+ self.remove_stack_folder(structured_output, version)
+ structured_output["remove_previous_stacks"] = {"exit_code": 0,
+ "message": format("Stack version {0} successfully removed!".format(version))}
+ self.put_structured_out(structured_output)
+
+ def remove_stack_folder(self, structured_output, version):
+ if version and version != '' and stack_root and stack_root != '':
+
+ Logger.info("Removing {0}/{1}".format(stack_root, version))
+ try:
+ Execute(('rm', '-f', stack_root + version),
+ sudo=True)
+ finally:
+ structured_output["remove_previous_stacks"] = {"exit_code": -1,
+ "message": "Failed to remove version {0}{1}".format(stack_root, version)}
+ self.put_structured_out(structured_output)
+
+ def get_packages_to_remove(self, version):
+ packages = []
+ formated_version = version.replace('.', '_').replace('-', '_')
+ all_installed_packages = []
+ allInstalledPackages(all_installed_packages)
+ all_installed_packages = [package[0] for package in all_installed_packages]
+ for package in all_installed_packages:
+ if formated_version in package and self.stack_tool_package not in package:
+ packages.append(package)
+ Logger.info("%s added to remove" % (package))
+ return packages
+
+ def check_no_symlink_to_version(self, structured_output, version):
+ files = os.listdir(stack_root_current)
+ for file in files:
+ if version in os.path.realpath(stack_root_current + file):
+ structured_output["remove_previous_stacks"] = {"exit_code": -1,
+ "message": "{0} contains symlink to version for remove! {1}".format(
+ stack_root_current, version)}
+ self.put_structured_out(structured_output)
+ raise Fail("{0} contains symlink to version for remove! {1}".format(stack_root_current, version))
+
+ def get_lower_versions(self, current_version):
+ versions = get_stack_versions(stack_root)
+ Logger.info("available versions: {0}".format(str(versions)))
+
+ lover_versions = []
+ for version in versions:
+ if self.compare(version, current_version) < 0 :
+ lover_versions.append(version)
+ Logger.info("version %s added to remove" % (version))
+ return lover_versions
+
+ def compare(self, version1, version2):
+ """
+ Compare version1 and version2
+ :param version1:
+ :param version2:
+ :return: Return negative if version1<version2, zero if version1==version2, positive if version1>version2
+ """
+ vesion1_sections = re.findall(r"[\w']+", version1)
+ vesion2_sections = re.findall(r"[\w']+", version2)
+ return cmp(vesion1_sections, vesion2_sections)
+
+if __name__ == "__main__":
+ RemovePreviousStacks().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
new file mode 100644
index 0000000..5a05cc2
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
@@ -0,0 +1,140 @@
+# !/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from mock.mock import patch
+from mock.mock import MagicMock
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from stacks.utils.RMFTestCase import *
+
+OLD_VERSION_STUB = '2.1.0.0-400'
+VERSION_STUB = '2.2.0.1-885'
+
+@patch.object(Logger, 'logger', new=MagicMock())
+class TestRemoveStackVersion(RMFTestCase):
+
+ @staticmethod
+ def _add_packages(arg):
+ arg.append(["pkg12_1_0_0_400", "1.0", "repo"])
+ arg.append(["pkg22_1_0_1_885", "2.0", "repo2"])
+ arg.append(["hdp-select2_1_0_1_885", "2.0", "repo2"])
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+ def test_normal_flow(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, ):
+
+ stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+
+ self.assertResourceCalled('Package', "pkg12_1_0_0_400", action=["remove"])
+ self.assertTrue(put_structured_out_mock.called)
+ self.assertEquals(put_structured_out_mock.call_args[0][0],
+ {'remove_previous_stacks': {'exit_code': 0,
+ 'message': 'Stack version 2.1.0.0-400 successfully removed!'}})
+ self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp2.1.0.0-400'),
+ sudo = True,
+ )
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+ def test_without_versions(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock ):
+
+ stack_versions_mock.return_value = [VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+ @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+ @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+ @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+ @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
+ @patch("os.listdir", new = MagicMock(return_value=["somefile" + OLD_VERSION_STUB]))
+ def test_symlink_exist(self,
+ write_actual_version_to_history_file_mock,
+ read_actual_version_from_history_file_mock,
+ stack_versions_mock,
+ put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, ):
+
+ stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+ allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+ list_ambari_managed_repos_mock.return_value = []
+
+ try:
+ self.executeScript("scripts/remove_previous_stacks.py",
+ classname="RemovePreviousStacks",
+ command="actionexecute",
+ config_file="remove_previous_stacks.json",
+ target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+ os_type=('Redhat', '6.4', 'Final')
+ )
+ self.fail("Should throw exception")
+ except Fail, e:
+ self.assertEquals(str(e), '/usr/hdp/current/ contains symlink to version for remove! 2.1.0.0-400')
+ pass # Expected
+
+ self.assertTrue(stack_versions_mock.called)
+ self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
new file mode 100644
index 0000000..cc4a626
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
@@ -0,0 +1,90 @@
+{
+ "configuration_attributes": {},
+ "roleCommand": "ACTIONEXECUTE",
+ "clusterName": "cc",
+ "hostname": "0b3.vm",
+ "passiveInfo": [],
+ "hostLevelParams": {
+ "agent_stack_retry_count": "5",
+ "agent_stack_retry_on_unavailability": "false",
+ "jdk_location": "http://0b3.vm:8080/resources/",
+ "ambari_db_rca_password": "mapred",
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "java_version": "8",
+ "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
+ "jce_name": "UnlimitedJCEPolicyJDK7.zip",
+ "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar",
+ "stack_version": "2.1",
+ "stack_name": "HDP",
+ "db_name": "ambari",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "jdk_name": "jdk-7u67-linux-x64.tar.gz",
+ "ambari_db_rca_username": "mapred",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"
+ },
+ "commandType": "SYSTEM",
+ "serviceName": "null",
+ "role": "remove_previous_stacks",
+ "forceRefreshConfigTags": [],
+ "taskId": 61,
+ "public_hostname": "0b3.vm",
+ "configurations": {
+ "cluster-env": {
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+ }
+ },
+ "commandParams": {
+ "command_timeout": "60",
+ "script_type": "PYTHON",
+ "repository_version": "2.2.0.1-885",
+ "script": "remove_previous_stacks.py",
+ "version": "2.2.0.1-885"
+ },
+ "commandId": "14-1",
+ "clusterHostInfo": {
+ "snamenode_host": [
+ "0b3.vm"
+ ],
+ "nm_hosts": [
+ "0b3.vm"
+ ],
+ "app_timeline_server_hosts": [
+ "0b3.vm"
+ ],
+ "all_ping_ports": [
+ "8670"
+ ],
+ "rm_host": [
+ "0b3.vm"
+ ],
+ "all_hosts": [
+ "0b3.vm"
+ ],
+ "slave_hosts": [
+ "0b3.vm"
+ ],
+ "namenode_host": [
+ "0b3.vm"
+ ],
+ "ambari_server_host": [
+ "0b3.vm"
+ ],
+ "zookeeper_hosts": [
+ "0b3.vm"
+ ],
+ "hs_host": [
+ "0b3.vm"
+ ]
+ },
+ "configurations": {
+ "cluster-env": {
+ "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+ "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+ },
+ "core-site": {
+ "io.compression.codecs": "com.hadoop.compression.lzo"
+ }
+ }
+}