You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2018/07/19 22:22:54 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-24318]
- Report Mpack Installation State When Installing Packages For Upgrade
(#1816)
This is an automated email from the ASF dual-hosted git repository.
jonathanhurley pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 4689c0d [AMBARI-24318] - Report Mpack Installation State When Installing Packages For Upgrade (#1816)
4689c0d is described below
commit 4689c0d03757156a4714b1aafe1c0804d1d141b6
Author: Jonathan Hurley <jo...@apache.org>
AuthorDate: Thu Jul 19 18:22:52 2018 -0400
[AMBARI-24318] - Report Mpack Installation State When Installing Packages For Upgrade (#1816)
---
.../test/python/resource_management/TestScript.py | 21 ---
.../libraries/functions/stack_select.py | 21 ---
.../libraries/functions/version_select_util.py | 133 ------------------
.../resource_management/libraries/script/script.py | 45 +++---
.../ambari/server/agent/HeartbeatProcessor.java | 23 ++--
.../upgrade/MpackInstallStateListener.java | 14 +-
.../system_action_definitions.xml | 10 --
.../custom_actions/scripts/mpack_packages.py | 32 +++--
.../scripts/remove_previous_stacks.py | 119 ----------------
.../src/test/python/TestVersionSelectUtil.py | 139 -------------------
.../custom_actions/TestRemoveStackVersion.py | 152 ---------------------
.../configs/remove_previous_stacks.json | 90 ------------
.../python/custom_actions/test_mpack_install.py | 39 ++++--
13 files changed, 97 insertions(+), 741 deletions(-)
diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py
index cc5c171..63968c0 100644
--- a/ambari-agent/src/test/python/resource_management/TestScript.py
+++ b/ambari-agent/src/test/python/resource_management/TestScript.py
@@ -86,27 +86,6 @@ class TestScript(RMFTestCase):
self.assertEquals({}, Script.structuredOut)
- @patch.object(Logger, "error", new = MagicMock())
- @patch.object(Script, "put_structured_out")
- @patch("resource_management.libraries.functions.version_select_util.get_component_version_from_symlink", new = MagicMock(return_value=None))
- @patch("resource_management.libraries.functions.stack_select.get_package_name", new = MagicMock(return_value="foo-package"))
- @patch("resource_management.libraries.functions.stack_select.unsafe_get_stack_versions", new = MagicMock(return_value=("",0,["2.6.0.0-1234"])))
- def test_save_version_structured_out_stack_select(self, pso_mock):
- """
- Tests that when writing out the version of the component to the structure output,
- if all else fails, we'll invoke the stack-select tool to see if there are any versions
- reported.
- :param pso_mock:
- :return:
- """
- script = Script()
- script.stroutfile = ''
- script.save_component_version_to_structured_out("start")
-
- self.assertEqual(pso_mock.call_count, 1)
- self.assertEquals(pso_mock.call_args[0][0], {'version':'2.6.0.0-1234'})
-
-
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index e204a2d..a50e7df 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -40,7 +40,6 @@ from resource_management.core import shell
from resource_management.core import sudo
from resource_management.core.shell import call
from resource_management.libraries.functions.version import format_stack_version
-from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
from resource_management.libraries.functions import stack_features
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions import upgrade_summary
@@ -472,26 +471,6 @@ def unsafe_get_stack_versions():
@deprecated(comment = "The stack-select tools are no longer used")
-def get_stack_versions(stack_root):
- """
- Gets list of stack versions installed on the host.
- By default a call to <stack-selector-tool> versions is made to get the list of installed stack versions.
- As a fallback list of installed versions is collected from stack version directories in stack install root.
- :param stack_root: Stack install root
- :return: Returns list of installed stack versions.
- """
- stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
- code, out = call((STACK_SELECT_PREFIX, stack_selector_path, 'versions'))
- versions = []
- if 0 == code:
- for line in out.splitlines():
- versions.append(line.rstrip('\n'))
- if not versions:
- versions = get_versions_from_stack_root(stack_root)
- return versions
-
-
-@deprecated(comment = "The stack-select tools are no longer used")
def get_stack_version_before_install(component_name):
"""
Works in the similar way to '<stack-selector-tool> status component',
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
deleted file mode 100644
index bc19b68..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-import os
-import re
-import tempfile
-
-from resource_management.core.logger import Logger
-from resource_management.core import shell
-from resource_management.libraries.functions import stack_tools
-from resource_management.libraries.functions.decorator import deprecated
-
-@deprecated(comment = "The stack-select tools are no longer used")
-def get_component_version_from_symlink(stack_name, component_name):
- """
- Gets the version of the specified component by invoking the stack-select tool to query for the
- version which is referenced by the symlink.
-
- :param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from
- the command-#.json file's ["stackSettings"]["stack_name"]
- :param component_name: Component name as a string necessary to get the version
- :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
- """
- version = None
- if not stack_name or not component_name:
- Logger.error("Could not determine component version because of the parameters is empty. " \
- "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
- return version
-
- stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
- if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path):
- tmpfile = tempfile.NamedTemporaryFile(delete=True)
- out, code = None, -1
- get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name, tmpfile.name)
-
- try:
- # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
- # ToDo: original problem looks strange
- with open(tmpfile.name, 'r') as f:
- code = shell.call(get_stack_comp_version_cmd, quiet=True)[0]
- out = f.read()
-
- if code != 0 or out is None:
- raise ValueError("Code is nonzero or output is empty")
-
- Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
- matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
- version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0]) > 0 else None
- Logger.debug("Version for component %s: %s" % (component_name, str(version)))
- except Exception, e:
- Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
- (component_name, get_stack_comp_version_cmd, str(code), str(out)))
- else:
- Logger.error("Could not find stack selector for stack: %s" % str(stack_name))
-
- return version
-
-@deprecated(comment = "The stack-select tools are no longer used")
-def get_component_version_with_stack_selector(stack_selector_path, component_name):
- """
- For specific cases where we deal with HDP add on services from a management pack, the version
- needs to be determined by using the specific stack selector itself.
- :param stack_selector_path: /usr/bin/hdf-select
- Comes from the service which calls for this function.
- :param component_name: Component name as a string necessary to get the version
- :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
- This function can be called by custom services, hence should not be removed
- """
- version = None
- out = None
- code = -1
- if not stack_selector_path:
- Logger.error("Stack selector path not provided")
- elif not os.path.exists(stack_selector_path):
- Logger.error("Stack selector path does not exist")
- elif not component_name:
- Logger.error("Component name not provided")
- else:
- tmpfile = tempfile.NamedTemporaryFile()
-
- get_stack_comp_version_cmd = ""
- try:
- # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
- with open(tmpfile.name, 'r') as file:
- get_stack_comp_version_cmd = '{0} status {1} > {2}' .format(stack_selector_path, component_name, tmpfile.name)
- code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
- out = file.read()
-
- if code != 0 or out is None:
- raise Exception("Code is nonzero or output is empty")
-
- Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
- matches = re.findall(r"([\d\.]+\-\d+)", out)
- version = matches[0] if matches and len(matches) > 0 else None
- except Exception, e:
- Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
- (component_name, get_stack_comp_version_cmd, str(code), str(out)))
- return version
-
-@deprecated(comment = "The stack-select tools are no longer used")
-def get_versions_from_stack_root(stack_root):
- """
- Given a stack install root, returns a list of stack versions currently installed.
- The list of installed stack versions is determined purely based on the stack version directories
- found in the stack install root.
- Because each stack name may have different logic, the input is a generic dictionary.
- :param stack_root: Stack install root directory
- :return: Returns list of installed stack versions
- """
- if stack_root is None or not os.path.exists(stack_root):
- return []
-
- installed_stack_versions = [f for f in os.listdir(stack_root) if os.path.isdir(os.path.join(stack_root, f))
- and re.match("([\d\.]+(-\d+)?)", f)]
- return installed_stack_versions
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index ddb5d1d..d1d249c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -48,7 +48,6 @@ from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
from resource_management.core.resources.packaging import Package
-from resource_management.libraries.functions import version_select_util
from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions import stack_tools
@@ -182,16 +181,11 @@ class Script(object):
return None
- def save_mpack_to_structured_out(self, command_name):
+ def save_mpack_to_structured_out(self, ):
"""
- Writes out information about the managment pack which was installed if this is an installation command.
- :param command_name: command name
+ Writes out information about the managment pack associated with the command.
:return: None
"""
- is_install_command = command_name is not None and command_name.lower() == "install"
- if not is_install_command:
- return
-
command_repository = CommandRepository(self.get_config()['repositoryFile'])
Logger.info("Reporting installation state for {0}".format(command_repository))
@@ -205,13 +199,11 @@ class Script(object):
self.put_structured_out({"mpack_installation": mpack_dictionary})
- def save_component_version_to_structured_out(self, command_name):
+ def save_component_version_to_structured_out(self):
"""
Saves the version of the component for this command to the structured out file. If the
command is an install command and the repository is trusted, then it will use the version of
the repository. Otherwise, it will consult the stack-select tool to read the symlink version.
-
- :param command_name: command name
:return: None
"""
from resource_management.libraries.functions import mpack_manager_helper
@@ -245,22 +237,38 @@ class Script(object):
component_type, mpack_name))
- def should_expose_component_version(self, command_name):
+ def should_write_mpack_information(self):
+ """
+ Gets whether this command should write out mpack information to the structured output.
+ This is usually only done in cases where the command is an install command.
+ However, scripts can override this function to provide their own handling.
+ :return: True if the mpack information should be written to structured output, False otherwise.
+ """
+ if self.is_hook():
+ return;
+
+ is_install_command = self.command_name is not None and self.command_name.lower() == "install"
+ if not is_install_command:
+ return False
+
+ return True
+
+
+ def should_expose_component_version(self):
"""
Analyzes config and given command to determine if stack version should be written
to structured out.
- :param command_name: command name
:return: True or False
"""
from resource_management.libraries.functions.default import default
- if command_name.lower() == "status":
+ if self.command_name.lower() == "status":
request_version = default("/commandParams/request_version", None)
if request_version is not None:
return True
else:
# Populate version only on base commands
version_reporting_commands = ["start", "install", "restart"]
- return command_name.lower() in version_reporting_commands
+ return self.command_name.lower() in version_reporting_commands
return False
@@ -356,10 +364,11 @@ class Script(object):
ex.pre_raise()
raise
finally:
- self.save_mpack_to_structured_out(self.command_name)
+ if self.should_write_mpack_information():
+ self.save_mpack_to_structured_out()
- if self.should_expose_component_version(self.command_name):
- self.save_component_version_to_structured_out(self.command_name)
+ if self.should_expose_component_version():
+ self.save_component_version_to_structured_out()
def execute_prefix_function(self, command_name, afix, env):
"""
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
index 4d0c072..991c82d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatProcessor.java
@@ -67,7 +67,6 @@ import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.UpgradeState;
import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
import org.apache.ambari.server.state.host.HostStatusUpdatesReceivedEvent;
-import org.apache.ambari.server.state.scheduler.RequestExecution;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpFailedEvent;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpSucceededEvent;
@@ -371,13 +370,19 @@ public class HeartbeatProcessor extends AbstractService{
}
}
+ // get the status of the command
+ HostRoleStatus status = HostRoleStatus.valueOf(report.getStatus());
+
@Nullable
JsonObject structuredOutputJson = null;
String structuredOutputString = report.getStructuredOut();
- if (StringUtils.isNotBlank(structuredOutputString)
- && !StringUtils.equals(structuredOutputString, "{}")) {
- JsonElement element = gson.fromJson(structuredOutputString, JsonElement.class);
- structuredOutputJson = element.getAsJsonObject();
+ // only try to parse thes structured output if the command is completed
+ if (status.isCompletedState()) {
+ if (StringUtils.isNotBlank(structuredOutputString)
+ && !StringUtils.equals(structuredOutputString, "{}")) {
+ JsonElement element = gson.fromJson(structuredOutputString, JsonElement.class);
+ structuredOutputJson = element.getAsJsonObject();
+ }
}
// If the report indicates the keytab file was successfully transferred to a host or removed
@@ -385,7 +390,7 @@ public class HeartbeatProcessor extends AbstractService{
if (Service.Type.KERBEROS.name().equalsIgnoreCase(report.getServiceName()) &&
Role.KERBEROS_CLIENT.name().equalsIgnoreCase(report.getRole()) &&
RoleCommand.CUSTOM_COMMAND.name().equalsIgnoreCase(report.getRoleCommand()) &&
- RequestExecution.Status.COMPLETED.name().equalsIgnoreCase(report.getStatus())) {
+ HostRoleStatus.COMPLETED == status) {
String customCommand = report.getCustomCommand();
@@ -456,7 +461,7 @@ public class HeartbeatProcessor extends AbstractService{
ServiceComponentHost scHost = svcComp.getServiceComponentHost(hostName);
String schName = scHost.getServiceComponentName();
- if (report.getStatus().equals(HostRoleStatus.COMPLETED.toString())) {
+ if (HostRoleStatus.COMPLETED == status) {
// Reading component version if it is present
ComponentVersionStructuredOut componentVersionStructuredOut = null;
if (null != structuredOutputJson) {
@@ -523,7 +528,7 @@ public class HeartbeatProcessor extends AbstractService{
scHost.handleEvent(new ServiceComponentHostOpSucceededEvent(schName,
hostName, now));
}
- } else if (report.getStatus().equals("FAILED")) {
+ } else if (HostRoleStatus.FAILED == status) {
if (structuredOutputJson != null) {
JsonElement upgradeStructedOutput = structuredOutputJson.get(
StructuredOutputType.UPGRADE_SUMMARY.getRoot());
@@ -550,7 +555,7 @@ public class HeartbeatProcessor extends AbstractService{
LOG.info("Received report for a command that is no longer active. " + report);
}
}
- } else if (report.getStatus().equals("IN_PROGRESS")) {
+ } else if (HostRoleStatus.IN_PROGRESS == status) {
scHost.handleEvent(new ServiceComponentHostOpInProgressEvent(schName,
hostName, now));
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java
index 42bb125..1ad1075 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/MpackInstallStateListener.java
@@ -244,7 +244,7 @@ public class MpackInstallStateListener {
if(null != installReportingElement) {
// try to parse the structured output of the install command
structuredOutput = m_gson.fromJson(installReportingElement,
- InstallCommandStructuredOutput.class);
+ InstallCommandStructuredOutput.class);
}
}
} catch (JsonSyntaxException jsonException) {
@@ -323,12 +323,6 @@ public class MpackInstallStateListener {
*/
private static class InstallCommandStructuredOutput {
/**
- * Either SUCCESS or FAIL
- */
- @SerializedName("package_installation_result")
- private String packageInstallationResult;
-
- /**
* The actual version returned, even when a failure during install occurs.
*/
@SerializedName("mpackName")
@@ -339,5 +333,11 @@ public class MpackInstallStateListener {
*/
@SerializedName("mpackId")
private Long mpackId = null;
+
+ /**
+ * The version of the mpack, including build number.
+ */
+ @SerializedName("mpackVersion")
+ private String mpackVersion = null;
}
}
diff --git a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
index ccea252..f542ef5 100644
--- a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
+++ b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
@@ -84,14 +84,4 @@
<targetType>ANY</targetType>
<permissions>CLUSTER.UPGRADE_DOWNGRADE_STACK</permissions>
</actionDefinition>
- <actionDefinition>
- <actionName>remove_previous_stacks</actionName>
- <actionType>SYSTEM</actionType>
- <inputs>version</inputs>
- <targetService/>
- <targetComponent/>
- <defaultTimeout>600</defaultTimeout>
- <description>Perform remove old stack version action</description>
- <targetType>ANY</targetType>
- </actionDefinition>
</actionDefinitions>
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/mpack_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/mpack_packages.py
index 98589ae..7a8c539 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/mpack_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/mpack_packages.py
@@ -42,7 +42,23 @@ class MpackPackages(Script):
self.repo_mgr = ManagerFactory.get()
self.repo_files = {}
+ def should_write_mpack_information(self):
+ """
+ Returns True always as this script should always write out mpack information.
+ :return: True
+ """
+ return True
+
def actionexecute(self, env):
+ """
+ Executes this script which performs the following work:
+ - Checks the transaction state of the package manager
+ - Attempts to install packages
+
+ If either is unsuccessful, this will raise a Fail() and the command will be failed.
+ :param env:
+ :return:
+ """
num_errors = 0
# Parse parameters
@@ -83,14 +99,6 @@ class MpackPackages(Script):
Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
- # Build structured output with initial values
- self.structured_output = {
- 'package_installation_result': 'FAIL',
- 'mpackId': command_repository.mpack_id
- }
-
- self.put_structured_out(self.structured_output)
-
try:
# check package manager non-completed transactions
if self.repo_mgr.check_uncompleted_transactions():
@@ -106,15 +114,15 @@ class MpackPackages(Script):
try:
ret_code = self.install_packages(package_list)
- if ret_code == 0:
- self.structured_output['package_installation_result'] = 'SUCCESS'
- self.put_structured_out(self.structured_output)
- else:
+ if ret_code != 0:
num_errors += 1
except Exception as err:
num_errors += 1
Logger.logger.exception("Could not install packages. Error: {0}".format(str(err)))
+ # save mmpack information to structured output
+ self.save_mpack_to_structured_out()
+
# Provide correct exit code
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
deleted file mode 100644
index 4ebd115..0000000
--- a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-import re
-
-import os
-from resource_management import Script, format, Package, Execute, Fail
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions import stack_tools
-from resource_management.libraries.functions.stack_select import get_stack_versions
-from ambari_commons.repo_manager import ManagerFactory
-
-CURRENT_ = "/current/"
-stack_root = Script.get_stack_root()
-stack_root_current = stack_root + CURRENT_
-
-
-class RemovePreviousStacks(Script):
-
-
- def actionexecute(self, env):
- config = Script.get_config()
- structured_output = {}
- version = config['commandParams']['version']
- self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
-
- versions_to_remove = self.get_lower_versions(version)
- self.pkg_provider = ManagerFactory.get()
-
- for low_version in versions_to_remove:
- self.remove_stack_version(structured_output, low_version)
-
- def remove_stack_version(self, structured_output, version):
- # check simlinks not refer to version for remove
- self.check_no_symlink_to_version(structured_output, version)
- packages_to_remove = self.get_packages_to_remove(version)
- for package in packages_to_remove:
- Package(package, action="remove")
- self.remove_stack_folder(structured_output, version)
- structured_output["remove_previous_stacks"] = {"exit_code": 0,
- "message": format("Stack version {0} successfully removed!".format(version))}
- self.put_structured_out(structured_output)
-
- def remove_stack_folder(self, structured_output, version):
- if version and version != '' and stack_root and stack_root != '':
-
- Logger.info("Removing {0}/{1}".format(stack_root, version))
- try:
- Execute(('rm', '-f', stack_root + version),
- sudo=True)
- finally:
- structured_output["remove_previous_stacks"] = {"exit_code": -1,
- "message": "Failed to remove version {0}{1}".format(stack_root, version)}
- self.put_structured_out(structured_output)
-
- def get_packages_to_remove(self, version):
- packages = []
- formated_version = version.replace('.', '_').replace('-', '_')
- all_installed_packages = self.pkg_provider.all_installed_packages()
-
- all_installed_packages = [package[0] for package in all_installed_packages]
- for package in all_installed_packages:
- if formated_version in package and self.stack_tool_package not in package:
- packages.append(package)
- Logger.info("%s added to remove" % (package))
- return packages
-
- def check_no_symlink_to_version(self, structured_output, version):
- files = os.listdir(stack_root_current)
- for file in files:
- if version in os.path.realpath(stack_root_current + file):
- structured_output["remove_previous_stacks"] = {"exit_code": -1,
- "message": "{0} contains symlink to version for remove! {1}".format(
- stack_root_current, version)}
- self.put_structured_out(structured_output)
- raise Fail("{0} contains symlink to version for remove! {1}".format(stack_root_current, version))
-
- def get_lower_versions(self, current_version):
- versions = get_stack_versions(stack_root)
- Logger.info("available versions: {0}".format(str(versions)))
-
- lover_versions = []
- for version in versions:
- if self.compare(version, current_version) < 0 :
- lover_versions.append(version)
- Logger.info("version %s added to remove" % (version))
- return lover_versions
-
- def compare(self, version1, version2):
- """
- Compare version1 and version2
- :param version1:
- :param version2:
- :return: Return negative if version1<version2, zero if version1==version2, positive if version1>version2
- """
- vesion1_sections = re.findall(r"[\w']+", version1)
- vesion2_sections = re.findall(r"[\w']+", version2)
- return cmp(vesion1_sections, vesion2_sections)
-
-if __name__ == "__main__":
- RemovePreviousStacks().execute()
diff --git a/ambari-server/src/test/python/TestVersionSelectUtil.py b/ambari-server/src/test/python/TestVersionSelectUtil.py
deleted file mode 100644
index 82985fa..0000000
--- a/ambari-server/src/test/python/TestVersionSelectUtil.py
+++ /dev/null
@@ -1,139 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from unittest import TestCase
-import os
-from mock.mock import patch, MagicMock
-
-from resource_management.core.logger import Logger
-
-class TestVersionSelectUtil(TestCase):
- """
- Class that tests the method of the version_select_util.py file.
- """
- def setUp(self):
- import imp
-
- Logger.logger = MagicMock()
-
- self.test_directory = os.path.dirname(os.path.abspath(__file__))
- test_file_path = os.path.join(self.test_directory, '../../../../ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py')
- with open(test_file_path, 'rb') as fp:
- self.module = imp.load_module('module', fp, test_file_path, ('.py', 'rb', imp.PY_SOURCE))
-
- @patch('__builtin__.open')
- @patch("resource_management.core.shell.call")
- @patch('os.path.exists')
- @patch("resource_management.libraries.functions.stack_tools.get_stack_tool")
- def test_get_component_version_from_symlink(self, get_stack_tool_mock, os_path_exists_mock, call_mock, open_mock):
- stack_expected_version = "2.2.1.0-2175"
-
- # Mock classes for reading from a file
- class MagicFile(object):
- allowed_names = set(["hadoop-hdfs-namenode",
- "hadoop-hdfs-datanode",
- "zookeeper-server",
- "zookeeper-client"
- ])
- def read(self, value):
- return (value + " - " + stack_expected_version) if value in self.allowed_names else ("ERROR: Invalid package - " + value)
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
- def __enter__(self):
- return self
- pass
-
- class MagicFile1(MagicFile):
- def read(self):
- return super(MagicFile1, self).read("hadoop-nonexistent-component-name")
- class MagicFile2(MagicFile):
- def read(self):
- return super(MagicFile2, self).read("hadoop-hdfs-namenode")
- class MagicFile3(MagicFile):
- def read(self):
- return super(MagicFile3, self).read("hadoop-hdfs-datanode")
-
- get_stack_tool_mock.side_effect = [("hdp-select", "/usr/bin/hdp-select", "hdp-select"),
- ("hdp-select", "/usr/bin/hdp-select", "hdp-select"),
- ("hdp-select", "/usr/bin/hdp-select", "hdp-select"),
- ("hdp-select", "/usr/bin/hdp-select", "hdp-select")]
- os_path_exists_mock.side_effect = [False, True, True, True]
- open_mock.side_effect = [MagicFile1(), MagicFile2(), MagicFile3()]
- call_mock.side_effect = [(0, "value will come from MagicFile"), ] * 3
-
- # Missing stack name
- version = self.module.get_component_version_from_symlink(None, "hadoop-hdfs-datanode")
- self.assertEquals(version, None)
- # Missing component name
- version = self.module.get_component_version_from_symlink("HDP", None)
- self.assertEquals(version, None)
-
- # Invalid stack name
- version = self.module.get_component_version_from_symlink("StackDoesNotExist", "hadoop-hdfs-datanode")
- self.assertEquals(version, None)
- # Invalid component name
- version = self.module.get_component_version_from_symlink("HDP", "hadoop-nonexistent-component-name")
- self.assertEquals(version, None)
-
- # Pass
- version = self.module.get_component_version_from_symlink("HDP", "hadoop-hdfs-namenode")
- self.assertEquals(version, stack_expected_version)
- version = self.module.get_component_version_from_symlink("HDP", "hadoop-hdfs-datanode")
- self.assertEquals(version, stack_expected_version)
-
- @patch('__builtin__.open')
- @patch("resource_management.core.shell.call")
- @patch('os.path.exists')
- @patch("resource_management.libraries.functions.stack_tools.get_stack_tool")
- def test_get_component_version_no_build_ids(self, get_stack_tool_mock, os_path_exists_mock, call_mock, open_mock):
- stack_expected_version = "2.2.1.0"
-
- # Mock classes for reading from a file
- class MagicFile(object):
- allowed_names = set(["hive-server2",
- "zookeeper-server"])
- def read(self, value):
- return (value + " - " + stack_expected_version) if value in self.allowed_names else ("ERROR: Invalid package - " + value)
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
- def __enter__(self):
- return self
- pass
-
- class MagicFile1(MagicFile):
- def read(self):
- return super(MagicFile1, self).read("hive-server2")
- class MagicFile2(MagicFile):
- def read(self):
- return super(MagicFile2, self).read("zookeeper-server")
-
- get_stack_tool_mock.side_effect = [("hdp-select", "/usr/bin/hdp-select", "hdp-select"),
- ("hdp-select", "/usr/bin/hdp-select", "hdp-select")]
- os_path_exists_mock.side_effect = [True, True]
- open_mock.side_effect = [MagicFile1(), MagicFile2()]
- call_mock.side_effect = [(0, "value will come from MagicFile"), ] * 2
-
- # Pass
- version = self.module.get_component_version_from_symlink("HDP", "hive-server2")
- self.assertEquals(version, stack_expected_version)
- version = self.module.get_component_version_from_symlink("HDP", "zookeeper-server")
- self.assertEquals(version, stack_expected_version)
diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
deleted file mode 100644
index ed3dfad..0000000
--- a/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# !/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-
-from mock.mock import patch
-from mock.mock import MagicMock
-from resource_management.core.logger import Logger
-from resource_management.core.exceptions import Fail
-from stacks.utils.RMFTestCase import *
-
-OLD_VERSION_STUB = '2.1.0.0-400'
-VERSION_STUB = '2.2.0.1-885'
-
-
-@patch.object(Logger, 'logger', new=MagicMock())
-class TestRemoveStackVersion(RMFTestCase):
-
- @staticmethod
- def _add_packages():
- return [
- ["pkg12_1_0_0_400", "1.0", "repo"],
- ["pkg22_1_0_1_885", "2.0", "repo2"],
- ["hdp-select2_1_0_1_885", "2.0", "repo2"]
- ]
-
- @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
- @patch("ambari_commons.repo_manager.ManagerFactory.get")
- @patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
- @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
- @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
- @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
- def test_normal_flow(self,
- write_actual_version_to_history_file_mock,
- read_actual_version_from_history_file_mock,
- stack_versions_mock,
- put_structured_out_mock, get_provider_mock, list_ambari_managed_repos_mock):
- m = MagicMock()
- m.all_installed_packages.side_effect = TestRemoveStackVersion._add_packages
- get_provider_mock.return_value = m
- stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
- list_ambari_managed_repos_mock.return_value = []
-
- self.executeScript("scripts/remove_previous_stacks.py",
- classname="RemovePreviousStacks",
- command="actionexecute",
- config_file="remove_previous_stacks.json",
- target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
- os_type=('Redhat', '6.4', 'Final')
- )
- self.assertTrue(stack_versions_mock.called)
- self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
-
- self.assertResourceCalled('Package', "pkg12_1_0_0_400", action=["remove"])
- self.assertTrue(put_structured_out_mock.called)
- self.assertEquals(put_structured_out_mock.call_args[0][0],
- {'remove_previous_stacks': {'exit_code': 0,
- 'message': 'Stack version 2.1.0.0-400 successfully removed!'}})
- self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp2.1.0.0-400'),
- sudo = True,
- )
- self.assertNoMoreResources()
-
- @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
- @patch("ambari_commons.repo_manager.ManagerFactory.get")
- @patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
- @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
- @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
- @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
- def test_without_versions(self,
- write_actual_version_to_history_file_mock,
- read_actual_version_from_history_file_mock,
- stack_versions_mock,
- put_structured_out_mock, get_provider_mock, list_ambari_managed_repos_mock ):
-
- stack_versions_mock.return_value = [VERSION_STUB]
-
- m = MagicMock()
- m.all_installed_packages.side_effect = TestRemoveStackVersion._add_packages
- get_provider_mock.return_value = m
-
- list_ambari_managed_repos_mock.return_value = []
-
- self.executeScript("scripts/remove_previous_stacks.py",
- classname="RemovePreviousStacks",
- command="actionexecute",
- config_file="remove_previous_stacks.json",
- target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
- os_type=('Redhat', '6.4', 'Final')
- )
- self.assertTrue(stack_versions_mock.called)
- self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
- self.assertNoMoreResources()
-
- @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
- @patch("ambari_commons.repo_manager.ManagerFactory.get")
- @patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
- @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
- @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
- @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new = MagicMock(return_value="hdp-select"))
- @patch("os.listdir", new = MagicMock(return_value=["somefile" + OLD_VERSION_STUB]))
- def test_symlink_exist(self,
- write_actual_version_to_history_file_mock,
- read_actual_version_from_history_file_mock,
- stack_versions_mock,
- put_structured_out_mock, get_provider_mock, list_ambari_managed_repos_mock, ):
-
- stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
-
- m = MagicMock()
- m.all_installed_packages.side_effect = TestRemoveStackVersion._add_packages
- get_provider_mock.return_value = m
-
- list_ambari_managed_repos_mock.return_value = []
-
- try:
- self.executeScript("scripts/remove_previous_stacks.py",
- classname="RemovePreviousStacks",
- command="actionexecute",
- config_file="remove_previous_stacks.json",
- target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
- os_type=('Redhat', '6.4', 'Final')
- )
- self.fail("Should throw exception")
- except Fail, e:
- self.assertEquals(str(e), '/usr/hdp/current/ contains symlink to version for remove! 2.1.0.0-400')
- pass # Expected
-
- self.assertTrue(stack_versions_mock.called)
- self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
- self.assertNoMoreResources()
\ No newline at end of file
diff --git a/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
deleted file mode 100644
index eccd547..0000000
--- a/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
+++ /dev/null
@@ -1,90 +0,0 @@
-{
- "configurationAttributes": {},
- "roleCommand": "ACTIONEXECUTE",
- "clusterName": "cc",
- "hostname": "0b3.vm",
- "passiveInfo": [],
- "hostLevelParams": {
- "agent_stack_retry_count": "5",
- "agent_stack_retry_on_unavailability": "false",
- "jdk_location": "http://0b3.vm:8080/resources",
- "ambari_db_rca_password": "mapred",
- "java_home": "/usr/jdk64/jdk1.7.0_67",
- "java_version": "8",
- "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
- "jce_name": "UnlimitedJCEPolicyJDK7.zip",
- "oracle_jdbc_url": "http://0b3.vm:8080/resources/ojdbc6.jar",
- "stack_version": "2.1",
- "stack_name": "HDP",
- "db_name": "ambari",
- "ambari_db_rca_driver": "org.postgresql.Driver",
- "jdk_name": "jdk-7u67-linux-x64.tar.gz",
- "ambari_db_rca_username": "mapred",
- "db_driver_filename": "mysql-connector-java.jar",
- "mysql_jdbc_url": "http://0b3.vm:8080/resources/mysql-connector-java.jar"
- },
- "commandType": "SYSTEM",
- "serviceName": "null",
- "role": "remove_previous_stacks",
- "forceRefreshConfigTags": [],
- "taskId": 61,
- "public_hostname": "0b3.vm",
- "configurations": {
- "cluster-env": {
- "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
- "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
- }
- },
- "commandParams": {
- "command_timeout": "60",
- "script_type": "PYTHON",
- "repository_version": "2.2.0.1-885",
- "script": "remove_previous_stacks.py",
- "version": "2.2.0.1-885"
- },
- "commandId": "14-1",
- "clusterHostInfo": {
- "snamenode_host": [
- "0b3.vm"
- ],
- "nm_hosts": [
- "0b3.vm"
- ],
- "app_timeline_server_hosts": [
- "0b3.vm"
- ],
- "all_ping_ports": [
- "8670"
- ],
- "resourcemanager_hosts": [
- "0b3.vm"
- ],
- "all_hosts": [
- "0b3.vm"
- ],
- "slave_hosts": [
- "0b3.vm"
- ],
- "namenode_host": [
- "0b3.vm"
- ],
- "ambari_server_host": [
- "0b3.vm"
- ],
- "zookeeper_server_hosts": [
- "0b3.vm"
- ],
- "hs_host": [
- "0b3.vm"
- ]
- },
- "configurations": {
- "cluster-env": {
- "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
- "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
- },
- "core-site": {
- "io.compression.codecs": "com.hadoop.compression.lzo"
- }
- }
-}
diff --git a/ambari-server/src/test/python/custom_actions/test_mpack_install.py b/ambari-server/src/test/python/custom_actions/test_mpack_install.py
index 2ec7095..02bf7b2 100644
--- a/ambari-server/src/test/python/custom_actions/test_mpack_install.py
+++ b/ambari-server/src/test/python/custom_actions/test_mpack_install.py
@@ -143,19 +143,12 @@ class TestMpackPackages(RMFTestCase):
@patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
@patch("ambari_commons.repo_manager.ManagerFactory.get")
@patch("resource_management.libraries.script.Script.put_structured_out")
- @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
@patch("ambari_commons.shell.launch_subprocess")
def test_normal_flow_rhel(self,
subprocess_with_timeout,
- stack_versions_mock,
put_structured_out_mock,
get_provider,
list_ambari_managed_repos_mock):
- stack_versions_mock.side_effect = [
- [], # before installation attempt
- [VERSION_STUB]
- ]
-
from ambari_commons.os_check import OSConst
from ambari_commons.repo_manager import ManagerFactory
@@ -183,9 +176,35 @@ class TestMpackPackages(RMFTestCase):
)
self.assertTrue(put_structured_out_mock.called)
self.assertEquals(put_structured_out_mock.call_args[0][0],
- {'package_installation_result': 'SUCCESS',
- 'mpackId': 2
- })
+ {
+ 'mpack_installation':
+ {
+ 'mpackId': 2,
+ 'mpackName': 'HDPCORE',
+ 'mpackVersion': '1.0.0-b251'
+ }
+ })
+
+ self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.21-repo-hdpcore',
+ append_to_file = False,
+ base_url = 'http://repos.ambari.apache.org/hdp/HDP-UTILS-1.1.0.21',
+ action = ['create'],
+ components = [u'HDP-UTILS', 'main'],
+ repo_template = None,
+ repo_file_name = 'ambari-hdpcore-2',
+ mirror_list = None,
+ )
+
+
+ self.assertResourceCalled('Repository', 'HDPCORE-1.0.0-b251-repo-hdpcore',
+ append_to_file = True,
+ base_url = 'http://repos.ambari.apache.org/hdp/HDPCORE-1.0.0-b251',
+ action = ['create'],
+ components = [u'HDPCORE', 'main'],
+ repo_template = None,
+ repo_file_name = 'ambari-hdpcore-2',
+ mirror_list = None,
+ )
self.assertNoMoreResources()