You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/21 04:38:20 UTC
[20/23] ambari git commit: AMBARI-21530 - Service Checks During
Upgrades Should Use Desired Stack (jonathanhurley)
AMBARI-21530 - Service Checks During Upgrades Should Use Desired Stack (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/282c4e21
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/282c4e21
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/282c4e21
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 282c4e213056d351331adac498d4655b8ebc9251
Parents: 4078c48
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 19 22:04:07 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu Jul 20 08:09:38 2017 -0400
----------------------------------------------------------------------
.../AmbariCustomCommandExecutionHelper.java | 5 --
.../internal/UpgradeResourceProvider.java | 50 +++++++++++-----
.../2.1.0.2.0/package/scripts/historyserver.py | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 11 ++--
.../2.1.0.2.0/package/scripts/service_check.py | 6 +-
.../src/test/python/TestStackFeature.py | 61 ++++++++++++++++++++
6 files changed, 103 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 2eca76c..723a10d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -27,7 +27,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_CO
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
@@ -40,7 +39,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JD
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_GROUPS;
@@ -772,9 +770,6 @@ public class AmbariCustomCommandExecutionHelper {
}
commandParams.put(COMMAND_TIMEOUT, commandTimeout);
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
-
execCmd.setCommandParams(commandParams);
if (actionParameters != null) { // If defined
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 0b47ff1..de3225f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1337,6 +1337,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
/**
+ * Adds the hooks and service folders based on the effective stack ID and the
+ * name of the service from the wrapper.
+ *
+ * @param wrapper
+ * the stage wrapper to use when detemrining the service name.
+ * @param effectiveStackId
+ * the stack ID to use when getting the hooks and service folders.
+ * @param commandParams
+ * the params to update with the new values
+ * @throws AmbariException
+ */
+ private void applyRepositoryAssociatedParameters(StageWrapper wrapper, StackId effectiveStackId,
+ Map<String, String> commandParams) throws AmbariException {
+ if (CollectionUtils.isNotEmpty(wrapper.getTasks())
+ && wrapper.getTasks().get(0).getService() != null) {
+
+ AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
+
+ StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(),
+ effectiveStackId.getStackVersion());
+
+ String serviceName = wrapper.getTasks().get(0).getService();
+ ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(),
+ effectiveStackId.getStackVersion(), serviceName);
+
+ commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
+ commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+ }
+ }
+
+ /**
* Creates an action stage using the {@link #EXECUTE_TASK_ROLE} custom action
* to execute some Python command.
*
@@ -1389,21 +1420,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// the ru_execute_tasks invokes scripts - it needs information about where
// the scripts live and for that it should always use the target repository
// stack
- if (CollectionUtils.isNotEmpty(wrapper.getTasks())
- && wrapper.getTasks().get(0).getService() != null) {
-
- AmbariMetaInfo ambariMetaInfo = s_metaProvider.get();
-
- StackInfo stackInfo = ambariMetaInfo.getStack(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion());
-
- String serviceName = wrapper.getTasks().get(0).getService();
- ServiceInfo serviceInfo = ambariMetaInfo.getService(effectiveStackId.getStackName(),
- effectiveStackId.getStackVersion(), serviceName);
-
- params.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- params.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
- }
+ applyRepositoryAssociatedParameters(wrapper, effectiveStackId, params);
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
EXECUTE_TASK_ROLE, Collections.singletonList(filter), params);
@@ -1567,6 +1584,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
+ // add things like hooks and service folders based on effective repo
+ applyRepositoryAssociatedParameters(wrapper, effectiveStackId, commandParams);
+
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
"SERVICE_CHECK", filters, commandParams);
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
index 7405a4e..36417a8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -92,7 +92,7 @@ class HistoryServerDefault(HistoryServer):
env.set_params(params)
self.configure(env) # FOR SECURITY
- if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
+ if check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.version_for_stack_feature_checks):
# MC Hammer said, "Can't touch this"
resource_created = copy_to_hdfs(
"mapreduce",
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 5f400a3..a4f7ee9 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -66,21 +66,20 @@ tarball_map = default("/configurations/cluster-env/tarball_map", None)
config_path = os.path.join(stack_root, "current/hadoop-client/conf")
config_dir = os.path.realpath(config_path)
+# get the correct version to use for checking stack features
+version_for_stack_feature_checks = get_stack_feature_version(config)
+
# This is expected to be of the form #.#.#.#
stack_version_unformatted = config['hostLevelParams']['stack_version']
-stack_version_formatted_major = format_stack_version(stack_version_unformatted)
stack_version_formatted = functions.get_stack_version('hadoop-yarn-resourcemanager')
-stack_supports_ru = stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted_major)
-stack_supports_timeline_state_store = stack_version_formatted_major and check_stack_feature(StackFeature.TIMELINE_STATE_STORE, stack_version_formatted_major)
+stack_supports_ru = check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks)
+stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_STATE_STORE, version_for_stack_feature_checks)
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade.
# It cannot be used during the initial Cluser Install because the version is not yet known.
version = default("/commandParams/version", None)
-# get the correct version to use for checking stack features
-version_for_stack_feature_checks = get_stack_feature_version(config)
-
stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks)
stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks)
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
index a36e8cc..1193b60 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
@@ -95,11 +95,7 @@ class ServiceCheckDefault(ServiceCheck):
mode=params.smoke_hdfs_user_mode,
)
- if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major):
- path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
- else:
- path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar"
-
+ path_to_distributed_shell_jar = format("{stack_root}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
yarn_distrubuted_shell_check_params = ["yarn org.apache.hadoop.yarn.applications.distributedshell.Client",
"-shell_command", "ls", "-num_containers", "{number_of_nm}",
"-jar", "{path_to_distributed_shell_jar}", "-timeout", "300000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/282c4e21/ambari-server/src/test/python/TestStackFeature.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py
index 230734c..6e8bcec 100644
--- a/ambari-server/src/test/python/TestStackFeature.py
+++ b/ambari-server/src/test/python/TestStackFeature.py
@@ -21,10 +21,13 @@ limitations under the License.
from resource_management.core.logger import Logger
from resource_management.libraries.functions.stack_features import get_stack_feature_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.script import Script
from resource_management.core.exceptions import Fail
from unittest import TestCase
+import json
+
Logger.initialize_logger()
class TestStackFeature(TestCase):
@@ -115,6 +118,34 @@ class TestStackFeature(TestCase):
stack_feature_version = get_stack_feature_version(command_json)
self.assertEqual("2.5.9.9-9999", stack_feature_version)
+
+ def test_get_stack_feature(self):
+ """
+ Tests the stack feature version calculated during a STOP command in a downgrade.
+ :return:
+ """
+ command_json = TestStackFeature._get_cluster_upgrade_restart_json()
+ Script.config = command_json
+
+ Script.config["configurations"] = {}
+ Script.config["configurations"]["cluster-env"] = {}
+ Script.config["configurations"]["cluster-env"]["stack_features"] = {}
+ Script.config["configurations"]["cluster-env"]["stack_features"] = json.dumps(TestStackFeature._get_stack_feature_json())
+
+ stack_feature_version = get_stack_feature_version(command_json)
+ self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version))
+ self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version))
+ self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version))
+
+ command_json = TestStackFeature._get_cluster_install_command_json()
+ Script.config.update(command_json)
+
+ stack_feature_version = get_stack_feature_version(command_json)
+ self.assertTrue(check_stack_feature("stack-feature-1", stack_feature_version))
+ self.assertTrue(check_stack_feature("stack-feature-2", stack_feature_version))
+ self.assertFalse(check_stack_feature("stack-feature-3", stack_feature_version))
+
+
@staticmethod
def _get_cluster_install_command_json():
"""
@@ -221,4 +252,34 @@ class TestStackFeature(TestCase):
"version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
+ }
+
+ @staticmethod
+ def _get_stack_feature_json():
+ """
+ A STOP command during a downgrade.
+ :return:
+ """
+ return {
+ "HDP": {
+ "stack_features":[
+ {
+ "name":"stack-feature-1",
+ "description":"Stack Feature 1",
+ "min_version":"2.2.0.0"
+ },
+ {
+ "name":"stack-feature-2",
+ "description":"Stack Feature 2",
+ "min_version":"2.2.0.0",
+ "max_version":"2.6.0.0"
+ },
+ {
+ "name":"stack-feature-3",
+ "description":"Stack Feature 3",
+ "min_version":"2.2.0.0",
+ "max_version":"2.3.0.0"
+ }
+ ]
+ }
}
\ No newline at end of file