You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/12 18:43:26 UTC
ambari git commit: AMBARI-21451 - Expected Values Like original_stack
Are Missing On Downgrades (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 08f48c1eb -> a6ac40bc0
AMBARI-21451 - Expected Values Like original_stack Are Missing On Downgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a6ac40bc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a6ac40bc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a6ac40bc
Branch: refs/heads/branch-2.5
Commit: a6ac40bc07e4c0bec207d0786e4e870a6fe194f2
Parents: 08f48c1
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 12 13:30:16 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 12 14:43:15 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/stack_features.py | 28 +++--
.../actionmanager/ExecutionCommandWrapper.java | 3 +-
.../ambari/server/agent/ExecutionCommand.java | 14 +--
.../controller/ActionExecutionContext.java | 30 +++---
.../controller/AmbariActionExecutionHelper.java | 8 +-
.../ClusterStackVersionResourceProvider.java | 2 +-
.../internal/UpgradeResourceProvider.java | 107 +++++++++++--------
.../upgrades/FinalizeUpgradeAction.java | 18 +---
.../upgrades/UpgradeUserKerberosDescriptor.java | 41 +++----
.../ambari/server/state/UpgradeContext.java | 31 +++---
.../SPARK/1.2.1/package/scripts/params.py | 11 +-
.../SPARK/1.2.1/package/scripts/setup_spark.py | 6 +-
.../1.2.1/package/scripts/spark_service.py | 6 +-
.../ComponentVersionCheckActionTest.java | 1 -
.../upgrades/UpgradeActionTest.java | 2 -
.../UpgradeUserKerberosDescriptorTest.java | 19 ++--
.../src/test/python/TestStackFeature.py | 44 ++++++--
.../python/custom_actions/test_ru_set_all.py | 6 +-
.../test/python/stacks/2.0.6/configs/nn_eu.json | 2 +-
.../stacks/2.0.6/configs/nn_eu_standby.json | 2 +-
.../2.1/configs/hive-metastore-upgrade.json | 2 +-
.../python/stacks/2.2/configs/knox_upgrade.json | 2 +-
22 files changed, 203 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index 7811e26..2c66728 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -104,7 +104,10 @@ def get_stack_feature_version(config):
# something like 2.4.0.0-1234; represents the version for the command
# (or None if this is a cluster install and it hasn't been calculated yet)
- version = default("/commandParams/version", None)
+ # this is always guaranteed to be the correct version for the command, even in
+ # upgrade and downgrade scenarios
+ command_version = default("/commandParams/version", None)
+ command_stack = default("/commandParams/target_stack", None)
# something like 2.4.0.0-1234
# (or None if this is a cluster install and it hasn't been calculated yet)
@@ -114,13 +117,13 @@ def get_stack_feature_version(config):
upgrade_direction = default("/commandParams/upgrade_direction", None)
# start out with the value that's right 99% of the time
- version_for_stack_feature_checks = version if version is not None else stack_version
+ version_for_stack_feature_checks = command_version if command_version is not None else stack_version
# if this is not an upgrade, then we take the simple path
if upgrade_direction is None:
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2} -> {3}".format(
- stack_version, version, current_cluster_version, version_for_stack_feature_checks))
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format(
+ stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks))
return version_for_stack_feature_checks
@@ -129,15 +132,12 @@ def get_stack_feature_version(config):
is_stop_command = _is_stop_command(config)
if not is_stop_command:
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3} -> {4}".format(
- stack_version, version, current_cluster_version, upgrade_direction,
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4} -> {5}".format(
+ stack_version, current_cluster_version, command_stack, command_version, upgrade_direction,
version_for_stack_feature_checks))
return version_for_stack_feature_checks
- original_stack = default("/commandParams/original_stack", None)
- target_stack = default("/commandParams/target_stack", None)
-
# something like 2.5.0.0-5678 (or None)
downgrade_from_version = default("/commandParams/downgrade_from_version", None)
@@ -153,15 +153,13 @@ def get_stack_feature_version(config):
# UPGRADE
if current_cluster_version is not None:
version_for_stack_feature_checks = current_cluster_version
- elif original_stack is not None:
- version_for_stack_feature_checks = original_stack
else:
- version_for_stack_feature_checks = version if version is not None else stack_version
+ version_for_stack_feature_checks = command_version if command_version is not None else stack_version
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3}, original_stack={4}, target_stack={5}, downgrade_from_version={6}, stop_command={7} -> {8}".format(
- stack_version, version, current_cluster_version, upgrade_direction, original_stack,
- target_stack, downgrade_from_version, is_stop_command, version_for_stack_feature_checks))
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4}, stop_command={5} -> {6}".format(
+ stack_version, current_cluster_version, command_stack, command_version, upgrade_direction,
+ is_stop_command, version_for_stack_feature_checks))
return version_for_stack_feature_checks
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 28946e7..8875314 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -198,8 +198,9 @@ public class ExecutionCommandWrapper {
Map<String,String> commandParams = executionCommand.getCommandParams();
+ // set the version for the command if it's not already set
ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
- if (null != effectiveClusterVersion) {
+ if (null != effectiveClusterVersion && !commandParams.containsKey(KeyNames.VERSION)) {
commandParams.put(KeyNames.VERSION,
effectiveClusterVersion.getRepositoryVersion().getVersion());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 32fb37b..7948d30 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -67,7 +67,7 @@ public class ExecutionCommand extends AgentCommand {
private String role;
@SerializedName("hostLevelParams")
- private Map<String, String> hostLevelParams = new HashMap<String, String>();
+ private Map<String, String> hostLevelParams = new HashMap<>();
@SerializedName("roleParams")
private Map<String, String> roleParams = null;
@@ -77,7 +77,7 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("clusterHostInfo")
private Map<String, Set<String>> clusterHostInfo =
- new HashMap<String, Set<String>>();
+ new HashMap<>();
@SerializedName("configurations")
private Map<String, Map<String, String>> configurations;
@@ -92,7 +92,7 @@ public class ExecutionCommand extends AgentCommand {
private boolean forceRefreshConfigTagsBeforeExecution = false;
@SerializedName("commandParams")
- private Map<String, String> commandParams = new HashMap<String, String>();
+ private Map<String, String> commandParams = new HashMap<>();
@SerializedName("serviceName")
private String serviceName;
@@ -104,10 +104,10 @@ public class ExecutionCommand extends AgentCommand {
private String componentName;
@SerializedName("kerberosCommandParams")
- private List<Map<String, String>> kerberosCommandParams = new ArrayList<Map<String, String>>();
+ private List<Map<String, String>> kerberosCommandParams = new ArrayList<>();
@SerializedName("localComponents")
- private Set<String> localComponents = new HashSet<String>();
+ private Set<String> localComponents = new HashSet<>();
@SerializedName("availableServices")
private Map<String, String> availableServices = new HashMap<>();
@@ -149,7 +149,7 @@ public class ExecutionCommand extends AgentCommand {
}
public Map<String, Map<String, String>> getConfigurationCredentials() {
- return this.configurationCredentials;
+ return configurationCredentials;
}
public String getCommandId() {
@@ -434,6 +434,8 @@ public class ExecutionCommand extends AgentCommand {
String USER_GROUPS = "user_groups";
String NOT_MANAGED_HDFS_PATH_LIST = "not_managed_hdfs_path_list";
String VERSION = "version";
+ String SOURCE_STACK = "source_stack";
+ String TARGET_STACK = "target_stack";
String REFRESH_TOPOLOGY = "refresh_topology";
String HOST_SYS_PREPPED = "host_sys_prepped";
String MAX_DURATION_OF_RETRIES = "max_duration_for_retries";
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
index c361094..af506f2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.ambari.server.actionmanager.TargetHostType;
import org.apache.ambari.server.controller.internal.RequestOperationLevel;
import org.apache.ambari.server.controller.internal.RequestResourceFilter;
-import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
/**
* The context required to create tasks and stages for a custom action
@@ -42,7 +42,7 @@ public class ActionExecutionContext {
private String expectedComponentName;
private boolean hostsInMaintenanceModeExcluded = true;
private boolean allowRetry = false;
- private StackId stackId;
+ private RepositoryVersionEntity repositoryVersion;
/**
* {@code true} if slave/client component failures should be automatically
@@ -171,27 +171,29 @@ public class ActionExecutionContext {
}
/**
- * Gets the stack to use for generating stack-associated values for a command.
- * In some cases the cluster's stack is not the correct one to use, such as
- * when distributing a repository.
+ * Gets the stack/version to use for generating stack-associated values for a
+ * command. In some cases the cluster's stack is not the correct one to use,
+ * such as when distributing a repository.
*
- * @return the stackId the stack to use when generating stack-specific content
- * for the command.
+ * @return the repository for the stack/version to use when generating
+ * stack-specific content for the command.
+ *
+ * @return
*/
- public StackId getStackId() {
- return stackId;
+ public RepositoryVersionEntity getRepositoryVersion() {
+ return repositoryVersion;
}
/**
- * Sets the stack to use for generating stack-associated values for a command.
- * In some cases the cluster's stack is not the correct one to use, such as
- * when distributing a repository.
+ * Sets the stack/version to use for generating stack-associated values for a
+ * command. In some cases the cluster's stack is not the correct one to use,
+ * such as when distributing a repository.
*
* @param stackId
* the stackId to use for stack-based properties on the command.
*/
- public void setStackId(StackId stackId) {
- this.stackId = stackId;
+ public void setRepositoryVersion(RepositoryVersionEntity repositoryVersion) {
+ this.repositoryVersion = repositoryVersion;
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index f75fb41..0638910 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -553,10 +553,12 @@ public class AmbariActionExecutionHelper {
// set the host level params if not already set by whoever is creating this command
if (!hostLevelParams.containsKey(STACK_NAME) || !hostLevelParams.containsKey(STACK_VERSION)) {
- // see if the action context has a stack ID set to use, otherwise use the
+ // see if the action context has a repository set to use for the command, otherwise use the
// cluster's current stack ID
- StackId stackId = actionContext.getStackId() != null ? actionContext.getStackId()
- : cluster.getCurrentStackVersion();
+ StackId stackId = cluster.getCurrentStackVersion();
+ if (null != actionContext.getRepositoryVersion()) {
+ stackId = actionContext.getRepositoryVersion().getStackId();
+ }
hostLevelParams.put(STACK_NAME, stackId.getStackName());
hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index 9ea6083..633fe8c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -713,7 +713,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
INSTALL_PACKAGES_ACTION, Collections.singletonList(filter), params);
- actionContext.setStackId(stackId);
+ actionContext.setRepositoryVersion(repoVersion);
actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true)));
return actionContext;
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 1130026..858b7cf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -883,9 +883,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
// keep track of which stack to use when building commands - an express
// upgrade switches the stack half way through while other types move it in
// the beginning
- StackId effectiveStackId = upgradeContext.getTargetStackId();
+ RepositoryVersionEntity effectiveRepositoryVersion = upgradeContext.getTargetRepositoryVersion();
if(upgradeContext.getType() == UpgradeType.NON_ROLLING ) {
- effectiveStackId = upgradeContext.getSourceStackId();
+ effectiveRepositoryVersion = upgradeContext.getSourceRepositoryVersion();
}
for (UpgradeGroupHolder group : groups) {
@@ -895,7 +895,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
if (upgradeContext.getType() == UpgradeType.NON_ROLLING
&& UpdateStackGrouping.class.equals(group.groupClass)) {
- effectiveStackId = upgradeContext.getTargetStackId();
+ effectiveRepositoryVersion = upgradeContext.getTargetRepositoryVersion();
}
List<UpgradeItemEntity> itemEntities = new ArrayList<>();
@@ -919,7 +919,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
itemEntities.add(itemEntity);
injectVariables(configHelper, cluster, itemEntity);
- makeServerSideStage(upgradeContext, req, effectiveStackId, itemEntity,
+ makeServerSideStage(upgradeContext, req, effectiveRepositoryVersion, itemEntity,
(ServerSideActionTask) task, skippable, supportsAutoSkipOnFailure, allowRetry,
pack, configUpgradePack);
}
@@ -934,7 +934,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
injectVariables(configHelper, cluster, itemEntity);
// upgrade items match a stage
- createStage(upgradeContext, req, effectiveStackId, itemEntity, wrapper, skippable,
+ createStage(upgradeContext, req, effectiveRepositoryVersion, itemEntity, wrapper,
+ skippable,
supportsAutoSkipOnFailure, allowRetry);
}
}
@@ -1275,10 +1276,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* the upgrade context (not {@code null}).
* @param request
* the request to add the new stage to (not {@code null}).
- * @param effectiveStackId
- * the stack ID to use when building the command. This will determine
- * things like stack tools and version information added to the
- * command (not {@code null}).
+ * @param effectiveRepositoryVersion
+ * the stack/version to use when building the command. This will
+ * determine things like stack tools and version information added to
+ * the command (not {@code null}).
* @param entity
* the upgrade entity to add the new items to (not {@code null}).
* @param wrapper
@@ -1292,25 +1293,27 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
private void createStage(UpgradeContext context, RequestStageContainer request,
- StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable,
- boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException {
+ RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity,
+ StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure,
+ boolean allowRetry) throws AmbariException {
switch (wrapper.getType()) {
case CONFIGURE:
case START:
case STOP:
case RESTART:
- makeCommandStage(context, request, effectiveStackId, entity, wrapper, skippable,
+ makeCommandStage(context, request, effectiveRepositoryVersion, entity, wrapper, skippable,
supportsAutoSkipOnFailure,
allowRetry);
break;
case RU_TASKS:
- makeActionStage(context, request, effectiveStackId, entity, wrapper, skippable,
+ makeActionStage(context, request, effectiveRepositoryVersion, entity, wrapper, skippable,
supportsAutoSkipOnFailure,
allowRetry);
break;
case SERVICE_CHECK:
- makeServiceCheckStage(context, request, effectiveStackId, entity, wrapper, skippable,
+ makeServiceCheckStage(context, request, effectiveRepositoryVersion, entity, wrapper,
+ skippable,
supportsAutoSkipOnFailure, allowRetry);
break;
default:
@@ -1341,9 +1344,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* the upgrade context.
* @param request
* the request object to add the stage to.
- * @param effectiveStackId
- * the stack ID to use when generating content for the command. On
- * some upgrade types, this may change during the course of the
+ * @param effectiveRepositoryVersion
+ * the stack/version to use when generating content for the command.
+ * On some upgrade types, this may change during the course of the
* upgrade orchestration. An express upgrade changes this after
* stopping all services.
* @param entity
@@ -1360,8 +1363,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
private void makeActionStage(UpgradeContext context, RequestStageContainer request,
- StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable,
- boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException {
+ RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity,
+ StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure,
+ boolean allowRetry) throws AmbariException {
if (0 == wrapper.getHosts().size()) {
throw new AmbariException(
@@ -1369,13 +1373,14 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
Cluster cluster = context.getCluster();
+ StackId effectiveStackId = effectiveRepositoryVersion.getStackId();
// add each host to this stage
RequestResourceFilter filter = new RequestResourceFilter("", "",
new ArrayList<>(wrapper.getHosts()));
LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks());
- Map<String, String> params = getNewParameterMap(request, context);
+ Map<String, String> params = getNewParameterMap(request, context, effectiveRepositoryVersion);
params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks());
// Apply additional parameters to the command that come from the stage.
@@ -1403,7 +1408,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
EXECUTE_TASK_ROLE, Collections.singletonList(filter), params);
- actionContext.setStackId(effectiveStackId);
+ actionContext.setRepositoryVersion(effectiveRepositoryVersion);
// hosts in maintenance mode are excluded from the upgrade
actionContext.setMaintenanceModeHostExcluded(true);
@@ -1449,10 +1454,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* Upgrade Context
* @param request
* Container for stage
- * @param effectiveStackId
- * the stack ID to use when building the command. This will determine
- * things like stack tools and version information added to the
- * command (not {@code null}).
+ * @param effectiveRepositoryVersion
+ * the stack/version to use when building the command. This will
+ * determine things like stack tools and version information added to
+ * the command (not {@code null}).
* @param entity
* Upgrade Item
* @param wrapper
@@ -1464,10 +1469,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
private void makeCommandStage(UpgradeContext context, RequestStageContainer request,
- StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable,
- boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException {
+ RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity,
+ StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure,
+ boolean allowRetry) throws AmbariException {
Cluster cluster = context.getCluster();
+ StackId effectiveStackId = effectiveRepositoryVersion.getStackId();
List<RequestResourceFilter> filters = new ArrayList<>();
@@ -1490,7 +1497,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
break;
}
- Map<String, String> commandParams = getNewParameterMap(request, context);
+ Map<String, String> commandParams = getNewParameterMap(request, context,
+ effectiveRepositoryVersion);
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
@@ -1498,7 +1506,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
function, filters, commandParams);
- actionContext.setStackId(effectiveStackId);
+ actionContext.setRepositoryVersion(effectiveRepositoryVersion);
actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration));
actionContext.setRetryAllowed(allowRetry);
@@ -1540,7 +1548,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
private void makeServiceCheckStage(UpgradeContext context, RequestStageContainer request,
- StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable,
+ RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity,
+ StageWrapper wrapper, boolean skippable,
boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException {
List<RequestResourceFilter> filters = new ArrayList<>();
@@ -1550,8 +1559,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
}
Cluster cluster = context.getCluster();
+ StackId effectiveStackId = effectiveRepositoryVersion.getStackId();
- Map<String, String> commandParams = getNewParameterMap(request, context);
+ Map<String, String> commandParams = getNewParameterMap(request, context,
+ effectiveRepositoryVersion);
// Apply additional parameters to the command that come from the stage.
applyAdditionalParameters(wrapper, commandParams);
@@ -1559,7 +1570,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
"SERVICE_CHECK", filters, commandParams);
- actionContext.setStackId(effectiveStackId);
+ actionContext.setRepositoryVersion(effectiveRepositoryVersion);
actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration));
actionContext.setRetryAllowed(allowRetry);
actionContext.setAutoSkipFailures(context.isServiceCheckFailureAutoSkipped());
@@ -1586,7 +1597,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
stage.setStageId(stageId);
entity.setStageId(Long.valueOf(stageId));
- Map<String, String> requestParams = getNewParameterMap(request, context);
+ Map<String, String> requestParams = getNewParameterMap(request, context,
+ effectiveRepositoryVersion);
+
s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams, jsons);
request.addStages(Collections.singletonList(stage));
@@ -1599,10 +1612,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* upgrade context
* @param request
* upgrade request
- * @param effectiveStackId
- * the stack ID to use when building the command. This will determine
- * things like stack tools and version information added to the
- * command (not {@code null}).
+ * @param effectiveRepositoryVersion
+ * the stack/version to use when building the command. This will
+ * determine things like stack tools and version information added to
+ * the command (not {@code null}).
* @param entity
* a single of upgrade
* @param task
@@ -1617,14 +1630,18 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @throws AmbariException
*/
private void makeServerSideStage(UpgradeContext context, RequestStageContainer request,
- StackId effectiveStackId, UpgradeItemEntity entity, ServerSideActionTask task,
+ RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity,
+ ServerSideActionTask task,
boolean skippable, boolean supportsAutoSkipOnFailure, boolean allowRetry,
UpgradePack upgradePack, ConfigUpgradePack configUpgradePack)
throws AmbariException {
Cluster cluster = context.getCluster();
+ StackId effectiveStackId = effectiveRepositoryVersion.getStackId();
+
+ Map<String, String> commandParams = getNewParameterMap(request, context,
+ effectiveRepositoryVersion);
- Map<String, String> commandParams = getNewParameterMap(request, context);
commandParams.put(UpgradeContext.COMMAND_PARAM_UPGRADE_PACK, upgradePack.getName());
// Notice that this does not apply any params because the input does not specify a stage.
@@ -1702,7 +1719,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
Role.AMBARI_SERVER_ACTION.toString(), Collections.<RequestResourceFilter> emptyList(),
commandParams);
- actionContext.setStackId(effectiveStackId);
+ actionContext.setRepositoryVersion(effectiveRepositoryVersion);
actionContext.setTimeout(Short.valueOf((short) -1));
actionContext.setRetryAllowed(allowRetry);
actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped());
@@ -1743,11 +1760,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* following properties are already set:
* <ul>
* <li>{@link UpgradeContext#COMMAND_PARAM_CLUSTER_NAME}
+ * <li>{@link UpgradeContext#COMMAND_PARAM_SOURCE_STACK}
+ * <li>{@link UpgradeContext#COMMAND_PARAM_TARGET_STACK}
* <li>{@link UpgradeContext#COMMAND_PARAM_VERSION}
* <li>{@link UpgradeContext#COMMAND_PARAM_DIRECTION}
- * <li>{@link UpgradeContext#COMMAND_PARAM_ORIGINAL_STACK}
- * <li>{@link UpgradeContext#COMMAND_PARAM_TARGET_STACK}
- * <li>{@link UpgradeContext#COMMAND_DOWNGRADE_FROM_VERSION}
* <li>{@link UpgradeContext#COMMAND_PARAM_UPGRADE_TYPE}
* <li>{@link KeyNames#REFRESH_CONFIG_TAGS_BEFORE_EXECUTION} - necessary in
* order to have the commands contain the correct configurations. Otherwise,
@@ -1761,10 +1777,13 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
* @return the initialized parameter map.
*/
private Map<String, String> getNewParameterMap(RequestStageContainer requestStageContainer,
- UpgradeContext context) {
+ UpgradeContext context, RepositoryVersionEntity effectiveRepositoryVersion) {
Map<String, String> parameters = context.getInitializedCommandParameters();
+
parameters.put(UpgradeContext.COMMAND_PARAM_REQUEST_ID,
String.valueOf(requestStageContainer.getId()));
+
+ parameters.put(UpgradeContext.COMMAND_PARAM_VERSION, effectiveRepositoryVersion.getVersion());
return parameters;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
index 0e6f0c4..d531460 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.StackUpgradeFinishEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
@@ -73,26 +74,13 @@ public class FinalizeUpgradeAction extends AbstractServerAction {
public static final String CLUSTER_NAME_KEY = "cluster_name";
public static final String UPGRADE_DIRECTION_KEY = "upgrade_direction";
- public static final String VERSION_KEY = "version";
+ public static final String VERSION_KEY = KeyNames.VERSION;
+ public static final String TARGET_STACK_KEY = KeyNames.TARGET_STACK;
public static final String REQUEST_ID = "request_id";
public static final String PREVIOUS_UPGRADE_NOT_COMPLETED_MSG = "It is possible that a previous upgrade was not finalized. " +
"For this reason, Ambari will not remove any configs. Please ensure that all database records are correct.";
/**
- * The original "current" stack of the cluster before the upgrade started.
- * This is the same regardless of whether the current direction is
- * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
- */
- public static final String ORIGINAL_STACK_KEY = "original_stack";
-
- /**
- * The target upgrade stack before the upgrade started. This is the same
- * regardless of whether the current direction is {@link Direction#UPGRADE} or
- * {@link Direction#DOWNGRADE}.
- */
- public static final String TARGET_STACK_KEY = "target_stack";
-
- /**
* The Cluster that this ServerAction implementation is executing on
*/
@Inject
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
index 60d02a3..842da95 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
@@ -17,11 +17,16 @@
*/
package org.apache.ambari.server.serveraction.upgrades;
-import com.google.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentMap;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
@@ -37,10 +42,7 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentMap;
+import com.google.inject.Inject;
/**
* Update the user-defined Kerberos Descriptor to work with the current stack.
@@ -56,25 +58,6 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction {
* @see Direction
*/
private static final String UPGRADE_DIRECTION_KEY = "upgrade_direction";
-
- /**
- * The original "current" stack of the cluster before the upgrade started.
- * This is the same regardless of whether the current direction is
- * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
- *
- * @see Direction
- */
- private static final String ORIGINAL_STACK_KEY = "original_stack";
-
- /**
- * The target upgrade stack before the upgrade started. This is the same
- * regardless of whether the current direction is {@link Direction#UPGRADE} or
- * {@link Direction#DOWNGRADE}.
- *
- * @see Direction
- */
- private static final String TARGET_STACK_KEY = "target_stack";
-
private final static String KERBEROS_DESCRIPTOR_NAME = "kerberos_descriptor";
private final static String KERBEROS_DESCRIPTOR_BACKUP_NAME = "kerberos_descriptor_backup";
@@ -104,21 +87,21 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction {
HostRoleCommand hostRoleCommand = getHostRoleCommand();
String clusterName = hostRoleCommand.getExecutionCommandWrapper().getExecutionCommand().getClusterName();
Cluster cluster = clusters.getCluster(clusterName);
- List<String> messages = new ArrayList<String>();
- List<String> errorMessages = new ArrayList<String>();
+ List<String> messages = new ArrayList<>();
+ List<String> errorMessages = new ArrayList<>();
if (cluster != null) {
logMessage(messages, "Obtaining the user-defined Kerberos descriptor");
- TreeMap<String, String> foreignKeys = new TreeMap<String, String>();
+ TreeMap<String, String> foreignKeys = new TreeMap<>();
foreignKeys.put("cluster", String.valueOf(cluster.getClusterId()));
ArtifactEntity entity = artifactDAO.findByNameAndForeignKeys("kerberos_descriptor", foreignKeys);
KerberosDescriptor userDescriptor = (entity == null) ? null : kerberosDescriptorFactory.createInstance(entity.getArtifactData());
if (userDescriptor != null) {
- StackId originalStackId = getStackIdFromCommandParams(ORIGINAL_STACK_KEY);
- StackId targetStackId = getStackIdFromCommandParams(TARGET_STACK_KEY);
+ StackId originalStackId = cluster.getCurrentStackVersion();
+ StackId targetStackId = getStackIdFromCommandParams(KeyNames.TARGET_STACK);
if (isDowngrade()) {
restoreDescriptor(foreignKeys, messages, errorMessages);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index b97dc80..93e6393 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -62,21 +62,15 @@ public class UpgradeContext {
public static final String COMMAND_PARAM_UPGRADE_TYPE = "upgrade_type";
public static final String COMMAND_PARAM_TASKS = "tasks";
public static final String COMMAND_PARAM_STRUCT_OUT = "structured_out";
- public static final String COMMAND_DOWNGRADE_FROM_VERSION = "downgrade_from_version";
- /**
- * The original "current" stack of the cluster before the upgrade started.
- * This is the same regardless of whether the current direction is
- * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
- */
- public static final String COMMAND_PARAM_ORIGINAL_STACK = "original_stack";
+ public static final String COMMAND_PARAM_SOURCE_STACK = KeyNames.SOURCE_STACK;
+ public static final String COMMAND_PARAM_TARGET_STACK = KeyNames.TARGET_STACK;
- /**
- * The target upgrade stack before the upgrade started. This is the same
- * regardless of whether the current direction is {@link Direction#UPGRADE} or
- * {@link Direction#DOWNGRADE}.
- */
- public static final String COMMAND_PARAM_TARGET_STACK = "target_stack";
+ @Deprecated
+ @Experimental(
+ feature = ExperimentalFeature.STACK_UPGRADES_BETWEEN_VENDORS,
+ comment = "This isn't needed anymore, but many python classes still use it")
+ public static final String COMMAND_PARAM_DOWNGRADE_FROM_VERSION = "downgrade_from_version";
/**
* The cluster that the upgrade is for.
@@ -528,6 +522,7 @@ public class UpgradeContext {
* <ul>
* <li>{@link #COMMAND_PARAM_CLUSTER_NAME}
* <li>{@link #COMMAND_PARAM_DIRECTION}
+ * <li>{@link #COMMAND_PARAM_DOWNGRADE_FROM_VERSION}
* <li>{@link #COMMAND_PARAM_UPGRADE_TYPE}
* <li>{@link KeyNames#REFRESH_CONFIG_TAGS_BEFORE_EXECUTION} - necessary in
* order to have the commands contain the correct configurations. Otherwise,
@@ -542,8 +537,16 @@ public class UpgradeContext {
public Map<String, String> getInitializedCommandParameters() {
Map<String, String> parameters = new HashMap<>();
+ Direction direction = getDirection();
parameters.put(COMMAND_PARAM_CLUSTER_NAME, m_cluster.getClusterName());
- parameters.put(COMMAND_PARAM_DIRECTION, getDirection().name().toLowerCase());
+ parameters.put(COMMAND_PARAM_DIRECTION, direction.name().toLowerCase());
+
+ parameters.put(COMMAND_PARAM_SOURCE_STACK, m_fromRepositoryVersion.getStackId().getStackId());
+ parameters.put(COMMAND_PARAM_TARGET_STACK, m_toRepositoryVersion.getStackId().getStackId());
+
+ if (direction == Direction.DOWNGRADE) {
+ parameters.put(COMMAND_PARAM_DOWNGRADE_FROM_VERSION, m_fromRepositoryVersion.getVersion());
+ }
if (null != getType()) {
// use the serialized attributes of the enum to convert it to a string,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
index b54b565..5dbbaed 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
@@ -21,6 +21,7 @@ limitations under the License.
import socket
import status_params
from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions import Direction
from setup_spark import *
@@ -57,10 +58,8 @@ upgrade_direction = default("/commandParams/upgrade_direction", None)
java_home = config['hostLevelParams']['java_home']
stack_name = status_params.stack_name
stack_root = Script.get_stack_root()
-stack_version_unformatted = config['hostLevelParams']['stack_version']
-if upgrade_direction == Direction.DOWNGRADE:
- stack_version_unformatted = config['commandParams']['original_stack'].split("-")[1]
-stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+version_for_stack_feature_checks = get_stack_feature_version(config)
sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
@@ -71,7 +70,7 @@ spark_conf = '/etc/spark/conf'
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+if check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks):
hadoop_home = stack_select.get_hadoop_dir("home")
spark_conf = format("{stack_root}/current/{component_directory}/conf")
spark_log_dir = config['configurations']['spark-env']['spark_log_dir']
@@ -212,7 +211,7 @@ dfs_type = default("/commandParams/dfs_type", "")
# livy is only supported from HDP 2.5
has_livyserver = False
-if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and "livy-env" in config['configurations']:
+if check_stack_feature(StackFeature.SPARK_LIVY, version_for_stack_feature_checks) and "livy-env" in config['configurations']:
livy_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY_SERVER")
livy_conf = format("{stack_root}/current/{livy_component_directory}/conf")
livy_log_dir = config['configurations']['livy-env']['livy_log_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
index 4034532..bc86a67 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
@@ -114,11 +114,11 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
mode=0644
)
- effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
+ effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks
if effective_version:
effective_version = format_stack_version(effective_version)
- if effective_version and check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version):
+ if check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version):
File(os.path.join(params.spark_conf, 'java-opts'),
owner=params.spark_user,
group=params.spark_group,
@@ -130,7 +130,7 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
action="delete"
)
- if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
+ if params.spark_thrift_fairscheduler_content and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create spark-thrift-fairscheduler.xml
File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"),
owner=params.spark_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
index 31a296a..2838186 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
@@ -34,11 +34,11 @@ def spark_service(name, upgrade_type=None, action=None):
if action == 'start':
- effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
+ effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks
if effective_version:
effective_version = format_stack_version(effective_version)
- if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
+ if name == 'jobhistoryserver' and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# copy spark-hdp-assembly.jar to hdfs
copy_to_hdfs("spark", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
# create spark history directory
@@ -58,7 +58,7 @@ def spark_service(name, upgrade_type=None, action=None):
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
- if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
+ if check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.version_for_stack_feature_checks):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java
index 47d2a81..6675c58 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java
@@ -409,7 +409,6 @@ public class ComponentVersionCheckActionTest {
Map<String, String> commandParams = new HashMap<>();
commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade");
commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo);
- commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId());
commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId());
ExecutionCommand executionCommand = new ExecutionCommand();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
index c9c0dd0..037e47b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
@@ -677,7 +677,6 @@ public class UpgradeActionTest {
Map<String, String> commandParams = new HashMap<>();
commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade");
commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo);
- commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId());
commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId());
ExecutionCommand executionCommand = new ExecutionCommand();
@@ -860,7 +859,6 @@ public class UpgradeActionTest {
Map<String, String> commandParams = new HashMap<>();
commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade");
commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo);
- commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId());
commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId());
ExecutionCommand executionCommand = new ExecutionCommand();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
index 6b80623..0da9088 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
@@ -25,6 +25,11 @@ import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
import static org.junit.Assert.assertEquals;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.agent.ExecutionCommand;
@@ -33,6 +38,7 @@ import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper;
@@ -45,11 +51,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
/**
* Tests OozieConfigCalculation logic
*/
@@ -63,6 +64,7 @@ public class UpgradeUserKerberosDescriptorTest {
private ArtifactDAO artifactDAO;
private TreeMap<String, Field> fields = new TreeMap<>();
+ private StackId HDP_24 = new StackId("HDP", "2.4");
@Before
public void setup() throws Exception {
@@ -74,6 +76,7 @@ public class UpgradeUserKerberosDescriptorTest {
expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
expect(cluster.getClusterId()).andReturn(1l).atLeastOnce();
+ expect(cluster.getCurrentStackVersion()).andReturn(HDP_24).atLeastOnce();
replay(clusters, cluster);
prepareFields();
@@ -83,10 +86,9 @@ public class UpgradeUserKerberosDescriptorTest {
@Test
public void testUpgrade() throws Exception {
- Map<String, String> commandParams = new HashMap<String, String>();
+ Map<String, String> commandParams = new HashMap<>();
commandParams.put("clusterName", "c1");
commandParams.put("upgrade_direction", "UPGRADE");
- commandParams.put("original_stack", "HDP-2.4");
commandParams.put("target_stack", "HDP-2.5");
ExecutionCommand executionCommand = new ExecutionCommand();
@@ -141,10 +143,9 @@ public class UpgradeUserKerberosDescriptorTest {
@Test
public void testDowngrade() throws Exception {
- Map<String, String> commandParams = new HashMap<String, String>();
+ Map<String, String> commandParams = new HashMap<>();
commandParams.put("clusterName", "c1");
commandParams.put("upgrade_direction", "DOWNGRADE");
- commandParams.put("original_stack", "HDP-2.4");
commandParams.put("target_stack", "HDP-2.5");
ExecutionCommand executionCommand = new ExecutionCommand();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/TestStackFeature.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py
index 0116a7a..230734c 100644
--- a/ambari-server/src/test/python/TestStackFeature.py
+++ b/ambari-server/src/test/python/TestStackFeature.py
@@ -28,6 +28,32 @@ from unittest import TestCase
Logger.initialize_logger()
class TestStackFeature(TestCase):
+ """
+ EU Upgrade (HDP 2.5 to HDP 2.6)
+ - STOP
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.5
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.5.0.0-1237
+ - START
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.6
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.6.0.0-334
+
+ EU Downgrade (HDP 2.6 to HDP 2.5)
+ - STOP
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.6
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.6.0.0-334
+ - START
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.5
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.5.0.0-1237
+ """
+
def test_get_stack_feature_version_missing_params(self):
try:
stack_feature_version = get_stack_feature_version({})
@@ -122,7 +148,7 @@ class TestStackFeature(TestCase):
"current_version": "2.4.0.0-1234"
},
"commandParams": {
- "original_stack": "2.4",
+ "source_stack": "2.4",
"target_stack": "2.5",
"upgrade_direction": "upgrade",
"version": "2.5.9.9-9999"
@@ -143,8 +169,8 @@ class TestStackFeature(TestCase):
"current_version":"2.4.0.0-1234"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
"version":"2.4.0.0-1234",
"downgrade_from_version": "2.5.9.9-9999"
@@ -166,10 +192,10 @@ class TestStackFeature(TestCase):
"current_version":"2.4.0.0-1234"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
- "version":"2.4.0.0-1234",
+ "version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
}
@@ -189,10 +215,10 @@ class TestStackFeature(TestCase):
"custom_command":"STOP"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
- "version":"2.4.0.0-1234",
+ "version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
index e1a89a8..8e03b7f 100644
--- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
+++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
@@ -225,7 +225,7 @@ class TestRUSetAll(RMFTestCase):
# alter JSON for a downgrade from 2.3 to 2.2
json_payload['commandParams']['version'] = "2.2.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-1234"
- json_payload['commandParams']['original_stack'] = "HDP-2.2"
+ json_payload['commandParams']['source_stack'] = "HDP-2.2"
json_payload['commandParams']['target_stack'] = "HDP-2.3"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.2"
@@ -263,7 +263,7 @@ class TestRUSetAll(RMFTestCase):
json_payload['commandParams']['version'] = "2.3.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-5678"
- json_payload['commandParams']['original_stack'] = "HDP-2.3"
+ json_payload['commandParams']['source_stack'] = "HDP-2.3"
json_payload['commandParams']['target_stack'] = "HDP-2.3"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.3"
@@ -291,7 +291,7 @@ class TestRUSetAll(RMFTestCase):
# alter JSON for a downgrade from 2.2 to 2.2
json_payload['commandParams']['version'] = "2.2.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.2.0.0-5678"
- json_payload['commandParams']['original_stack'] = "HDP-2.2"
+ json_payload['commandParams']['source_stack'] = "HDP-2.2"
json_payload['commandParams']['target_stack'] = "HDP-2.2"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.2"
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
index 7f77d83..3aadf2c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
@@ -25,7 +25,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2844",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.2",
+ "source_stack": "HDP-2.2",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"desired_namenode_role": "standby",
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
index 87b18af..2d48ff6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
@@ -25,7 +25,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2844",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.2",
+ "source_stack": "HDP-2.2",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"desired_namenode_role": "standby",
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
index 99fcba0..021695b 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
@@ -13,7 +13,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2950",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.3",
+ "source_stack": "HDP-2.3",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"script_type": "PYTHON"
http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
index a9db11c..1805c3b 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
@@ -22,7 +22,7 @@
"upgrade_type": "rolling_upgrade",
"command_retry_max_attempt_count": "3",
"version": "2.3.0.0-2096",
- "original_stack": "HDP-2.3",
+ "source_stack": "HDP-2.3",
"command_retry_enabled": "false",
"command_timeout": "1200",
"target_stack": "HDP-2.3",