You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ab...@apache.org on 2017/07/14 12:38:15 UTC
[25/31] ambari git commit: AMBARI-21451 - Expected Values Like
original_stack Are Missing On Downgrades (jonathanhurley)
AMBARI-21451 - Expected Values Like original_stack Are Missing On Downgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f27f3aff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f27f3aff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f27f3aff
Branch: refs/heads/branch-feature-logsearch-ui
Commit: f27f3affbb4c7f49944dcefc7581ac228b103e3f
Parents: eb3d3ea
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 12 13:30:16 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 12 19:26:37 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/stack_features.py | 28 ++--
.../controller/ActionExecutionContext.java | 30 ++--
.../controller/AmbariActionExecutionHelper.java | 15 +-
.../ClusterStackVersionResourceProvider.java | 2 +-
.../upgrades/UpgradeUserKerberosDescriptor.java | 142 +++++++------------
.../ambari/server/state/UpgradeContext.java | 16 ++-
.../SPARK/1.2.1/package/scripts/params.py | 11 +-
.../SPARK/1.2.1/package/scripts/setup_spark.py | 6 +-
.../1.2.1/package/scripts/spark_service.py | 6 +-
.../UpgradeUserKerberosDescriptorTest.java | 59 ++++++--
.../src/test/python/TestStackFeature.py | 44 ++++--
.../test/python/stacks/2.0.6/configs/nn_eu.json | 2 +-
.../stacks/2.0.6/configs/nn_eu_standby.json | 2 +-
.../2.1/configs/hive-metastore-upgrade.json | 2 +-
.../python/stacks/2.2/configs/knox_upgrade.json | 2 +-
15 files changed, 199 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
index 576c138..24201dd 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py
@@ -105,7 +105,10 @@ def get_stack_feature_version(config):
# something like 2.4.0.0-1234; represents the version for the command
# (or None if this is a cluster install and it hasn't been calculated yet)
- version = default("/commandParams/version", None)
+ # this is always guaranteed to be the correct version for the command, even in
+ # upgrade and downgrade scenarios
+ command_version = default("/commandParams/version", None)
+ command_stack = default("/commandParams/target_stack", None)
# something like 2.4.0.0-1234
# (or None if this is a cluster install and it hasn't been calculated yet)
@@ -115,13 +118,13 @@ def get_stack_feature_version(config):
upgrade_direction = default("/commandParams/upgrade_direction", None)
# start out with the value that's right 99% of the time
- version_for_stack_feature_checks = version if version is not None else stack_version
+ version_for_stack_feature_checks = command_version if command_version is not None else stack_version
# if this is not an upgrade, then we take the simple path
if upgrade_direction is None:
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2} -> {3}".format(
- stack_version, version, current_cluster_version, version_for_stack_feature_checks))
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format(
+ stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks))
return version_for_stack_feature_checks
@@ -130,15 +133,12 @@ def get_stack_feature_version(config):
is_stop_command = _is_stop_command(config)
if not is_stop_command:
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3} -> {4}".format(
- stack_version, version, current_cluster_version, upgrade_direction,
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4} -> {5}".format(
+ stack_version, current_cluster_version, command_stack, command_version, upgrade_direction,
version_for_stack_feature_checks))
return version_for_stack_feature_checks
- original_stack = default("/commandParams/original_stack", None)
- target_stack = default("/commandParams/target_stack", None)
-
# something like 2.5.0.0-5678 (or None)
downgrade_from_version = default("/commandParams/downgrade_from_version", None)
@@ -154,15 +154,13 @@ def get_stack_feature_version(config):
# UPGRADE
if current_cluster_version is not None:
version_for_stack_feature_checks = current_cluster_version
- elif original_stack is not None:
- version_for_stack_feature_checks = format_stack_version(original_stack)
else:
- version_for_stack_feature_checks = version if version is not None else stack_version
+ version_for_stack_feature_checks = command_version if command_version is not None else stack_version
Logger.info(
- "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3}, original_stack={4}, target_stack={5}, downgrade_from_version={6}, stop_command={7} -> {8}".format(
- stack_version, version, current_cluster_version, upgrade_direction, original_stack,
- target_stack, downgrade_from_version, is_stop_command, version_for_stack_feature_checks))
+ "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4}, stop_command={5} -> {6}".format(
+ stack_version, current_cluster_version, command_stack, command_version, upgrade_direction,
+ is_stop_command, version_for_stack_feature_checks))
return version_for_stack_feature_checks
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
index 34d6db9..5d71869 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java
@@ -27,7 +27,7 @@ import org.apache.ambari.server.actionmanager.TargetHostType;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.controller.internal.RequestOperationLevel;
import org.apache.ambari.server.controller.internal.RequestResourceFilter;
-import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
/**
* The context required to create tasks and stages for a custom action
@@ -44,7 +44,7 @@ public class ActionExecutionContext {
private String expectedComponentName;
private boolean hostsInMaintenanceModeExcluded = true;
private boolean allowRetry = false;
- private StackId stackId;
+ private RepositoryVersionEntity repositoryVersion;
private List<ExecutionCommandVisitor> m_visitors = new ArrayList<>();
@@ -175,27 +175,29 @@ public class ActionExecutionContext {
}
/**
- * Gets the stack to use for generating stack-associated values for a command.
- * In some cases the cluster's stack is not the correct one to use, such as
- * when distributing a repository.
+ * Gets the stack/version to use for generating stack-associated values for a
+ * command. In some cases the cluster's stack is not the correct one to use,
+ * such as when distributing a repository.
*
- * @return the stackId the stack to use when generating stack-specific content
- * for the command.
+ * @return the repository for the stack/version to use when generating
+ * stack-specific content for the command.
+ *
+ * @return
*/
- public StackId getStackId() {
- return stackId;
+ public RepositoryVersionEntity getRepositoryVersion() {
+ return repositoryVersion;
}
/**
- * Sets the stack to use for generating stack-associated values for a command.
- * In some cases the cluster's stack is not the correct one to use, such as
- * when distributing a repository.
+ * Sets the stack/version to use for generating stack-associated values for a
+ * command. In some cases the cluster's stack is not the correct one to use,
+ * such as when distributing a repository.
*
* @param stackId
* the stackId to use for stack-based properties on the command.
*/
- public void setStackId(StackId stackId) {
- this.stackId = stackId;
+ public void setRepositoryVersion(RepositoryVersionEntity repositoryVersion) {
+ this.repositoryVersion = repositoryVersion;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 391daa9..55356c7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -539,13 +539,18 @@ public class AmbariActionExecutionHelper {
// if the repo is null, see if any values from the context should go on the
// host params and then return
if (null == repositoryVersion) {
- if (null != actionContext.getStackId()) {
- StackId stackId = actionContext.getStackId();
+ // see if the action context has a repository set to use for the command
+ if (null != actionContext.getRepositoryVersion()) {
+ StackId stackId = actionContext.getRepositoryVersion().getStackId();
hostLevelParams.put(STACK_NAME, stackId.getStackName());
hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
}
return;
+ } else {
+ StackId stackId = repositoryVersion.getStackId();
+ hostLevelParams.put(STACK_NAME, stackId.getStackName());
+ hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
}
JsonObject rootJsonObject = new JsonObject();
@@ -569,11 +574,5 @@ public class AmbariActionExecutionHelper {
}
hostLevelParams.put(REPO_INFO, rootJsonObject.toString());
-
- // set the host level params if not already set by whoever is creating this command
- if (!hostLevelParams.containsKey(STACK_NAME) || !hostLevelParams.containsKey(STACK_VERSION)) {
- hostLevelParams.put(STACK_NAME, repositoryVersion.getStackName());
- hostLevelParams.put(STACK_VERSION, repositoryVersion.getStackVersion());
- }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index c4fce8a..9ecea95 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -613,7 +613,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(),
INSTALL_PACKAGES_ACTION, Collections.singletonList(filter), roleParams);
- actionContext.setStackId(stackId);
+ actionContext.setRepositoryVersion(repoVersion);
actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true)));
repoVersionHelper.addCommandRepository(actionContext, osFamily, repoVersion, repoInfo);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
index 59690a3..78aaa77 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java
@@ -22,6 +22,8 @@ import java.util.List;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.ambari.annotations.Experimental;
+import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
@@ -29,10 +31,10 @@ import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
-import org.apache.ambari.server.serveraction.AbstractServerAction;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper;
@@ -48,34 +50,9 @@ import com.google.inject.Inject;
*
* @see org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper
*/
-public class UpgradeUserKerberosDescriptor extends AbstractServerAction {
+public class UpgradeUserKerberosDescriptor extends AbstractUpgradeServerAction {
private static final Logger LOG = LoggerFactory.getLogger(UpgradeUserKerberosDescriptor.class);
- /**
- * The upgrade direction.
- *
- * @see Direction
- */
- private static final String UPGRADE_DIRECTION_KEY = "upgrade_direction";
-
- /**
- * The original "current" stack of the cluster before the upgrade started.
- * This is the same regardless of whether the current direction is
- * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
- *
- * @see Direction
- */
- private static final String ORIGINAL_STACK_KEY = "original_stack";
-
- /**
- * The target upgrade stack before the upgrade started. This is the same
- * regardless of whether the current direction is {@link Direction#UPGRADE} or
- * {@link Direction#DOWNGRADE}.
- *
- * @see Direction
- */
- private static final String TARGET_STACK_KEY = "target_stack";
-
private final static String KERBEROS_DESCRIPTOR_NAME = "kerberos_descriptor";
private final static String KERBEROS_DESCRIPTOR_BACKUP_NAME = "kerberos_descriptor_backup";
@@ -108,70 +85,73 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction {
List<String> messages = new ArrayList<>();
List<String> errorMessages = new ArrayList<>();
- if (cluster != null) {
- logMessage(messages, "Obtaining the user-defined Kerberos descriptor");
+ UpgradeContext upgradeContext = getUpgradeContext(cluster);
- TreeMap<String, String> foreignKeys = new TreeMap<>();
- foreignKeys.put("cluster", String.valueOf(cluster.getClusterId()));
+ logMessage(messages, "Obtaining the user-defined Kerberos descriptor");
- ArtifactEntity entity = artifactDAO.findByNameAndForeignKeys("kerberos_descriptor", foreignKeys);
- KerberosDescriptor userDescriptor = (entity == null) ? null : kerberosDescriptorFactory.createInstance(entity.getArtifactData());
+ TreeMap<String, String> foreignKeys = new TreeMap<>();
+ foreignKeys.put("cluster", String.valueOf(cluster.getClusterId()));
- if (userDescriptor != null) {
- StackId originalStackId = getStackIdFromCommandParams(ORIGINAL_STACK_KEY);
- StackId targetStackId = getStackIdFromCommandParams(TARGET_STACK_KEY);
+ ArtifactEntity entity = artifactDAO.findByNameAndForeignKeys("kerberos_descriptor", foreignKeys);
+ KerberosDescriptor userDescriptor = (entity == null) ? null : kerberosDescriptorFactory.createInstance(entity.getArtifactData());
- if (isDowngrade()) {
- restoreDescriptor(foreignKeys, messages, errorMessages);
- } else {
- backupDescriptor(foreignKeys, messages, errorMessages);
+ if (userDescriptor != null) {
+
+ @Experimental(
+ feature = ExperimentalFeature.PATCH_UPGRADES,
+ comment = "This needs to be correctly done per-service")
+
+ StackId originalStackId = cluster.getCurrentStackVersion();
+ StackId targetStackId = upgradeContext.getRepositoryVersion().getStackId();
+
+ if (upgradeContext.getDirection() == Direction.DOWNGRADE) {
+ restoreDescriptor(foreignKeys, messages, errorMessages);
+ } else {
+ backupDescriptor(foreignKeys, messages, errorMessages);
- KerberosDescriptor newDescriptor = null;
- KerberosDescriptor previousDescriptor = null;
+ KerberosDescriptor newDescriptor = null;
+ KerberosDescriptor previousDescriptor = null;
- if (targetStackId == null) {
- logErrorMessage(messages, errorMessages, "The new stack version information was not found.");
- } else {
- logMessage(messages, String.format("Obtaining new stack Kerberos descriptor for %s.", targetStackId.toString()));
- newDescriptor = ambariMetaInfo.getKerberosDescriptor(targetStackId.getStackName(), targetStackId.getStackVersion());
+ if (targetStackId == null) {
+ logErrorMessage(messages, errorMessages, "The new stack version information was not found.");
+ } else {
+ logMessage(messages, String.format("Obtaining new stack Kerberos descriptor for %s.", targetStackId.toString()));
+ newDescriptor = ambariMetaInfo.getKerberosDescriptor(targetStackId.getStackName(), targetStackId.getStackVersion());
- if (newDescriptor == null) {
- logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the new stack version, %s, was not found.", targetStackId.toString()));
- }
+ if (newDescriptor == null) {
+ logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the new stack version, %s, was not found.", targetStackId.toString()));
}
+ }
- if (originalStackId == null) {
- logErrorMessage(messages, errorMessages, "The previous stack version information was not found.");
- } else {
- logMessage(messages, String.format("Obtaining previous stack Kerberos descriptor for %s.", originalStackId.toString()));
- previousDescriptor = ambariMetaInfo.getKerberosDescriptor(originalStackId.getStackName(), originalStackId.getStackVersion());
+ if (originalStackId == null) {
+ logErrorMessage(messages, errorMessages, "The previous stack version information was not found.");
+ } else {
+ logMessage(messages, String.format("Obtaining previous stack Kerberos descriptor for %s.", originalStackId.toString()));
+ previousDescriptor = ambariMetaInfo.getKerberosDescriptor(originalStackId.getStackName(), originalStackId.getStackVersion());
- if (newDescriptor == null) {
- logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the previous stack version, %s, was not found.", originalStackId.toString()));
- }
+ if (newDescriptor == null) {
+ logErrorMessage(messages, errorMessages, String.format("The Kerberos descriptor for the previous stack version, %s, was not found.", originalStackId.toString()));
}
+ }
- if (errorMessages.isEmpty()) {
- logMessage(messages, "Updating the user-specified Kerberos descriptor.");
+ if (errorMessages.isEmpty()) {
+ logMessage(messages, "Updating the user-specified Kerberos descriptor.");
- KerberosDescriptor updatedDescriptor = KerberosDescriptorUpdateHelper.updateUserKerberosDescriptor(
- previousDescriptor,
- newDescriptor,
- userDescriptor);
+ KerberosDescriptor updatedDescriptor = KerberosDescriptorUpdateHelper.updateUserKerberosDescriptor(
+ previousDescriptor,
+ newDescriptor,
+ userDescriptor);
- logMessage(messages, "Storing updated user-specified Kerberos descriptor.");
+ logMessage(messages, "Storing updated user-specified Kerberos descriptor.");
- entity.setArtifactData(updatedDescriptor.toMap());
- artifactDAO.merge(entity);
+ entity.setArtifactData(updatedDescriptor.toMap());
+ artifactDAO.merge(entity);
- logMessage(messages, "Successfully updated the user-specified Kerberos descriptor.");
- }
+ logMessage(messages, "Successfully updated the user-specified Kerberos descriptor.");
}
- } else {
- logMessage(messages, "A user-specified Kerberos descriptor was not found. No updates are necessary.");
}
} else {
- logErrorMessage(messages, errorMessages, String.format("The cluster named %s was not found.", clusterName));
+ logMessage(messages, "A user-specified Kerberos descriptor was not found. No updates are necessary.");
}
if (!errorMessages.isEmpty()) {
@@ -181,24 +161,6 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction {
return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", StringUtils.join(messages, "\n"), StringUtils.join(errorMessages, "\n"));
}
- /**
- * Determines if upgrade direction is {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
- *
- * @return {@code true} if {@link Direction#DOWNGRADE}; {@code false} if {@link Direction#UPGRADE}
- */
- private boolean isDowngrade() {
- return Direction.DOWNGRADE.name().equalsIgnoreCase(getCommandParameterValue(UPGRADE_DIRECTION_KEY));
- }
-
- private StackId getStackIdFromCommandParams(String commandParamKey) {
- String stackId = getCommandParameterValue(commandParamKey);
- if (stackId == null) {
- return null;
- } else {
- return new StackId(stackId);
- }
- }
-
private void logMessage(List<String> messages, String message) {
LOG.info(message);
messages.add(message);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
index 3ecf64d..1695bd3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java
@@ -99,7 +99,13 @@ public class UpgradeContext {
public static final String COMMAND_PARAM_TASKS = "tasks";
public static final String COMMAND_PARAM_STRUCT_OUT = "structured_out";
- /**
+ @Deprecated
+ @Experimental(
+ feature = ExperimentalFeature.PATCH_UPGRADES,
+ comment = "This isn't needed anymore, but many python classes still use it")
+ public static final String COMMAND_PARAM_DOWNGRADE_FROM_VERSION = "downgrade_from_version";
+
+ /*
* The cluster that the upgrade is for.
*/
final private Cluster m_cluster;
@@ -744,6 +750,7 @@ public class UpgradeContext {
* <ul>
* <li>{@link #COMMAND_PARAM_CLUSTER_NAME}
* <li>{@link #COMMAND_PARAM_DIRECTION}
+ * <li>{@link #COMMAND_PARAM_DOWNGRADE_FROM_VERSION}
* <li>{@link #COMMAND_PARAM_UPGRADE_TYPE}
* <li>{@link KeyNames#REFRESH_CONFIG_TAGS_BEFORE_EXECUTION} - necessary in
* order to have the commands contain the correct configurations. Otherwise,
@@ -758,8 +765,13 @@ public class UpgradeContext {
public Map<String, String> getInitializedCommandParameters() {
Map<String, String> parameters = new HashMap<>();
+ Direction direction = getDirection();
parameters.put(COMMAND_PARAM_CLUSTER_NAME, m_cluster.getClusterName());
- parameters.put(COMMAND_PARAM_DIRECTION, getDirection().name().toLowerCase());
+ parameters.put(COMMAND_PARAM_DIRECTION, direction.name().toLowerCase());
+
+ if (direction == Direction.DOWNGRADE) {
+ parameters.put(COMMAND_PARAM_DOWNGRADE_FROM_VERSION, m_repositoryVersion.getVersion());
+ }
if (null != getType()) {
// use the serialized attributes of the enum to convert it to a string,
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
index 74fd76a..93b4944 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
@@ -23,6 +23,7 @@ import status_params
from setup_spark import *
from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import get_stack_feature_version
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.libraries.functions.get_stack_version import get_stack_version
@@ -56,10 +57,8 @@ upgrade_direction = default("/commandParams/upgrade_direction", None)
java_home = config['hostLevelParams']['java_home']
stack_name = status_params.stack_name
stack_root = Script.get_stack_root()
-stack_version_unformatted = config['hostLevelParams']['stack_version']
-if upgrade_direction == Direction.DOWNGRADE:
- stack_version_unformatted = config['commandParams']['original_stack'].split("-")[1]
-stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+version_for_stack_feature_checks = get_stack_feature_version(config)
sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
@@ -70,7 +69,7 @@ spark_conf = '/etc/spark/conf'
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+if check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks):
hadoop_home = stack_select.get_hadoop_dir("home")
spark_conf = format("{stack_root}/current/{component_directory}/conf")
spark_log_dir = config['configurations']['spark-env']['spark_log_dir']
@@ -211,7 +210,7 @@ dfs_type = default("/commandParams/dfs_type", "")
# livy is only supported from HDP 2.5
has_livyserver = False
-if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and "livy-env" in config['configurations']:
+if check_stack_feature(StackFeature.SPARK_LIVY, version_for_stack_feature_checks) and "livy-env" in config['configurations']:
livy_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY_SERVER")
livy_conf = format("{stack_root}/current/{livy_component_directory}/conf")
livy_log_dir = config['configurations']['livy-env']['livy_log_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
index 50c1555..53c8f9e 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
@@ -118,11 +118,11 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
mode=0644
)
- effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
+ effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks
if effective_version:
effective_version = format_stack_version(effective_version)
- if effective_version and check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version):
+ if check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version):
File(os.path.join(params.spark_conf, 'java-opts'),
owner=params.spark_user,
group=params.spark_group,
@@ -134,7 +134,7 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
action="delete"
)
- if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
+ if params.spark_thrift_fairscheduler_content and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create spark-thrift-fairscheduler.xml
File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"),
owner=params.spark_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
index 31a296a..2838186 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py
@@ -34,11 +34,11 @@ def spark_service(name, upgrade_type=None, action=None):
if action == 'start':
- effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
+ effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks
if effective_version:
effective_version = format_stack_version(effective_version)
- if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
+ if name == 'jobhistoryserver' and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# copy spark-hdp-assembly.jar to hdfs
copy_to_hdfs("spark", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
# create spark history directory
@@ -58,7 +58,7 @@ def spark_service(name, upgrade_type=None, action=None):
# Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not
# need to copy the tarball, otherwise, copy it.
- if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted):
+ if check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.version_for_stack_feature_checks):
resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
if resource_created:
params.HdfsResource(None, action="execute")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
index ed92955..86f6d3b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java
@@ -36,11 +36,17 @@ import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.ArtifactDAO;
import org.apache.ambari.server.orm.entities.ArtifactEntity;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.orm.entities.UpgradeEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeContext;
+import org.apache.ambari.server.state.UpgradeContextFactory;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper;
+import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.easymock.Capture;
import org.easymock.EasyMock;
import org.junit.Before;
@@ -58,23 +64,34 @@ import org.powermock.modules.junit4.PowerMockRunner;
public class UpgradeUserKerberosDescriptorTest {
private Clusters clusters;
private Cluster cluster;
+ private UpgradeEntity upgrade;
+ private UpgradeContext upgradeContext;
private AmbariMetaInfo ambariMetaInfo;
private KerberosDescriptorFactory kerberosDescriptorFactory;
private ArtifactDAO artifactDAO;
+ private UpgradeContextFactory upgradeContextFactory;
private TreeMap<String, Field> fields = new TreeMap<>();
+ private StackId HDP_24 = new StackId("HDP", "2.4");
@Before
public void setup() throws Exception {
clusters = EasyMock.createMock(Clusters.class);
cluster = EasyMock.createMock(Cluster.class);
+ upgrade = EasyMock.createNiceMock(UpgradeEntity.class);
kerberosDescriptorFactory = EasyMock.createNiceMock(KerberosDescriptorFactory.class);
ambariMetaInfo = EasyMock.createMock(AmbariMetaInfo.class);
artifactDAO = EasyMock.createNiceMock(ArtifactDAO.class);
+ upgradeContextFactory = EasyMock.createNiceMock(UpgradeContextFactory.class);
+ upgradeContext = EasyMock.createNiceMock(UpgradeContext.class);
expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
expect(cluster.getClusterId()).andReturn(1l).atLeastOnce();
- replay(clusters, cluster);
+ expect(cluster.getCurrentStackVersion()).andReturn(HDP_24).atLeastOnce();
+ expect(cluster.getUpgradeInProgress()).andReturn(upgrade).atLeastOnce();
+ expect(upgradeContextFactory.create(cluster, upgrade)).andReturn(upgradeContext).atLeastOnce();
+
+ replay(clusters, cluster, upgradeContextFactory, upgrade);
prepareFields();
@@ -82,12 +99,16 @@ public class UpgradeUserKerberosDescriptorTest {
@Test
public void testUpgrade() throws Exception {
+ StackId stackId = new StackId("HDP", "2.5");
+ RepositoryVersionEntity repositoryVersion = EasyMock.createNiceMock(RepositoryVersionEntity.class);
+ expect(repositoryVersion.getStackId()).andReturn(stackId).atLeastOnce();
+
+ expect(upgradeContext.getDirection()).andReturn(Direction.UPGRADE).atLeastOnce();
+ expect(upgradeContext.getRepositoryVersion()).andReturn(repositoryVersion).atLeastOnce();
+ replay(repositoryVersion, upgradeContext);
Map<String, String> commandParams = new HashMap<>();
commandParams.put("clusterName", "c1");
- commandParams.put("upgrade_direction", "UPGRADE");
- commandParams.put("original_stack", "HDP-2.4");
- commandParams.put("target_stack", "HDP-2.5");
ExecutionCommand executionCommand = new ExecutionCommand();
executionCommand.setCommandParams(commandParams);
@@ -140,12 +161,16 @@ public class UpgradeUserKerberosDescriptorTest {
@Test
public void testDowngrade() throws Exception {
+ StackId stackId = new StackId("HDP", "2.5");
+ RepositoryVersionEntity repositoryVersion = EasyMock.createNiceMock(RepositoryVersionEntity.class);
+ expect(repositoryVersion.getStackId()).andReturn(stackId).atLeastOnce();
+
+ expect(upgradeContext.getDirection()).andReturn(Direction.DOWNGRADE).atLeastOnce();
+ expect(upgradeContext.getRepositoryVersion()).andReturn(repositoryVersion).atLeastOnce();
+ replay(repositoryVersion, upgradeContext);
Map<String, String> commandParams = new HashMap<>();
commandParams.put("clusterName", "c1");
- commandParams.put("upgrade_direction", "DOWNGRADE");
- commandParams.put("original_stack", "HDP-2.4");
- commandParams.put("target_stack", "HDP-2.5");
ExecutionCommand executionCommand = new ExecutionCommand();
executionCommand.setCommandParams(commandParams);
@@ -188,12 +213,19 @@ public class UpgradeUserKerberosDescriptorTest {
}
private void prepareFields() throws NoSuchFieldException {
- String[] fieldsNames = {"artifactDAO","clusters","ambariMetaInfo","kerberosDescriptorFactory"};
- for(String fieldName : fieldsNames)
- {
- Field clustersField = UpgradeUserKerberosDescriptor.class.getDeclaredField(fieldName);
- clustersField.setAccessible(true);
- fields.put(fieldName, clustersField);
+ String[] fieldsNames = { "artifactDAO", "clusters", "ambariMetaInfo",
+ "kerberosDescriptorFactory", "m_upgradeContextFactory" };
+
+ for (String fieldName : fieldsNames) {
+ try {
+ Field clustersField = UpgradeUserKerberosDescriptor.class.getDeclaredField(fieldName);
+ clustersField.setAccessible(true);
+ fields.put(fieldName, clustersField);
+ } catch( NoSuchFieldException noSuchFieldException ){
+ Field clustersField = UpgradeUserKerberosDescriptor.class.getSuperclass().getDeclaredField(fieldName);
+ clustersField.setAccessible(true);
+ fields.put(fieldName, clustersField);
+ }
}
}
private void injectFields(UpgradeUserKerberosDescriptor action) throws IllegalAccessException {
@@ -201,5 +233,6 @@ public class UpgradeUserKerberosDescriptorTest {
fields.get("clusters").set(action, clusters);
fields.get("ambariMetaInfo").set(action, ambariMetaInfo);
fields.get("kerberosDescriptorFactory").set(action, kerberosDescriptorFactory);
+ fields.get("m_upgradeContextFactory").set(action, upgradeContextFactory);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/python/TestStackFeature.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py
index 0116a7a..230734c 100644
--- a/ambari-server/src/test/python/TestStackFeature.py
+++ b/ambari-server/src/test/python/TestStackFeature.py
@@ -28,6 +28,32 @@ from unittest import TestCase
Logger.initialize_logger()
class TestStackFeature(TestCase):
+ """
+ EU Upgrade (HDP 2.5 to HDP 2.6)
+ - STOP
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.5
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.5.0.0-1237
+ - START
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.6
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.6.0.0-334
+
+ EU Downgrade (HDP 2.6 to HDP 2.5)
+ - STOP
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.6
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.6.0.0-334
+ - START
+ hostLevelParams/stack_name = HDP
+ hostLevelParams/stack_version = 2.5
+ hostLevelParams/current_version = 2.5.0.0-1237
+ commandParams/version = 2.5.0.0-1237
+ """
+
def test_get_stack_feature_version_missing_params(self):
try:
stack_feature_version = get_stack_feature_version({})
@@ -122,7 +148,7 @@ class TestStackFeature(TestCase):
"current_version": "2.4.0.0-1234"
},
"commandParams": {
- "original_stack": "2.4",
+ "source_stack": "2.4",
"target_stack": "2.5",
"upgrade_direction": "upgrade",
"version": "2.5.9.9-9999"
@@ -143,8 +169,8 @@ class TestStackFeature(TestCase):
"current_version":"2.4.0.0-1234"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
"version":"2.4.0.0-1234",
"downgrade_from_version": "2.5.9.9-9999"
@@ -166,10 +192,10 @@ class TestStackFeature(TestCase):
"current_version":"2.4.0.0-1234"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
- "version":"2.4.0.0-1234",
+ "version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
}
@@ -189,10 +215,10 @@ class TestStackFeature(TestCase):
"custom_command":"STOP"
},
"commandParams":{
- "original_stack":"2.4",
- "target_stack":"2.5",
+ "source_stack":"2.5",
+ "target_stack":"2.4",
"upgrade_direction":"downgrade",
- "version":"2.4.0.0-1234",
+ "version":"2.5.9.9-9999",
"downgrade_from_version":"2.5.9.9-9999"
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
index 7f77d83..3aadf2c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json
@@ -25,7 +25,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2844",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.2",
+ "source_stack": "HDP-2.2",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"desired_namenode_role": "standby",
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
index 87b18af..2d48ff6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json
@@ -25,7 +25,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2844",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.2",
+ "source_stack": "HDP-2.2",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"desired_namenode_role": "standby",
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
index 99fcba0..021695b 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
@@ -13,7 +13,7 @@
"upgrade_type": "nonrolling_upgrade",
"version": "2.3.2.0-2950",
"forceRefreshConfigTagsBeforeExecution": "*",
- "original_stack": "HDP-2.3",
+ "source_stack": "HDP-2.3",
"command_timeout": "1200",
"target_stack": "HDP-2.3",
"script_type": "PYTHON"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f27f3aff/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
index a9db11c..1805c3b 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json
@@ -22,7 +22,7 @@
"upgrade_type": "rolling_upgrade",
"command_retry_max_attempt_count": "3",
"version": "2.3.0.0-2096",
- "original_stack": "HDP-2.3",
+ "source_stack": "HDP-2.3",
"command_retry_enabled": "false",
"command_timeout": "1200",
"target_stack": "HDP-2.3",