You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/07/22 13:31:12 UTC
ambari git commit: AMBARI-17836 - EU POST Call Takes More Than 3
Minutes on 900 Node Cluster (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk cabf68299 -> 3ac13b5c7
AMBARI-17836 - EU POST Call Takes More Than 3 Minutes on 900 Node Cluster (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ac13b5c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ac13b5c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ac13b5c
Branch: refs/heads/trunk
Commit: 3ac13b5c793a05da245722f122a6445af2ba1169
Parents: cabf682
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Jul 21 12:36:12 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Jul 22 09:29:57 2016 -0400
----------------------------------------------------------------------
.../src/main/python/ambari_agent/ActionQueue.py | 10 -
.../server/actionmanager/ActionScheduler.java | 3 +-
.../actionmanager/ExecutionCommandWrapper.java | 210 ++++++++++++-------
.../ambari/server/agent/ExecutionCommand.java | 32 +--
.../server/configuration/Configuration.java | 2 +-
.../controller/AmbariActionExecutionHelper.java | 40 ++--
.../AmbariCustomCommandExecutionHelper.java | 184 ++++++++--------
.../AmbariManagementControllerImpl.java | 8 +-
.../internal/ClientConfigResourceProvider.java | 110 +++++-----
.../internal/UpgradeResourceProvider.java | 2 +-
.../ambari/server/state/ConfigHelper.java | 8 +-
.../ambari/server/state/UpgradeHelper.java | 20 +-
.../AmbariCustomCommandExecutionHelperTest.java | 8 +-
.../AmbariManagementControllerImplTest.java | 15 +-
.../ClientConfigResourceProviderTest.java | 78 +++++--
.../UpgradeResourceProviderHDP22Test.java | 119 ++---------
16 files changed, 422 insertions(+), 427 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index 1e7b1b6..c8696ef 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -407,16 +407,6 @@ class ActionQueue(threading.Thread):
# let ambari know that configuration tags were applied
configHandler = ActualConfigHandler(self.config, self.configTags)
- #update
- if command.has_key('forceRefreshConfigTags') and len(command['forceRefreshConfigTags']) > 0 :
- forceRefreshConfigTags = command['forceRefreshConfigTags']
- logger.info("Got refresh additional component tags command")
-
- for configTag in forceRefreshConfigTags :
- configHandler.update_component_tag(command['role'], configTag, command['configurationTags'][configTag])
-
- roleResult['customCommand'] = self.CUSTOM_COMMAND_RESTART # force restart for component to evict stale_config on server side
- command['configurationTags'] = configHandler.read_actual_component(command['role'])
if command.has_key('configurationTags'):
configHandler.write_actual(command['configurationTags'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
index 205ef9f..90ffa5b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
@@ -523,8 +523,7 @@ class ActionScheduler implements Runnable {
private boolean isStageHasBackgroundCommandsOnly(Stage s, String host) {
for (ExecutionCommandWrapper c : s.getExecutionCommands(host)) {
- if(c.getExecutionCommand().getCommandType() != AgentCommandType.BACKGROUND_EXECUTION_COMMAND)
- {
+ if (c.getCommandType() != AgentCommandType.BACKGROUND_EXECUTION_COMMAND) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index ef12c3a..2f6ab02 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -23,17 +23,17 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.ClusterNotFoundException;
+import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.ExecutionCommand;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.utils.StageUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
@@ -54,6 +54,9 @@ public class ExecutionCommandWrapper {
@Inject
ConfigHelper configHelper;
+ @Inject
+ private Gson gson;
+
@AssistedInject
public ExecutionCommandWrapper(@Assisted String jsonExecutionCommand) {
this.jsonExecutionCommand = jsonExecutionCommand;
@@ -64,110 +67,155 @@ public class ExecutionCommandWrapper {
this.executionCommand = executionCommand;
}
- @SuppressWarnings("serial")
+ /**
+ * Gets the execution command by either de-serializing the backing JSON
+ * command or returning the encapsulated instance which has already been
+ * de-serialized.
+ * <p/>
+ * If the {@link ExecutionCommand} has configuration tags which need to be
+ * refreshed, then this method will lookup the appropriate configuration tags
+ * before building the final configurations to set ont he command. Therefore,
+ * the {@link ExecutionCommand} is allowed to have no configuration tags as
+ * long as it has been instructed to set updated ones at execution time.
+ *
+ * @return
+ * @see ExecutionCommand#setForceRefreshConfigTagsBeforeExecution(Set)
+ */
public ExecutionCommand getExecutionCommand() {
if (executionCommand != null) {
return executionCommand;
- } else if (jsonExecutionCommand != null) {
- executionCommand = StageUtils.getGson().fromJson(jsonExecutionCommand, ExecutionCommand.class);
-
- if (executionCommand.getConfigurationTags() != null
- &&
- !executionCommand.getConfigurationTags().isEmpty()) {
-
- // For a configuration type, both tag and an actual configuration can be stored
- // Configurations from the tag is always expanded and then over-written by the actual
- // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==>
- // global:{a1:A2,b1:B1,c1:C1}
- Long clusterId = hostRoleCommandDAO.findByPK(
- executionCommand.getTaskId()).getStage().getClusterId();
-
- try {
- Cluster cluster = clusters.getClusterById(clusterId);
- Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags();
-
- // Execution commands have config-tags already set during their creation. However, these
- // tags become stale at runtime when other ExecutionCommands run and change the desired
- // configs (like ConfigureAction). Hence an ExecutionCommand can specify which config-types
- // should be refreshed at runtime. Specifying <code>*</code> will result in all config-type
- // tags to be refreshed to the latest cluster desired-configs.
- Set<String> refreshConfigTagsBeforeExecution = executionCommand.getForceRefreshConfigTagsBeforeExecution();
- if (refreshConfigTagsBeforeExecution != null && !refreshConfigTagsBeforeExecution.isEmpty()) {
- AmbariManagementController managementController = AmbariServer.getController();
- Map<String, Map<String, String>> configTags = managementController.findConfigurationTagsWithOverrides(
- cluster, executionCommand.getHostname());
- for (String refreshConfigTag : refreshConfigTagsBeforeExecution) {
- if ("*".equals(refreshConfigTag)) {
- // if forcing a refresh of *, then clear out any existing
- // configurations so that all of the new configurations are
- // forcefully applied
- LOG.debug("ExecutionCommandWrapper.getExecutionCommand: refreshConfigTag set to {}, so clearing config for full refresh.", refreshConfigTag);
- executionCommand.getConfigurations().clear();
- configurationTags = configTags;
- executionCommand.setConfigurationTags(configTags);
- break;
- } else if (configurationTags.containsKey(refreshConfigTag) && configTags.containsKey(refreshConfigTag)) {
- configurationTags.put(refreshConfigTag, configTags.get(refreshConfigTag));
- }
- }
- }
+ }
+
+ if( null == jsonExecutionCommand ){
+ throw new RuntimeException(
+ "Invalid ExecutionCommandWrapper, both object and string"
+ + " representations are null");
+ }
+
+ try {
+ executionCommand = gson.fromJson(jsonExecutionCommand, ExecutionCommand.class);
+
+ // sanity; if no configurations, just initialize to prevent NPEs
+ if (null == executionCommand.getConfigurations()) {
+ executionCommand.setConfigurations(new TreeMap<String, Map<String, String>>());
+ }
+
+ Map<String, Map<String, String>> configurations = executionCommand.getConfigurations();
+
+ // For a configuration type, both tag and an actual configuration can be stored
+ // Configurations from the tag is always expanded and then over-written by the actual
+ // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==>
+ // global:{a1:A2,b1:B1,c1:C1}
+ Long clusterId = hostRoleCommandDAO.findByPK(
+ executionCommand.getTaskId()).getStage().getClusterId();
+
+ Cluster cluster = clusters.getClusterById(clusterId);
+
+ // Execution commands may have config-tags already set during their creation.
+ // However, these tags become stale at runtime when other
+ // ExecutionCommands run and change the desired configs (like
+ // ConfigureAction). Hence an ExecutionCommand can specify which
+ // config-types should be refreshed at runtime. Specifying <code>*</code>
+ // will result in all config-type tags to be refreshed to the latest
+ // cluster desired-configs. Additionally, there may be no configuration
+ // tags set but refresh might be set to *. In this case, they should still
+ // be refreshed with the latest.
+ boolean refreshConfigTagsBeforeExecution = executionCommand.getForceRefreshConfigTagsBeforeExecution();
+ if (refreshConfigTagsBeforeExecution) {
+ Map<String, Map<String, String>> configurationTags = configHelper.getEffectiveDesiredTags(
+ cluster, executionCommand.getHostname());
+
+ // then clear out any existing configurations so that all of the new
+ // configurations are forcefully applied
+ LOG.debug("Refreshing all configuration tags before execution");
- Map<String, Map<String, String>> configProperties = configHelper
+ configurations.clear();
+ executionCommand.setConfigurationTags(configurationTags);
+ }
+
+ // now that the tags have been updated (if necessary), fetch the
+ // configurations
+ Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags();
+ if (null != configurationTags && !configurationTags.isEmpty()) {
+ Map<String, Map<String, String>> configProperties = configHelper
.getEffectiveConfigProperties(cluster, configurationTags);
- // Apply the configurations saved with the Execution Cmd on top of
- // derived configs - This will take care of all the hacks
- for (Map.Entry<String, Map<String, String>> entry : configProperties.entrySet()) {
- String type = entry.getKey();
- Map<String, String> allLevelMergedConfig = entry.getValue();
+ // Apply the configurations saved with the Execution Cmd on top of
+ // derived configs - This will take care of all the hacks
+ for (Map.Entry<String, Map<String, String>> entry : configProperties.entrySet()) {
+ String type = entry.getKey();
+ Map<String, String> allLevelMergedConfig = entry.getValue();
- if (executionCommand.getConfigurations().containsKey(type)) {
- Map<String, String> mergedConfig =
- configHelper.getMergedConfig(allLevelMergedConfig,
- executionCommand.getConfigurations().get(type));
+ if (configurations.containsKey(type)) {
+ Map<String, String> mergedConfig = configHelper.getMergedConfig(allLevelMergedConfig,
+ configurations.get(type));
- executionCommand.getConfigurations().get(type).clear();
- executionCommand.getConfigurations().get(type).putAll(mergedConfig);
+ configurations.get(type).clear();
+ configurations.get(type).putAll(mergedConfig);
- } else {
- executionCommand.getConfigurations().put(type, new HashMap<String, String>());
- executionCommand.getConfigurations().get(type).putAll(allLevelMergedConfig);
- }
+ } else {
+ configurations.put(type, new HashMap<String, String>());
+ configurations.get(type).putAll(allLevelMergedConfig);
}
+ }
- Map<String, Map<String, Map<String, String>>> configAttributes = configHelper.getEffectiveConfigAttributes(cluster,
- executionCommand.getConfigurationTags());
+ Map<String, Map<String, Map<String, String>>> configAttributes = configHelper.getEffectiveConfigAttributes(
+ cluster, executionCommand.getConfigurationTags());
- for (Map.Entry<String, Map<String, Map<String, String>>> attributesOccurrence : configAttributes.entrySet()) {
- String type = attributesOccurrence.getKey();
- Map<String, Map<String, String>> attributes = attributesOccurrence.getValue();
+ for (Map.Entry<String, Map<String, Map<String, String>>> attributesOccurrence : configAttributes.entrySet()) {
+ String type = attributesOccurrence.getKey();
+ Map<String, Map<String, String>> attributes = attributesOccurrence.getValue();
- if (executionCommand.getConfigurationAttributes() != null) {
- if (!executionCommand.getConfigurationAttributes().containsKey(type)) {
- executionCommand.getConfigurationAttributes().put(type, new TreeMap<String, Map<String, String>>());
- }
- configHelper.cloneAttributesMap(attributes, executionCommand.getConfigurationAttributes().get(type));
+ if (executionCommand.getConfigurationAttributes() != null) {
+ if (!executionCommand.getConfigurationAttributes().containsKey(type)) {
+ executionCommand.getConfigurationAttributes().put(type,
+ new TreeMap<String, Map<String, String>>());
+ }
+ configHelper.cloneAttributesMap(attributes,
+ executionCommand.getConfigurationAttributes().get(type));
}
- }
-
- } catch (AmbariException e) {
- throw new RuntimeException(e);
}
}
+ } catch (ClusterNotFoundException cnfe) {
+ // it's possible that there are commands without clusters; in such cases,
+ // just return the de-serialized command and don't try to read configs
+ LOG.warn(
+ "Unable to lookup the cluster byt ID; assuming that there is no cluster and therefore no configs for this execution command: {}",
+ cnfe.getMessage());
return executionCommand;
- } else {
+ } catch (AmbariException e) {
+ throw new RuntimeException(e);
+ }
+
+ return executionCommand;
+ }
+
+ /**
+ * Gets the type of command by deserializing the JSON and invoking
+ * {@link ExecutionCommand#getCommandType()}.
+ *
+ * @return
+ */
+ public AgentCommandType getCommandType() {
+ if (executionCommand != null) {
+ return executionCommand.getCommandType();
+ }
+
+ if (null == jsonExecutionCommand) {
throw new RuntimeException(
- "Invalid ExecutionCommandWrapper, both object and string"
- + " representations are null");
+ "Invalid ExecutionCommandWrapper, both object and string" + " representations are null");
}
+
+ return gson.fromJson(jsonExecutionCommand,
+ ExecutionCommand.class).getCommandType();
}
public String getJson() {
if (jsonExecutionCommand != null) {
return jsonExecutionCommand;
} else if (executionCommand != null) {
- jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand);
+ jsonExecutionCommand = gson.toJson(executionCommand);
return jsonExecutionCommand;
} else {
throw new RuntimeException(
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 28de8ed..29737ee 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -88,11 +88,8 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("configurationTags")
private Map<String, Map<String, String>> configurationTags;
- @SerializedName("forceRefreshConfigTags")
- private Set<String> forceRefreshConfigTags = new HashSet<String>();
-
@SerializedName("forceRefreshConfigTagsBeforeExecution")
- private Set<String> forceRefreshConfigTagsBeforeExecution = new HashSet<String>();
+ private boolean forceRefreshConfigTagsBeforeExecution = false;
@SerializedName("commandParams")
private Map<String, String> commandParams = new HashMap<String, String>();
@@ -233,27 +230,16 @@ public class ExecutionCommand extends AgentCommand {
public void setConfigurations(Map<String, Map<String, String>> configurations) {
this.configurations = configurations;
}
- /**
- * @return Returns the set of config-types that have to be propagated to actual-config of component of given custom command, if command is successfully finished.
- */
- public Set<String> getForceRefreshConfigTags() {
- return forceRefreshConfigTags;
- }
-
- public void setForceRefreshConfigTags(Set<String> forceRefreshConfigTags) {
- this.forceRefreshConfigTags = forceRefreshConfigTags;
- }
/**
- * Comma separated list of config-types whose tags have be refreshed
- * at runtime before being executed. If all config-type tags have to be
- * refreshed, "*" can be specified.
+ * Gets whether configuration tags shoudl be refreshed right before the
+ * command is scheduled.
*/
- public Set<String> getForceRefreshConfigTagsBeforeExecution() {
+ public boolean getForceRefreshConfigTagsBeforeExecution() {
return forceRefreshConfigTagsBeforeExecution;
}
- public void setForceRefreshConfigTagsBeforeExecution(Set<String> forceRefreshConfigTagsBeforeExecution) {
+ public void setForceRefreshConfigTagsBeforeExecution(boolean forceRefreshConfigTagsBeforeExecution) {
this.forceRefreshConfigTagsBeforeExecution = forceRefreshConfigTagsBeforeExecution;
}
@@ -274,7 +260,7 @@ public class ExecutionCommand extends AgentCommand {
for (Map.Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) {
serviceVersionMap.put(entry.getKey(), entry.getValue().getVersion());
}
- this.availableServices = serviceVersionMap;
+ availableServices = serviceVersionMap;
}
public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
@@ -379,7 +365,6 @@ public class ExecutionCommand extends AgentCommand {
String AMBARI_DB_RCA_USERNAME = "ambari_db_rca_username";
String AMBARI_DB_RCA_PASSWORD = "ambari_db_rca_password";
String COMPONENT_CATEGORY = "component_category";
- String REFRESH_ADITIONAL_COMPONENT_TAGS = "forceRefreshConfigTags";
String USER_LIST = "user_list";
String GROUP_LIST = "group_list";
String NOT_MANAGED_HDFS_PATH_LIST = "not_managed_hdfs_path_list";
@@ -393,9 +378,8 @@ public class ExecutionCommand extends AgentCommand {
String LOG_OUTPUT = "log_output";
/**
- * Comma separated list of config-types whose tags have be refreshed
- * at runtime before being executed. If all config-type tags have to be
- * refreshed, "*" can be specified.
+ * A boolean indicating whether configuration tags should be refreshed
+ * before sending the command.
*/
String REFRESH_CONFIG_TAGS_BEFORE_EXECUTION = "forceRefreshConfigTagsBeforeExecution";
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 399f26c..9655bf9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2634,7 +2634,7 @@ public class Configuration {
* @return the location of the template file, or {@code null} if not defined.
*/
public String getAlertTemplateFile() {
- return properties.getProperty(ALERT_TEMPLATE_FILE);
+ return StringUtils.strip(properties.getProperty(ALERT_TEMPLATE_FILE));
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 9c2db1c..5cc807c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -28,7 +28,6 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TY
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
-import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -369,6 +368,10 @@ public class AmbariActionExecutionHelper {
// create tasks for each host
for (String hostName : targetHosts) {
+ // ensure that any tags that need to be refreshed are extracted from the
+ // context and put onto the execution command
+ Map<String, String> actionParameters = actionContext.getParameters();
+
stage.addHostRoleExecutionCommand(hostName, Role.valueOf(actionContext.getActionName()),
RoleCommand.ACTIONEXECUTE,
new ServiceComponentHostOpInProgressEvent(actionContext.getActionName(), hostName,
@@ -401,10 +404,22 @@ public class AmbariActionExecutionHelper {
execCmd.setConfigurations(new TreeMap<String, Map<String, String>>());
execCmd.setConfigurationAttributes(new TreeMap<String, Map<String, Map<String, String>>>());
- // !!! ensure that the config tags are added to this command so that the
- // configurations can be populated from the tags before the command is
- // sent
- Map<String, Map<String, String>> configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName);
+ // if the command should fetch brand new configuration tags before
+ // execution, then we don't need to fetch them now
+ if (null != actionParameters && !actionParameters.isEmpty()) {
+ if (actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)) {
+ execCmd.setForceRefreshConfigTagsBeforeExecution(true);
+ }
+ }
+
+ // when building complex orchestration ahead of time (such as when
+ // performing ugprades), fetching configuration tags can take a very long
+ // time - if it's not needed, then don't do it
+ Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>();
+ if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) {
+ configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName);
+ }
+
execCmd.setConfigurationTags(configTags);
execCmd.setCommandParams(commandParams);
@@ -431,7 +446,7 @@ public class AmbariActionExecutionHelper {
roleParams = new TreeMap<String, String>();
}
- roleParams.putAll(actionContext.getParameters());
+ roleParams.putAll(actionParameters);
SecretReference.replaceReferencesWithPasswords(roleParams, cluster);
@@ -447,19 +462,6 @@ public class AmbariActionExecutionHelper {
execCmd.setRoleParams(roleParams);
- // ensure that any tags that need to be refreshed are extracted from the
- // context and put onto the execution command
- Map<String, String> actionParameters = actionContext.getParameters();
- if (null != actionParameters && !actionParameters.isEmpty()) {
- if (actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)) {
- String[] split = StringUtils.split(
- actionParameters.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION));
- Set<String> configsToRefresh = new HashSet<String>(Arrays.asList(split));
-
- execCmd.setForceRefreshConfigTagsBeforeExecution(configsToRefresh);
- }
- }
-
if (null != cluster) {
// Generate localComponents
for (ServiceComponentHost sch : cluster.getServiceComponentHosts(hostName)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 2174a64..8c8ae10 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -18,6 +18,33 @@
package org.apache.ambari.server.controller;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
+
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
@@ -54,6 +81,7 @@ import org.apache.ambari.server.state.CommandScriptDefinition;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.CustomCommandDefinition;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
@@ -83,33 +111,6 @@ import com.google.gson.JsonObject;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_COMMAND;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
-
/**
* Helper class containing logic to process custom command execution requests .
* This class has special support needed for SERVICE_CHECK and DECOMMISSION.
@@ -338,8 +339,23 @@ public class AmbariCustomCommandExecutionHelper {
new TreeMap<String, Map<String, String>>();
Map<String, Map<String, Map<String, String>>> configurationAttributes =
new TreeMap<String, Map<String, Map<String, String>>>();
- Map<String, Map<String, String>> configTags =
- managementController.findConfigurationTagsWithOverrides(cluster, hostName);
+ Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>();
+
+ ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName,
+ componentName).getExecutionCommand();
+
+ // if the command should fetch brand new configuration tags before
+ // execution, then we don't need to fetch them now
+ if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){
+ execCmd.setForceRefreshConfigTagsBeforeExecution(true);
+ }
+
+ // when building complex orchestration ahead of time (such as when
+ // performing ugprades), fetching configuration tags can take a very long
+ // time - if it's not needed, then don't do it
+ if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) {
+ configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName);
+ }
HostRoleCommand cmd = stage.getHostRoleCommand(hostName, componentName);
if (cmd != null) {
@@ -347,9 +363,6 @@ public class AmbariCustomCommandExecutionHelper {
cmd.setCustomCommandName(commandName);
}
- ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName,
- componentName).getExecutionCommand();
-
//set type background
if(customCommandDefinition != null && customCommandDefinition.isBackground()){
execCmd.setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND);
@@ -359,14 +372,6 @@ public class AmbariCustomCommandExecutionHelper {
execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
- if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)){
- execCmd.setForceRefreshConfigTags(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)));
- }
-
- if(actionExecutionContext.getParameters() != null && actionExecutionContext.getParameters().containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){
- execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionExecutionContext.getParameters().get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)));
- }
-
execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
Map<String, String> hostLevelParams = new TreeMap<String, String>();
@@ -378,15 +383,17 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(STACK_NAME, stackId.getStackName());
hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());
- Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster);
+ Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
+
+ Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs);
String userList = gson.toJson(userSet);
hostLevelParams.put(USER_LIST, userList);
- Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster);
+ Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs);
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster);
+ Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -458,8 +465,17 @@ public class AmbariCustomCommandExecutionHelper {
private void applyCustomCommandBackendLogic(Cluster cluster, String serviceName, String componentName, String commandName, String hostname) throws AmbariException {
switch (commandName) {
case "RESTART":
- LOG.info("Updating desired state on RESTART for the service [{}], service component [{}]", serviceName, componentName);
- cluster.getService(serviceName).getServiceComponent(componentName).getServiceComponentHost(hostname).setDesiredState(State.STARTED);
+ ServiceComponentHost serviceComponentHost = cluster.getService(
+ serviceName).getServiceComponent(componentName).getServiceComponentHost(hostname);
+
+ State currentDesiredState = serviceComponentHost.getDesiredState();
+ if (currentDesiredState != State.STARTED) {
+ LOG.info("Updating desired state to {} on RESTART for {}/{} because it was {}",
+ State.STARTED, serviceName, componentName, currentDesiredState);
+
+ serviceComponentHost.setDesiredState(State.STARTED);
+ }
+
break;
default:
LOG.debug("No backend operations needed for the custom command: {}", commandName);
@@ -467,21 +483,6 @@ public class AmbariCustomCommandExecutionHelper {
}
}
- /**
- * Splits the passed comma separated value and returns it as set.
- *
- * @param commaSeparatedTags separated list
- *
- * @return set of items or null
- */
- private Set<String> parseAndValidateComponentsMapping(String commaSeparatedTags) {
- Set<String> retVal = null;
- if(commaSeparatedTags != null && !commaSeparatedTags.trim().isEmpty()){
- Collections.addAll(retVal = new HashSet<String>(), commaSeparatedTags.split(","));
- }
- return retVal;
- }
-
private void findHostAndAddServiceCheckAction(final ActionExecutionContext actionExecutionContext,
final RequestResourceFilter resourceFilter, Stage stage) throws AmbariException {
@@ -606,18 +607,27 @@ public class AmbariCustomCommandExecutionHelper {
new TreeMap<String, Map<String, String>>();
Map<String, Map<String, Map<String, String>>> configurationAttributes =
new TreeMap<String, Map<String, Map<String, String>>>();
- Map<String, Map<String, String>> configTags =
- managementController.findConfigurationTagsWithOverrides(cluster, hostname);
+ Map<String, Map<String, String>> configTags = new TreeMap<String, Map<String, String>>();
ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostname,
smokeTestRole).getExecutionCommand();
+ // if the command should fetch brand new configuration tags before
+ // execution, then we don't need to fetch them now
+ if(actionParameters != null && actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){
+ execCmd.setForceRefreshConfigTagsBeforeExecution(true);
+ }
+
+ // when building complex orchestration ahead of time (such as when
+ // performing ugprades), fetching configuration tags can take a very long
+ // time - if it's not needed, then don't do it
+ if (!execCmd.getForceRefreshConfigTagsBeforeExecution()) {
+ configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostname);
+ }
+
execCmd.setConfigurations(configurations);
execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
- if(actionParameters != null && actionParameters.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)){
- execCmd.setForceRefreshConfigTagsBeforeExecution(parseAndValidateComponentsMapping(actionParameters.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION)));
- }
// Generate cluster host info
execCmd.setClusterHostInfo(
@@ -631,15 +641,15 @@ public class AmbariCustomCommandExecutionHelper {
Map<String, String> commandParams = new TreeMap<String, String>();
//Propagate HCFS service type info
- Iterator<Service> it = cluster.getServices().values().iterator();
- while(it.hasNext()) {
- ServiceInfo serviceInfoInstance = ambariMetaInfo.getService(stackId.getStackName(),stackId.getStackVersion(), it.next().getName());
- LOG.info("Iterating service type Instance in addServiceCheckAction:: " + serviceInfoInstance.getName());
- if(serviceInfoInstance.getServiceType() != null) {
- LOG.info("Adding service type info in addServiceCheckAction:: " + serviceInfoInstance.getServiceType());
- commandParams.put("dfs_type",serviceInfoInstance.getServiceType());
- break;
- }
+ Map<String, ServiceInfo> serviceInfos = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion());
+ for (ServiceInfo serviceInfoInstance : serviceInfos.values()) {
+ if (serviceInfoInstance.getServiceType() != null) {
+ LOG.debug("Adding {} to command parameters for {}", serviceInfoInstance.getServiceType(),
+ serviceInfoInstance.getName());
+
+ commandParams.put("dfs_type", serviceInfoInstance.getServiceType());
+ break;
+ }
}
String commandTimeout = configs.getDefaultAgentTaskTimeout(false);
@@ -993,10 +1003,6 @@ public class AmbariCustomCommandExecutionHelper {
extraParams.put(componentName, requestParams.get(componentName));
}
- if(requestParams.containsKey(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS)){
- actionExecutionContext.getParameters().put(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS, requestParams.get(KeyNames.REFRESH_ADITIONAL_COMPONENT_TAGS));
- }
-
// If command should be retried upon failure then add the option and also the default duration for retry
if (requestParams.containsKey(KeyNames.COMMAND_RETRY_ENABLED)) {
extraParams.put(KeyNames.COMMAND_RETRY_ENABLED, requestParams.get(KeyNames.COMMAND_RETRY_ENABLED));
@@ -1186,17 +1192,16 @@ public class AmbariCustomCommandExecutionHelper {
clusterHostInfoJson = StageUtils.getGson().toJson(clusterHostInfo);
//Propogate HCFS service type info to command params
- Iterator<Service> it = cluster.getServices().values().iterator();
- while(it.hasNext()) {
- ServiceInfo serviceInfoInstance = ambariMetaInfo.getService(stackId.getStackName(),stackId.getStackVersion(), it.next().getName());
- LOG.info("Iterating service type Instance in getCommandJson:: " + serviceInfoInstance.getName());
- if(serviceInfoInstance.getServiceType() != null) {
- LOG.info("Adding service type info in getCommandJson:: " + serviceInfoInstance.getServiceType());
- commandParamsStage.put("dfs_type",serviceInfoInstance.getServiceType());
- break;
- }
+ Map<String, ServiceInfo> serviceInfos = ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion());
+ for (ServiceInfo serviceInfoInstance : serviceInfos.values()) {
+ if (serviceInfoInstance.getServiceType() != null) {
+ LOG.debug("Adding {} to command parameters for {}", serviceInfoInstance.getServiceType(),
+ serviceInfoInstance.getName());
+
+ commandParamsStage.put("dfs_type", serviceInfoInstance.getServiceType());
+ break;
+ }
}
-
}
String hostParamsStageJson = StageUtils.getGson().toJson(hostParamsStage);
@@ -1229,7 +1234,8 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled());
hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount());
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster);
+ Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
+ Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 066acab..aba45bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2281,15 +2281,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
String packageList = gson.toJson(packages);
hostParams.put(PACKAGE_LIST, packageList);
- Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster);
+ Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
+
+ Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs);
String userList = gson.toJson(userSet);
hostParams.put(USER_LIST, userList);
- Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster);
+ Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs);
String groupList = gson.toJson(groupSet);
hostParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster);
+ Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index af24a69..c390c86 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -17,10 +17,44 @@
*/
package org.apache.ambari.server.controller.internal;
-import com.google.gson.Gson;
-import com.google.inject.assistedinject.Assisted;
-import com.google.inject.assistedinject.AssistedInject;
-import com.google.inject.persist.Transactional;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
@@ -54,43 +88,10 @@ import org.apache.ambari.server.utils.StageUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_COUNT;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AGENT_STACK_RETRY_ON_UNAVAILABILITY;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_HOME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JCE_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_VERSION;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
+import com.google.gson.Gson;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+import com.google.inject.persist.Transactional;
/**
* Resource provider for client config resources.
@@ -194,7 +195,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
getComponent(stackId.getStackName(), stackId.getStackVersion(), serviceName, componentName);
packageFolder = managementController.getAmbariMetaInfo().
getService(stackId.getStackName(), stackId.getStackVersion(), serviceName).getServicePackageFolder();
-
+
String commandScript = componentInfo.getCommandScript().getScript();
List<ClientConfigFileDefinition> clientConfigFiles = componentInfo.getClientConfigFiles();
@@ -204,7 +205,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
String resourceDirPath = configs.getResourceDirPath();
String packageFolderAbsolute = resourceDirPath + File.separator + packageFolder;
-
+
String commandScriptAbsolute = packageFolderAbsolute + File.separator + commandScript;
@@ -336,15 +337,15 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
String packageList = gson.toJson(packages);
hostLevelParams.put(PACKAGE_LIST, packageList);
- Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster);
+ Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs);
String userList = gson.toJson(userSet);
hostLevelParams.put(USER_LIST, userList);
- Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster);
+ Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs);
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster);
+ Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId,PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -484,13 +485,14 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
commandLineThread.join(timeout);
logStreamThread.join(timeout);
Integer returnCode = commandLineThread.getReturnCode();
- if (returnCode == null)
+ if (returnCode == null) {
throw new TimeoutException();
- else if (returnCode != 0)
+ } else if (returnCode != 0) {
throw new ExecutionException(String.format("Execution of \"%s\" returned %d.", commandLine, returnCode),
new Throwable(logStream.getOutput()));
- else
+ } else {
return commandLineThread.returnCode;
+ }
} catch (InterruptedException ex) {
commandLineThread.interrupt();
Thread.currentThread().interrupt();
@@ -505,7 +507,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
private Integer returnCode;
public void setReturnCode(Integer exit) {
- this.returnCode = exit;
+ returnCode = exit;
}
public Integer getReturnCode() {
@@ -517,6 +519,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
}
+ @Override
public void run() {
try {
setReturnCode(process.waitFor());
@@ -533,14 +536,15 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
private StringBuilder output;
public LogStreamReader(InputStream is) {
- this.reader = new BufferedReader(new InputStreamReader(is));
- this.output = new StringBuilder("");
+ reader = new BufferedReader(new InputStreamReader(is));
+ output = new StringBuilder("");
}
public String getOutput() {
- return this.output.toString();
+ return output.toString();
}
+ @Override
public void run() {
try {
String line = reader.readLine();
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index c390f44..55e9dbd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1703,7 +1703,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
*/
private Map<String, String> getNewParameterMap(RequestStageContainer requestStageContainer) {
Map<String, String> parameters = new HashMap<String, String>();
- parameters.put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, "*");
+ parameters.put(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION, "true");
parameters.put(COMMAND_PARAM_REQUEST_ID, String.valueOf(requestStageContainer.getId()));
return parameters;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 216d850..16edc74 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -35,8 +35,6 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ambari.annotations.TransactionalLock;
-import org.apache.ambari.annotations.TransactionalLock.LockArea;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
@@ -590,9 +588,9 @@ public class ConfigHelper {
return result;
}
- public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType, Cluster cluster) throws AmbariException {
+ public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType,
+ Cluster cluster, Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
- Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
Map<String, Config> actualConfigs = new HashMap<>();
Set<String> result = new HashSet<String>();
@@ -1417,7 +1415,7 @@ public class ConfigHelper {
public void run() {
staleConfigCacheLock.writeLock().lock();
try {
- ConfigHelper.this.staleConfigCacheDesiredConfigs = m_desiredConfigs;
+ staleConfigCacheDesiredConfigs = m_desiredConfigs;
if (null == m_keysToInvalidate || m_keysToInvalidate.isEmpty()) {
staleConfigsCache.invalidateAll();
} else {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 8dcd163..6cefa79 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -71,6 +71,7 @@ import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
+import com.google.inject.persist.Transactional;
/**
* Class to assist with upgrading a cluster.
@@ -749,12 +750,21 @@ public class UpgradeHelper {
}
/**
- * Transitions all affected components to upgrading state. Transition is performed
- * only for components that advertise their version. Service component desired
- * version is set to one passed as an argument
- * @param version desired version (like 2.2.1.0-1234) for upgrade
- * @param targetServices targets for upgrade
+ * Transitions all affected components to {@link UpgradeState#IN_PROGRESS}.
+ * Transition is performed only for components that advertise their version.
+ * Additionally sets the service component desired version to the specified
+ * argument.
+ * <p/>
+ * Because this iterates over all of the components on every host and updates
+ * the upgrade state individually, we wrap this method inside of a transaction
+ * to prevent 1000's of transactions from being opened and committed.
+ *
+ * @param version
+ * desired version (like 2.2.1.0-1234) for upgrade
+ * @param targetServices
+ * targets for upgrade
*/
+ @Transactional
public void putComponentsToUpgradingState(String version,
Map<Service, Set<ServiceComponent>> targetServices) throws AmbariException {
// TODO: generalize method?
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index 7a5f377..e82900d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -148,7 +148,7 @@ public class AmbariCustomCommandExecutionHelperTest {
ExecuteActionRequest actionRequest = new ExecuteActionRequest("c1", "REFRESHQUEUES",
new HashMap<String, String>() {
{
- put("forceRefreshConfigTags", "capacity-scheduler");
+ put("forceRefreshConfigTagsBeforeExecution", "true");
}
}, false);
actionRequest.getResourceFilters().add(new RequestResourceFilter("YARN", "RESOURCEMANAGER", Collections.singletonList("c1-c6401")));
@@ -169,11 +169,7 @@ public class AmbariCustomCommandExecutionHelperTest {
Assert.assertEquals(1, commands.size());
ExecutionCommand command = commands.get(0).getExecutionCommand();
-
- Assert.assertNotNull(command.getForceRefreshConfigTags());
- Assert.assertEquals(1, command.getForceRefreshConfigTags().size());
- Assert.assertEquals("capacity-scheduler",
- command.getForceRefreshConfigTags().iterator().next());
+ Assert.assertEquals(true, command.getForceRefreshConfigTagsBeforeExecution());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
index 37a2e63..e54a117 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
@@ -18,8 +18,6 @@
package org.apache.ambari.server.controller;
-import javax.persistence.RollbackException;
-import junit.framework.Assert;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOST_SYS_PREPPED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JAVA_VERSION;
@@ -59,6 +57,8 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import javax.persistence.RollbackException;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
import org.apache.ambari.server.HostNotFoundException;
@@ -89,6 +89,7 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.ConfigImpl;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.MaintenanceState;
import org.apache.ambari.server.state.PropertyInfo;
@@ -119,6 +120,8 @@ import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.util.Modules;
+import junit.framework.Assert;
+
/**
* AmbariManagementControllerImpl unit tests
*/
@@ -1992,8 +1995,11 @@ public class AmbariManagementControllerImplTest {
RepositoryVersionEntity repositoryVersionEntity = createNiceMock(RepositoryVersionEntity.class);
ConfigHelper configHelper = createNiceMock(ConfigHelper.class);
+ Map<String, DesiredConfig> desiredConfigs = new HashMap<>();
+
expect(cluster.getClusterName()).andReturn(clusterName);
expect(cluster.getDesiredStackVersion()).andReturn(stackId);
+ expect(cluster.getDesiredConfigs()).andReturn(desiredConfigs);
expect(stackId.getStackName()).andReturn(SOME_STACK_NAME).anyTimes();
expect(stackId.getStackVersion()).andReturn(SOME_STACK_VERSION).anyTimes();
expect(configuration.getMySQLJarName()).andReturn(MYSQL_JAR);
@@ -2009,8 +2015,9 @@ public class AmbariManagementControllerImplTest {
expect(clusterVersionDAO.findByClusterAndStateCurrent(clusterName)).andReturn(clusterVersionEntity).anyTimes();
expect(clusterVersionEntity.getRepositoryVersion()).andReturn(repositoryVersionEntity).anyTimes();
expect(repositoryVersionEntity.getVersion()).andReturn("1234").anyTimes();
- expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH,
- cluster)).andReturn(notManagedHdfsPathSet);
+ expect(configHelper.getPropertyValuesWithPropertyType(stackId,
+ PropertyInfo.PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs)).andReturn(
+ notManagedHdfsPathSet);
replay(manager, clusters, cluster, injector, stackId, configuration, clusterVersionDAO, clusterVersionEntity,
repositoryVersionEntity, configHelper);
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
index 9603226..f0bddf8 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java
@@ -18,15 +18,60 @@
package org.apache.ambari.server.controller.internal;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertFalse;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.controller.*;
-import org.apache.ambari.server.controller.spi.*;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.RequestStatusResponse;
+import org.apache.ambari.server.controller.ServiceComponentHostRequest;
+import org.apache.ambari.server.controller.ServiceComponentHostResponse;
+import org.apache.ambari.server.controller.spi.Predicate;
+import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.controller.utilities.PredicateBuilder;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
-import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.stack.StackManager;
+import org.apache.ambari.server.state.ClientConfigFileDefinition;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.CommandScriptDefinition;
+import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.DesiredConfig;
+import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.ServiceOsSpecific;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.utils.StageUtils;
+import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -34,17 +79,6 @@ import org.powermock.api.easymock.PowerMock;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.util.*;
-
-import org.apache.ambari.server.stack.StackManager;
-
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertFalse;
-
/**
* TaskResourceProvider tests.
*/
@@ -224,7 +258,7 @@ public class ClientConfigResourceProviderTest {
Map<String, String> returnConfigMap = new HashMap<String, String>();
returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT);
returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
-
+
// set expectations
expect(managementController.getConfigHelper()).andReturn(configHelper);
expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
@@ -291,7 +325,7 @@ public class ClientConfigResourceProviderTest {
expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes();
Set<String> userSet = new HashSet<String>();
userSet.add("hdfs");
- expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet);
+ expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet);
PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes();
PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject());
expect(mockFile.getParent()).andReturn("");
@@ -422,11 +456,11 @@ public class ClientConfigResourceProviderTest {
Set<ServiceComponentHostResponse> responses = new LinkedHashSet<ServiceComponentHostResponse>();
responses.add(shr1);
-
+
Map<String, String> returnConfigMap = new HashMap<String, String>();
returnConfigMap.put(Configuration.SERVER_TMP_DIR_KEY, Configuration.SERVER_TMP_DIR_DEFAULT);
returnConfigMap.put(Configuration.AMBARI_PYTHON_WRAP_KEY, Configuration.AMBARI_PYTHON_WRAP_DEFAULT);
-
+
// set expectations
expect(managementController.getConfigHelper()).andReturn(configHelper);
expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
@@ -494,7 +528,7 @@ public class ClientConfigResourceProviderTest {
expect(serviceInfo.getOsSpecifics()).andReturn(new HashMap<String, ServiceOsSpecific>()).anyTimes();
Set<String> userSet = new HashSet<String>();
userSet.add("hdfs");
- expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster)).andReturn(userSet);
+ expect(configHelper.getPropertyValuesWithPropertyType(stackId, PropertyInfo.PropertyType.USER, cluster, desiredConfigMap)).andReturn(userSet);
PowerMock.expectNew(File.class, new Class<?>[]{String.class}, anyObject(String.class)).andReturn(mockFile).anyTimes();
PowerMock.createNiceMockAndExpectNew(PrintWriter.class, anyObject());
expect(mockFile.getParent()).andReturn("");
@@ -532,9 +566,9 @@ public class ClientConfigResourceProviderTest {
runtime, process);
PowerMock.verifyAll();
}
-
-
-
+
+
+
@Test
public void testDeleteResources() throws Exception {
Resource.Type type = Resource.Type.ClientConfig;
http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac13b5c/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java
index 115b518..139702c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderHDP22Test.java
@@ -18,9 +18,9 @@
package org.apache.ambari.server.controller.internal;
import static org.easymock.EasyMock.anyLong;
+import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -66,7 +66,6 @@ import org.apache.ambari.server.security.authorization.RoleAuthorization;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.ConfigImpl;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostState;
@@ -78,24 +77,21 @@ import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.topology.TopologyManager;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.ambari.server.view.ViewRegistry;
-import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
-import com.google.gson.Gson;
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.persist.PersistService;
-import com.google.inject.util.Modules;
import org.junit.runner.RunWith;
import org.powermock.api.easymock.PowerMock;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
+import com.google.common.collect.ImmutableMap;
+import com.google.gson.Gson;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
+
/**
* UpgradeResourceDefinition tests.
*/
@@ -110,81 +106,22 @@ public class UpgradeResourceProviderHDP22Test {
private Clusters clusters;
private OrmTestHelper helper;
private AmbariManagementController amc;
- private ConfigHelper configHelper;
private StackDAO stackDAO;
private TopologyManager topologyManager;
private static final String configTagVersion1 = "version1";
private static final String configTagVersion2 = "version2";
- @SuppressWarnings("serial")
- private static final Map<String, String> configTagVersion1Properties = new HashMap<String, String>() {
- {
- put("hive.server2.thrift.port", "10000");
- }
- };
- private static final Map<String, String> configTagVersion2Properties = new HashMap<String, String>() {
- {
- put("hive.server2.thrift.port", "10010");
- }
- };
+ private static final Map<String, String> configTagVersion1Properties = new ImmutableMap.Builder<String, String>().put(
+ "hive.server2.thrift.port", "10000").build();
+
+ private static final Map<String, String> configTagVersion2Properties = new ImmutableMap.Builder<String, String>().put(
+ "hive.server2.thrift.port", "10010").build();
- @SuppressWarnings({ "serial", "unchecked" })
@Before
public void before() throws Exception {
- // setup the config helper for placeholder resolution
- configHelper = EasyMock.createNiceMock(ConfigHelper.class);
-
- expect(configHelper.getPlaceholderValueFromDesiredConfigurations(EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn(
- "placeholder-rendered-properly").anyTimes();
-
- expect(configHelper.getDefaultProperties(EasyMock.anyObject(StackId.class), EasyMock.anyObject(Cluster.class))).andReturn(
- new HashMap<String, Map<String, String>>()).anyTimes();
-
- expect(configHelper.getEffectiveConfigAttributes(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn(
- new HashMap<String, Map<String, Map<String, String>>>()).anyTimes();
-
- expect(configHelper.getEffectiveDesiredTags(EasyMock.anyObject(Cluster.class), EasyMock.eq("h1"))).andReturn(new HashMap<String, Map<String, String>>() {
- {
- put("hive-site", new HashMap<String, String>() {
- {
- put("tag", configTagVersion1);
- }
- });
- }
- }).times(3);
-
- expect(configHelper.getEffectiveDesiredTags(EasyMock.anyObject(Cluster.class), EasyMock.eq("h1"))).andReturn(new HashMap<String, Map<String, String>>() {
- {
- put("hive-site", new HashMap<String, String>() {
- {
- put("tag", configTagVersion2);
- }
- });
- }
- }).times(2);
-
- expect(configHelper.getEffectiveConfigProperties(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn(
- new HashMap<String, Map<String, String>>() {
- {
- put("hive-site", configTagVersion1Properties);
- }
- }).times(1);
-
- expect(configHelper.getEffectiveConfigProperties(EasyMock.anyObject(Cluster.class), EasyMock.anyObject(Map.class))).andReturn(
- new HashMap<String, Map<String, String>>() {
- {
- put("hive-site", configTagVersion2Properties);
- }
- }).times(2);
-
- expect(configHelper.getMergedConfig(EasyMock.anyObject(Map.class),
- EasyMock.anyObject(Map.class))).andReturn(new HashMap<String, String>()).anyTimes();
-
- EasyMock.replay(configHelper);
-
// create an injector which will inject the mocks
- injector = Guice.createInjector(Modules.override(new InMemoryDefaultTestModule()).with(new MockModule()));
+ injector = Guice.createInjector(new InMemoryDefaultTestModule());
injector.getInstance(GuiceJpaInitializer.class);
@@ -272,10 +209,6 @@ public class UpgradeResourceProviderHDP22Test {
injector = null;
}
- private void setupConfigHelper() {
-
- }
-
/**
* Tests upgrades from HDP-2.2.x to HDP-2.2.y
*
@@ -374,8 +307,8 @@ public class UpgradeResourceProviderHDP22Test {
String executionCommandJson = new String(ece.getCommand());
Map<String, Object> commandMap = gson.<Map<String, Object>> fromJson(executionCommandJson, Map.class);
- // ensure that the latest tag is being used and that "*" is forcing a
- // refresh - this is absolutely required for upgrades
+ // ensure that the latest tag is being used - this is absolutely required
+ // for upgrades
Set<String> roleCommandsThatMustHaveRefresh = new HashSet<String>();
roleCommandsThatMustHaveRefresh.add("SERVICE_CHECK");
roleCommandsThatMustHaveRefresh.add("RESTART");
@@ -385,15 +318,10 @@ public class UpgradeResourceProviderHDP22Test {
if (roleCommandsThatMustHaveRefresh.contains(roleCommand)) {
assertTrue(commandMap.containsKey(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION));
Object object = commandMap.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION);
- assertTrue(object instanceof List);
-
- @SuppressWarnings("unchecked")
- List<String> tags = (List<String>) commandMap.get(KeyNames.REFRESH_CONFIG_TAGS_BEFORE_EXECUTION);
- assertEquals(1, tags.size());
- assertEquals("*", tags.get(0));
+ assertTrue(Boolean.valueOf(object.toString()));
ExecutionCommandWrapperFactory ecwFactory = injector.getInstance(ExecutionCommandWrapperFactory.class);
- ExecutionCommandWrapper executionCommandWrapper = ecwFactory.createFromJson(executionCommandJson);
+ ExecutionCommandWrapper executionCommandWrapper = ecwFactory.createFromJson(executionCommandJson);
ExecutionCommand executionCommand = executionCommandWrapper.getExecutionCommand();
Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags();
assertEquals(configTagVersion2, configurationTags.get("hive-site").get("tag"));
@@ -410,17 +338,4 @@ public class UpgradeResourceProviderHDP22Test {
private UpgradeResourceProvider createProvider(AmbariManagementController amc) {
return new UpgradeResourceProvider(amc);
}
-
- /**
- *
- */
- private class MockModule implements Module {
- /**
- *
- */
- @Override
- public void configure(Binder binder) {
- binder.bind(ConfigHelper.class).toInstance(configHelper);
- }
- }
}