You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by be...@apache.org on 2018/06/29 08:23:12 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-24162]
Support for cluster-settings (remove cluster-env) (benyoka) (#1627)
This is an automated email from the ASF dual-hosted git repository.
benyoka pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 48ddb09 [AMBARI-24162] Support for cluster-settings (remove cluster-env) (benyoka) (#1627)
48ddb09 is described below
commit 48ddb0901e2b12ee881698acf8a7b3de5c02799c
Author: benyoka <be...@users.noreply.github.com>
AuthorDate: Fri Jun 29 10:23:08 2018 +0200
[AMBARI-24162] Support for cluster-settings (remove cluster-env) (benyoka) (#1627)
* AMBARI-24162 copy cluster settings from cluster-env + setting fixes (benyoka)
* AMBARI-24162 fixint tests WIP (benyoka)
* AMBARI-24162 fix unit tests #2 (benyoka)
* AMBARI-24162 write new unit tests (benyoka)
* AMBARI-24162 review commits and unit test fixes (benyoka)
* AMBARI-24162 fix blueprint export and blueprint unit tests (benyoka)
---
.../controller/AmbariManagementControllerImpl.java | 10 +-
.../internal/BlueprintConfigurationProcessor.java | 48 --------
.../internal/BlueprintResourceProvider.java | 32 +++---
.../apache/ambari/server/state/ConfigHelper.java | 7 +-
.../ambari/server/topology/AmbariContext.java | 44 ++++---
.../ambari/server/topology/BlueprintFactory.java | 8 +-
.../ambari/server/topology/BlueprintImpl.java | 2 +-
.../server/topology/ClusterTopologyImpl.java | 52 +++++++++
.../org/apache/ambari/server/topology/Setting.java | 20 ++++
.../ambari/server/topology/SettingFactory.java | 7 +-
.../topology/tasks/ConfigureClusterTask.java | 1 +
.../validators/StackConfigTypeValidator.java | 7 ++
.../src/main/resources/cluster-settings.xml | 21 ++++
.../BlueprintConfigurationProcessorTest.java | 101 +++-------------
.../ambari/server/topology/AmbariContextTest.java | 128 ++++++++++++++-------
.../topology/ClusterConfigurationRequestTest.java | 18 ++-
.../topology/ClusterDeployWithStartOnlyTest.java | 55 +++++----
.../topology/ClusterDeploymentTestCommon.java | 38 ++++++
...terInstallWithoutStartOnComponentLevelTest.java | 48 +++++---
.../topology/ClusterInstallWithoutStartTest.java | 51 +++++---
.../server/topology/ClusterTopologyImplTest.java | 50 +++++++-
.../ambari/server/topology/SettingFactoryTest.java | 27 +++++
.../apache/ambari/server/topology/SettingTest.java | 35 ++++++
.../server/topology/TopologyManagerTest.java | 9 +-
24 files changed, 524 insertions(+), 295 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index aa92bac..31dea50 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2588,15 +2588,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
(commandParams.get(CLUSTER_PHASE_PROPERTY).equals(CLUSTER_PHASE_INITIAL_INSTALL) ||
commandParams.get(CLUSTER_PHASE_PROPERTY).equals(CLUSTER_PHASE_INITIAL_START))) {
String retryEnabledStr =
- configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.CLUSTER_ENV,
- ConfigHelper.CLUSTER_ENV_RETRY_ENABLED);
+ cluster.getClusterSetting(ConfigHelper.COMMAND_RETRY_ENABLED).getClusterSettingValue();
String commandsStr =
- configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.CLUSTER_ENV,
- ConfigHelper.CLUSTER_ENV_RETRY_COMMANDS);
+ cluster.getClusterSetting(ConfigHelper.COMMANDS_TO_RETRY).getClusterSettingValue();
String retryMaxTimeStr =
- configHelper.getValueFromDesiredConfigurations(cluster,
- ConfigHelper.CLUSTER_ENV,
- ConfigHelper.CLUSTER_ENV_RETRY_MAX_TIME_IN_SEC);
+ cluster.getClusterSetting(ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC).getClusterSettingValue();
if (StringUtils.isNotEmpty(retryEnabledStr)) {
retryEnabled = Boolean.TRUE.toString().equals(retryEnabledStr);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 8275cfc..5c45712 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -74,18 +74,6 @@ public class BlueprintConfigurationProcessor {
*/
public static final Pattern HOST_GROUP_PLACEHOLDER_PATTERN = Pattern.compile("%HOSTGROUP::(\\S+?)%");
- private final static String COMMAND_RETRY_ENABLED_PROPERTY_NAME = "command_retry_enabled";
-
- private final static String COMMANDS_TO_RETRY_PROPERTY_NAME = "commands_to_retry";
-
- private final static String COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME = "command_retry_max_time_in_sec";
-
- private final static String COMMAND_RETRY_ENABLED_DEFAULT = "true";
-
- private final static String COMMANDS_TO_RETRY_DEFAULT = "INSTALL,START";
-
- private final static String COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT = "600";
-
private final static String CLUSTER_ENV_CONFIG_TYPE_NAME = "cluster-env";
private final static String HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES = "hbase.coprocessor.master.classes";
@@ -449,7 +437,6 @@ public class BlueprintConfigurationProcessor {
// Explicitly set any properties that are required but not currently provided in the stack definition.
setStackToolsAndFeatures(clusterConfig, configTypesUpdated);
- setRetryConfiguration(clusterConfig, configTypesUpdated);
setupHDFSProxyUsers(clusterConfig, configTypesUpdated);
addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getStack());
@@ -2956,41 +2943,6 @@ public class BlueprintConfigurationProcessor {
}
/**
- * This method ensures that Ambari command retry is enabled if not explicitly overridden in
- * cluster-env by the Blueprint or Cluster Creation template. The new dynamic provisioning model
- * requires that retry be enabled for most multi-node clusters, to this method sets reasonable defaults
- * in order to preserve backwards compatibility and to simplify Blueprint creation.
- *
- * If the retry-specific properties in cluster-env are not set, then the config processor
- * will set these values to defaults in cluster-env.
- *
- * @param configuration cluster configuration
- */
- private static void setRetryConfiguration(Configuration configuration, Set<String> configTypesUpdated) {
- boolean wasUpdated = false;
-
- if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_ENABLED_PROPERTY_NAME) == null) {
- configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_ENABLED_PROPERTY_NAME, COMMAND_RETRY_ENABLED_DEFAULT);
- wasUpdated = true;
- }
-
- if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMANDS_TO_RETRY_PROPERTY_NAME) == null) {
- configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMANDS_TO_RETRY_PROPERTY_NAME, COMMANDS_TO_RETRY_DEFAULT);
- wasUpdated = true;
- }
-
- if (configuration.getPropertyValue(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME) == null) {
- configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_PROPERTY_NAME, COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT);
- wasUpdated = true;
- }
-
- if (wasUpdated) {
- configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME);
- }
- }
-
-
- /**
* Sets the read-only properties for stack features & tools, overriding
* anything provided in the blueprint.
*
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 17aed41..22f367c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -393,23 +393,29 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
if(config instanceof BlueprintConfigEntity) {
Map<String, String> properties = JsonUtils.fromJson(config.getConfigData(), new TypeReference<Map<String, String>>(){});
- // TODO: use multiple mpacks
- BlueprintMpackInstanceEntity mpack =
- ((BlueprintConfigEntity)config).getBlueprintEntity().getMpackInstances().iterator().next();
- StackInfo metaInfoStack;
-
- try {
- metaInfoStack = ambariMetaInfo.getStack(mpack.getMpackName(), mpack.getMpackVersion());
- } catch (AmbariException e) {
- throw new NoSuchResourceException(e.getMessage());
+ // TODO: use multiple mpacks, + make it work with blueprints with no mpack
+ Collection<BlueprintMpackInstanceEntity> mpackInstances = ((BlueprintConfigEntity) config).getBlueprintEntity().getMpackInstances();
+ if (mpackInstances.isEmpty()) {
+ LOG.warn("This blueprint does not specify any mpacks/stacks. Configurations cannot be retrieved.");
}
+ else {
+ BlueprintMpackInstanceEntity mpack =
+ ((BlueprintConfigEntity)config).getBlueprintEntity().getMpackInstances().iterator().next();
+ StackInfo metaInfoStack;
+
+ try {
+ metaInfoStack = ambariMetaInfo.getStack(mpack.getMpackName(), mpack.getMpackVersion());
+ } catch (AmbariException e) {
+ throw new NoSuchResourceException(e.getMessage());
+ }
- Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes =
- metaInfoStack.getConfigPropertiesTypes(type);
+ Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes =
+ metaInfoStack.getConfigPropertiesTypes(type);
- SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1L);
+ SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1L);
- configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+ configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+ }
} else {
Map<String, Object> properties = JsonUtils.fromJson(config.getConfigData(), new TypeReference<Map<String, Object>>(){});
configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index f918cdb..e15a87a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -105,15 +105,14 @@ public class ConfigHelper {
*/
public static final String HBASE_SITE = "hbase-site";
public static final String HDFS_SITE = "hdfs-site";
- public static final String HIVE_SITE = "hive-site";
public static final String YARN_SITE = "yarn-site";
public static final String CLUSTER_ENV = "cluster-env";
public static final String CLUSTER_SETTINGS = "cluster-settings";
public static final String CLUSTER_ENV_ALERT_REPEAT_TOLERANCE = "alerts_repeat_tolerance";
- public static final String CLUSTER_ENV_RETRY_ENABLED = "command_retry_enabled";
+ public static final String COMMAND_RETRY_ENABLED = "command_retry_enabled";
public static final String SERVICE_CHECK_TYPE = "service_check_type";
- public static final String CLUSTER_ENV_RETRY_COMMANDS = "commands_to_retry";
- public static final String CLUSTER_ENV_RETRY_MAX_TIME_IN_SEC = "command_retry_max_time_in_sec";
+ public static final String COMMANDS_TO_RETRY = "commands_to_retry";
+ public static final String COMMAND_RETRY_MAX_TIME_IN_SEC = "command_retry_max_time_in_sec";
public static final String COMMAND_RETRY_MAX_TIME_IN_SEC_DEFAULT = "600";
public static final String CLUSTER_ENV_STACK_NAME_PROPERTY = "stack_name";
public static final String CLUSTER_ENV_STACK_FEATURES_PROPERTY = "stack_features";
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 9071920..021449d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -18,13 +18,13 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
@@ -50,7 +50,6 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.ClusterRequest;
import org.apache.ambari.server.controller.ConfigGroupRequest;
-import org.apache.ambari.server.controller.ConfigurationRequest;
import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.controller.ServiceComponentHostRequest;
@@ -59,7 +58,6 @@ import org.apache.ambari.server.controller.ServiceGroupRequest;
import org.apache.ambari.server.controller.ServiceGroupResponse;
import org.apache.ambari.server.controller.ServiceRequest;
import org.apache.ambari.server.controller.ServiceResponse;
-import org.apache.ambari.server.controller.internal.AbstractResourceProvider;
import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
import org.apache.ambari.server.controller.internal.ConfigGroupResourceProvider;
import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
@@ -90,6 +88,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
@@ -222,21 +221,21 @@ public class AmbariContext {
public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) {
Set<StackId> stackIds = topology.getStackIds();
- createAmbariClusterResource(clusterName, stackIds, securityType);
+ createAmbariClusterResource(clusterName, stackIds, securityType, topology.getSetting().getClusterSettings());
createAmbariServiceAndComponentResources(topology, clusterName);
}
- private void createAmbariClusterResource(String clusterName, Set<StackId> stackIds, SecurityType securityType) {
+ private void createAmbariClusterResource(String clusterName, Set<StackId> stackIds, SecurityType securityType,
+ Map<String, String> clusterSettings) {
String stackInfo = stackIds.iterator().next().toString(); // temporary
final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType, stackInfo, null);
-
try {
RetryHelper.executeWithRetry(() -> {
getController().createCluster(clusterRequest);
return null;
});
- addDefaultClusterSettings(clusterName);
+ addClusterSettings(clusterName, clusterSettings);
} catch (AmbariException e) {
LOG.error("Failed to create Cluster resource: ", e);
if (e.getCause() instanceof DuplicateResourceException) {
@@ -248,10 +247,17 @@ public class AmbariContext {
}
// FIXME temporarily add default cluster settings -- should be provided by ClusterImpl itself
- private void addDefaultClusterSettings(String clusterName) throws AmbariException {
+ private void addClusterSettings(String clusterName, Map<String, String> clusterSettings) throws AmbariException {
Cluster cluster = getController().getClusters().getCluster(clusterName);
- for (PropertyInfo p : getController().getAmbariMetaInfo().getClusterProperties()) {
- cluster.addClusterSetting(p.getName(), p.getValue());
+ Map<String, String> defaultClusterSettings =
+ getController().getAmbariMetaInfo().getClusterProperties().stream()
+ .collect(toMap(PropertyInfo::getName, PropertyInfo::getValue));
+
+ // Override default settings with those coming from blueprint / cluster template
+ defaultClusterSettings.putAll(clusterSettings);
+
+ for (Map.Entry<String, String> setting : defaultClusterSettings.entrySet()) {
+ cluster.addClusterSetting(setting.getKey(), setting.getValue());
}
}
@@ -268,8 +274,16 @@ public class AmbariContext {
.collect(toSet());
Set<ServiceComponentRequest> componentRequests = topology.getComponents()
- .map(c -> new ServiceComponentRequest(clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.componentName(), c.componentName(),
- topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName()))) // FIXME settings by service type or name?
+ .map( c ->
+ new ServiceComponentRequest(
+ clusterName,
+ c.effectiveServiceGroupName(),
+ c.effectiveServiceName(),
+ c.componentName(),
+ c.componentName(),
+ State.INIT.name(),
+ topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName()))
+ ) // FIXME settings by service type or name?
.collect(toSet());
try {
@@ -399,7 +413,7 @@ public class AmbariContext {
}
public void downloadMissingMpacks(Set<MpackInstance> mpacks) {
- ResourceProvider resourceProvider = getClusterController().ensureResourceProvider(Resource.Type.Mpack);
+ ResourceProvider resourceProvider = getClusterController().ensureResourceProvider(Resource.Type.Mpack);
new DownloadMpacksTask(resourceProvider, ambariMetaInfo.get()).downloadMissingMpacks(mpacks);
}
@@ -472,10 +486,6 @@ public class AmbariContext {
}
}
- //todo: non topology type shouldn't be returned
- public List<ConfigurationRequest> createConfigurationRequests(Map<String, Object> clusterProperties) {
- return AbstractResourceProvider.getConfigurationRequests("Clusters", clusterProperties);
- }
public void setConfigurationOnCluster(final ClusterRequest clusterRequest) {
try {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
index 0f13539..9fc08b7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology;
+import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.toSet;
import static org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BLUEPRINT_NAME_PROPERTY_ID;
import static org.apache.ambari.server.controller.internal.BlueprintResourceProvider.COMPONENT_MPACK_INSTANCE_PROPERTY;
@@ -57,6 +58,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableSet;
/**
* Create a Blueprint instance.
@@ -115,7 +117,11 @@ public class BlueprintFactory {
Collection<HostGroup> hostGroups = processHostGroups(properties);
Configuration configuration = configFactory.getConfiguration((Collection<Map<String, String>>)
properties.get(CONFIGURATION_PROPERTY_ID));
- Setting setting = SettingFactory.getSetting((Collection<Map<String, Object>>) properties.get(SETTING_PROPERTY_ID));
+
+ Map<String, Object> settingProperties = properties.entrySet().stream()
+ .filter(entry -> SETTING_PROPERTY_ID.equals(entry.getKey()) || entry.getKey().startsWith(SETTING_PROPERTY_ID + "/"))
+ .collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
+ Setting setting = SettingFactory.getSetting(ImmutableSet.of(settingProperties));
return new BlueprintImpl(name, hostGroups, stackIds, mpackInstances, configuration, securityConfiguration, setting);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
index 7e60cb9..676077a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java
@@ -91,7 +91,7 @@ public class BlueprintImpl implements Blueprint {
hostGroups.put(hostGroup.getName(), hostGroup);
}
this.configuration = configuration;
- this.setting = setting != null ? setting : new Setting(ImmutableMap.of());
+ this.setting = setting != null ? setting : new Setting(new HashMap<>());
repoSettings = processRepoSettings();
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
index 428571b..8e2a044 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java
@@ -43,6 +43,7 @@ import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.controller.internal.StackDefinition;
import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.StackId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -98,6 +99,56 @@ public class ClusterTopologyImpl implements ClusterTopology {
checkForDuplicateHosts(topologyRequest.getHostGroupInfo());
registerHostGroupInfo(topologyRequest.getHostGroupInfo());
+ adjustTopology();
+ }
+
+ /**
+ * This is to collect configurations formerly (in Ambari 2.x) belonging to cluster-env and already migrated to
+ * cluster settings. Eventually all configurations from cluster-env should be migrated and this collection
+ * should be removed.
+ */
+ private static final Set<String> SAFE_TO_REMOVE_FROM_CLUSTER_ENV = ImmutableSet.of(
+ ConfigHelper.COMMAND_RETRY_ENABLED,
+ ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC,
+ ConfigHelper.COMMANDS_TO_RETRY
+ );
+
+ /**
+ * This method adjusts cluster topologies coming from the Ambari 2.x blueprint structure for Ambari
+ * 3.x.
+ * Currently it extract configuration from cluster-env and transforms it into cluster settings.
+ */
+ private void adjustTopology() {
+ Set<PropertyInfo> clusterProperties = ambariContext.getController().getAmbariMetaInfo().getClusterProperties();
+ Set<String> clusterSettingPropertyNames = clusterProperties.stream().map(PropertyInfo::getName).collect(toSet());
+ Map<String, String> clusterEnv =
+ configuration.getFullProperties().getOrDefault(ConfigHelper.CLUSTER_ENV, ImmutableMap.of());
+
+ Set<String> propertiesToConvert = Sets.intersection(clusterEnv.keySet(), clusterSettingPropertyNames);
+ Set<String> remainingProperties = Sets.difference(clusterEnv.keySet(), clusterSettingPropertyNames);
+ LOG.info("Will convert {} properties from cluster-env to cluster settings, leave {} as is. Properties to convert: {}," +
+ " remaining properties: {}", propertiesToConvert.size(), remainingProperties.size(), propertiesToConvert,
+ remainingProperties);
+
+ // Get cluster_settings from setting or create if not exists
+ Map<String, String> clusterSettings = setting
+ .getProperties()
+ .computeIfAbsent( Setting.SETTING_NAME_CLUSTER_SETTINGS, __ -> {
+ Set<Map<String, String>> set = new HashSet<>();
+ set.add(new HashMap<>());
+ return set;
+ })
+ .iterator()
+ .next();
+
+ // convert cluster-env to cluster settings
+ propertiesToConvert.forEach( prop -> clusterSettings.put(prop, clusterEnv.get(prop)));
+
+ // Ideally, all converted properties should be removed from cluster-env. Since legacy Ambari code sometimes
+ // still uses cluster-env only remove properties that are known to have been migrated.
+ Sets.intersection(propertiesToConvert, SAFE_TO_REMOVE_FROM_CLUSTER_ENV).forEach(
+ prop -> configuration.removeProperty(ConfigHelper.CLUSTER_ENV, prop)
+ );
}
public ClusterTopologyImpl(
@@ -124,6 +175,7 @@ public class ClusterTopologyImpl implements ClusterTopology {
checkForDuplicateHosts(request.getHostGroupInfo());
registerHostGroupInfo(request.getHostGroupInfo());
+ adjustTopology();
}
public ClusterTopologyImpl withAdditionalComponents(Map<String, Set<ResolvedComponent>> additionalComponents) throws InvalidTopologyException {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java
index cf13473..1973d83 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java
@@ -18,7 +18,9 @@
package org.apache.ambari.server.topology;
+import static java.util.Collections.emptySet;
import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
import java.util.List;
import java.util.Map;
@@ -35,6 +37,7 @@ public class Setting {
*/
private final Map<String, Set<Map<String, String>>> properties;
+ static final String SETTING_NAME_CLUSTER_SETTINGS = "cluster_settings";
static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings";
static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings";
static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings";
@@ -160,6 +163,23 @@ public class Setting {
.collect(toList());
}
+
+ /**
+ * Extracts and returns the cluster settings.
+ * Currently the settings under the categories "cluster_settings" and "recovery_settings".
+ * Settings are flattened from sets of maps to a simple map.
+ * @return cluster settings
+ */
+ Map<String, String> getClusterSettings() {
+ Set<String> settingsToExtract =
+ ImmutableSet.of(SETTING_NAME_CLUSTER_SETTINGS, SETTING_NAME_RECOVERY_SETTINGS);
+ return settingsToExtract.stream()
+ .flatMap( settingCategory ->
+ properties.getOrDefault(settingCategory, emptySet()).stream())
+ .flatMap( map -> map.entrySet().stream() )
+ .collect(toMap(e -> e.getKey(), e -> e.getValue()));
+ }
+
/**
* Get whether the specified component in the service is enabled
* for auto start.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java
index ff00282..2d65ba1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java
@@ -73,13 +73,16 @@ public class SettingFactory {
for (Map.Entry<String, Object> entry : settingMap.entrySet()) {
final String[] propertyNames = entry.getKey().split("/");
Set<Map<String, String>> settingValue;
+ String settingCategory;
if (entry.getValue() instanceof Set) {
+ settingCategory = propertyNames[propertyNames.length - 1];
settingValue = (Set<Map<String, String>>) entry.getValue();
}
else if (propertyNames.length > 1){
+ settingCategory = propertyNames[0];
Map<String, String> property = new HashMap<>();
property.put(propertyNames[1], String.valueOf(entry.getValue()));
- settingValue = properties.get(propertyNames[0]);
+ settingValue = properties.get(settingCategory);
if (settingValue == null) {
settingValue = new HashSet<>();
}
@@ -88,7 +91,7 @@ public class SettingFactory {
else {
throw new IllegalArgumentException("Invalid setting schema: " + String.valueOf(entry.getValue()));
}
- properties.put(propertyNames[0], settingValue);
+ properties.put(settingCategory, settingValue);
}
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
index 0f92b3b..b00a729 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
@@ -46,6 +46,7 @@ public class ConfigureClusterTask implements Callable<Boolean> {
private static final long DEFAULT_TIMEOUT = TimeUnit.MINUTES.toMillis(30);
private static final long REPEAT_DELAY = TimeUnit.SECONDS.toMillis(1);
+ // TODO: use cluster settings instead
private static final String TIMEOUT_PROPERTY_NAME = "cluster_configure_task_timeout";
private static final Logger LOG = LoggerFactory.getLogger(ConfigureClusterTask.class);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
index d10f8cc9..4f538d2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java
@@ -17,11 +17,14 @@ package org.apache.ambari.server.topology.validators;
import java.util.HashSet;
import java.util.Set;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.InvalidTopologyException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableSet;
+
/**
* Validates whether incoming config types (form the blueprint or the cluster creation template) are valid.
* A configuration type is considered valid if the stack based on which the cluster is to be created contains such a
@@ -30,6 +33,9 @@ import org.slf4j.LoggerFactory;
public class StackConfigTypeValidator implements TopologyValidator {
private static final Logger LOGGER = LoggerFactory.getLogger(StackConfigTypeValidator.class);
+ private static final Set<String> GLOBAL_CONFIG_TYPES = ImmutableSet.of(
+ ConfigHelper.CLUSTER_ENV);
+
@Override
public ClusterTopology validate(ClusterTopology topology) throws InvalidTopologyException {
@@ -45,6 +51,7 @@ public class StackConfigTypeValidator implements TopologyValidator {
// remove all "valid" config types from the incoming set
incomingConfigTypes.removeAll(stackConfigTypes);
+ incomingConfigTypes.removeAll(GLOBAL_CONFIG_TYPES);
if (!incomingConfigTypes.isEmpty()) {
// there are config types in the request that are not in the stack
diff --git a/ambari-server/src/main/resources/cluster-settings.xml b/ambari-server/src/main/resources/cluster-settings.xml
index 6eb81dd..2331a8d 100644
--- a/ambari-server/src/main/resources/cluster-settings.xml
+++ b/ambari-server/src/main/resources/cluster-settings.xml
@@ -329,4 +329,25 @@ gpgcheck=0
</description>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>command_retry_enabled</name>
+ <value>true</value>
+ <description>If flag is set to true, ambari will retry failed commands.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>commands_to_retry</name>
+ <value>INSTALL,START</value>
+ <description>Specifies what kinds of commands should be retried on failure
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>command_retry_max_time_in_sec</name>
+ <value>600</value>
+ <description>Sets the timeframe in which a failed command should be retried.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 8e4f80c..14f1d41 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -48,6 +48,7 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.KerberosHelper;
@@ -78,6 +79,7 @@ import org.apache.ambari.server.topology.InvalidTopologyException;
import org.apache.ambari.server.topology.ResolvedComponent;
import org.apache.ambari.server.topology.SecurityConfiguration;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
+import org.apache.ambari.server.topology.Setting;
import org.apache.commons.lang.StringUtils;
import org.easymock.EasyMock;
import org.easymock.EasyMockRule;
@@ -157,11 +159,18 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
@Mock(type = MockType.NICE)
private SecurityConfigurationFactory securityFactory;
+ @Mock
+ private AmbariMetaInfo metaInfo;
+
+ private Setting setting;
+
@Before
public void init() throws Exception {
expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes();
expect(bp.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes();
expect(bp.getName()).andReturn("test-bp").anyTimes();
+ setting = new Setting(new HashMap<>());
+ expect(bp.getSetting()).andReturn(setting).anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).atLeastOnce();
expect(stack.getVersion()).andReturn(STACK_VERSION).atLeastOnce();
@@ -183,8 +192,9 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(configHelper.getDefaultStackProperties(
EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes();
- stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties);
+ stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties);
+ expect(metaInfo.getClusterProperties()).andReturn(ImmutableSet.of()).anyTimes();
expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once();
expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes();
@@ -195,6 +205,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes();
expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes();
expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes();
expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes();
Set<String> properties = new HashSet<>();
properties.add("core-site/hadoop.security.auth_to_local");
@@ -2172,13 +2183,11 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
assertEquals("128m", updatedVal2);
assertEquals("Incorrect number of config types updated",
- 3, configTypesUpdated.size());
+ 2, configTypesUpdated.size());
assertTrue("Expected config type not updated",
configTypesUpdated.contains("oozie-env"));
assertTrue("Expected config type not updated",
configTypesUpdated.contains("yarn-site"));
- assertTrue("Expected config type not updated",
- configTypesUpdated.contains("cluster-env"));
}
@Test
@@ -2735,79 +2744,6 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
}
@Test
- public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception {
- Map<String, Map<String, String>> configProperties =
- new HashMap<>();
-
- HashMap<String, String> clusterEnvProperties = new HashMap<>();
- configProperties.put("cluster-env", clusterEnvProperties);
-
- Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap());
-
- TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet());
- ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY);
-
- BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
-
- Set<String> updatedConfigTypes =
- updater.doUpdateForClusterCreate();
-
- // after update, verify that the retry properties for commands and installs are set as expected
- assertEquals("Incorrect number of properties added to cluster-env for retry",
- 3, clusterEnvProperties.size());
- assertEquals("command_retry_enabled was not set to the expected default",
- "true", clusterEnvProperties.get("command_retry_enabled"));
- assertEquals("commands_to_retry was not set to the expected default",
- "INSTALL,START", clusterEnvProperties.get("commands_to_retry"));
- assertEquals("command_retry_max_time_in_sec was not set to the expected default",
- "600", clusterEnvProperties.get("command_retry_max_time_in_sec"));
-
- assertEquals("Incorrect number of config types updated by this operation",
- 1, updatedConfigTypes.size());
-
- assertTrue("Expected type not included in the updated set",
- updatedConfigTypes.contains("cluster-env"));
- }
-
- @Test
- public void testDoUpdateForClusterVerifyRetrySettingsCustomized() throws Exception {
- Map<String, Map<String, String>> configProperties =
- new HashMap<>();
-
- HashMap<String, String> clusterEnvProperties = new HashMap<>();
- configProperties.put("cluster-env", clusterEnvProperties);
-
- clusterEnvProperties.put("command_retry_enabled", "false");
- clusterEnvProperties.put("commands_to_retry", "TEST");
- clusterEnvProperties.put("command_retry_max_time_in_sec", "1");
-
-
- Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap());
-
- TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet());
- ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY);
-
- BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
-
- Set<String> updatedConfigTypes =
- updater.doUpdateForClusterCreate();
-
- // after update, verify that the retry properties for commands and installs are set as expected
- // in this case, the customer-provided overrides should be honored, rather than the retry defaults
- assertEquals("Incorrect number of properties added to cluster-env for retry",
- 3, clusterEnvProperties.size());
- assertEquals("command_retry_enabled was not set to the expected default",
- "false", clusterEnvProperties.get("command_retry_enabled"));
- assertEquals("commands_to_retry was not set to the expected default",
- "TEST", clusterEnvProperties.get("commands_to_retry"));
- assertEquals("command_retry_max_time_in_sec was not set to the expected default",
- "1", clusterEnvProperties.get("command_retry_max_time_in_sec"));
-
- assertEquals("Incorrect number of config types updated",
- 0, updatedConfigTypes.size());
- }
-
- @Test
public void testDoUpdateForClusterWithNameNodeHAEnabledSpecifyingHostNamesDirectly() throws Exception {
final String expectedNameService = "mynameservice";
final String expectedHostName = "c6401.apache.ambari.org";
@@ -5350,9 +5286,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// verify that correct configuration types were listed as updated in the returned set
assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes,
- 3, updatedConfigTypes.size());
- assertTrue("Expected config type not found in updated set",
- updatedConfigTypes.contains("cluster-env"));
+ 2, updatedConfigTypes.size());
assertTrue("Expected config type not found in updated set",
updatedConfigTypes.contains("hdfs-site"));
assertTrue("Expected config type not found in updated set",
@@ -5424,9 +5358,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
// verify that correct configuration types were listed as updated in the returned set
assertEquals("Incorrect number of updated config types returned, set = " + updatedConfigTypes,
- 2, updatedConfigTypes.size());
- assertTrue("Expected config type 'cluster-env' not found in updated set",
- updatedConfigTypes.contains("cluster-env"));
+ 1, updatedConfigTypes.size());
assertTrue("Expected config type 'hdfs-site' not found in updated set",
updatedConfigTypes.contains("hdfs-site"));
}
@@ -8217,7 +8149,8 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport {
throws InvalidTopologyException {
- replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters, cluster);
+ replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters,
+ cluster, metaInfo);
Map<String, HostGroupInfo> hostGroupInfo = new HashMap<>();
Map<String, HostGroup> allHostGroups = new HashMap<>();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
index 97da23b..71e2280 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
@@ -18,7 +18,9 @@
package org.apache.ambari.server.topology;
+import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.emptySet;
+import static java.util.stream.Collectors.toMap;
import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.capture;
@@ -34,7 +36,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -42,6 +43,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.ambari.annotations.Experimental;
@@ -77,18 +79,25 @@ import org.apache.ambari.server.state.ConfigFactory;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
+import org.apache.commons.lang3.tuple.Pair;
import org.easymock.Capture;
+import org.easymock.CaptureType;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
import com.google.inject.util.Providers;
/**
@@ -98,6 +107,8 @@ import com.google.inject.util.Providers;
@Experimental(
feature = ExperimentalFeature.UNIT_TEST_REQUIRED,
comment = "Add test cases for mpacks and multiple/bad versions")
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(AmbariContext.class)
public class AmbariContextTest {
private static final String BP_NAME = "testBP";
@@ -133,7 +144,7 @@ public class AmbariContextTest {
private static final Host host1 = createNiceMock(Host.class);
private static final Host host2 = createNiceMock(Host.class);
private static final ConfigFactory configFactory = createNiceMock(ConfigFactory.class);
- private static final Service mockService1 = createStrictMock(Service.class);
+// private static final Service mockService1 = createStrictMock(Service.class);
private static final Collection<String> blueprintServices = new HashSet<>();
private static final Map<Long, ConfigGroup> configGroups = new HashMap<>();
@@ -141,44 +152,27 @@ public class AmbariContextTest {
private Configuration group1Configuration = null;
private static final Set<String> group1Hosts = ImmutableSet.of(HOST1, HOST2);
+ private Capture<String> clusterSettingKeys;
+ private Capture<String> clusterSettingValues;
+
private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = EasyMock.newCapture();
- private Setting setting = createNiceMock(Setting.class);
+ private Setting setting;
@Before
public void setUp() throws Exception {
reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo,
hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters,
- cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory);
+ cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory, metaInfo);
// "inject" context state
- Class<AmbariContext> clazz = AmbariContext.class;
- Field f = clazz.getDeclaredField("controller");
- f.setAccessible(true);
- f.set(context, Providers.of(controller));
-
- f = clazz.getDeclaredField("clusterController");
- f.setAccessible(true);
- f.set(null, clusterController);
-
- f = clazz.getDeclaredField("hostResourceProvider");
- f.setAccessible(true);
- f.set(null, hostResourceProvider);
-
- f = clazz.getDeclaredField("serviceGroupResourceProvider");
- f.setAccessible(true);
- f.set(null, serviceGroupResourceProvider);
-
- f = clazz.getDeclaredField("serviceResourceProvider");
- f.setAccessible(true);
- f.set(null, serviceResourceProvider);
-
- f = clazz.getDeclaredField("componentResourceProvider");
- f.setAccessible(true);
- f.set(null, componentResourceProvider);
-
- f = clazz.getDeclaredField("hostComponentResourceProvider");
- f.setAccessible(true);
- f.set(null, hostComponentResourceProvider);
+ Whitebox.setInternalState(context, "controller", Providers.of(controller));
+ Whitebox.setInternalState(context, "ambariMetaInfo", Providers.of(metaInfo));
+ Whitebox.setInternalState(AmbariContext.class, "clusterController", clusterController);
+ Whitebox.setInternalState(AmbariContext.class, "hostResourceProvider", hostResourceProvider);
+ Whitebox.setInternalState(AmbariContext.class, "serviceGroupResourceProvider", serviceGroupResourceProvider);
+ Whitebox.setInternalState(AmbariContext.class, "serviceResourceProvider", serviceResourceProvider);
+ Whitebox.setInternalState(AmbariContext.class, "componentResourceProvider", componentResourceProvider);
+ Whitebox.setInternalState(AmbariContext.class, "hostComponentResourceProvider", hostComponentResourceProvider);
// bp configuration
Map<String, Map<String, String>> bpProperties = new HashMap<>();
@@ -252,8 +246,18 @@ public class AmbariContextTest {
builderFor("service2", "s2Component1").stackId(STACK_ID).buildPartial()
)).anyTimes();
expect(topology.getConfiguration()).andReturn(bpConfiguration).anyTimes();
+
+ setting = new Setting(
+ map(
+ Pair.of("recovery_settings",
+ newHashSet(
+ map(Pair.of("recovery_enabled", "true")))),
+ Pair.of("service_settings",
+ newHashSet(
+ map(Pair.of("name", "service1"), Pair.of("recovery_enabled", "true"))))
+ )
+ );
expect(topology.getSetting()).andReturn(setting).anyTimes();
- expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
@@ -265,7 +269,13 @@ public class AmbariContextTest {
expect(controller.getClusters()).andReturn(clusters).anyTimes();
expect(controller.getConfigHelper()).andReturn(configHelper).anyTimes();
expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes();
- expect(metaInfo.getClusterProperties()).andReturn(emptySet()).anyTimes();
+ expect(metaInfo.getClusterProperties()).andReturn(
+ Sets.newHashSet(
+ property("command_retry_enabled", "true"),
+ property("commands_to_retry", "INSTALL,START"),
+ property("command_retry_max_time_in_sec", "600")
+ )
+ ).anyTimes();
expect(clusters.getCluster(CLUSTER_ID)).andReturn(cluster).anyTimes();
expect(clusters.getCluster(CLUSTER_NAME)).andReturn(cluster).anyTimes();
@@ -276,8 +286,11 @@ public class AmbariContextTest {
Map<String, Host> clusterHosts = ImmutableMap.of(HOST1, host1, HOST2, host2);
expect(clusters.getHostsForCluster(CLUSTER_NAME)).andReturn(clusterHosts).anyTimes();
+ clusterSettingKeys = EasyMock.newCapture(CaptureType.ALL);
+ clusterSettingValues = EasyMock.newCapture(CaptureType.ALL);
expect(cluster.getClusterId()).andReturn(CLUSTER_ID).anyTimes();
expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+ expect(cluster.addClusterSetting(capture(clusterSettingKeys), capture(clusterSettingValues))).andReturn(null).anyTimes();
expect(host1.getHostId()).andReturn(1L).anyTimes();
expect(host2.getHostId()).andReturn(2L).anyTimes();
@@ -287,20 +300,19 @@ public class AmbariContextTest {
expect(configGroup1.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_1)).anyTimes();
expect(configGroup2.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_2)).anyTimes();
-
}
@After
public void tearDown() throws Exception {
verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo,
- hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters,
- cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory);
+ hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster,
+ group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory);
}
private void replayAll() {
replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo,
- hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters,
- cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory);
+ hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster,
+ group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory);
}
@Test
@@ -395,6 +407,21 @@ public class AmbariContextTest {
assertEquals("STARTED", startProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID));
assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
installPredicateCapture.getValue());
+
+ // verify that cluster settings has been configured with values coming from both defaults and the topology
+ Map<String, String> recordedClusterSettings = IntStream.range(0, clusterSettingKeys.getValues().size())
+ .boxed()
+ .map(i -> Pair.of(clusterSettingKeys.getValues().get(i), clusterSettingValues.getValues().get(i)))
+ .collect(toMap(p -> p.getLeft(), p -> p.getRight()));
+
+ assertEquals(
+ ImmutableMap.of(
+ "command_retry_enabled", "true",
+ "commands_to_retry", "INSTALL,START",
+ "command_retry_max_time_in_sec", "600",
+ "recovery_enabled", "true"),
+ recordedClusterSettings
+ );
}
@Test
@@ -684,14 +711,31 @@ public class AmbariContextTest {
assertFalse(topologyResolved);
}
+ private static final <K, V> Map<K, V> map(Pair<K, V>... keyValuePairs) {
+ Map<K, V> map = new HashMap<>(keyValuePairs.length);
+ for (Pair<K, V> kv: keyValuePairs) {
+ map.put(kv.getLeft(), kv.getRight());
+ }
+ return map;
+ }
+
+ private static PropertyInfo property(String name, String value) {
+ PropertyInfo info = new PropertyInfo();
+ info.setName(name);
+ info.setValue(value);
+ return info;
+ }
+
@Test
public void testProvisionCluster_downloadMissingMpack() throws Exception {
PowerMock.mockStatic(AmbariContext.class);
expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes();
+ PowerMock.replay(AmbariContext.class);
// given
MpackInstance mpack1 = new MpackInstance("HDPCORE", "HDPCORE", "1.0.0.0", "http://mpacks.org/hdpcore", Configuration.createEmpty());
MpackInstance mpack2 = new MpackInstance("HDF", "HDF", "3.3.0", "http://mpacks.org/hdf", Configuration.createEmpty());
+ reset(metaInfo);
expect(metaInfo.getStack(mpack1.getStackId())).andReturn(null);
expect(metaInfo.getStack(mpack2.getStackId())).andThrow(new StackAccessException("Testing missing stack"));
@@ -703,8 +747,8 @@ public class AmbariContextTest {
.andReturn(new RequestStatusImpl(null, null, null))
.once();
- PowerMock.replay(AmbariContext.class);
- replay(clusterController, metaInfo, mpackResourceProvider);
+ replay(mpackResourceProvider);
+ replayAll();
// when
context.downloadMissingMpacks(ImmutableSet.of(mpack1, mpack2));
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index d32bc92..2e67538 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -143,7 +143,7 @@ public class ClusterConfigurationRequestTest {
Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", null);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(2, updatedConfigTypes.size());
+ assertEquals(1, updatedConfigTypes.size());
}
/**
@@ -158,7 +158,7 @@ public class ClusterConfigurationRequestTest {
"defaultTestValue", null);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(2, updatedConfigTypes.size());
+ assertEquals(1, updatedConfigTypes.size());
}
@@ -174,7 +174,7 @@ public class ClusterConfigurationRequestTest {
"defaultTestValue", null);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(1, updatedConfigTypes.size());
+ assertEquals(0, updatedConfigTypes.size());
}
/**
@@ -188,7 +188,7 @@ public class ClusterConfigurationRequestTest {
Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos("testPropertyValue", null, null);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(1, updatedConfigTypes.size());
+ assertEquals(0, updatedConfigTypes.size());
}
@Test
@@ -202,7 +202,7 @@ public class ClusterConfigurationRequestTest {
Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", kerberosConfig);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(1, updatedConfigTypes.size());
+ assertEquals(0, updatedConfigTypes.size());
}
@Test
@@ -216,7 +216,7 @@ public class ClusterConfigurationRequestTest {
Capture<? extends Set<String>> captureUpdatedConfigTypes = testProcessWithKerberos(null, "defaultTestValue", kerberosConfig);
Set<String> updatedConfigTypes = captureUpdatedConfigTypes.getValue();
- assertEquals(1, updatedConfigTypes.size());
+ assertEquals(0, updatedConfigTypes.size());
}
private Capture<? extends Set<String>> testProcessWithKerberos(String blueprintPropertyValue, String
@@ -277,9 +277,7 @@ public class ClusterConfigurationRequestTest {
expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes();
expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes();
- expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections
- .emptyList()).anyTimes();
- Set<ServiceResponse> services = IntStream.range(0, SERVICE_NAMES.size()).boxed().map(
+ Set<ServiceResponse> services = IntStream.range(0, SERVICE_NAMES.size()).boxed().map(
serviceId -> new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "CORE", (long)serviceId, SERVICE_NAMES.get(serviceId),
null, null, null, null, false, false, false, false, false)
).collect(toSet());
@@ -352,8 +350,6 @@ public class ClusterConfigurationRequestTest {
expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes();
expect(ambariContext.getClusterName(1L)).andReturn(CLUSTER_NAME).anyTimes();
- expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections
- .emptyList()).anyTimes();
Set<ServiceResponse> services = IntStream.range(0, serviceNames.size()).boxed().
map(
serviceId -> new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "CORE", (long)serviceId, SERVICE_NAMES.get(serviceId),
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
index 5f0e4e3..73cab9d 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java
@@ -17,7 +17,11 @@
*/
package org.apache.ambari.server.topology;
+import static java.util.Collections.emptySet;
import static java.util.stream.Collectors.toSet;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service;
import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor;
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyLong;
@@ -31,12 +35,10 @@ import static org.easymock.EasyMock.isA;
import static org.easymock.EasyMock.newCapture;
import static org.junit.Assert.assertEquals;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -69,7 +71,6 @@ import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTask;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
-import org.apache.ambari.server.topology.validators.TopologyValidator;
import org.easymock.Capture;
import org.easymock.EasyMockRule;
import org.easymock.EasyMockSupport;
@@ -88,12 +89,11 @@ import org.powermock.reflect.Whitebox;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ AmbariServer.class })
+@PrepareForTest({ AmbariContext.class, AmbariServer.class })
public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
- private static final String CLUSTER_NAME = "test-cluster";
- private static final long CLUSTER_ID = 1;
private static final String BLUEPRINT_NAME = "test-bp";
private static final String STACK_NAME = "test-stack";
private static final String STACK_VERSION = "test-stack-version";
@@ -181,6 +181,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
@Mock
private ComponentResolver componentResolver;
+ @Mock(type = MockType.NICE)
+ private ClusterTopology clusterTopology;
+
private final Configuration stackConfig = new Configuration(new HashMap<>(),
new HashMap<>());
private final Configuration bpConfiguration = new Configuration(new HashMap<>(),
@@ -211,8 +214,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
private String predicate = "Hosts/host_name=foo";
- private List<TopologyValidator> topologyValidators = new ArrayList<>();
-
private Capture<Map<String, Object>> configRequestPropertiesCapture;
private Capture<Map<String, Object>> configRequestPropertiesCapture2;
private Capture<Map<String, Object>> configRequestPropertiesCapture3;
@@ -270,7 +271,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes();
expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes();
expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes();
- expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes();
expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes();
expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes();
@@ -303,9 +303,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes();
expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes();
- expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes();
- expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes();
- expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes();
+ expect(stack.getDependenciesForComponent(anyString())).andReturn(emptySet()).anyTimes();
+ expect(stack.getExcludedConfigurationTypes("service1")).andReturn(emptySet()).anyTimes();
+ expect(stack.getExcludedConfigurationTypes("service2")).andReturn(emptySet()).anyTimes();
expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes();
expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes();
expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes();
@@ -319,7 +319,7 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(request.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes();
expect(request.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes();
expect(request.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes();
- expect(request.getSecurityConfiguration()).andReturn(null).anyTimes();
+ expect(request.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes();
expect(request.shouldValidateTopology()).andReturn(true).anyTimes();
expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes();
expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes();
@@ -361,7 +361,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
PowerMock.mockStatic(AmbariServer.class);
expect(AmbariServer.getController()).andReturn(managementController).anyTimes();
PowerMock.replay(AmbariServer.class);
- expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes();
+ PowerMock.mockStatic(AmbariContext.class);
+ expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes();
+ PowerMock.replay(AmbariContext.class);
expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes();
expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes();
RequestStatus completedStatus = createNiceMock(RequestStatus.class);
@@ -372,7 +374,11 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes();
expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+ expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes();
+ expect(ambariContext.getServices(anyString())).andReturn(
+ Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes();
expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes();
+ expect(ambariContext.getController()).andReturn(managementController).anyTimes();
//todo: don't ignore param
ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE));
expectLastCall().once();
@@ -380,20 +386,23 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes();
expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes();
expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes();
- // so only INITIAL config
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))).
- andReturn(Collections.singletonList(configurationRequest));
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))).
- andReturn(Collections.singletonList(configurationRequest2)).once();
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))).
- andReturn(Collections.singletonList(configurationRequest3)).once();
// INSTALL task expectation
-
expect(ambariContext.createAmbariTask(anyLong(), anyLong(), eq("component3"),
anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommandInstallComponent3).times(3);
expect(ambariContext.createAmbariTask(anyLong(), anyLong(), eq("component4"),
anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommandInstallComponent4).times(2);
+ expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology);
+
+ expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
+ expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes();
+ expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes();
+ expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes();
+ expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes();
+ expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes();
+ expect(clusterTopology.getStack()).andReturn(stack).anyTimes();
+ expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes();
expect(hostRoleCommandInstallComponent3.getTaskId()).andReturn(1L).atLeastOnce();
expect(hostRoleCommandInstallComponent3.getRole()).andReturn(Role.valueOf("component3")).atLeastOnce();
@@ -421,7 +430,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
expect(hostRoleCommandStartComponent2.getRole()).andReturn(Role.NAMENODE).atLeastOnce();
expect(hostRoleCommandStartComponent2.getStatus()).andReturn(HostRoleStatus.COMPLETED).atLeastOnce();
-
ambariContext.setConfigurationOnCluster(capture(updateClusterConfigRequestCapture));
expectLastCall().times(3);
ambariContext.persistInstallStateForUI(CLUSTER_NAME, STACK_ID);
@@ -456,4 +464,5 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport {
LogicalRequest request = topologyManager.getRequest(1);
assertEquals(request.getHostRequests().size(), 3);
}
+
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java
new file mode 100644
index 0000000..b23d88e
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeploymentTestCommon.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import org.apache.ambari.server.controller.ServiceResponse;
+
+/**
+ * Common functionality for cluster deployment tests
+ */
+public class ClusterDeploymentTestCommon {
+
+ static final String CLUSTER_NAME = "test-cluster";
+ static final long CLUSTER_ID = 1;
+
+ /**
+ * @return a {@link ServiceResponse} instance for tests
+ */
+ static ServiceResponse service(String serviceName, long serviceId) {
+ return new ServiceResponse(CLUSTER_ID, CLUSTER_NAME, 1L, "service-group-1", serviceId, serviceName, null, null,
+ null, null, true, true, true, true, true);
+ }
+}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
index 9ea744f..65a8f6f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java
@@ -20,6 +20,9 @@ package org.apache.ambari.server.topology;
import static java.util.stream.Collectors.toSet;
import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service;
import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor;
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyLong;
@@ -32,12 +35,10 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.isA;
import static org.easymock.EasyMock.newCapture;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -46,6 +47,7 @@ import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.ClusterRequest;
@@ -70,7 +72,6 @@ import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTask;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
-import org.apache.ambari.server.topology.validators.TopologyValidator;
import org.easymock.Capture;
import org.easymock.EasyMockRule;
import org.easymock.EasyMockSupport;
@@ -89,12 +90,11 @@ import org.powermock.reflect.Whitebox;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ AmbariServer.class })
+@PrepareForTest({ AmbariContext.class, AmbariServer.class })
public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupport {
- private static final String CLUSTER_NAME = "test-cluster";
- private static final long CLUSTER_ID = 1;
private static final String BLUEPRINT_NAME = "test-bp";
private static final String STACK_NAME = "test-stack";
private static final String STACK_VERSION = "test-stack-version";
@@ -165,6 +165,8 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
private ArtifactResourceProvider artifactResourceProvider;
@Mock(type = MockType.NICE)
private MpackResourceProvider mpackResourceProvider;
+ @Mock
+ private AmbariMetaInfo ambariMetaInfo;
@Mock(type = MockType.NICE)
@@ -178,6 +180,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
@Mock
private ComponentResolver componentResolver;
+ @Mock(type = MockType.NICE)
+ private ClusterTopology clusterTopology;
+
private final Configuration stackConfig = new Configuration(new HashMap<>(),
new HashMap<>());
private final Configuration bpConfiguration = new Configuration(new HashMap<>(),
@@ -208,8 +213,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
private String predicate = "Hosts/host_name=foo";
- private List<TopologyValidator> topologyValidators = new ArrayList<>();
-
private Capture<Map<String, Object>> configRequestPropertiesCapture;
private Capture<Map<String, Object>> configRequestPropertiesCapture2;
private Capture<Map<String, Object>> configRequestPropertiesCapture3;
@@ -269,6 +272,8 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes();
expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes();
expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes();
+ expect(ambariContext.getServices(anyString())).andReturn(
+ Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes();
expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes();
expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes();
@@ -365,7 +370,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
PowerMock.mockStatic(AmbariServer.class);
expect(AmbariServer.getController()).andReturn(managementController).anyTimes();
PowerMock.replay(AmbariServer.class);
- expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes();
+
+ PowerMock.mockStatic(AmbariContext.class);
+ expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes();
+ PowerMock.replay(AmbariContext.class);
expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes();
expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes();
RequestStatus completedStatus = createNiceMock(RequestStatus.class);
@@ -376,7 +384,11 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes();
expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+ expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
+ expect(ambariMetaInfo.getClusterProperties()).andReturn(Sets.newHashSet()).anyTimes();
+
expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes();
+ expect(ambariContext.getController()).andReturn(managementController).anyTimes();
//todo: don't ignore param
ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE));
expectLastCall().once();
@@ -384,13 +396,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes();
expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes();
expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes();
- // so only INITIAL config
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))).
- andReturn(Collections.singletonList(configurationRequest));
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))).
- andReturn(Collections.singletonList(configurationRequest2)).once();
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))).
- andReturn(Collections.singletonList(configurationRequest3)).once();
// INSTALL task expectation
expect(ambariContext.createAmbariTask(anyLong(), anyLong(), anyString(),
anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommand).times(7);
@@ -417,6 +422,17 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp
persistedState.persistLogicalRequest((LogicalRequest) anyObject(), anyLong());
expectLastCall().once();
+ expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology);
+ expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
+ expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes();
+ expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes();
+ expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_AND_START).anyTimes();
+ expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes();
+ expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes();
+ expect(clusterTopology.getStack()).andReturn(stack).anyTimes();
+ expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes();
+
replayAll();
Whitebox.setInternalState(topologyManager, "executor", executor);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
index 295817b..525e691 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
@@ -20,6 +20,9 @@ package org.apache.ambari.server.topology;
import static java.util.stream.Collectors.toSet;
import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_ID;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.CLUSTER_NAME;
+import static org.apache.ambari.server.topology.ClusterDeploymentTestCommon.service;
import static org.apache.ambari.server.topology.StackComponentResolverTest.builderFor;
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyLong;
@@ -32,12 +35,10 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.isA;
import static org.easymock.EasyMock.newCapture;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -47,6 +48,7 @@ import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.ClusterRequest;
@@ -71,7 +73,6 @@ import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTask;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
-import org.apache.ambari.server.topology.validators.TopologyValidator;
import org.easymock.Capture;
import org.easymock.EasyMockRule;
import org.easymock.EasyMockSupport;
@@ -90,13 +91,13 @@ import org.powermock.reflect.Whitebox;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ AmbariServer.class })
+@PrepareForTest({ AmbariContext.class, AmbariServer.class })
public class ClusterInstallWithoutStartTest extends EasyMockSupport {
- private static final String CLUSTER_NAME = "test-cluster";
- private static final long CLUSTER_ID = 1;
+
private static final String BLUEPRINT_NAME = "test-bp";
private static final String STACK_NAME = "test-stack";
private static final String STACK_VERSION = "test-stack-version";
@@ -168,7 +169,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
private ArtifactResourceProvider artifactResourceProvider;
@Mock(type = MockType.NICE)
private MpackResourceProvider mpackResourceProvider;
-
+ @Mock
+ private AmbariMetaInfo ambariMetaInfo;
@Mock(type = MockType.NICE)
private ComponentInfo serviceComponentInfo;
@@ -181,6 +183,9 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
@Mock
private ComponentResolver componentResolver;
+ @Mock(type = MockType.NICE)
+ private ClusterTopology clusterTopology;
+
private final Configuration stackConfig = new Configuration(new HashMap<>(),
new HashMap<>());
private final Configuration bpConfiguration = new Configuration(new HashMap<>(),
@@ -211,8 +216,6 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
private String predicate = "Hosts/host_name=foo";
- private List<TopologyValidator> topologyValidators = new ArrayList<>();
-
private Capture<ClusterTopology> clusterTopologyCapture;
private Capture<Map<String, Object>> configRequestPropertiesCapture;
private Capture<Map<String, Object>> configRequestPropertiesCapture2;
@@ -273,6 +276,8 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes();
expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes();
expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes();
+ expect(ambariContext.getServices(anyString())).andReturn(
+ Sets.newHashSet(service("service1", 1L), service("service2", 2L))).anyTimes();
expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes();
expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes();
@@ -368,7 +373,11 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
PowerMock.mockStatic(AmbariServer.class);
expect(AmbariServer.getController()).andReturn(managementController).anyTimes();
PowerMock.replay(AmbariServer.class);
- expect(ambariContext.getClusterController()).andReturn(clusterController).anyTimes();
+
+ PowerMock.mockStatic(AmbariContext.class);
+ expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes();
+ PowerMock.replay(AmbariContext.class);
+
expect(clusterController.ensureResourceProvider(Resource.Type.Mpack)).andReturn(mpackResourceProvider).anyTimes();
expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes();
RequestStatus completedStatus = createNiceMock(RequestStatus.class);
@@ -379,7 +388,11 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(clusters.getClusterById(anyLong())).andReturn(cluster).anyTimes();
expect(cluster.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
+ expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes();
+ expect(ambariMetaInfo.getClusterProperties()).andReturn(Sets.newHashSet()).anyTimes();
+
expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes();
+ expect(ambariContext.getController()).andReturn(managementController).anyTimes();
//todo: don't ignore param
ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE));
expectLastCall().once();
@@ -387,13 +400,6 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes();
expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes();
expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes();
- // so only INITIAL config
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))).
- andReturn(Collections.singletonList(configurationRequest));
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))).
- andReturn(Collections.singletonList(configurationRequest2)).once();
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))).
- andReturn(Collections.singletonList(configurationRequest3)).once();
// INSTALL task expectation
expect(ambariContext.createAmbariTask(anyLong(), anyLong(), anyString(),
anyString(), eq(AmbariContext.TaskType.INSTALL), anyBoolean())).andReturn(hostRoleCommand).atLeastOnce();
@@ -418,6 +424,17 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport {
persistedState.persistLogicalRequest((LogicalRequest) anyObject(), anyLong());
expectLastCall().once();
+ expect(ambariContext.createClusterTopology(request)).andReturn(clusterTopology);
+ expect(clusterTopology.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes();
+ expect(clusterTopology.getClusterId()).andReturn(CLUSTER_ID).anyTimes();
+ expect(clusterTopology.getSetting()).andReturn(new Setting(ImmutableMap.of())).anyTimes();
+ expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_ONLY).anyTimes();
+ expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes();
+ expect(clusterTopology.getConfiguration()).andReturn(bpConfiguration).anyTimes();
+ expect(clusterTopology.getStack()).andReturn(stack).anyTimes();
+ expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(new StackId(STACK_NAME, STACK_VERSION))).anyTimes();
+
replayAll();
Whitebox.setInternalState(topologyManager, "executor", executor);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
index bab1b0c..074d87f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
@@ -32,9 +32,12 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.ProvisionAction;
import org.apache.ambari.server.controller.internal.StackDefinition;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.commons.lang3.tuple.Pair;
@@ -52,6 +55,8 @@ import org.junit.runner.RunWith;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
/**
* Unit tests for ClusterTopologyImpl.
@@ -106,14 +111,23 @@ public class ClusterTopologyImplTest extends EasyMockSupport {
"group4", ImmutableSet.of(
builderFor("any_service", "component5"))
);
+ private final AmbariManagementController controller = mock(AmbariManagementController.class);
+ private final AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class);
private BlueprintBasedClusterProvisionRequest provisionRequest;
private Configuration bpconfiguration;
@Before
public void setUp() throws Exception {
- bpconfiguration = Configuration.createEmpty();
-
+ bpconfiguration = new Configuration(
+ Maps.newHashMap(ImmutableMap.of(
+ "cluster-env",
+ Maps.newHashMap(ImmutableMap.of(
+ "commands_to_retry", "INSTALL",
+ "command_retry_max_time_in_sec", "500",
+ "unknown_property_that_should_not_become_cluster_setting", "some_value"))
+ )),
+ new HashMap<>());
HostGroupInfo group1Info = new HostGroupInfo("group1");
HostGroupInfo group2Info = new HostGroupInfo("group2");
HostGroupInfo group3Info = new HostGroupInfo("group3");
@@ -158,6 +172,7 @@ public class ClusterTopologyImplTest extends EasyMockSupport {
expect(stack.getServicesForComponent("ZOOKEEPER_CLIENT")).andAnswer(() -> Stream.of(Pair.of(STACK_ID, aServiceWith(aComponent("ZOOKEEPER_CLIENT"))))).anyTimes();
expect(ambariContext.composeStacks(STACK_IDS)).andReturn(stack).anyTimes();
+ expect(ambariContext.getController()).andReturn(controller).anyTimes();
expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes();
expect(blueprint.getHostGroups()).andReturn(hostGroupMap).anyTimes();
@@ -167,12 +182,20 @@ public class ClusterTopologyImplTest extends EasyMockSupport {
expect(hostGroup.getName()).andReturn(name).anyTimes();
expect(blueprint.getHostGroup(name)).andReturn(hostGroup).anyTimes();
}
+ expect(blueprint.getSetting()).andReturn(new Setting(new HashMap<>()));
expect(group1.getConfiguration()).andReturn(configuration).anyTimes();
expect(group2.getConfiguration()).andReturn(configuration).anyTimes();
expect(group3.getConfiguration()).andReturn(configuration).anyTimes();
expect(group4.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes();
+ expect(metaInfo.getClusterProperties()).andReturn(
+ Sets.newHashSet(
+ propertyInfo("command_retry_enabled", "true"),
+ propertyInfo("commands_to_retry", "INSTALL,START"),
+ propertyInfo("command_retry_max_time_in_sec", "600"))).anyTimes();
+
replayAll();
provisionRequest = new BlueprintBasedClusterProvisionRequest(ambariContext, null, blueprint, new TestTopologyRequest());
@@ -262,6 +285,29 @@ public class ClusterTopologyImplTest extends EasyMockSupport {
assertFalse(topology.isComponentHadoopCompatible("ZOOKEEPER_CLIENT"));
}
+ @Test
+ public void testAdjustTopology() throws Exception {
+ ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, provisionRequest, resolvedComponents);
+ Map<String, String> clusterSettings = topology.getSetting().getClusterSettings();
+ assertEquals(
+ ImmutableMap.of(
+ "commands_to_retry", "INSTALL",
+ "command_retry_max_time_in_sec", "500"),
+ clusterSettings
+ );
+ assertEquals(
+ ImmutableMap.of("unknown_property_that_should_not_become_cluster_setting", "some_value"),
+ topology.getConfiguration().getFullProperties().get("cluster-env")
+ );
+ }
+
+ private static PropertyInfo propertyInfo(String name, String value) {
+ PropertyInfo info = new PropertyInfo();
+ info.setName(name);
+ info.setValue(value);
+ return info;
+ }
+
private ServiceInfo aHCFSWith(ComponentInfo... components) {
ServiceInfo service = aServiceWith(components);
service.setServiceType(ServiceInfo.HADOOP_COMPATIBLE_FS);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java
index 94c412d..e8a775e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java
@@ -30,6 +30,9 @@ import java.util.Set;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
/**
* Test the SettingFactory class
*/
@@ -57,6 +60,29 @@ public class SettingFactoryTest {
}
/**
+ * Test creating setting defined in {"settings/recovery_settings" -> { ... } } style maps (typically coming from
+ * REST API).
+ */
+ @Test
+ public void testGetSettingFromSlashedPropertymaps() {
+ Set<Map<String, String>> clusterSettings =
+ ImmutableSet.of(
+ ImmutableMap.of("command_retry_enabled", "true",
+ "commands_to_retry", "INSTALL,START"));
+
+ Map<String, Object> propertyMap = ImmutableMap.of(
+ "settings/recovery_settings",
+ ImmutableSet.of(ImmutableMap.of("recovery_enabled", "true")),
+ "settings/cluster_settings", clusterSettings
+ );
+ Setting setting = SettingFactory.getSetting(ImmutableSet.of(propertyMap));
+
+ assertEquals("true", setting.getRecoveryEnabled("any service", "any component"));
+ assertEquals(clusterSettings, setting.getProperties().get("cluster_settings"));
+
+ }
+
+ /**
* {
* "recovery_settings":[
* {
@@ -134,4 +160,5 @@ public class SettingFactoryTest {
return setting;
}
+
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java
index ecd5acb..a4ee1a6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java
@@ -28,6 +28,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.state.ConfigHelper;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -193,4 +194,38 @@ public class SettingTest {
assertFalse(setting.shouldSkipFailure());
}
+ @Test
+ public void testGetClusterSettings() throws Exception {
+ Setting setting = new Setting(
+ ImmutableMap.of(
+ "cluster_settings",
+ ImmutableSet.of(
+ ImmutableMap.of(
+ ConfigHelper.COMMAND_RETRY_ENABLED, "true"),
+ ImmutableMap.of(
+ ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC, "600",
+ ConfigHelper.COMMANDS_TO_RETRY, "INSTALL,START")),
+ "recovery_settings",
+ ImmutableSet.of(
+ ImmutableMap.of(
+ Setting.SETTING_NAME_RECOVERY_ENABLED, "true",
+ "recovery_type", "AUTO_START"),
+ ImmutableMap.of(
+ "recovery_lifetime_max_count", "1024",
+ "recovery_max_count", "6"))
+ ));
+
+ Map<String, String> expectedClusterSettings = ImmutableMap.<String, String>builder()
+ .put(ConfigHelper.COMMAND_RETRY_ENABLED, "true")
+ .put(ConfigHelper.COMMAND_RETRY_MAX_TIME_IN_SEC, "600")
+ .put(ConfigHelper.COMMANDS_TO_RETRY, "INSTALL,START")
+ .put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true")
+ .put("recovery_type", "AUTO_START")
+ .put("recovery_lifetime_max_count", "1024")
+ .put("recovery_max_count", "6")
+ .build();
+
+ assertEquals(expectedClusterSettings, setting.getClusterSettings());
+ }
+
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
index e549ce4..8dc4520 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
@@ -396,19 +396,13 @@ public class TopologyManagerTest {
expect(ambariContext.getClusterId(CLUSTER_NAME)).andReturn(CLUSTER_ID).anyTimes();
expect(ambariContext.getClusterName(CLUSTER_ID)).andReturn(CLUSTER_NAME).anyTimes();
// cluster configuration task run() isn't executed by mock executor
- // so only INITIAL config
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture))).
- andReturn(Collections.singletonList(configurationRequest)).anyTimes();
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture2))).
- andReturn(Collections.singletonList(configurationRequest2)).anyTimes();
- expect(ambariContext.createConfigurationRequests(capture(configRequestPropertiesCapture3))).
- andReturn(Collections.singletonList(configurationRequest3)).anyTimes();
ambariContext.setConfigurationOnCluster(capture(updateClusterConfigRequestCapture));
expectLastCall().anyTimes();
ambariContext.persistInstallStateForUI(CLUSTER_NAME, STACK_ID);
expectLastCall().anyTimes();
expect(ambariContext.getServices(anyString())).andReturn(services).anyTimes();
+ expect(ambariContext.getController()).andReturn(controller).anyTimes();
expect(resourceProvider.createResources((anyObject()))).andReturn(new RequestStatusImpl(null, null, null)).anyTimes(); // persist raw request
expect(clusterController.ensureResourceProvider(anyObject(Resource.Type.class))).andReturn(resourceProvider);
@@ -433,6 +427,7 @@ public class TopologyManagerTest {
return null;
}).
anyTimes();
+ expect(metaInfo.getClusterProperties()).andReturn(new HashSet<>());
expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes();
mockStatic(AmbariServer.class);
expect(AmbariServer.getController()).andReturn(controller).anyTimes();