You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/04/17 20:18:17 UTC
[23/34] ambari git commit: AMBARI-20628. Ambari doesn't set
properties correctly (magyari_sandor)
AMBARI-20628. Ambari doesn't set properties correctly (magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bf637950
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bf637950
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bf637950
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: bf637950901f2b0f213c1cc149f5e49b8a8d0968
Parents: c57300a
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Thu Apr 13 17:04:14 2017 +0200
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Fri Apr 14 10:20:09 2017 +0200
----------------------------------------------------------------------
.../StackAdvisorBlueprintProcessor.java | 55 ++++----------------
.../topology/ClusterConfigurationRequest.java | 8 +--
.../StackAdvisorBlueprintProcessorTest.java | 2 -
3 files changed, 16 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 0abcc14..d306e25 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -32,10 +32,8 @@ import org.apache.ambari.server.controller.internal.ConfigurationTopologyExcepti
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.topology.AdvisedConfiguration;
-import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
-import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.slf4j.Logger;
@@ -75,13 +73,13 @@ public class StackAdvisorBlueprintProcessor {
/**
* Recommend configurations by the stack advisor, then store the results in cluster topology.
* @param clusterTopology cluster topology instance
- * @param existingConfigurations Existing configurations of cluster
+ * @param userProvidedConfigurations User configurations of cluster provided in Blueprint + Cluster template
*/
- public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map<String, String>> existingConfigurations) throws ConfigurationTopologyException {
+ public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map<String, String>> userProvidedConfigurations) throws ConfigurationTopologyException {
StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, StackAdvisorRequestType.CONFIGURATIONS);
try {
RecommendationResponse response = stackAdvisorHelper.recommend(request);
- addAdvisedConfigurationsToTopology(response, clusterTopology, existingConfigurations);
+ addAdvisedConfigurationsToTopology(response, clusterTopology, userProvidedConfigurations);
} catch (StackAdvisorException e) {
throw new ConfigurationTopologyException(RECOMMENDATION_FAILED, e);
} catch (IllegalArgumentException e) {
@@ -94,7 +92,7 @@ public class StackAdvisorBlueprintProcessor {
Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,
- hgHostsMap);
+ hgHostsMap);
return StackAdvisorRequest.StackAdvisorRequestBuilder
.forStack(stack.getName(), stack.getVersion())
.forServices(new ArrayList<>(clusterTopology.getBlueprint().getServices()))
@@ -167,7 +165,7 @@ public class StackAdvisorBlueprintProcessor {
}
private void addAdvisedConfigurationsToTopology(RecommendationResponse response,
- ClusterTopology topology, Map<String, Map<String, String>> existingConfigurations) {
+ ClusterTopology topology, Map<String, Map<String, String>> userProvidedConfigurations) {
Preconditions.checkArgument(response.getRecommendations() != null,
"Recommendation response is empty.");
Preconditions.checkArgument(response.getRecommendations().getBlueprint() != null,
@@ -175,67 +173,36 @@ public class StackAdvisorBlueprintProcessor {
Preconditions.checkArgument(response.getRecommendations().getBlueprint().getConfigurations() != null,
"Configurations are missing from the recommendation blueprint response.");
- Map<String, Map<String, String>> userProvidedProperties = getUserProvidedProperties(topology, existingConfigurations);
Map<String, BlueprintConfigurations> recommendedConfigurations =
response.getRecommendations().getBlueprint().getConfigurations();
for (Map.Entry<String, BlueprintConfigurations> configEntry : recommendedConfigurations.entrySet()) {
String configType = configEntry.getKey();
BlueprintConfigurations blueprintConfig = filterBlueprintConfig(configType, configEntry.getValue(),
- userProvidedProperties, topology);
+ userProvidedConfigurations, topology);
topology.getAdvisedConfigurations().put(configType, new AdvisedConfiguration(
blueprintConfig.getProperties(), blueprintConfig.getPropertyAttributes()));
}
}
/**
- * Gather user defined properties. (keep that only which is not included in the stack defaults or it overrides the stack default value)
- */
- private Map<String, Map<String, String>> getUserProvidedProperties(ClusterTopology topology, Map<String, Map<String, String>> existingConfigurations) {
- Map<String, Map<String, String>> userProvidedProperties = Maps.newHashMap();
- Blueprint blueprint = topology.getBlueprint();
- Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
- Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
-
- for (Map.Entry<String, Map<String, String>> configGroup : existingConfigurations.entrySet()) {
- String configType = configGroup.getKey();
- Map<String, String> configsToAdd = Maps.newHashMap();
- for (Map.Entry<String, String> configProp : configGroup.getValue().entrySet()) {
- if (stackDefaultProps.containsKey(configType) && stackDefaultProps.get(configType).containsKey(configProp.getKey())) {
- String originalValue = stackDefaultProps.get(configType).get(configProp.getKey());
- if (originalValue != null && !originalValue.equals(configProp.getValue())) {
- configsToAdd.put(configProp.getKey(), configProp.getValue());
- }
- } else {
- configsToAdd.put(configProp.getKey(), configProp.getValue());
- }
- }
- if (!configsToAdd.isEmpty()) {
- userProvidedProperties.put(configGroup.getKey(), configsToAdd);
- }
- }
-
- return userProvidedProperties;
- }
-
- /**
- * Remove user defined properties from stack advisor output in case of ONLY_STACK_DEFAULTS_APPLY or
+ * Remove user defined properties from Stack Advisor output in case of ONLY_STACK_DEFAULTS_APPLY or
* ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES.
*/
private BlueprintConfigurations filterBlueprintConfig(String configType, BlueprintConfigurations config,
- Map<String, Map<String, String>> userProvidedProperties,
+ Map<String, Map<String, String>> userProvidedConfigurations,
ClusterTopology topology) {
if (topology.getConfigRecommendationStrategy() == ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY ||
topology.getConfigRecommendationStrategy() == ConfigRecommendationStrategy
.ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES) {
- if (userProvidedProperties.containsKey(configType)) {
+ if (userProvidedConfigurations.containsKey(configType)) {
BlueprintConfigurations newConfig = new BlueprintConfigurations();
Map<String, String> filteredProps = Maps.filterKeys(config.getProperties(),
- Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+ Predicates.not(Predicates.in(userProvidedConfigurations.get(configType).keySet())));
newConfig.setProperties(Maps.newHashMap(filteredProps));
if (config.getPropertyAttributes() != null) {
Map<String, ValueAttributesInfo> filteredAttributes = Maps.filterKeys(config.getPropertyAttributes(),
- Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+ Predicates.not(Predicates.in(userProvidedConfigurations.get(configType).keySet())));
newConfig.setPropertyAttributes(Maps.newHashMap(filteredAttributes));
}
return newConfig;
http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 5913f4b..0e7d70b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -137,17 +137,19 @@ public class ClusterConfigurationRequest {
// this will update the topo cluster config and all host group configs in the cluster topology
Set<String> updatedConfigTypes = new HashSet<>();
- Configuration clusterConfiguration = clusterTopology.getConfiguration();
- Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
+ Map<String, Map<String, String>> userProvidedConfigurations = clusterTopology.getConfiguration().getFullProperties(1);
try {
if (configureSecurity) {
+ Configuration clusterConfiguration = clusterTopology.getConfiguration();
+ Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties();
updatedConfigTypes.addAll(configureKerberos(clusterConfiguration, existingConfigurations));
}
// obtain recommended configurations before config updates
if (!ConfigRecommendationStrategy.NEVER_APPLY.equals(this.clusterTopology.getConfigRecommendationStrategy())) {
- stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology, existingConfigurations);
+ // get merged properties form Blueprint & cluster template (this doesn't contains stack default values)
+ stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology, userProvidedConfigurations);
}
updatedConfigTypes.addAll(configurationProcessor.doUpdateForClusterCreate());
http://git-wip-us.apache.org/repos/asf/ambari/blob/bf637950/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
index 49f070a..3c5f8ed 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
@@ -165,9 +165,7 @@ public class StackAdvisorBlueprintProcessorTest {
underTest.adviseConfiguration(clusterTopology, props);
// THEN
assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey1"));
- assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey3"));
assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey2"));
- assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey3"));
assertEquals("dummyValue", advisedConfigurations.get("core-site").getProperties().get("dummyKey1"));
assertEquals(Boolean.toString(true), advisedConfigurations.get("core-site")
.getPropertyValueAttributes().get("dummyKey2").getDelete());