You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2016/02/01 23:36:35 UTC
ambari git commit: AMBARI-14722. Blueprints Stack advisor should
consider user provided inputs on ONLY_STACK_DEFAULTS_APPLY strategy. (Oliver
Szabo via rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk d456b1580 -> 9991ad88b
AMBARI-14722. Blueprints Stack advisor should consider user provided inputs on ONLY_STACK_DEFAULTS_APPLY strategy. (Oliver Szabo via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9991ad88
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9991ad88
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9991ad88
Branch: refs/heads/trunk
Commit: 9991ad88b8a226fe6fce8b28b17feec314112fa0
Parents: d456b15
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Mon Feb 1 17:35:46 2016 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Mon Feb 1 17:35:46 2016 -0500
----------------------------------------------------------------------
.../StackAdvisorBlueprintProcessor.java | 66 +++++++++++-
.../StackAdvisorBlueprintProcessorTest.java | 100 ++++++++++++++++++-
2 files changed, 160 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9991ad88/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
index 337ad06..ea9fffb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.api.services.stackadvisor;
import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -36,8 +37,13 @@ import org.apache.ambari.server.api.services.stackadvisor.recommendations.Recomm
import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BlueprintConfigurations;
import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.orm.entities.BlueprintConfiguration;
+import org.apache.ambari.server.state.ValueAttributesInfo;
import org.apache.ambari.server.topology.AdvisedConfiguration;
+import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.ClusterTopology;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
+import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.HostGroupInfo;
import org.slf4j.Logger;
@@ -161,13 +167,71 @@ public class StackAdvisorBlueprintProcessor {
Preconditions.checkArgument(response.getRecommendations().getBlueprint().getConfigurations() != null,
"Configurations are missing from the recommendation blueprint response.");
+ Map<String, Map<String, String>> userProvidedProperties = getUserProvidedProperties(topology);
Map<String, BlueprintConfigurations> recommendedConfigurations =
response.getRecommendations().getBlueprint().getConfigurations();
for (Map.Entry<String, BlueprintConfigurations> configEntry : recommendedConfigurations.entrySet()) {
String configType = configEntry.getKey();
- BlueprintConfigurations blueprintConfig = configEntry.getValue();
+ BlueprintConfigurations blueprintConfig = filterBlueprintConfig(configType, configEntry.getValue(),
+ userProvidedProperties, topology);
topology.getAdvisedConfigurations().put(configType, new AdvisedConfiguration(
blueprintConfig.getProperties(), blueprintConfig.getPropertyAttributes()));
}
}
+
+ /**
+ * Gather user defined properties. (keep that only which is not included in the stack defaults or it overrides the stack default value)
+ */
+ private Map<String, Map<String, String>> getUserProvidedProperties(ClusterTopology topology) {
+ Map<String, Map<String, String>> userProvidedProperties = Maps.newHashMap();
+ Blueprint blueprint = topology.getBlueprint();
+ Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices());
+
+ Map<String, Map<String, String>> stackDefaultProps = stackDefaults.getProperties();
+ Map<String, Map<String, String>> fullConfig = topology.getConfiguration().getFullProperties();
+
+ for (Map.Entry<String, Map<String, String>> configGroup : fullConfig.entrySet()) {
+ String configType = configGroup.getKey();
+ Map<String, String> configsToAdd = Maps.newHashMap();
+ for (Map.Entry<String, String> configProp : configGroup.getValue().entrySet()) {
+ if (stackDefaultProps.containsKey(configType) && stackDefaultProps.get(configType).containsKey(configProp.getKey())) {
+ String originalValue = stackDefaultProps.get(configType).get(configProp.getKey());
+ if (originalValue != null && !originalValue.equals(configProp.getValue())) {
+ configsToAdd.put(configProp.getKey(), configProp.getValue());
+ }
+ } else {
+ configsToAdd.put(configProp.getKey(), configProp.getValue());
+ }
+ }
+ if (!configsToAdd.isEmpty()) {
+ userProvidedProperties.put(configGroup.getKey(), configsToAdd);
+ }
+ }
+
+ return userProvidedProperties;
+ }
+
+ /**
+ * Remove user defined properties from stack advisor output in case of it applies only on the stack defaults
+ */
+ private BlueprintConfigurations filterBlueprintConfig(String configType, BlueprintConfigurations config,
+ Map<String, Map<String, String>> userProvidedProperties,
+ ClusterTopology topology) {
+ if (topology.getConfigRecommendationStrategy() == ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY) {
+ if (userProvidedProperties.containsKey(configType)) {
+ BlueprintConfigurations newConfig = new BlueprintConfigurations();
+ Map<String, String> filteredProps = Maps.filterKeys(config.getProperties(),
+ Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+ newConfig.setProperties(Maps.newHashMap(filteredProps));
+
+ if (config.getPropertyAttributes() != null) {
+ Map<String, ValueAttributesInfo> filteredAttributes = Maps.filterKeys(config.getPropertyAttributes(),
+ Predicates.not(Predicates.in(userProvidedProperties.get(configType).keySet())));
+ newConfig.setPropertyAttributes(Maps.newHashMap(filteredAttributes));
+ }
+ return newConfig;
+ }
+ }
+ return config;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/9991ad88/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
index 60a8dde..c4e694b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
@@ -20,7 +20,7 @@ package org.apache.ambari.server.api.services.stackadvisor;
import com.google.common.collect.Maps;
import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
-import org.apache.ambari.server.topology.Component;
+import org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroup;
import static org.easymock.EasyMock.anyObject;
@@ -29,6 +29,7 @@ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
@@ -77,21 +78,93 @@ public class StackAdvisorBlueprintProcessorTest {
expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
+ expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY);
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(stack.getVersion()).andReturn("2.3").anyTimes();
expect(stack.getName()).andReturn("HDP").anyTimes();
+ expect(stack.getConfiguration(Arrays.asList("HDFS", "YARN", "HIVE"))).andReturn(createStackDefaults()).anyTimes();
expect(blueprint.getServices()).andReturn(Arrays.asList("HDFS", "YARN", "HIVE")).anyTimes();
expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes();
expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes();
expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse());
- expect(configuration.getFullProperties()).andReturn(createProps());
+ expect(configuration.getFullProperties()).andReturn(createProps()).anyTimes();
+
+ replay(clusterTopology, blueprint, stack, hostGroup, configuration, stackAdvisorHelper);
+ // WHEN
+ underTest.adviseConfiguration(clusterTopology);
+ // THEN
+ assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey1"));
+ assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey3"));
+ assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey2"));
+ assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey3"));
+ assertEquals("dummyValue", advisedConfigurations.get("core-site").getProperties().get("dummyKey1"));
+ assertEquals(Boolean.toString(true), advisedConfigurations.get("core-site")
+ .getPropertyValueAttributes().get("dummyKey2").getDelete());
+ }
+
+ @Test
+ public void testAdviseConfigurationWithOnlyStackDefaultsApply() throws StackAdvisorException, ConfigurationTopologyException {
+ // GIVEN
+ Map<String, AdvisedConfiguration> advisedConfigurations = new HashMap<String, AdvisedConfiguration>();
+ expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
+ expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
+ expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
+ expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY);
+ expect(blueprint.getStack()).andReturn(stack).anyTimes();
+ expect(stack.getVersion()).andReturn("2.3").anyTimes();
+ expect(stack.getName()).andReturn("HDP").anyTimes();
+ expect(stack.getConfiguration(Arrays.asList("HDFS", "YARN", "HIVE"))).andReturn(createStackDefaults()).anyTimes();
+ expect(blueprint.getServices()).andReturn(Arrays.asList("HDFS", "YARN", "HIVE")).anyTimes();
+ expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes();
+ expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes();
+ expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse());
+ expect(configuration.getFullProperties()).andReturn(createProps()).anyTimes();
+
+ replay(clusterTopology, blueprint, stack, hostGroup, configuration, stackAdvisorHelper);
+ // WHEN
+ underTest.adviseConfiguration(clusterTopology);
+ // THEN
+ assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey1"));
+ assertFalse(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey3"));
+ assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey2"));
+ assertFalse(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey3"));
+ assertEquals("dummyValue", advisedConfigurations.get("core-site").getProperties().get("dummyKey1"));
+ assertEquals(Boolean.toString(true), advisedConfigurations.get("core-site")
+ .getPropertyValueAttributes().get("dummyKey2").getDelete());
+ }
+
+ @Test
+ public void testAdviseConfigurationWithOnlyStackDefaultsApplyWhenNoUserInputForDefault() throws StackAdvisorException, ConfigurationTopologyException {
+ // GIVEN
+ Map<String, Map<String, String>> props = createProps();
+ props.get("core-site").put("dummyKey3", "stackDefaultValue");
+ Map<String, AdvisedConfiguration> advisedConfigurations = new HashMap<String, AdvisedConfiguration>();
+ expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
+ expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
+ expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
+ expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY);
+ expect(blueprint.getStack()).andReturn(stack).anyTimes();
+ expect(stack.getVersion()).andReturn("2.3").anyTimes();
+ expect(stack.getName()).andReturn("HDP").anyTimes();
+ expect(stack.getConfiguration(Arrays.asList("HDFS", "YARN", "HIVE"))).andReturn(createStackDefaults()).anyTimes();
+ expect(blueprint.getServices()).andReturn(Arrays.asList("HDFS", "YARN", "HIVE")).anyTimes();
+ expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes();
+ expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes();
+ expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse());
+ expect(configuration.getFullProperties()).andReturn(props).anyTimes();
replay(clusterTopology, blueprint, stack, hostGroup, configuration, stackAdvisorHelper);
// WHEN
underTest.adviseConfiguration(clusterTopology);
// THEN
assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey1"));
+ assertTrue(advisedConfigurations.get("core-site").getProperties().containsKey("dummyKey3"));
assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey2"));
+ assertTrue(advisedConfigurations.get("core-site").getPropertyValueAttributes().containsKey("dummyKey3"));
assertEquals("dummyValue", advisedConfigurations.get("core-site").getProperties().get("dummyKey1"));
assertEquals(Boolean.toString(true), advisedConfigurations.get("core-site")
.getPropertyValueAttributes().get("dummyKey2").getDelete());
@@ -157,6 +230,7 @@ public class StackAdvisorBlueprintProcessorTest {
Map<String, Map<String, String>> props = Maps.newHashMap();
Map<String, String> siteProps = Maps.newHashMap();
siteProps.put("myprop", "myvalue");
+ siteProps.put("dummyKey3", "userinput");
props.put("core-site", siteProps);
return props;
}
@@ -178,6 +252,18 @@ public class StackAdvisorBlueprintProcessorTest {
return hostGroupInfoMap;
}
+ private Configuration createStackDefaults() {
+ Map<String, Map<String, String>> stackDefaultProps =
+ new HashMap<String, Map<String, String>>();
+ Map<String, String> coreSiteDefault = new HashMap<String, String>();
+ coreSiteDefault.put("dummyKey3", "stackDefaultValue");
+ stackDefaultProps.put("core-site", coreSiteDefault);
+
+ Map<String, Map<String, Map<String, String>>> stackDefaultAttributes =
+ new HashMap<String, Map<String, Map<String, String>>>();
+ return new Configuration(stackDefaultProps, stackDefaultAttributes);
+ }
+
private RecommendationResponse createRecommendationResponse() {
RecommendationResponse response = new RecommendationResponse();
RecommendationResponse.Recommendation recommendations = new RecommendationResponse.Recommendation();
@@ -188,11 +274,15 @@ public class StackAdvisorBlueprintProcessorTest {
new RecommendationResponse.BlueprintConfigurations();
Map<String, String> properties = new HashMap<String, String>();
properties.put("dummyKey1", "dummyValue");
+ properties.put("dummyKey3", "dummyValue-override");
blueprintConfig.setProperties(properties);
Map<String, ValueAttributesInfo> propAttributes = new HashMap<String, ValueAttributesInfo>();
- ValueAttributesInfo valueAttributesInfo = new ValueAttributesInfo();
- valueAttributesInfo.setDelete("true");
- propAttributes.put("dummyKey2", valueAttributesInfo);
+ ValueAttributesInfo valueAttributesInfo1 = new ValueAttributesInfo();
+ ValueAttributesInfo valueAttributesInfo2 = new ValueAttributesInfo();
+ valueAttributesInfo1.setDelete("true");
+ valueAttributesInfo2.setDelete("true");
+ propAttributes.put("dummyKey2", valueAttributesInfo1);
+ propAttributes.put("dummyKey3", valueAttributesInfo2);
blueprintConfig.setPropertyAttributes(propAttributes);
blueprintConfigurationsMap.put("core-site", blueprintConfig);
blueprint.setConfigurations(blueprintConfigurationsMap);