You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2016/02/05 17:08:39 UTC
ambari git commit: AMBARI-14745. Ambari server throws error when
unused configs are present in blueprint. (Oliver Szabo via rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk 1eb846c15 -> cd35e806c
AMBARI-14745. Ambari server throws error when unused configs are present in blueprint. (Oliver Szabo via rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cd35e806
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cd35e806
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cd35e806
Branch: refs/heads/trunk
Commit: cd35e806c0cfd63056afea61d2168e74cf1c1ae8
Parents: 1eb846c
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Fri Feb 5 11:07:51 2016 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Fri Feb 5 11:08:23 2016 -0500
----------------------------------------------------------------------
.../topology/ClusterConfigurationRequest.java | 28 ++++++++
.../ambari/server/topology/Configuration.java | 15 ++++
.../ClusterConfigurationRequestTest.java | 75 +++++++++++++++++---
.../ClusterInstallWithoutStartTest.java | 2 +
.../server/topology/ConfigurationTest.java | 14 ++++
.../server/topology/TopologyManagerTest.java | 2 +
6 files changed, 128 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 70fa880..c9120de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -30,6 +30,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurati
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,11 +75,38 @@ public class ClusterConfigurationRequest {
// set initial configuration (not topology resolved)
this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology);
this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor;
+ removeOrphanConfigTypes(clusterTopology);
if (setInitial) {
setConfigurationsOnCluster(clusterTopology, TopologyManager.INITIAL_CONFIG_TAG, Collections.<String>emptySet());
}
}
+ /**
+ * Remove config-types, if there is no any services related to them (except cluster-env and global).
+ */
+ private void removeOrphanConfigTypes(ClusterTopology clusterTopology) {
+ Configuration configuration = clusterTopology.getConfiguration();
+ Collection<String> configTypes = configuration.getAllConfigTypes();
+ for (String configType : configTypes) {
+ if (!configType.equals("cluster-env") && !configType.equals("global")) {
+ String service = clusterTopology.getBlueprint().getStack().getServiceForConfigType(configType);
+ if (!clusterTopology.getBlueprint().getServices().contains(service)) {
+ configuration.removeConfigType(configType);
+ LOG.info("Not found any service for config type '{}'. It will be removed from configuration.", configType);
+ Map<String, HostGroupInfo> hostGroupInfoMap = clusterTopology.getHostGroupInfo();
+ if (MapUtils.isNotEmpty(hostGroupInfoMap)) {
+ for (Map.Entry<String, HostGroupInfo> hostGroupInfo : hostGroupInfoMap.entrySet()) {
+ if (hostGroupInfo.getValue().getConfiguration() != null) {
+ hostGroupInfo.getValue().getConfiguration().removeConfigType(configType);
+ LOG.info("Not found any service for config type '{}'. It will be removed from host group scoped configuration.", configType);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology topology, boolean setInitial, StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor, boolean configureSecurity) {
this(ambariContext, topology, setInitial, stackAdvisorBlueprintProcessor);
this.configureSecurity = configureSecurity;
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 108ff74..79281b4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -344,4 +344,19 @@ public class Configuration {
public void setParentConfiguration(Configuration parent) {
parentConfiguration = parent;
}
+
+ /**
+ * Remove all occurrences of a config type
+ */
+ public void removeConfigType(String configType) {
+ if (properties != null && properties.containsKey(configType)) {
+ properties.remove(configType);
+ }
+ if (attributes != null && attributes.containsKey(configType)) {
+ attributes.remove(configType);
+ }
+ if (parentConfiguration != null) {
+ parentConfiguration.removeConfigType(configType);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index 2e7bd3b..ece1287 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -28,6 +28,17 @@ import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.collect.Maps;
+import org.easymock.EasyMock;
import org.easymock.Capture;
import org.easymock.CaptureType;
import org.easymock.EasyMockRule;
@@ -40,14 +51,6 @@ import org.powermock.api.easymock.PowerMock;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
@@ -56,6 +59,8 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.newCapture;
import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.easymock.EasyMock.capture;
import static org.junit.Assert.assertEquals;
@@ -184,6 +189,7 @@ public class ClusterConfigurationRequestTest {
expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
+ expect(stack.getServiceForConfigType(anyString())).andReturn("KERBEROS").anyTimes();
expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType")
).anyTimes();
expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes();
@@ -308,4 +314,57 @@ public class ClusterConfigurationRequestTest {
}
+ @Test
+ public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() {
+ // GIVEN
+ Configuration configuration = createConfigurations();
+ Set<String> services = new HashSet<String>();
+ services.add("HDFS");
+ services.add("RANGER");
+ Map<String, HostGroupInfo> hostGroupInfoMap = Maps.newHashMap();
+ HostGroupInfo hg1 = new HostGroupInfo("hg1");
+ hg1.setConfiguration(createConfigurations());
+ hostGroupInfoMap.put("hg1", hg1);
+
+ expect(topology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(topology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap);
+ expect(blueprint.getStack()).andReturn(stack).anyTimes();
+ expect(blueprint.getServices()).andReturn(services).anyTimes();
+ expect(stack.getServiceForConfigType("hdfs-site")).andReturn("HDFS").anyTimes();
+ expect(stack.getServiceForConfigType("admin-properties")).andReturn("RANGER").anyTimes();
+ expect(stack.getServiceForConfigType("yarn-site")).andReturn("YARN").anyTimes();
+
+ EasyMock.replay(stack, blueprint, topology);
+ // WHEN
+ new ClusterConfigurationRequest(ambariContext, topology, false, stackAdvisorBlueprintProcessor);
+ // THEN
+ assertFalse(configuration.getFullProperties().containsKey("yarn-site"));
+ assertFalse(configuration.getFullAttributes().containsKey("yarn-site"));
+ assertTrue(configuration.getFullAttributes().containsKey("hdfs-site"));
+ assertTrue(configuration.getFullProperties().containsKey("cluster-env"));
+ assertTrue(configuration.getFullProperties().containsKey("global"));
+ assertFalse(hg1.getConfiguration().getFullAttributes().containsKey("yarn-site"));
+ verify(stack, blueprint, topology);
+ }
+
+ private Configuration createConfigurations() {
+ Map<String, Map<String, String>> firstLevelConfig = Maps.newHashMap();
+ firstLevelConfig.put("hdfs-site", new HashMap<String, String>());
+ firstLevelConfig.put("yarn-site", new HashMap<String, String>());
+ firstLevelConfig.put("cluster-env", new HashMap<String, String>());
+ firstLevelConfig.put("global", new HashMap<String, String>());
+
+ Map<String, Map<String, Map<String, String>>> firstLevelAttributes = Maps.newHashMap();
+ firstLevelAttributes.put("hdfs-site", new HashMap<String, Map<String, String>>());
+
+ Map<String, Map<String, String>> secondLevelConfig = Maps.newHashMap();
+ secondLevelConfig.put("admin-properties", new HashMap<String, String>());
+ Map<String, Map<String, Map<String, String>>> secondLevelAttributes = Maps.newHashMap();
+ secondLevelAttributes.put("admin-properties", new HashMap<String, Map<String, String>>());
+ secondLevelAttributes.put("yarn-site", new HashMap<String, Map<String, String>>());
+
+ Configuration secondLevelConf = new Configuration(secondLevelConfig, secondLevelAttributes);
+ return new Configuration(firstLevelConfig, firstLevelAttributes, secondLevelConf);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
index dd66b1b..156580a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
@@ -251,6 +251,8 @@ public class ClusterInstallWithoutStartTest {
expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
+ expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+ expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.<String>emptySet()).anyTimes();
expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.<String>emptySet()).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
index e971e03..c4d70b5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -346,6 +346,20 @@ public class ConfigurationTest {
assertNull(configuration.getPropertyValue("type1", "XXXXX"));
}
+ @Test
+ public void testRemoveConfigTypes() {
+ Configuration configuration = createConfigurationWithParents_PropsOnly();
+ configuration.removeConfigType("type1");
+ assertNull(configuration.getProperties().get("type1"));
+ }
+
+ @Test
+ public void testRemoveConfigTypesForAttributes() {
+ Configuration configuration = createConfigurationWithParents_PropsOnly();
+ configuration.removeConfigType("type1");
+ assertNull(configuration.getAttributes().get("type1"));
+ }
+
private Configuration createConfigurationWithParents_PropsOnly() {
// parents parent config properties
Map<String, Map<String, String>> parentParentProperties = new HashMap<String, Map<String, String>>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd35e806/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
index 7810f92..69c1935 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java
@@ -228,6 +228,8 @@ public class TopologyManagerTest {
expect(stack.getComponents()).andReturn(serviceComponents).anyTimes();
expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes();
expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes();
+ expect(stack.getServiceForConfigType("service1-site")).andReturn("service1");
+ expect(stack.getServiceForConfigType("service2-site")).andReturn("service2");
expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();