You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by js...@apache.org on 2015/06/05 23:30:38 UTC
[1/2] ambari git commit: AMBARI-11737. Fix setting of blueprint host
group scoped configuration
Repository: ambari
Updated Branches:
refs/heads/trunk 7eaf73c30 -> 4afac3006
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
index 7dcb7be..1a234da 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.topology;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -29,25 +30,35 @@ import java.util.Set;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigGroupRequest;
import org.apache.ambari.server.controller.ServiceComponentRequest;
import org.apache.ambari.server.controller.ServiceRequest;
import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
+import org.apache.ambari.server.controller.internal.ConfigGroupResourceProvider;
import org.apache.ambari.server.controller.internal.HostComponentResourceProvider;
import org.apache.ambari.server.controller.internal.HostResourceProvider;
import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.controller.predicate.EqualsPredicate;
+import org.apache.ambari.server.controller.spi.ClusterController;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.Request;
+import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.easymock.Capture;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.createStrictMock;
import static org.easymock.EasyMock.expect;
@@ -64,24 +75,44 @@ import static org.junit.Assert.assertTrue;
//todo: switch over to EasyMockSupport
public class AmbariContextTest {
+ private static final String BP_NAME = "testBP";
private static final String CLUSTER_NAME = "testCluster";
private static final String STACK_NAME = "testStack";
private static final String STACK_VERSION = "testVersion";
+ private static final String HOST_GROUP_1 = "group1";
+ private static final String HOST_GROUP_2 = "group2";
+ private static final String HOST1 = "host1";
+ private static final String HOST2 = "host2";
+ StackId stackId = new StackId(STACK_NAME, STACK_VERSION);
private static final AmbariContext context = new AmbariContext();
private static final AmbariManagementController controller = createStrictMock(AmbariManagementController.class);
+ private static final ClusterController clusterController = createStrictMock(ClusterController.class);
private static final HostResourceProvider hostResourceProvider = createStrictMock(HostResourceProvider.class);
private static final ServiceResourceProvider serviceResourceProvider = createStrictMock(ServiceResourceProvider.class);
private static final ComponentResourceProvider componentResourceProvider = createStrictMock(ComponentResourceProvider.class);
private static final HostComponentResourceProvider hostComponentResourceProvider = createStrictMock(HostComponentResourceProvider.class);
+ private static final ConfigGroupResourceProvider configGroupResourceProvider = createStrictMock(ConfigGroupResourceProvider.class);
private static final ClusterTopology topology = createNiceMock(ClusterTopology.class);
private static final Blueprint blueprint = createNiceMock(Blueprint.class);
private static final Stack stack = createNiceMock(Stack.class);
private static final Clusters clusters = createStrictMock(Clusters.class);
private static final Cluster cluster = createStrictMock(Cluster.class);
+ private static final HostGroupInfo group1Info = createNiceMock(HostGroupInfo.class);
+ private static final ConfigHelper configHelper = createNiceMock(ConfigHelper.class);
+ private static final ConfigGroup configGroup1 = createMock(ConfigGroup.class);
+ private static final ConfigGroup configGroup2 = createMock(ConfigGroup.class);
+ private static final Host host1 = createNiceMock(Host.class);
+ private static final Host host2 = createNiceMock(Host.class);
private static final Collection<String> blueprintServices = new HashSet<String>();
private static final Map<String, Service> clusterServices = new HashMap<String, Service>();
+ private static final Map<Long, ConfigGroup> configGroups = new HashMap<Long, ConfigGroup>();
+ private Configuration bpConfiguration = null;
+ private Configuration group1Configuration = null;
+ private static final Collection<String> group1Hosts = Arrays.asList(HOST1, HOST2);
+
+ private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<Set<ConfigGroupRequest>>();
@Before
public void setUp() throws Exception {
@@ -91,6 +122,10 @@ public class AmbariContextTest {
f.setAccessible(true);
f.set(null, controller);
+ f = clazz.getDeclaredField("clusterController");
+ f.setAccessible(true);
+ f.set(null, clusterController);
+
f = clazz.getDeclaredField("hostResourceProvider");
f.setAccessible(true);
f.set(null, hostResourceProvider);
@@ -107,34 +142,80 @@ public class AmbariContextTest {
f.setAccessible(true);
f.set(null, hostComponentResourceProvider);
+ // bp configuration
+ Map<String, Map<String, String>> bpProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> bpType1Props = new HashMap<String, String>();
+ bpProperties.put("type1", bpType1Props);
+ bpType1Props.put("prop1", "val1");
+ bpType1Props.put("prop2", "val2");
+ bpConfiguration = new Configuration(bpProperties, null);
+
+ // host group 1 configuration
+ Map<String, Map<String, String>> group1Properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> type1Props = new HashMap<String, String>();
+ group1Properties.put("type1", type1Props);
+ type1Props.put("prop1", "val1.2");
+ type1Props.put("prop3", "val3");
+ group1Configuration = new Configuration(group1Properties, null, bpConfiguration);
+
+ // config type -> service mapping
+ Map<String, String> configTypeServiceMapping = new HashMap<String, String>();
+ configTypeServiceMapping.put("type1", "service1");
+
+ // config groups
+ configGroups.put(1L, configGroup1);
+ configGroups.put(2L, configGroup2);
+
blueprintServices.add("service1");
blueprintServices.add("service2");
expect(topology.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
expect(topology.getBlueprint()).andReturn(blueprint).anyTimes();
+ expect(topology.getHostGroupInfo()).andReturn(Collections.singletonMap(HOST_GROUP_1, group1Info)).anyTimes();
+ expect(blueprint.getName()).andReturn(BP_NAME).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(blueprint.getServices()).andReturn(blueprintServices).anyTimes();
- expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2"));
- expect(blueprint.getComponents("service2")).andReturn(Collections.singleton("s2Component1"));
+ expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes();
+ expect(blueprint.getComponents("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes();
+ expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes();
expect(stack.getName()).andReturn(STACK_NAME).anyTimes();
expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes();
+ for (Map.Entry<String, String> entry : configTypeServiceMapping.entrySet()) {
+ expect(stack.getServiceForConfigType(entry.getKey())).andReturn(entry.getValue()).anyTimes();
+ }
+
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+ expect(controller.getConfigHelper()).andReturn(configHelper).anyTimes();
+
+ expect(clusters.getCluster(CLUSTER_NAME)).andReturn(cluster).anyTimes();
+ expect(clusters.getHost(HOST1)).andReturn(host1).anyTimes();
+ expect(clusters.getHost(HOST2)).andReturn(host2).anyTimes();
+
+ expect(group1Info.getConfiguration()).andReturn(group1Configuration).anyTimes();
+ expect(group1Info.getHostNames()).andReturn(group1Hosts).anyTimes();
+
+ expect(configGroup1.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_1)).anyTimes();
+ expect(configGroup2.getName()).andReturn(String.format("%s:%s", BP_NAME, HOST_GROUP_2)).anyTimes();
}
@After
public void tearDown() throws Exception {
- verify(controller, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
- hostComponentResourceProvider, topology, blueprint, stack, clusters, cluster);
+ verify(controller, clusterController, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
+ hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters,
+ cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2);
- reset(controller, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
- hostComponentResourceProvider, topology, blueprint, stack, clusters, cluster);
+ reset(controller, clusterController, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
+ hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters,
+ cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2);
}
private void replayAll() {
- replay(controller, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
- hostComponentResourceProvider, topology, blueprint, stack, clusters, cluster);
+ replay(controller, clusterController, hostResourceProvider, serviceResourceProvider, componentResourceProvider,
+ hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters,
+ cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2);
}
@Test
@@ -226,4 +307,59 @@ public class AmbariContextTest {
assertEquals(new EqualsPredicate<String>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME),
installPredicateCapture.getValue());
}
+
+ @Test
+ public void testRegisterHostWithConfigGroup_createNewConfigGroup() throws Exception {
+ // test specific expectations
+ expect(cluster.getConfigGroups()).andReturn(Collections.<Long, ConfigGroup>emptyMap()).once();
+ expect(clusterController.ensureResourceProvider(Resource.Type.ConfigGroup)).andReturn(configGroupResourceProvider).once();
+ //todo: for now not using return value so just returning null
+ expect(configGroupResourceProvider.createResources(capture(configGroupRequestCapture))).andReturn(null).once();
+ configHelper.moveDeprecatedGlobals(stackId, group1Configuration.getFullProperties(1), CLUSTER_NAME);
+ // replay all mocks
+ replayAll();
+
+ // test
+ context.registerHostWithConfigGroup(HOST1, topology, HOST_GROUP_1);
+
+ // assertions
+ Set<ConfigGroupRequest> configGroupRequests = configGroupRequestCapture.getValue();
+ assertEquals(1, configGroupRequests.size());
+ ConfigGroupRequest configGroupRequest = configGroupRequests.iterator().next();
+ assertEquals(CLUSTER_NAME, configGroupRequest.getClusterName());
+ assertEquals("testBP:group1", configGroupRequest.getGroupName());
+ assertEquals("service1", configGroupRequest.getTag());
+ assertEquals("Host Group Configuration", configGroupRequest.getDescription());
+ Collection<String> requestHosts = configGroupRequest.getHosts();
+ requestHosts.retainAll(group1Hosts);
+ assertEquals(group1Hosts.size(), requestHosts.size());
+
+ Map<String, Config> requestConfig = configGroupRequest.getConfigs();
+ assertEquals(1, requestConfig.size());
+ Config type1Config = requestConfig.get("type1");
+ //todo: other properties such as cluster name are not currently being explicitly set on config
+ assertEquals("type1", type1Config.getType());
+ assertEquals("group1", type1Config.getTag());
+ Map<String, String> requestProps = type1Config.getProperties();
+ assertEquals(3, requestProps.size());
+ // 1.2 is overridden value
+ assertEquals("val1.2", requestProps.get("prop1"));
+ assertEquals("val2", requestProps.get("prop2"));
+ assertEquals("val3", requestProps.get("prop3"));
+ }
+
+ @Test
+ public void testRegisterHostWithConfigGroup_registerWithExistingConfigGroup() throws Exception {
+ // test specific expectations
+ expect(cluster.getConfigGroups()).andReturn(configGroups).once();
+
+ configGroup1.addHost(host1);
+ configGroup1.persist();
+
+ // replay all mocks
+ replayAll();
+
+ // test
+ context.registerHostWithConfigGroup(HOST1, topology, HOST_GROUP_1);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
new file mode 100644
index 0000000..1b9734e
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurationTest.java
@@ -0,0 +1,469 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Configuration unit tests.
+ */
+public class ConfigurationTest {
+
+ private static final Map<String, Map<String, String>> EMPTY_PROPERTIES = new HashMap<String, Map<String, String>>();
+ private static final Map<String, Map<String, Map<String, String>>> EMPTY_ATTRIBUTES = new HashMap<String, Map<String, Map<String, String>>>();
+
+ @Test
+ public void testGetProperties_noParent() {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProperties1 = new HashMap<String, String>();
+ typeProperties1.put("prop1", "val1");
+ typeProperties1.put("prop2", "val2");
+ Map<String, String> typeProperties2 = new HashMap<String, String>();
+ typeProperties2.put("prop1", "val1");
+ typeProperties2.put("prop3", "val3");
+
+ properties.put("type1", typeProperties1);
+ properties.put("type2", typeProperties2);
+
+ Configuration configuration = new Configuration(properties, EMPTY_ATTRIBUTES);
+ assertEquals(properties, configuration.getProperties());
+ assertEquals(EMPTY_ATTRIBUTES, configuration.getAttributes());
+ }
+
+ @Test
+ public void testGetFullProperties_noParent() {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProperties1 = new HashMap<String, String>();
+ typeProperties1.put("prop1", "val1");
+ typeProperties1.put("prop2", "val2");
+ Map<String, String> typeProperties2 = new HashMap<String, String>();
+ typeProperties2.put("prop1", "val1");
+ typeProperties2.put("prop3", "val3");
+
+ properties.put("type1", typeProperties1);
+ properties.put("type2", typeProperties2);
+
+ Configuration configuration = new Configuration(properties, EMPTY_ATTRIBUTES);
+ assertEquals(properties, configuration.getFullProperties());
+ assertEquals(EMPTY_ATTRIBUTES, configuration.getAttributes());
+ }
+
+ @Test
+ public void testGetProperties_withParent() {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProperties1 = new HashMap<String, String>();
+ typeProperties1.put("prop1", "val1");
+ typeProperties1.put("prop2", "val2");
+ Map<String, String> typeProperties2 = new HashMap<String, String>();
+ typeProperties2.put("prop1", "val1");
+ typeProperties2.put("prop3", "val3");
+
+ properties.put("type1", typeProperties1);
+ properties.put("type2", typeProperties2);
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> parentTypeProperties1 = new HashMap<String, String>();
+ parentTypeProperties1.put("prop5", "val5");
+ Map<String, String> parentTypeProperties3 = new HashMap<String, String>();
+ parentTypeProperties3.put("prop6", "val6");
+
+ parentProperties.put("type1", parentTypeProperties1);
+ parentProperties.put("type3", parentTypeProperties3);
+
+ Configuration parentConfiguration = new Configuration(parentProperties, EMPTY_ATTRIBUTES);
+
+ Configuration configuration = new Configuration(properties, EMPTY_ATTRIBUTES, parentConfiguration);
+ // parent should not be reflected in getProperties() result
+ assertEquals(properties, configuration.getProperties());
+ assertEquals(EMPTY_ATTRIBUTES, configuration.getAttributes());
+ }
+
+ @Test
+ public void testGetFullProperties_withParent() {
+ Configuration configuration = createConfigurationWithParentsPropsOnly();
+ // get prop maps prior to calling getFullProperties
+ Map<String, Map<String, String>> leafProperties = configuration.getProperties();
+ Map<String, Map<String, String>> parentProperties = configuration.getParentConfiguration().getProperties();
+ Map<String, Map<String, String>> parentParentProperties = configuration.getParentConfiguration().getParentConfiguration().getProperties();
+
+ // test
+ // all parents should be reflected in getFullProperties() result
+ Map<String, Map<String, String>> fullProperties = configuration.getFullProperties();
+
+ // type1, type2, type3, type4
+ assertEquals(4, fullProperties.size());
+ // type1
+ Map<String, String> type1Props = fullProperties.get("type1");
+ assertEquals(5, type1Props.size());
+ assertEquals("val1.3", type1Props.get("prop1"));
+ assertEquals("val2.2", type1Props.get("prop2"));
+ assertEquals("val3.1", type1Props.get("prop3"));
+ assertEquals("val6.2", type1Props.get("prop6"));
+ assertEquals("val9.3", type1Props.get("prop9"));
+
+ //type2
+ Map<String, String> type2Props = fullProperties.get("type2");
+ assertEquals(2, type2Props.size());
+ assertEquals("val4.3", type2Props.get("prop4"));
+ assertEquals("val5.1", type2Props.get("prop5"));
+
+ //type3
+ Map<String, String> type3Props = fullProperties.get("type3");
+ assertEquals(2, type3Props.size());
+ assertEquals("val7.3", type3Props.get("prop7"));
+ assertEquals("val8.2", type3Props.get("prop8"));
+
+ //type4
+ Map<String, String> type4Props = fullProperties.get("type4");
+ assertEquals(2, type4Props.size());
+ assertEquals("val10.3", type4Props.get("prop10"));
+ assertEquals("val11.3", type4Props.get("prop11"));
+
+ // ensure that underlying property map is not modified in getFullProperties
+ assertEquals(leafProperties, configuration.getProperties());
+ assertEquals(parentProperties, configuration.getParentConfiguration().getProperties());
+ assertEquals(parentParentProperties, configuration.getParentConfiguration().getParentConfiguration().getProperties());
+
+ assertEquals(EMPTY_ATTRIBUTES, configuration.getAttributes());
+
+ Collection<String> configTypes = configuration.getAllConfigTypes();
+ assertEquals(4, configTypes.size());
+ assertTrue(configTypes.containsAll(Arrays.asList("type1", "type2", "type3", "type4")));
+ }
+
+ @Test
+ public void testGetFullProperties_withParent_specifyDepth() {
+ Configuration configuration = createConfigurationWithParentsPropsOnly();
+ // get prop maps prior to calling getFullProperties
+ Map<String, Map<String, String>> leafProperties = configuration.getProperties();
+ Map<String, Map<String, String>> parentProperties = configuration.getParentConfiguration().getProperties();
+ Map<String, Map<String, String>> parentParentProperties = configuration.getParentConfiguration().getParentConfiguration().getProperties();
+
+ // test
+ // specify a depth of 1 which means to include only 1 level up the parent chain
+ Map<String, Map<String, String>> fullProperties = configuration.getFullProperties(1);
+
+ // type1, type2, type3, type4
+ assertEquals(4, fullProperties.size());
+ // type1
+ Map<String, String> type1Props = fullProperties.get("type1");
+ assertEquals(4, type1Props.size());
+ assertEquals("val1.3", type1Props.get("prop1"));
+ assertEquals("val2.2", type1Props.get("prop2"));
+ assertEquals("val6.2", type1Props.get("prop6"));
+ assertEquals("val9.3", type1Props.get("prop9"));
+
+ //type2
+ Map<String, String> type2Props = fullProperties.get("type2");
+ assertEquals(1, type2Props.size());
+ assertEquals("val4.3", type2Props.get("prop4"));
+
+ //type3
+ Map<String, String> type3Props = fullProperties.get("type3");
+ assertEquals(2, type3Props.size());
+ assertEquals("val7.3", type3Props.get("prop7"));
+ assertEquals("val8.2", type3Props.get("prop8"));
+
+ //type4
+ Map<String, String> type4Props = fullProperties.get("type4");
+ assertEquals(2, type4Props.size());
+ assertEquals("val10.3", type4Props.get("prop10"));
+ assertEquals("val11.3", type4Props.get("prop11"));
+
+ // ensure that underlying property maps are not modified in getFullProperties
+ assertEquals(leafProperties, configuration.getProperties());
+ assertEquals(parentProperties, configuration.getParentConfiguration().getProperties());
+ assertEquals(parentParentProperties, configuration.getParentConfiguration().getParentConfiguration().getProperties());
+
+ assertEquals(EMPTY_ATTRIBUTES, configuration.getAttributes());
+ }
+
+ @Test
+ public void testGetAttributes_noParent() {
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Map<String, Map<String, String>> attributeProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> properties1 = new HashMap<String, String>();
+ properties1.put("prop1", "val1");
+ properties1.put("prop2", "val2");
+ Map<String, String> properties2 = new HashMap<String, String>();
+ properties2.put("prop1", "val3");
+ attributeProperties.put("attribute1", properties1);
+ attributeProperties.put("attribute2", properties2);
+
+ attributes.put("type1", attributeProperties);
+
+ //test
+ Configuration configuration = new Configuration(EMPTY_PROPERTIES, attributes);
+ // assert attributes
+ assertEquals(attributes, configuration.getAttributes());
+ // assert empty properties
+ assertEquals(EMPTY_PROPERTIES, configuration.getProperties());
+ }
+
+ @Test
+ public void testGetFullAttributes_withParent() {
+ Configuration configuration = createConfigurationWithParentsAttributesOnly();
+ Map<String, Map<String, Map<String, String>>> leafAttributes = configuration.getAttributes();
+ Map<String, Map<String, Map<String, String>>> parentAttributes = configuration.getParentConfiguration().getAttributes();
+ Map<String, Map<String, Map<String, String>>> parentParentAttributes = configuration.getParentConfiguration().getParentConfiguration().getAttributes();
+ // test
+ // all parents should be reflected in getFullAttributes() result
+ Map<String, Map<String, Map<String, String>>> fullAttributes = configuration.getFullAttributes();
+ assertEquals(2, fullAttributes.size());
+
+ // type 1
+ Map<String, Map<String, String>> type1Attributes = fullAttributes.get("type1");
+ // attribute1, attribute2, attribute3, attribute4
+ assertEquals(4, type1Attributes.size());
+ // attribute1
+ Map<String, String> attribute1Properties = type1Attributes.get("attribute1");
+ assertEquals(5, attribute1Properties.size());
+ assertEquals("val1.3", attribute1Properties.get("prop1"));
+ assertEquals("val2.2", attribute1Properties.get("prop2"));
+ assertEquals("val3.1", attribute1Properties.get("prop3"));
+ assertEquals("val6.2", attribute1Properties.get("prop6"));
+ assertEquals("val9.3", attribute1Properties.get("prop9"));
+
+ //attribute2
+ Map<String, String> attribute2Properties = type1Attributes.get("attribute2");
+ assertEquals(2, attribute2Properties.size());
+ assertEquals("val4.3", attribute2Properties.get("prop4"));
+ assertEquals("val5.1", attribute2Properties.get("prop5"));
+
+ //attribute3
+ Map<String, String> attribute3Properties = type1Attributes.get("attribute3");
+ assertEquals(2, attribute3Properties.size());
+ assertEquals("val7.3", attribute3Properties.get("prop7"));
+ assertEquals("val8.2", attribute3Properties.get("prop8"));
+
+ //attribute4
+ Map<String, String> attribute4Properties = type1Attributes.get("attribute4");
+ assertEquals(2, attribute4Properties.size());
+ assertEquals("val10.3", attribute4Properties.get("prop10"));
+ assertEquals("val11.3", attribute4Properties.get("prop11"));
+
+ // type 2
+ Map<String, Map<String, String>> type2Attributes = fullAttributes.get("type2");
+ // attribute100, attribute101
+ assertEquals(2, type2Attributes.size());
+
+ Map<String, String> attribute100Properties = type2Attributes.get("attribute100");
+ assertEquals(3, attribute100Properties.size());
+ assertEquals("val100.3", attribute100Properties.get("prop100"));
+ assertEquals("val101.1", attribute100Properties.get("prop101"));
+ assertEquals("val102.3", attribute100Properties.get("prop102"));
+
+ Map<String, String> attribute101Properties = type2Attributes.get("attribute101");
+ assertEquals(2, attribute101Properties.size());
+ assertEquals("val100.2", attribute101Properties.get("prop100"));
+ assertEquals("val101.1", attribute101Properties.get("prop101"));
+
+ // ensure that underlying attribute maps are not modified in getFullProperties
+ assertEquals(leafAttributes, configuration.getAttributes());
+ assertEquals(parentAttributes, configuration.getParentConfiguration().getAttributes());
+ assertEquals(parentParentAttributes, configuration.getParentConfiguration().getParentConfiguration().getAttributes());
+
+ assertEquals(EMPTY_PROPERTIES, configuration.getProperties());
+
+ Collection<String> configTypes = configuration.getAllConfigTypes();
+ assertEquals(2, configTypes.size());
+ assertTrue(configTypes.containsAll(Arrays.asList("type1", "type2")));
+ }
+
+ @Test
+ public void testGetPropertyValue() {
+ Configuration configuration = createConfigurationWithParentsPropsOnly();
+
+ assertEquals("val1.3", configuration.getPropertyValue("type1", "prop1"));
+ assertEquals("val2.2", configuration.getPropertyValue("type1", "prop2"));
+ assertEquals("val3.1", configuration.getPropertyValue("type1", "prop3"));
+ assertEquals("val4.3", configuration.getPropertyValue("type2", "prop4"));
+ assertEquals("val5.1", configuration.getPropertyValue("type2", "prop5"));
+ assertEquals("val6.2", configuration.getPropertyValue("type1", "prop6"));
+ assertEquals("val7.3", configuration.getPropertyValue("type3", "prop7"));
+ assertEquals("val8.2", configuration.getPropertyValue("type3", "prop8"));
+ assertEquals("val10.3", configuration.getPropertyValue("type4", "prop10"));
+ assertEquals("val11.3", configuration.getPropertyValue("type4", "prop11"));
+ }
+
+ @Test
+ public void testGetAttributeValue() {
+ Configuration configuration = createConfigurationWithParentsAttributesOnly();
+
+ assertEquals("val1.3", configuration.getAttributeValue("type1", "prop1", "attribute1"));
+ assertEquals("val2.2", configuration.getAttributeValue("type1", "prop2", "attribute1"));
+ assertEquals("val3.1", configuration.getAttributeValue("type1", "prop3", "attribute1"));
+ assertEquals("val4.3", configuration.getAttributeValue("type1", "prop4", "attribute2"));
+ assertEquals("val5.1", configuration.getAttributeValue("type1", "prop5", "attribute2"));
+ assertEquals("val6.2", configuration.getAttributeValue("type1", "prop6", "attribute1"));
+ assertEquals("val7.3", configuration.getAttributeValue("type1", "prop7", "attribute3"));
+ assertEquals("val8.2", configuration.getAttributeValue("type1", "prop8", "attribute3"));
+ assertEquals("val100.3", configuration.getAttributeValue("type2", "prop100", "attribute100"));
+ assertEquals("val101.1", configuration.getAttributeValue("type2", "prop101", "attribute100"));
+ assertEquals("val102.3", configuration.getAttributeValue("type2", "prop102", "attribute100"));
+ assertEquals("val100.2", configuration.getAttributeValue("type2", "prop100", "attribute101"));
+ assertEquals("val101.1", configuration.getAttributeValue("type2", "prop101", "attribute101"));
+ }
+
+ private Configuration createConfigurationWithParentsPropsOnly() {
+ // parents parent config properties
+ Map<String, Map<String, String>> parentParentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> parentParentTypeProperties1 = new HashMap<String, String>();
+ parentParentTypeProperties1.put("prop1", "val1.1");
+ parentParentTypeProperties1.put("prop2", "val2.1");
+ parentParentTypeProperties1.put("prop3", "val3.1");
+ Map<String, String> parentParentTypeProperties2 = new HashMap<String, String>();
+ parentParentTypeProperties2.put("prop4", "val4.1");
+ parentParentTypeProperties2.put("prop5", "val5.1");
+
+ parentParentProperties.put("type1", parentParentTypeProperties1);
+ parentParentProperties.put("type2", parentParentTypeProperties2);
+ Configuration parentParentConfiguration = new Configuration(parentParentProperties, EMPTY_ATTRIBUTES);
+
+ // parent config properties
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> parentTypeProperties1 = new HashMap<String, String>(); // override
+ parentTypeProperties1.put("prop1", "val1.2"); // override parent
+ parentTypeProperties1.put("prop2", "val2.2"); // override parent
+ parentTypeProperties1.put("prop6", "val6.2"); // new
+ Map<String, String> parentTypeProperties3 = new HashMap<String, String>(); // new
+ parentTypeProperties3.put("prop7", "val7.2"); // new
+ parentTypeProperties3.put("prop8", "val8.2"); // new
+
+ parentProperties.put("type1", parentTypeProperties1);
+ parentProperties.put("type3", parentTypeProperties3);
+ Configuration parentConfiguration = new Configuration(parentProperties, EMPTY_ATTRIBUTES, parentParentConfiguration);
+
+ // leaf config properties
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProperties1 = new HashMap<String, String>();
+ typeProperties1.put("prop1", "val1.3"); // overrides both parent and parents parent
+ typeProperties1.put("prop9", "val9.3"); // new
+ Map<String, String> typeProperties2 = new HashMap<String, String>(); // overrides
+ typeProperties2.put("prop4", "val4.3"); // overrides parents parent value
+ Map<String, String> typeProperties3 = new HashMap<String, String>(); // overrides
+ typeProperties3.put("prop7", "val7.3"); // overrides parents parent value
+ Map<String, String> typeProperties4 = new HashMap<String, String>(); // new
+ typeProperties4.put("prop10", "val10.3"); // new
+ typeProperties4.put("prop11", "val11.3"); // new
+
+ properties.put("type1", typeProperties1);
+ properties.put("type2", typeProperties2);
+ properties.put("type3", typeProperties3);
+ properties.put("type4", typeProperties4);
+ return new Configuration(properties, EMPTY_ATTRIBUTES, parentConfiguration);
+ }
+
+ private Configuration createConfigurationWithParentsAttributesOnly() {
+ // parents parent config attributes.
+ Map<String, Map<String, Map<String, String>>> parentParentAttributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Map<String, Map<String, String>> parentParentTypeAttributes1 = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> parentParentTypeAttributes2 = new HashMap<String, Map<String, String>>();
+ parentParentAttributes.put("type1", parentParentTypeAttributes1);
+ parentParentAttributes.put("type2", parentParentTypeAttributes2);
+
+ Map<String, String> parentParentAttributeProperties1 = new HashMap<String, String>();
+ parentParentAttributeProperties1.put("prop1", "val1.1");
+ parentParentAttributeProperties1.put("prop2", "val2.1");
+ parentParentAttributeProperties1.put("prop3", "val3.1");
+ Map<String, String> parentParentAttributeProperties2 = new HashMap<String, String>();
+ parentParentAttributeProperties2.put("prop4", "val4.1");
+ parentParentAttributeProperties2.put("prop5", "val5.1");
+
+ parentParentTypeAttributes1.put("attribute1", parentParentAttributeProperties1);
+ parentParentTypeAttributes1.put("attribute2", parentParentAttributeProperties2);
+
+ Map<String, String> parentParentAttributeProperties100 = new HashMap<String, String>();
+ parentParentAttributeProperties100.put("prop100", "val100.1");
+ parentParentAttributeProperties100.put("prop101", "val101.1");
+
+ Map<String, String> parentParentAttributeProperties101 = new HashMap<String, String>();
+ parentParentAttributeProperties101.put("prop100", "val100.1");
+ parentParentAttributeProperties101.put("prop101", "val101.1");
+
+ parentParentTypeAttributes2.put("attribute100", parentParentAttributeProperties100);
+ parentParentTypeAttributes2.put("attribute101", parentParentAttributeProperties101);
+ Configuration parentParentConfiguration = new Configuration(EMPTY_PROPERTIES,
+ new HashMap<String, Map<String, Map<String, String>>>(parentParentAttributes));
+
+ // parent config attributes
+ Map<String, Map<String, Map<String, String>>> parentAttributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Map<String, Map<String, String>> parentTypeAttributes1 = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> parentTypeAttributes2 = new HashMap<String, Map<String, String>>();
+ parentAttributes.put("type1", parentTypeAttributes1);
+ parentAttributes.put("type2", parentTypeAttributes2);
+
+ Map<String, String> parentAttributeProperties1 = new HashMap<String, String>(); // override
+ parentAttributeProperties1.put("prop1", "val1.2"); // override parent
+ parentAttributeProperties1.put("prop2", "val2.2"); // override parent
+ parentAttributeProperties1.put("prop6", "val6.2"); // new
+ Map<String, String> parentAttributeProperties3 = new HashMap<String, String>(); // new
+ parentAttributeProperties3.put("prop7", "val7.2"); // new
+ parentAttributeProperties3.put("prop8", "val8.2"); // new
+
+ parentTypeAttributes1.put("attribute1", parentAttributeProperties1);
+ parentTypeAttributes1.put("attribute3", parentAttributeProperties3);
+
+ Map<String, String> parentAttributeProperties101 = new HashMap<String, String>();
+ parentAttributeProperties101.put("prop100", "val100.2");
+ parentTypeAttributes2.put("attribute101", parentAttributeProperties101);
+ Configuration parentConfiguration = new Configuration(EMPTY_PROPERTIES,
+ new HashMap<String, Map<String, Map<String, String>>>(parentAttributes), parentParentConfiguration);
+
+ // leaf config attributes
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Map<String, Map<String, String>> typeAttributes1 = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> typeAttributes2 = new HashMap<String, Map<String, String>>();
+ attributes.put("type1", typeAttributes1);
+ attributes.put("type2", typeAttributes2);
+
+ Map<String, String> attributeProperties1 = new HashMap<String, String>();
+ attributeProperties1.put("prop1", "val1.3"); // overrides both parent and parents parent
+ attributeProperties1.put("prop9", "val9.3"); // new
+ Map<String, String> attributeProperties2 = new HashMap<String, String>(); // overrides
+ attributeProperties2.put("prop4", "val4.3"); // overrides parents parent value
+ Map<String, String> attributeProperties3 = new HashMap<String, String>(); // overrides
+ attributeProperties3.put("prop7", "val7.3"); // overrides parents parent value
+ Map<String, String> attributeProperties4 = new HashMap<String, String>(); // new
+ attributeProperties4.put("prop10", "val10.3"); // new
+ attributeProperties4.put("prop11", "val11.3"); // new
+
+ typeAttributes1.put("attribute1", attributeProperties1);
+ typeAttributes1.put("attribute2", attributeProperties2);
+ typeAttributes1.put("attribute3", attributeProperties3);
+ typeAttributes1.put("attribute4", attributeProperties4);
+
+ Map<String, String> attributeProperties100 = new HashMap<String, String>(); // overrides parents parent
+ attributeProperties100.put("prop100", "val100.3"); // overrides parents parent
+ attributeProperties100.put("prop102", "val102.3"); // new
+
+ typeAttributes1.put("attribute1", attributeProperties1);
+ typeAttributes2.put("attribute100", attributeProperties100);
+ return new Configuration(EMPTY_PROPERTIES,
+ new HashMap<String, Map<String, Map<String, String>>>(attributes), parentConfiguration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
index bc23b35..f4ded70 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.server.topology;
import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.PropertyInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -143,9 +144,9 @@ public class RequiredPasswordValidatorTest {
expect(stack.getServiceForComponent("component3")).andReturn("service2").anyTimes();
expect(stack.getServiceForComponent("component4")).andReturn("service3").anyTimes();
- expect(stack.getRequiredConfigurationProperties("service1", "PASSWORD")).andReturn(service1RequiredPwdConfigs).anyTimes();
- expect(stack.getRequiredConfigurationProperties("service2", "PASSWORD")).andReturn(service2RequiredPwdConfigs).anyTimes();
- expect(stack.getRequiredConfigurationProperties("service3", "PASSWORD")).andReturn(service3RequiredPwdConfigs).anyTimes();
+ expect(stack.getRequiredConfigurationProperties("service1", PropertyInfo.PropertyType.PASSWORD)).andReturn(service1RequiredPwdConfigs).anyTimes();
+ expect(stack.getRequiredConfigurationProperties("service2", PropertyInfo.PropertyType.PASSWORD)).andReturn(service2RequiredPwdConfigs).anyTimes();
+ expect(stack.getRequiredConfigurationProperties("service3", PropertyInfo.PropertyType.PASSWORD)).andReturn(service3RequiredPwdConfigs).anyTimes();
replay(topology, blueprint, stack, group1, group2);
}
[2/2] ambari git commit: AMBARI-11737. Fix setting of blueprint host
group scoped configuration
Posted by js...@apache.org.
AMBARI-11737. Fix setting of blueprint host group scoped configuration
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4afac300
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4afac300
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4afac300
Branch: refs/heads/trunk
Commit: 4afac3006c4d9f7c102d15752382597a3da6effe
Parents: 7eaf73c
Author: John Speidel <js...@hortonworks.com>
Authored: Fri Jun 5 16:34:20 2015 -0400
Committer: John Speidel <js...@hortonworks.com>
Committed: Fri Jun 5 17:30:32 2015 -0400
----------------------------------------------------------------------
.../BlueprintConfigurationProcessor.java | 184 ++++----
.../server/controller/internal/Stack.java | 28 +-
.../ambari/server/topology/AmbariContext.java | 30 +-
.../ambari/server/topology/Configuration.java | 221 +++++++--
.../topology/RequiredPasswordValidator.java | 13 +-
.../BlueprintConfigurationProcessorTest.java | 449 +++++++++++++++++-
.../server/controller/internal/StackTest.java | 85 ++++
.../server/topology/AmbariContextTest.java | 152 +++++-
.../server/topology/ConfigurationTest.java | 469 +++++++++++++++++++
.../topology/RequiredPasswordValidatorTest.java | 7 +-
10 files changed, 1455 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index ababc29..ca7b2b1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -76,9 +76,9 @@ public class BlueprintConfigurationProcessor {
/**
* Updaters that preserve the original property value, functions
- * as a placeholder for DB-related properties that need to be
- * removed from export, but do not require an update during
- * cluster creation
+ * as a placeholder for DB-related properties that need to be
+ * removed from export, but do not require an update during
+ * cluster creation
*/
private static Map<String, Map<String, PropertyUpdater>> removePropertyUpdaters =
new HashMap<String, Map<String, PropertyUpdater>>();
@@ -133,11 +133,6 @@ public class BlueprintConfigurationProcessor {
new DependencyNotEqualsFilter("hive.server2.authentication", "hive-site", "NONE"),
new HDFSNameNodeHAFilter() };
- /**
- * Configuration properties to be updated
- */
- //private Map<String, Map<String, String>> properties;
-
private ClusterTopology clusterTopology;
@@ -155,7 +150,7 @@ public class BlueprintConfigurationProcessor {
String propertyName = updaterEntry.getKey();
PropertyUpdater updater = updaterEntry.getValue();
- // topo cluster scoped configuration which also includes all default and BP properties
+ // cluster scoped configuration which also includes all default and BP properties
Map<String, Map<String, String>> clusterProps = clusterTopology.getConfiguration().getFullProperties();
Map<String, String> typeMap = clusterProps.get(type);
if (typeMap != null && typeMap.containsKey(propertyName)) {
@@ -178,20 +173,23 @@ public class BlueprintConfigurationProcessor {
return requiredHostGroups;
}
-
/**
* Update properties for cluster creation. This involves updating topology related properties with
* concrete topology information.
*/
public void doUpdateForClusterCreate() throws ConfigurationTopologyException {
Configuration clusterConfig = clusterTopology.getConfiguration();
- Map<String, Map<String, String>> clusterProps = clusterConfig.getFullProperties();
Map<String, HostGroupInfo> groupInfoMap = clusterTopology.getHostGroupInfo();
// filter out any properties that should not be included, based on the dependencies
// specified in the stacks, and the filters defined in this class
- doFilterPriorToClusterUpdate(clusterProps);
+ doFilterPriorToClusterUpdate(clusterConfig);
+ // this needs to be called after doFilterPriorToClusterUpdate() to ensure that the returned
+ // set of properties (copy) doesn't include the removed properties. If an updater
+ // removes a property other than the property it is registered for then we will
+ // have an issue as it won't be removed from the clusterProps map as it is a copy.
+ Map<String, Map<String, String>> clusterProps = clusterConfig.getFullProperties();
for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) {
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) {
String type = entry.getKey();
@@ -209,7 +207,7 @@ public class BlueprintConfigurationProcessor {
// host group configs
for (HostGroupInfo groupInfo : groupInfoMap.values()) {
Configuration hgConfig = groupInfo.getConfiguration();
- Map<String, Map<String, String>> hgConfigProps = hgConfig.getProperties();
+ Map<String, Map<String, String>> hgConfigProps = hgConfig.getFullProperties(1);
Map<String, String> hgTypeMap = hgConfigProps.get(type);
if (hgTypeMap != null && hgTypeMap.containsKey(propertyName)) {
hgConfig.setProperty(type, propertyName, updater.updateForClusterCreate(
@@ -237,16 +235,14 @@ public class BlueprintConfigurationProcessor {
clusterConfig.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", nnHostIterator.next());
}
}
- setMissingConfigurations(clusterProps);
+ setMissingConfigurations(clusterConfig);
}
/**
* Update properties for blueprint export.
* This involves converting concrete topology information to host groups.
*/
- //todo: use cluster topology
public void doUpdateForBlueprintExport() {
-
// HA configs are only processed in cluster configuration, not HG configurations
if (clusterTopology.isNameNodeHAEnabled()) {
doNameNodeHAUpdate();
@@ -260,22 +256,26 @@ public class BlueprintConfigurationProcessor {
doOozieServerHAUpdate();
}
- Collection<Map<String, Map<String, String>>> allConfigs = new ArrayList<Map<String, Map<String, String>>>();
- allConfigs.add(clusterTopology.getConfiguration().getFullProperties());
+ Collection<Configuration> allConfigs = new ArrayList<Configuration>();
+ allConfigs.add(clusterTopology.getConfiguration());
for (HostGroupInfo groupInfo : clusterTopology.getHostGroupInfo().values()) {
- // don't use full properties, only the properties specified in the host group config
- allConfigs.add(groupInfo.getConfiguration().getProperties());
+ Configuration hgConfiguration = groupInfo.getConfiguration();
+ if (! hgConfiguration.getFullProperties(1).isEmpty()) {
+ // create new configuration which only contains properties specified in host group and BP host group
+ allConfigs.add(new Configuration(hgConfiguration.getProperties(), null,
+ new Configuration(hgConfiguration.getParentConfiguration().getProperties(), null)));
+ }
}
- for (Map<String, Map<String, String>> properties : allConfigs) {
- doSingleHostExportUpdate(singleHostTopologyUpdaters, properties);
- doSingleHostExportUpdate(dbHostTopologyUpdaters, properties);
+ for (Configuration configuration : allConfigs) {
+ doSingleHostExportUpdate(singleHostTopologyUpdaters, configuration);
+ doSingleHostExportUpdate(dbHostTopologyUpdaters, configuration);
- doMultiHostExportUpdate(multiHostTopologyUpdaters, properties);
+ doMultiHostExportUpdate(multiHostTopologyUpdaters, configuration);
- doRemovePropertyExport(removePropertyUpdaters, properties);
+ doRemovePropertyExport(removePropertyUpdaters, configuration);
- doFilterPriorToExport(properties);
+ doFilterPriorToExport(configuration);
}
}
@@ -287,44 +287,35 @@ public class BlueprintConfigurationProcessor {
* not be included in a collection (a Blueprint export in this case),
* then the property is removed prior to the export.
*
- *
- * @param properties config properties to process for filtering
+ * @param configuration configuration being processed
*/
- private void doFilterPriorToExport(Map<String, Map<String, String>> properties) {
- for (String configType : properties.keySet()) {
- Map<String, String> configPropertiesPerType =
- properties.get(configType);
-
- Set<String> propertiesToExclude = new HashSet<String>();
- for (String propertyName : configPropertiesPerType.keySet()) {
- if (shouldPropertyBeExcludedForBlueprintExport(propertyName, configPropertiesPerType.get(propertyName), configType, clusterTopology)) {
- propertiesToExclude.add(propertyName);
- }
- }
-
- if (!propertiesToExclude.isEmpty()) {
- for (String propertyName : propertiesToExclude) {
- configPropertiesPerType.remove(propertyName);
+ private void doFilterPriorToExport(Configuration configuration) {
+ Map<String, Map<String, String>> properties = configuration.getFullProperties();
+ for (Map.Entry<String, Map<String, String>> configEntry : properties.entrySet()) {
+ String type = configEntry.getKey();
+ Map<String, String> typeProperties = configEntry.getValue();
+
+ for (Map.Entry<String, String> propertyEntry : typeProperties.entrySet()) {
+ String propertyName = propertyEntry.getKey();
+ String propertyValue = propertyEntry.getValue();
+ if (shouldPropertyBeExcludedForBlueprintExport(propertyName, propertyValue, type, clusterTopology)) {
+ configuration.removeProperty(type, propertyName);
}
}
}
}
- private void doFilterPriorToClusterUpdate(Map<String, Map<String, String>> properties) {
- for (String configType : properties.keySet()) {
- Map<String, String> configPropertiesPerType =
- properties.get(configType);
-
- Set<String> propertiesToExclude = new HashSet<String>();
- for (String propertyName : configPropertiesPerType.keySet()) {
- if (shouldPropertyBeExcludedForClusterUpdate(propertyName, configPropertiesPerType.get(propertyName), configType, this.clusterTopology)) {
- propertiesToExclude.add(propertyName);
- }
- }
-
- if (!propertiesToExclude.isEmpty()) {
- for (String propertyName : propertiesToExclude) {
- configPropertiesPerType.remove(propertyName);
+ private void doFilterPriorToClusterUpdate(Configuration configuration) {
+ // getFullProperties returns a copy so changes to it are not reflected in config properties
+ Map<String, Map<String, String>> properties = configuration.getFullProperties();
+ for (Map.Entry<String, Map<String, String>> configEntry : properties.entrySet()) {
+ String configType = configEntry.getKey();
+ Map<String, String> configPropertiesPerType = configEntry.getValue();
+
+ for (Map.Entry<String, String> propertyEntry : configPropertiesPerType.entrySet()) {
+ String propName = propertyEntry.getKey();
+ if (shouldPropertyBeExcludedForClusterUpdate(propName, propertyEntry.getValue(), configType, clusterTopology)) {
+ configuration.removeProperty(configType, propName);
}
}
}
@@ -449,18 +440,20 @@ public class BlueprintConfigurationProcessor {
* be removed from the configuration that will be available in
* the exported Blueprint.
*
- * @param updaters set of updaters for properties that should
- * always be removed during a Blueprint export
+ * @param updaters set of updaters for properties that should
+ * always be removed during a Blueprint export
+ * @param configuration configuration being processed
*/
private void doRemovePropertyExport(Map<String, Map<String, PropertyUpdater>> updaters,
- Map<String, Map<String, String>> properties) {
+ Configuration configuration) {
+ Map<String, Map<String, String>> properties = configuration.getProperties();
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
Map<String, String> typeProperties = properties.get(type);
if ( (typeProperties != null) && (typeProperties.containsKey(propertyName)) ) {
- typeProperties.remove(propertyName);
+ configuration.removeProperty(type, propertyName);
}
}
}
@@ -478,7 +471,7 @@ public class BlueprintConfigurationProcessor {
// perform a single host update on these dynamically generated property names
if (highAvailabilityUpdaters.get("hdfs-site").size() > 0) {
- doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
+ doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration());
}
}
@@ -494,7 +487,7 @@ public class BlueprintConfigurationProcessor {
// perform a single host update on these dynamically generated property names
if (highAvailabilityUpdaters.get("yarn-site").size() > 0) {
- doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
+ doSingleHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration());
}
}
@@ -508,7 +501,7 @@ public class BlueprintConfigurationProcessor {
Map<String, Map<String, PropertyUpdater>> highAvailabilityUpdaters = createMapOfOozieServerHAUpdaters();
if (highAvailabilityUpdaters.get("oozie-site").size() > 0) {
- doMultiHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration().getFullProperties());
+ doMultiHostExportUpdate(highAvailabilityUpdaters, clusterTopology.getConfiguration());
}
}
@@ -601,6 +594,7 @@ public class BlueprintConfigurationProcessor {
* @return true if Oozie HA is enabled
* false if Oozie HA is not enabled
*/
+ //todo: pass in configuration
static boolean isOozieServerHAEnabled(Map<String, Map<String, String>> configProperties) {
return configProperties.containsKey("oozie-site") && configProperties.get("oozie-site").containsKey("oozie.services.ext")
&& configProperties.get("oozie-site").get("oozie.services.ext").contains("org.apache.oozie.service.ZKLocksService");
@@ -720,7 +714,11 @@ public class BlueprintConfigurationProcessor {
* @return true if the given property should be excluded
* false if the given property should be included
*/
- private static boolean shouldPropertyBeExcludedForClusterUpdate(String propertyName, String propertyValue, String propertyType, ClusterTopology topology) {
+ private static boolean shouldPropertyBeExcludedForClusterUpdate(String propertyName,
+ String propertyValue,
+ String propertyType,
+ ClusterTopology topology) {
+
for(PropertyFilter filter : clusterUpdatePropertyFilters) {
try {
if (!filter.isPropertyIncluded(propertyName, propertyValue, propertyType, topology)) {
@@ -737,15 +735,14 @@ public class BlueprintConfigurationProcessor {
return false;
}
-
-
/**
* Update single host topology configuration properties for blueprint export.
*
- * @param updaters registered updaters
+ * @param updaters registered updaters
+ * @param configuration configuration being processed
*/
- private void doSingleHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) {
-
+ private void doSingleHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Configuration configuration) {
+ Map<String, Map<String, String>> properties = configuration.getFullProperties();
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
@@ -761,8 +758,8 @@ public class BlueprintConfigurationProcessor {
//todo: need to use regular expression to avoid matching a host which is a superset.
if (propValue.contains(host)) {
matchedHost = true;
- typeProperties.put(propertyName, propValue.replace(
- host, "%HOSTGROUP::" + groupInfo.getHostGroupName() + "%"));
+ configuration.setProperty(type, propertyName,
+ propValue.replace(host, "%HOSTGROUP::" + groupInfo.getHostGroupName() + "%"));
break;
}
}
@@ -780,7 +777,7 @@ public class BlueprintConfigurationProcessor {
! isSpecialNetworkAddress(propValue) &&
! isUndefinedAddress(propValue)) {
- typeProperties.remove(propertyName);
+ configuration.removeProperty(type, propertyName);
}
}
}
@@ -829,9 +826,11 @@ public class BlueprintConfigurationProcessor {
/**
* Update multi host topology configuration properties for blueprint export.
*
- * @param updaters registered updaters
+ * @param updaters registered updaters
+ * @param configuration configuration being processed
*/
- private void doMultiHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Map<String, Map<String, String>> properties) {
+ private void doMultiHostExportUpdate(Map<String, Map<String, PropertyUpdater>> updaters, Configuration configuration) {
+ Map<String, Map<String, String>> properties = configuration.getFullProperties();
for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaters.entrySet()) {
String type = entry.getKey();
for (String propertyName : entry.getValue().keySet()) {
@@ -869,7 +868,7 @@ public class BlueprintConfigurationProcessor {
if (inBrackets) {
sb.append(']');
}
- typeProperties.put(propertyName, sb.toString());
+ configuration.setProperty(type, propertyName, sb.toString());
}
}
}
@@ -1994,8 +1993,10 @@ public class BlueprintConfigurationProcessor {
/**
* Explicitly set any properties that are required but not currently provided in the stack definition.
+ *
+ * @param configuration configuration where properties are to be added
*/
- void setMissingConfigurations(Map<String, Map<String, String>> mapClusterConfigurations) {
+ void setMissingConfigurations(Configuration configuration) {
// AMBARI-5206
final Map<String , String> userProps = new HashMap<String , String>();
@@ -2019,18 +2020,18 @@ public class BlueprintConfigurationProcessor {
userProps.put("falcon_user", "falcon-env");
}
-
String proxyUserHosts = "hadoop.proxyuser.%s.hosts";
String proxyUserGroups = "hadoop.proxyuser.%s.groups";
+ Map<String, Map<String, String>> existingProperties = configuration.getFullProperties();
for (String property : userProps.keySet()) {
String configType = userProps.get(property);
- Map<String, String> configs = mapClusterConfigurations.get(configType);
+ Map<String, String> configs = existingProperties.get(configType);
if (configs != null) {
String user = configs.get(property);
if (user != null && !user.isEmpty()) {
- ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserHosts, user), "*");
- ensureProperty(mapClusterConfigurations, "core-site", String.format(proxyUserGroups, user), "users");
+ ensureProperty(configuration, "core-site", String.format(proxyUserHosts, user), "*");
+ ensureProperty(configuration, "core-site", String.format(proxyUserGroups, user), "users");
}
} else {
LOG.debug("setMissingConfigurations: no user configuration found for type = " + configType +
@@ -2044,19 +2045,14 @@ public class BlueprintConfigurationProcessor {
* Ensure that the specified property exists.
* If not, set a default value.
*
- * @param type config type
- * @param property property name
- * @param defaultValue default value
+ * @param configuration configuration being processed
+ * @param type config type
+ * @param property property name
+ * @param defaultValue default value
*/
- private void ensureProperty(Map<String, Map<String, String>> mapClusterConfigurations, String type, String property, String defaultValue) {
- Map<String, String> properties = mapClusterConfigurations.get(type);
- if (properties == null) {
- properties = new HashMap<String, String>();
- mapClusterConfigurations.put(type, properties);
- }
-
- if (! properties.containsKey(property)) {
- properties.put(property, defaultValue);
+ private void ensureProperty(Configuration configuration, String type, String property, String defaultValue) {
+ if (configuration.getPropertyValue(type, property) == null) {
+ configuration.setProperty(type, property, defaultValue);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
index ba01d68..7f911e9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java
@@ -337,21 +337,27 @@ public class Stack {
}
/**
- * Get required config properties for the specified service and configuration type.
+ * Get required config properties for the specified service which belong to the specified property type.
*
- * @param service service name
- * @param type configuration type
+ * @param service service name
+ * @param propertyType property type
*
- * @return collection of required properties for the given service and type
+ * @return collection of required properties for the given service and property type
*/
- //todo: change type to PropertyInfo.PropertyType
- public Collection<ConfigProperty> getRequiredConfigurationProperties(String service, String type) {
- Collection<ConfigProperty> requiredConfigs = new HashSet<ConfigProperty>();
- Map<String, ConfigProperty> configProperties = requiredServiceConfigurations.get(service).get(type);
- if (configProperties != null) {
- requiredConfigs.addAll(configProperties.values());
+ public Collection<ConfigProperty> getRequiredConfigurationProperties(String service, PropertyInfo.PropertyType propertyType) {
+ Collection<ConfigProperty> matchingProperties = new HashSet<ConfigProperty>();
+ Map<String, Map<String, ConfigProperty>> requiredProperties = requiredServiceConfigurations.get(service);
+ if (requiredProperties != null) {
+ for (Map.Entry<String, Map<String, ConfigProperty>> typePropertiesEntry : requiredProperties.entrySet()) {
+ for (ConfigProperty configProperty : typePropertiesEntry.getValue().values()) {
+ if (configProperty.getPropertyTypes().contains(propertyType)) {
+ matchingProperties.add(configProperty);
+ }
+ }
+
+ }
}
- return requiredConfigs;
+ return matchingProperties;
}
public boolean isPasswordProperty(String service, String type, String propertyName) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 0de4b00..091018a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -52,6 +52,7 @@ import org.apache.ambari.server.controller.internal.RequestImpl;
import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.controller.predicate.EqualsPredicate;
+import org.apache.ambari.server.controller.spi.ClusterController;
import org.apache.ambari.server.controller.spi.Predicate;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.utilities.ClusterControllerHelper;
@@ -76,6 +77,7 @@ public class AmbariContext {
private static PersistedState persistedState = new PersistedStateImpl();
private static AmbariManagementController controller;
+ private static ClusterController clusterController;
//todo: task id's. Use existing mechanism for getting next task id sequence
private final static AtomicLong nextTaskId = new AtomicLong(10000);
@@ -260,13 +262,20 @@ public class AmbariContext {
return getController().getActionManager().getNextRequestId();
}
- public synchronized AmbariManagementController getController() {
+ public synchronized static AmbariManagementController getController() {
if (controller == null) {
controller = AmbariServer.getController();
}
return controller;
}
+ public synchronized static ClusterController getClusterController() {
+ if (clusterController == null) {
+ clusterController = ClusterControllerHelper.getClusterController();
+ }
+ return clusterController;
+ }
+
public static void init(HostRoleCommandFactory factory) {
hostRoleCommandFactory = factory;
}
@@ -415,22 +424,23 @@ public class AmbariContext {
* and the hosts associated with the host group are assigned to the config group.
*/
private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String groupName) {
-
- //HostGroupEntity entity = hostGroup.getEntity();
Map<String, Map<String, Config>> groupConfigs = new HashMap<String, Map<String, Config>>();
-
- Stack stack = topology.getBlueprint().getHostGroup(groupName).getStack();
+ Stack stack = topology.getBlueprint().getStack();
// get the host-group config with cluster creation template overrides
Configuration topologyHostGroupConfig = topology.
getHostGroupInfo().get(groupName).getConfiguration();
+ // only get user provided configuration for host group which includes only CCT/HG and BP/HG properties
+ Map<String, Map<String, String>> userProvidedGroupProperties =
+ topologyHostGroupConfig.getFullProperties(1);
+
+ //todo: doesn't belong here.
//handling backwards compatibility for group configs
- //todo: doesn't belong here
- convertGlobalProperties(topology, topologyHostGroupConfig.getProperties());
+ convertGlobalProperties(topology, userProvidedGroupProperties);
- // iterate over topo host group configs which were defined in CCT/HG and BP/HG only, no parent configs
- for (Map.Entry<String, Map<String, String>> entry : topologyHostGroupConfig.getProperties().entrySet()) {
+ // iterate over topo host group configs which were defined in
+ for (Map.Entry<String, Map<String, String>> entry : userProvidedGroupProperties.entrySet()) {
String type = entry.getKey();
String service = stack.getServiceForConfigType(type);
Config config = new ConfigImpl(type);
@@ -461,7 +471,7 @@ public class AmbariContext {
// get the config group provider and create config group resource
ConfigGroupResourceProvider configGroupProvider = (ConfigGroupResourceProvider)
- ClusterControllerHelper.getClusterController().ensureResourceProvider(Resource.Type.ConfigGroup);
+ getClusterController().ensureResourceProvider(Resource.Type.ConfigGroup);
try {
configGroupProvider.createResources(Collections.singleton(request));
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
index 2447b8b..b7b9343 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java
@@ -28,12 +28,28 @@ import java.util.Map;
* Configuration for a topology entity such as a blueprint, hostgroup or cluster.
*/
public class Configuration {
-
+ /**
+ * properties for this configuration instance
+ */
private Map<String, Map<String, String>> properties;
+
+ /**
+ * attributes for this configuration instance
+ */
private Map<String, Map<String, Map<String, String>>> attributes;
+ /**
+ * parent configuration
+ */
private Configuration parentConfiguration;
+ /**
+ * Constructor.
+ *
+ * @param properties properties
+ * @param attributes attributes
+ * @param parentConfiguration parent configuration
+ */
public Configuration(Map<String, Map<String, String>> properties,
Map<String, Map<String, Map<String, String>>> attributes,
Configuration parentConfiguration) {
@@ -52,6 +68,12 @@ public class Configuration {
// }
}
+ /**
+ * Configuration.
+ *
+ * @param properties properties
+ * @param attributes attributes
+ */
public Configuration(Map<String, Map<String, String>> properties,
Map<String, Map<String, Map<String, String>>> attributes) {
@@ -59,17 +81,39 @@ public class Configuration {
this.attributes = attributes;
}
+ /**
+ * Get the properties for this instance only; parent properties are not included.
+ *
+ * @return map of properties for this configuration instance keyed by config type
+ */
public Map<String, Map<String, String>> getProperties() {
return properties;
}
+ /**
+ * Get a complete merged map of properties including this instance and the entire parent hierarchy.
+ * Properties are merged so that children override the same property specified in it's parent hierarchy.
+ * This result is re-calculated for each request in case a parent value has changed so the result
+ * should be cached if possible.
+ *
+ * @return complete map of merged properties keyed by config type
+ */
public Map<String, Map<String, String>> getFullProperties() {
return getFullProperties(Integer.MAX_VALUE);
}
- //re-calculated each time in case parent properties changed
+ /**
+ * Get a merged map of properties including this instance and n levels of the parent hierarchy.
+ * Properties are merged so that children override the same property specified in it's parent hierarchy.
+ * This result is re-calculated for each request in case a parent value has changed so the result
+ * should be cached if possible.
+ *
+ * @param depthLimit the number of parent levels to include in the results. Specifying 0 is the same
+ * as calling {@link #getProperties()}
+ *
+ * @return map of merged properties keyed by config type
+ */
public Map<String, Map<String, String>> getFullProperties(int depthLimit) {
-
if (depthLimit == 0) {
return new HashMap<String, Map<String, String>>(properties);
}
@@ -80,7 +124,7 @@ public class Configuration {
for (Map.Entry<String, Map<String, String>> entry : properties.entrySet()) {
String configType = entry.getKey();
- Map<String, String> typeProps = entry.getValue();
+ Map<String, String> typeProps = new HashMap<String, String>(entry.getValue());
if (mergedProperties.containsKey(configType)) {
mergedProperties.get(configType).putAll(typeProps);
@@ -91,12 +135,23 @@ public class Configuration {
return mergedProperties;
}
+ /**
+ * Get the attributes for this instance only; parent attributes aren't included.
+ *
+ * @return map of attributes {configType -> {attributeName -> {propName, attributeValue}}}
+ */
public Map<String, Map<String, Map<String, String>>> getAttributes() {
return attributes;
}
- //re-calculate each time in case parent properties changed
- // attribute structure is very confusing. {type -> {attributeName -> {propName, attributeValue}}}
+ /**
+ * Get a complete merged map of attributes including this instance and the entire parent hierarchy.
+ * Attributes are merged so that children override the same attribute specified in it's parent hierarchy.
+ * This result is re-calculated for each request in case a parent value has changed so the result
+ * should be cached if possible.
+ *
+ * @return complete map of merged attributes {configType -> {attributeName -> {propName, attributeValue}}}
+ */
public Map<String, Map<String, Map<String, String>>> getFullAttributes() {
Map<String, Map<String, Map<String, String>>> mergedAttributeMap = parentConfiguration == null ?
new HashMap<String, Map<String, Map<String, String>>>() :
@@ -104,11 +159,14 @@ public class Configuration {
for (Map.Entry<String, Map<String, Map<String, String>>> typeEntry : attributes.entrySet()) {
String type = typeEntry.getKey();
+ Map<String, Map<String, String>> typeAttributes =
+ new HashMap<String, Map<String, String>>(typeEntry.getValue());
+
if (! mergedAttributeMap.containsKey(type)) {
- mergedAttributeMap.put(type, typeEntry.getValue());
+ mergedAttributeMap.put(type, typeAttributes);
} else {
Map<String, Map<String, String>> mergedAttributes = mergedAttributeMap.get(type);
- for (Map.Entry<String, Map<String, String>> attributeEntry : typeEntry.getValue().entrySet()) {
+ for (Map.Entry<String, Map<String, String>> attributeEntry : typeAttributes.entrySet()) {
String attribute = attributeEntry.getKey();
if (! mergedAttributes.containsKey(attribute)) {
mergedAttributes.put(attribute, attributeEntry.getValue());
@@ -121,54 +179,107 @@ public class Configuration {
}
}
}
-
- mergedAttributeMap.putAll(attributes);
-
return mergedAttributeMap;
}
- public Collection<String> getAllConfigTypes() {
- Collection<String> allTypes = new HashSet<String>();
- for (String type : getFullProperties().keySet()) {
- allTypes.add(type);
- }
-
- for (String type : getFullAttributes().keySet()) {
- allTypes.add(type);
+ /**
+ * Get the requested property value from the full merged set of properties.
+ *
+ * @param configType configuration type
+ * @param propertyName property name
+ *
+ * @return requested property value or null if property isn't set in configuration hierarchy
+ */
+ public String getPropertyValue(String configType, String propertyName) {
+ String value = null;
+ if (properties.containsKey(configType) && properties.get(configType).containsKey(propertyName)) {
+ value = properties.get(configType).get(propertyName);
+ } else if (parentConfiguration != null) {
+ value = parentConfiguration.getPropertyValue(configType, propertyName);
}
- return allTypes;
- }
-
- public Configuration getParentConfiguration() {
- return parentConfiguration;
- }
-
- public void setParentConfiguration(Configuration parent) {
- parentConfiguration = parent;
+ return value;
}
- public String getPropertyValue(String configType, String propertyName) {
- return properties.containsKey(configType) ?
- properties.get(configType).get(propertyName) : null;
- }
+ /**
+ * Get the requested attribute value from the full merged set of attributes.
+ *
+ * @param configType configuration type
+ * @param propertyName attribute property name
+ * @param attributeName attribute name
+ *
+ * @return requested attribute value or null if the attribute isn't set in configuration hierarchy
+ */
+ public String getAttributeValue(String configType, String propertyName, String attributeName) {
+ String value = null;
+ if (attributes.containsKey(configType) &&
+ attributes.get(configType).containsKey(attributeName) &&
+ attributes.get(configType).get(attributeName).containsKey(propertyName)) {
+
+ value = attributes.get(configType).get(attributeName).get(propertyName);
+ } else if (parentConfiguration != null) {
+ value = parentConfiguration.getAttributeValue(configType, propertyName, attributeName);
+ }
- public boolean containsProperty(String configType, String propertyName) {
- return properties.containsKey(configType) && properties.get(configType).containsKey(propertyName);
+ return value;
}
+ /**
+ * Set a property on the configuration.
+ * The property will be set on this instance so it will override any value specified in
+ * the parent hierarchy.
+ *
+ * @param configType configuration type
+ * @param propertyName property name
+ * @param value property value
+ *
+ * @return the previous value of the property or null if it didn't exist
+ */
public String setProperty(String configType, String propertyName, String value) {
+ String previousValue = getPropertyValue(configType, propertyName);
Map<String, String> typeProperties = properties.get(configType);
if (typeProperties == null) {
typeProperties = new HashMap<String, String>();
properties.put(configType, typeProperties);
}
+ typeProperties.put(propertyName, value);
+ return previousValue;
+ }
- return typeProperties.put(propertyName, value);
+ /**
+ * Remove a property from the configuration hierarchy.
+ *
+ * @param configType configuration type
+ * @param propertyName property name
+ *
+ * @return the previous value of the removed property or null if it didn't exist in the
+ * configuration hierarchy
+ */
+ public String removeProperty(String configType, String propertyName) {
+ String previousValue = null;
+ if (properties.containsKey(configType) && properties.get(configType).containsKey(propertyName)) {
+ previousValue = properties.get(configType).remove(propertyName);
+ } else if (parentConfiguration != null) {
+ previousValue = parentConfiguration.removeProperty(configType, propertyName);
+ }
+ return previousValue;
}
- // attribute structure is very confusing: {type -> {attributeName -> {propName, attributeValue}}}
+ /**
+ * Set an attribute on the hierarchy.
+ * The attribute will be set on this instance so it will override any value specified in
+ * the parent hierarchy.
+ *
+ * @param configType configuration type
+ * @param propertyName attribute property name
+ * @param attributeName attribute name
+ * @param attributeValue attribute property value
+ *
+ * @return the previous value of the attribute or null if it didn't exist
+ */
public String setAttribute(String configType, String propertyName, String attributeName, String attributeValue) {
+ String previousValue = getAttributeValue(configType, propertyName, attributeName);
+
Map<String, Map<String, String>> typeAttributes = attributes.get(configType);
if (typeAttributes == null) {
typeAttributes = new HashMap<String, Map<String, String>>();
@@ -181,7 +292,43 @@ public class Configuration {
typeAttributes.put(attributeName, attributes);
}
- return attributes.put(propertyName, attributeValue);
+ attributes.put(propertyName, attributeValue);
+ return previousValue;
}
+ /**
+ * Get the complete set of configuration types represented in both full properties and full attributes.
+ *
+ * @return collection of all represented configuration types
+ */
+ public Collection<String> getAllConfigTypes() {
+ Collection<String> allTypes = new HashSet<String>();
+ for (String type : getFullProperties().keySet()) {
+ allTypes.add(type);
+ }
+
+ for (String type : getFullAttributes().keySet()) {
+ allTypes.add(type);
+ }
+
+ return allTypes;
+ }
+
+ /**
+ * Get the parent configuration.
+ *
+ * @return the parent configuration or null if no parent is set
+ */
+ public Configuration getParentConfiguration() {
+ return parentConfiguration;
+ }
+
+ /**
+ * Set the parent configuration.
+ *
+ * @param parent parent configuration to set
+ */
+ public void setParentConfiguration(Configuration parent) {
+ parentConfiguration = parent;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/main/java/org/apache/ambari/server/topology/RequiredPasswordValidator.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/RequiredPasswordValidator.java
index 252b78b..e26de3f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/RequiredPasswordValidator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/RequiredPasswordValidator.java
@@ -20,6 +20,7 @@
package org.apache.ambari.server.topology;
import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.PropertyInfo;
import java.util.Collection;
import java.util.HashMap;
@@ -44,16 +45,12 @@ public class RequiredPasswordValidator implements TopologyValidator {
* default is specified via 'default_password'
*/
public void validate(ClusterTopology topology) throws InvalidTopologyException {
- String errStr = "Missing required password properties. Specify a value for these " +
- "properties in the cluster or host group configurations or include 'default_password' field in request. ";
- if (topology == null) {
- throw new InvalidTopologyException(errStr);
- }
-
Map<String, Map<String, Collection<String>>> missingPasswords = validateRequiredPasswords(topology);
if (! missingPasswords.isEmpty()) {
- throw new InvalidTopologyException(errStr + missingPasswords);
+ throw new InvalidTopologyException("Missing required password properties. Specify a value for these " +
+ "properties in the cluster or host group configurations or include 'default_password' field in request. " +
+ missingPasswords);
}
}
@@ -94,7 +91,7 @@ public class RequiredPasswordValidator implements TopologyValidator {
if (processedServices.add(serviceName)) {
//todo: do I need to subtract excluded configs?
Collection<Stack.ConfigProperty> requiredProperties =
- stack.getRequiredConfigurationProperties(serviceName, "PASSWORD");
+ stack.getRequiredConfigurationProperties(serviceName, PropertyInfo.PropertyType.PASSWORD);
for (Stack.ConfigProperty property : requiredProperties) {
String category = property.getType();
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 83ed594..b049fd3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -67,7 +67,6 @@ public class BlueprintConfigurationProcessorTest {
private final Map<String, Collection<String>> serviceComponents = new HashMap<String, Collection<String>>();
private final Blueprint bp = createNiceMock(Blueprint.class);
- //private final AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class);
private final ServiceInfo serviceInfo = createNiceMock(ServiceInfo.class);
private final Stack stack = createNiceMock(Stack.class);
private final AmbariContext ambariConext = createNiceMock(AmbariContext.class);
@@ -114,6 +113,7 @@ public class BlueprintConfigurationProcessorTest {
Collection<String> hiveComponents = new HashSet<String>();
hiveComponents.add("MYSQL_SERVER");
hiveComponents.add("HIVE_METASTORE");
+ hiveComponents.add("HIVE_SERVER");
serviceComponents.put("HIVE", hiveComponents);
Collection<String> falconComponents = new HashSet<String>();
@@ -139,6 +139,10 @@ public class BlueprintConfigurationProcessorTest {
oozieComponents.add("OOZIE_CLIENT");
serviceComponents.put("OOZIE", oozieComponents);
+ Collection<String> hbaseComponents = new HashSet<String>();
+ hbaseComponents.add("HBASE_MASTER");
+ serviceComponents.put("HBASE", hbaseComponents);
+
for (Map.Entry<String, Collection<String>> entry : serviceComponents.entrySet()) {
String service = entry.getKey();
for (String component : entry.getValue()) {
@@ -188,6 +192,95 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
+ public void testDoUpdateForBlueprintExport_SingleHostProperty_specifiedInParentConfig() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> yarnSiteProps = new HashMap<String, String>();
+ yarnSiteProps.put("yarn.resourcemanager.hostname", "testhost");
+ properties.put("yarn-site", yarnSiteProps);
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> parentYarnSiteProps = new HashMap<String, String>();
+ parentYarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "testhost");
+ parentProperties.put("yarn-site", parentYarnSiteProps);
+
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("SECONDARY_NAMENODE");
+ hgComponents.add("RESOURCEMANAGER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost"));
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add("DATANODE");
+ hgComponents2.add("HDFS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2, Collections.singleton("testhost2"));
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+ configProcessor.doUpdateForBlueprintExport();
+
+ assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
+ assertEquals("%HOSTGROUP::group1%", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address"));
+ }
+
+ @Test
+ public void testDoUpdateForBlueprintExport_SingleHostProperty_hostGroupConfiguration() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> typeProps = new HashMap<String, String>();
+ typeProps.put("yarn.resourcemanager.hostname", "testhost");
+ properties.put("yarn-site", typeProps);
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("SECONDARY_NAMENODE");
+ hgComponents.add("RESOURCEMANAGER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost"));
+
+ Collection<String> hgComponents2 = new HashSet<String>();
+ hgComponents2.add("DATANODE");
+ hgComponents2.add("HDFS_CLIENT");
+
+ Map<String, Map<String, String>> group2Properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> group2YarnSiteProps = new HashMap<String, String>();
+ group2YarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "testhost");
+ group2Properties.put("yarn-site", group2YarnSiteProps);
+ // host group config -> BP config -> cluster scoped config
+ Configuration group2BPConfiguration = new Configuration(Collections.<String, Map<String, String>>emptyMap(),
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig);
+
+ Configuration group2Configuration = new Configuration(group2Properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), group2BPConfiguration);
+
+ // set config on hostgroup
+ TestHostGroup group2 = new TestHostGroup("group2", hgComponents2,
+ Collections.singleton("testhost2"), group2Configuration);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+ configProcessor.doUpdateForBlueprintExport();
+
+ assertEquals("%HOSTGROUP::group1%", properties.get("yarn-site").get("yarn.resourcemanager.hostname"));
+ assertEquals("%HOSTGROUP::group1%",
+ group2Configuration.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address"));
+ }
+
+ @Test
public void testDoUpdateForBlueprintExport_SingleHostProperty__withPort() throws Exception {
Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
Map<String, String> typeProps = new HashMap<String, String>();
@@ -501,14 +594,14 @@ public class BlueprintConfigurationProcessorTest {
configProcessor.doUpdateForBlueprintExport();
- assertEquals("Exported properties map was not of the expected size",
- 1,typeProps.size());
- assertEquals("ranger-yarn-plugin-properties config type was not properly exported",
- 1, properties.get("ranger-yarn-plugin-properties").size());
+ assertEquals("Exported properties map was not of the expected size", 1,
+ properties.get("custom-test-properties").size());
+ assertEquals("ranger-yarn-plugin-properties config type was not properly exported", 1,
+ properties.get("ranger-yarn-plugin-properties").size());
// verify that the following password properties matching the "*_PASSWORD" rule have been excluded
assertFalse("Password property should have been excluded",
- properties.get("ranger-yarn-plugin-properties").containsKey("REPOSITORY_CONFIG_PASSWORD"));
+ properties.get("ranger-yarn-plugin-properties").containsKey("REPOSITORY_CONFIG_PASSWORD"));
assertFalse("Password property should have been excluded",
properties.get("ranger-yarn-plugin-properties").containsKey("SSL_KEYSTORE_PASSWORD"));
assertFalse("Password property should have been excluded",
@@ -1574,6 +1667,146 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
+ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_providedInParent() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> yarnSiteProps = new HashMap<String, String>();
+ yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost");
+ properties.put("yarn-site", yarnSiteProps);
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> parentYarnSiteProps = new HashMap<String, String>();
+ parentYarnSiteProps.put("yarn.resourcemanager.resource-tracker.address", "localhost");
+ parentProperties.put("yarn-site", parentYarnSiteProps);
+
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+ Collection<String> group1Components = new HashSet<String>();
+ group1Components.add("NAMENODE");
+ group1Components.add("SECONDARY_NAMENODE");
+ group1Components.add("RESOURCEMANAGER");
+ TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost"));
+
+ Collection<String> group2Components = new HashSet<String>();
+ group2Components.add("DATANODE");
+ group2Components.add("HDFS_CLIENT");
+ TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2"));
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+
+ updater.doUpdateForClusterCreate();
+
+ assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
+ assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.resource-tracker.address"));
+ }
+
+ @Test
+ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_hostGroupConfig() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> yarnSiteProps = new HashMap<String, String>();
+ yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost");
+ properties.put("yarn-site", yarnSiteProps);
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> group1Components = new HashSet<String>();
+ group1Components.add("NAMENODE");
+ group1Components.add("SECONDARY_NAMENODE");
+ group1Components.add("RESOURCEMANAGER");
+ TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost"));
+
+ Collection<String> group2Components = new HashSet<String>();
+ group2Components.add("DATANODE");
+ group2Components.add("HDFS_CLIENT");
+
+ Map<String, Map<String, String>> group2Properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> group2YarnSiteProperties = new HashMap<String, String>();
+ group2YarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", "localhost");
+ group2Properties.put("yarn-site", group2YarnSiteProperties);
+ // group 2 host group configuration
+ // HG config -> BP HG config -> cluster scoped config
+ Configuration group2BPConfig = new Configuration(Collections.<String, Map<String, String>>emptyMap(),
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig);
+
+ Configuration group2Config = new Configuration(group2Properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), group2BPConfig);
+ // set config on HG
+ TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2"), group2Config);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+
+ updater.doUpdateForClusterCreate();
+
+ assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
+ assertEquals("testhost", group2Config.getProperties().get("yarn-site").get("yarn.resourcemanager.resource-tracker.address"));
+ }
+
+ @Test
+ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_BPHostGroupConfig() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> yarnSiteProps = new HashMap<String, String>();
+ yarnSiteProps.put("yarn.resourcemanager.hostname", "localhost");
+ properties.put("yarn-site", yarnSiteProps);
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> group1Components = new HashSet<String>();
+ group1Components.add("NAMENODE");
+ group1Components.add("SECONDARY_NAMENODE");
+ group1Components.add("RESOURCEMANAGER");
+ TestHostGroup group1 = new TestHostGroup("group1", group1Components, Collections.singleton("testhost"));
+
+ Collection<String> group2Components = new HashSet<String>();
+ group2Components.add("DATANODE");
+ group2Components.add("HDFS_CLIENT");
+
+ Map<String, Map<String, String>> group2BPProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> group2YarnSiteProperties = new HashMap<String, String>();
+ group2YarnSiteProperties.put("yarn.resourcemanager.resource-tracker.address", "localhost");
+ group2BPProperties.put("yarn-site", group2YarnSiteProperties);
+ // group 2 host group configuration
+ // HG config -> BP HG config -> cluster scoped config
+ Configuration group2BPConfig = new Configuration(group2BPProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), clusterConfig);
+
+ // can't set parent here because it is reset in cluster topology
+ Configuration group2Config = new Configuration(new HashMap<String, Map<String, String>>(),
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ // set config on HG
+ TestHostGroup group2 = new TestHostGroup("group2", group2Components, Collections.singleton("testhost2"), group2Config);
+
+ Collection<TestHostGroup> hostGroups = new HashSet<TestHostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ // todo: set as BP hostgroup
+ topology.getHostGroupInfo().get("group2").getConfiguration().setParentConfiguration(group2BPConfig);
+
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology);
+
+ updater.doUpdateForClusterCreate();
+
+ assertEquals("testhost", clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
+ assertEquals("testhost", group2Config.getProperties().get("yarn-site").get("yarn.resourcemanager.resource-tracker.address"));
+ }
+
+ @Test
public void testDoUpdateForClusterCreate_SingleHostProperty__MissingComponent() throws Exception {
Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
Map<String, String> typeProps = new HashMap<String, String>();
@@ -3507,7 +3740,7 @@ public class BlueprintConfigurationProcessorTest {
// customized stack calls for this test only
// simulate the case of the stack object throwing a RuntimeException, to indicate a config error
- expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("unexpected error!!"));
+ expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).atLeastOnce();
expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce();
Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap());
@@ -3605,9 +3838,9 @@ public class BlueprintConfigurationProcessorTest {
updater.doUpdateForClusterCreate();
assertTrue("hive.server2.authentication.kerberos.keytab should have been included in configuration",
- hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.keytab"));
+ hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.keytab"));
assertTrue("hive.server2.authentication.kerberos.principal should have been included in configuration",
- hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.principal"));
+ hiveSiteProperties.containsKey("hive.server2.authentication.kerberos.principal"));
}
@Test
@@ -3669,7 +3902,7 @@ public class BlueprintConfigurationProcessorTest {
updater.doUpdateForClusterCreate();
assertFalse("hbase.coprocessor.regionserver.classes should have been filtered out of configuration",
- hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes"));
+ hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes"));
}
@@ -3732,7 +3965,7 @@ public class BlueprintConfigurationProcessorTest {
updater.doUpdateForClusterCreate();
assertTrue("hbase.coprocessor.regionserver.classes should have been included in configuration",
- hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes"));
+ hbaseSiteProperties.containsKey("hbase.coprocessor.regionserver.classes"));
}
@@ -4449,6 +4682,188 @@ public class BlueprintConfigurationProcessorTest {
assertTrue(requiredGroups.containsAll(Arrays.asList("group1", "group2")));
}
+ @Test
+ public void testAllDefaultUserAndGroupProxyPropertiesSet() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+ Map<String, String> hiveEnvProperties = new HashMap<String, String>();
+ Map<String, String> hbaseEnvProperties = new HashMap<String, String>();
+ Map<String, String> falconEnvProperties = new HashMap<String, String>();
+
+ properties.put("oozie-env", oozieEnvProperties);
+ properties.put("hive-env", hiveEnvProperties);
+ properties.put("hbase-env", hbaseEnvProperties);
+ properties.put("falcon-env", falconEnvProperties);
+
+ oozieEnvProperties.put("oozie_user", "test-oozie-user");
+
+ hiveEnvProperties.put("hive_user", "test-hive-user");
+ hiveEnvProperties.put("hcat_user", "test-hcat-user");
+
+ hbaseEnvProperties.put("hbase_user", "test-hbase-user");
+
+ falconEnvProperties.put("falcon_user", "test-falcon-user");
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("OOZIE_SERVER");
+ hgComponents1.add("HIVE_SERVER");
+ hgComponents1.add("HBASE_MASTER");
+ hgComponents1.add("FALCON_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ configProcessor.doUpdateForClusterCreate();
+
+ assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-oozie-user.hosts"));
+ assertEquals("users", properties.get("core-site").get("hadoop.proxyuser.test-oozie-user.groups"));
+
+ assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hive-user.hosts"));
+ assertEquals("users", properties.get("core-site").get("hadoop.proxyuser.test-hive-user.groups"));
+
+ assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hcat-user.hosts"));
+ assertEquals("users", properties.get("core-site").get("hadoop.proxyuser.test-hcat-user.groups"));
+
+ assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-hbase-user.hosts"));
+ assertEquals("users", properties.get("core-site").get("hadoop.proxyuser.test-hbase-user.groups"));
+
+ assertEquals("*", properties.get("core-site").get("hadoop.proxyuser.test-falcon-user.hosts"));
+ assertEquals("users", properties.get("core-site").get("hadoop.proxyuser.test-falcon-user.groups"));
+ }
+
+ @Test
+ public void testRelevantDefaultUserAndGroupProxyPropertiesSet() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+ Map<String, String> falconEnvProperties = new HashMap<String, String>();
+
+ properties.put("oozie-env", oozieEnvProperties);
+ properties.put("falcon-env", falconEnvProperties);
+
+ oozieEnvProperties.put("oozie_user", "test-oozie-user");
+
+ falconEnvProperties.put("falcon_user", "test-falcon-user");
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("OOZIE_SERVER");
+ hgComponents1.add("FALCON_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ configProcessor.doUpdateForClusterCreate();
+
+ Map<String, String> coreSiteProperties = properties.get("core-site");
+ assertEquals(4, coreSiteProperties.size());
+
+ assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.hosts"));
+ assertEquals("users", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.groups"));
+
+ assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.hosts"));
+ assertEquals("users", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.groups"));
+ }
+
+ @Test
+ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> coreSiteProperties = new HashMap<String, String>();
+ Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+ Map<String, String> falconEnvProperties = new HashMap<String, String>();
+
+ properties.put("core-site", coreSiteProperties);
+ properties.put("oozie-env", oozieEnvProperties);
+ properties.put("falcon-env", falconEnvProperties);
+
+ coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.hosts", "testOozieHostsVal");
+ coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.groups", "testOozieGroupsVal");
+
+ oozieEnvProperties.put("oozie_user", "test-oozie-user");
+
+ falconEnvProperties.put("falcon_user", "test-falcon-user");
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("OOZIE_SERVER");
+ hgComponents1.add("FALCON_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ configProcessor.doUpdateForClusterCreate();
+
+ assertEquals(4, coreSiteProperties.size());
+
+ assertEquals("testOozieHostsVal", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.hosts"));
+ assertEquals("testOozieGroupsVal", coreSiteProperties.get("hadoop.proxyuser.test-oozie-user.groups"));
+
+ assertEquals("*", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.hosts"));
+ assertEquals("users", coreSiteProperties.get("hadoop.proxyuser.test-falcon-user.groups"));
+ }
+
+ @Test
+ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided2() throws Exception {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> falconEnvProperties = new HashMap<String, String>();
+ properties.put("falcon-env", falconEnvProperties);
+ falconEnvProperties.put("falcon_user", "test-falcon-user");
+
+ Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>();
+ Map<String, String> oozieEnvProperties = new HashMap<String, String>();
+ parentProperties.put("oozie-env", oozieEnvProperties);
+ oozieEnvProperties.put("oozie_user", "test-oozie-user");
+ Map<String, String> coreSiteProperties = new HashMap<String, String>();
+ parentProperties.put("core-site", coreSiteProperties);
+ coreSiteProperties.put("hadoop.proxyuser.test-oozie-user.hosts", "testOozieHostsVal");
+
+ Configuration parentClusterConfig = new Configuration(parentProperties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+ Configuration clusterConfig = new Configuration(properties,
+ Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig);
+
+ Collection<String> hgComponents1 = new HashSet<String>();
+ hgComponents1.add("OOZIE_SERVER");
+ hgComponents1.add("FALCON_SERVER");
+ TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1"));
+
+ Collection<TestHostGroup> hostGroups = Collections.singletonList(group1);
+
+ ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups);
+ BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
+
+ configProcessor.doUpdateForClusterCreate();
+
+ Map<String, String> leafConfigCoreSiteProps = properties.get("core-site");
+ // because "hadoop.proxyuser.test-oozie-user.hosts" is provided in the parent config, it shouldn't be added
+ assertEquals(3, leafConfigCoreSiteProps.size());
+
+ // ensure that explicitly set value is unchanged
+ assertEquals("testOozieHostsVal", clusterConfig.getPropertyValue("core-site", "hadoop.proxyuser.test-oozie-user.hosts"));
+
+ assertEquals("users", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-oozie-user.groups"));
+
+ assertEquals("*", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-falcon-user.hosts"));
+ assertEquals("users", leafConfigCoreSiteProps.get("hadoop.proxyuser.test-falcon-user.groups"));
+ }
+
+
private static String createExportedAddress(String expectedPortNum, String expectedHostGroupName) {
return createExportedHostName(expectedHostGroupName, expectedPortNum);
}
@@ -4481,7 +4896,7 @@ public class BlueprintConfigurationProcessorTest {
HostGroupInfo groupInfo = new HostGroupInfo(hostGroup.name);
groupInfo.addHosts(hostGroup.hosts);
//todo: HG configs
- groupInfo.setConfiguration(EMPTY_CONFIG);
+ groupInfo.setConfiguration(hostGroup.configuration);
//create host group which is set on topology
allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", stack,
@@ -4515,11 +4930,21 @@ public class BlueprintConfigurationProcessorTest {
private String name;
private Collection<String> components;
private Collection<String> hosts;
+ private Configuration configuration;
public TestHostGroup(String name, Collection<String> components, Collection<String> hosts) {
this.name = name;
this.components = components;
this.hosts = hosts;
+ this.configuration = new Configuration(Collections.<String, Map<String, String>>emptyMap(),
+ Collections.<String, Map<String, Map<String, String>>>emptyMap());
+ }
+
+ public TestHostGroup(String name, Collection<String> components, Collection<String> hosts, Configuration configuration) {
+ this.name = name;
+ this.components = components;
+ this.hosts = hosts;
+ this.configuration = configuration;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4afac300/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java
index a8b466d..65f94aa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackTest.java
@@ -35,6 +35,8 @@ import org.easymock.EasyMock;
import org.easymock.EasyMockSupport;
import org.junit.Test;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -44,6 +46,7 @@ import static org.easymock.EasyMock.expect;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
import static org.powermock.api.easymock.PowerMock.createNiceMock;
import static org.powermock.api.easymock.PowerMock.replay;
@@ -145,4 +148,86 @@ public class StackTest {
}
+ @Test
+ public void testGetRequiredProperties_serviceAndPropertyType() throws Exception {
+ AmbariManagementController controller = createNiceMock(AmbariManagementController.class);
+ AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class);
+ Capture<Set<StackServiceRequest>> stackServiceRequestCapture = new Capture<Set<StackServiceRequest>>();
+ StackServiceResponse stackServiceResponse = createNiceMock(StackServiceResponse.class);
+ Capture<Set<StackServiceComponentRequest>> stackComponentRequestCapture = new Capture<Set<StackServiceComponentRequest>>();
+ StackServiceComponentResponse stackComponentResponse = createNiceMock(StackServiceComponentResponse.class);
+ Capture<Set<StackConfigurationRequest>> stackConfigurationRequestCapture = new Capture<Set<StackConfigurationRequest>>();
+ Capture<Set<StackLevelConfigurationRequest>> stackLevelConfigurationRequestCapture = new Capture<Set<StackLevelConfigurationRequest>>();
+ StackConfigurationResponse stackConfigurationResponse = EasyMock.createNiceMock(StackConfigurationResponse.class);
+ StackConfigurationResponse stackConfigurationResponse2 = EasyMock.createNiceMock(StackConfigurationResponse.class);
+
+ expect(controller.getStackServices(capture(stackServiceRequestCapture))).
+ andReturn(Collections.singleton(stackServiceResponse)).anyTimes();
+
+ expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes();
+
+ expect(stackServiceResponse.getServiceName()).andReturn("service1").anyTimes();
+ expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.<String>emptySet());
+
+ expect(controller.getStackComponents(capture(stackComponentRequestCapture))).
+ andReturn(Collections.singleton(stackComponentResponse)).anyTimes();
+
+ expect(stackComponentResponse.getComponentName()).andReturn("component1").anyTimes();
+ expect(stackComponentResponse.getComponentCategory()).andReturn("test-site.xml").anyTimes();
+
+ expect(controller.getStackConfigurations(capture(stackConfigurationRequestCapture))).
+ andReturn(new HashSet<StackConfigurationResponse>(Arrays.asList(
+ stackConfigurationResponse, stackConfigurationResponse2))).anyTimes();
+
+ // no stack level configs for this test
+ expect(controller.getStackLevelConfigurations(capture(stackLevelConfigurationRequestCapture))).
+ andReturn(Collections.<StackConfigurationResponse>emptySet()).anyTimes();
+
+ expect(stackConfigurationResponse.getPropertyName()).andReturn("prop1").anyTimes();
+ expect(stackConfigurationResponse.getPropertyValue()).andReturn(null).anyTimes();
+ expect(stackConfigurationResponse.getType()).andReturn("test-site.xml").anyTimes();
+ expect(stackConfigurationResponse.getPropertyType()).andReturn(
+ Collections.singleton(PropertyInfo.PropertyType.PASSWORD)).anyTimes();
+ expect(stackConfigurationResponse.getPropertyAttributes()).andReturn(Collections.<String, String>emptyMap()).anyTimes();
+ expect(stackConfigurationResponse.isRequired()).andReturn(true).anyTimes();
+
+ // not a PASSWORD property type so shouldn't be returned
+ expect(stackConfigurationResponse2.getPropertyName()).andReturn("prop2").anyTimes();
+ expect(stackConfigurationResponse2.getPropertyValue()).andReturn(null).anyTimes();
+ expect(stackConfigurationResponse2.getType()).andReturn("test-site.xml").anyTimes();
+ expect(stackConfigurationResponse2.getPropertyType()).andReturn(
+ Collections.singleton(PropertyInfo.PropertyType.USER)).anyTimes();
+ expect(stackConfigurationResponse2.getPropertyAttributes()).andReturn(Collections.<String, String>emptyMap()).anyTimes();
+ expect(stackConfigurationResponse2.isRequired()).andReturn(true).anyTimes();
+
+ expect(metaInfo.getComponentDependencies("test", "1.0", "service1", "component1")).
+ andReturn(Collections.<DependencyInfo>emptyList()).anyTimes();
+
+ replay(controller, stackServiceResponse, stackComponentResponse, stackConfigurationResponse,
+ stackConfigurationResponse2, metaInfo);
+
+ // test
+ Stack stack = new Stack("test", "1.0", controller);
+ // get required password properties
+ Collection<Stack.ConfigProperty> requiredPasswordProperties = stack.getRequiredConfigurationProperties(
+ "service1", PropertyInfo.PropertyType.PASSWORD);
+
+ // assertions
+ assertEquals(1, requiredPasswordProperties.size());
+ Stack.ConfigProperty requiredPasswordConfigProperty = requiredPasswordProperties.iterator().next();
+ assertEquals("test-site", requiredPasswordConfigProperty.getType());
+ assertEquals("prop1", requiredPasswordConfigProperty.getName());
+ assertTrue(requiredPasswordConfigProperty.getPropertyTypes().contains(PropertyInfo.PropertyType.PASSWORD));
+
+ StackServiceRequest stackServiceRequest = stackServiceRequestCapture.getValue().iterator().next();
+ assertEquals("test", stackServiceRequest.getStackName());
+ assertEquals("1.0", stackServiceRequest.getStackVersion());
+
+ StackServiceComponentRequest stackComponentRequest = stackComponentRequestCapture.getValue().iterator().next();
+ assertEquals("service1", stackComponentRequest.getServiceName());
+ assertEquals("test", stackComponentRequest.getStackName());
+ assertEquals("1.0", stackComponentRequest.getStackVersion());
+ assertNull(stackComponentRequest.getComponentName());
+ }
+
}