You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rn...@apache.org on 2015/01/21 18:03:59 UTC
ambari git commit: AMBARI-9223. Add properties_attributes support to
exported Blueprints. (rnettleton)
Repository: ambari
Updated Branches:
refs/heads/trunk f5672d009 -> bc6f2c785
AMBARI-9223. Add properties_attributes support to exported Blueprints. (rnettleton)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bc6f2c78
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bc6f2c78
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bc6f2c78
Branch: refs/heads/trunk
Commit: bc6f2c78552e5a21eaff72919d9b55d1dd79a290
Parents: f5672d0
Author: Bob Nettleton <rn...@hortonworks.com>
Authored: Wed Jan 21 12:02:59 2015 -0500
Committer: Bob Nettleton <rn...@hortonworks.com>
Committed: Wed Jan 21 12:03:48 2015 -0500
----------------------------------------------------------------------
.../query/render/ClusterBlueprintRenderer.java | 37 ++++-
.../BlueprintConfigurationProcessor.java | 2 +-
.../render/ClusterBlueprintRendererTest.java | 165 ++++++++++++++++++-
.../BlueprintConfigurationProcessorTest.java | 23 +++
4 files changed, 213 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc6f2c78/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
index a584490..a5cdfe0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
@@ -239,10 +239,10 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
*
* @return cluster configuration
*/
- private List<Map<String, Map<String, String>>> processConfigurations(TreeNode<Resource> clusterNode,
+ private List<Map<String, Map<String, Map<String, ?>>>> processConfigurations(TreeNode<Resource> clusterNode,
Collection<HostGroupImpl> hostGroups) {
- List<Map<String, Map<String, String>>> configList = new ArrayList<Map<String, Map<String, String>>>();
+ List<Map<String, Map<String, Map<String, ?>>>> configList = new ArrayList<Map<String, Map<String, Map<String, ?>>>>();
Map<String, Object> desiredConfigMap = clusterNode.getObject().getPropertiesMap().get("Clusters/desired_configs");
TreeNode<Resource> configNode = clusterNode.getChild("configurations");
@@ -255,7 +255,16 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
properties = updater.doUpdateForBlueprintExport(hostGroups);
- configList.add(properties);
+
+ // build up maps for properties and property attributes
+ Map<String, Map<String, ?>> typeMap =
+ new HashMap<String, Map<String, ?>>();
+ typeMap.put("properties", properties.get(configuration.getType()));
+ if ((configuration.getPropertyAttributes() != null) && !configuration.getPropertyAttributes().isEmpty()) {
+ typeMap.put("properties_attributes", configuration.getPropertyAttributes());
+ }
+
+ configList.add(Collections.singletonMap(configuration.getType(), typeMap));
}
}
return configList;
@@ -566,6 +575,12 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
private Map<String, String> properties = new HashMap<String, String>();
/**
+ * Attributes for the properties in the cluster configuration.
+ */
+ private Map<String, ?> propertyAttributes =
+ new HashMap<String, Object>();
+
+ /**
* Constructor.
*
* @param configNode configuration node
@@ -579,6 +594,9 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
// property map type is currently <String, Object>
properties = (Map) configNode.getObject().getPropertiesMap().get("properties");
+ // get the property attributes set in this configuration
+ propertyAttributes = (Map) configNode.getObject().getPropertiesMap().get("properties_attributes");
+
if (properties != null) {
stripRequiredProperties(properties);
} else {
@@ -617,13 +635,23 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
return properties;
}
+ /**
+ * Get property attributes.
+ *
+ * @return map of property attributes
+ */
+ public Map<String, ?> getPropertyAttributes() {
+ return propertyAttributes;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Configuration that = (Configuration) o;
- return tag.equals(that.tag) && type.equals(that.type) && properties.equals(that.properties);
+ return tag.equals(that.tag) && type.equals(that.type) && properties.equals(that.properties)
+ && propertyAttributes.equals(that.propertyAttributes);
}
@Override
@@ -631,6 +659,7 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
int result = type.hashCode();
result = 31 * result + tag.hashCode();
result = 31 * result + properties.hashCode();
+ result = 31 * result + propertyAttributes.hashCode();
return result;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc6f2c78/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 8f84f62..e7219e9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -891,7 +891,7 @@ public class BlueprintConfigurationProcessor {
indexOfEnd = matcher.end();
}
- if (indexOfEnd < (origValue.length() - 1)) {
+ if ((indexOfEnd > -1) && (indexOfEnd < (origValue.length() - 1))) {
suffix = origValue.substring(indexOfEnd);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc6f2c78/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
index 61ea1ad..15280b9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.api.query.render;
+import junit.framework.Assert;
import org.apache.ambari.server.api.query.QueryInfo;
import org.apache.ambari.server.api.resources.ClusterResourceDefinition;
import org.apache.ambari.server.api.resources.HostComponentResourceDefinition;
@@ -30,24 +31,23 @@ import org.apache.ambari.server.api.util.TreeNodeImpl;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.ResourceImpl;
import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackInfo;
import org.junit.Test;
-import java.lang.reflect.Field;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -177,17 +177,162 @@ public class ClusterBlueprintRendererTest {
}
}
+ @Test
+ public void testFinalizeResultWithAttributes() throws Exception{
+
+ AmbariManagementController controller = createMock(AmbariManagementController.class);
+ AmbariMetaInfo stackInfo = createNiceMock(AmbariMetaInfo.class);
+ ServiceInfo hdfsService = new ServiceInfo();
+ hdfsService.setName("HDFS");
+ ServiceInfo mrService = new ServiceInfo();
+ mrService.setName("MAPREDUCE");
+
+ expect(controller.getAmbariMetaInfo()).andReturn(stackInfo).atLeastOnce();
+ expect(stackInfo.getStack("HDP", "1.3.3")).andReturn(new StackInfo()).atLeastOnce();
+
+ replay(controller, stackInfo);
+
+ Result result = new ResultImpl(true);
+ Map<String, Object> testDesiredConfigMap =
+ new HashMap<String, Object>();
+
+ DesiredConfig testDesiredConfig =
+ new DesiredConfig();
+
+ testDesiredConfig.setTag("test-tag-one");
+ testDesiredConfigMap.put("test-type-one", testDesiredConfig);
+
+ createClusterResultTree(result.getResultTree(), testDesiredConfigMap);
+
+ ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(controller);
+ Result blueprintResult = renderer.finalizeResult(result);
+
+ TreeNode<Resource> blueprintTree = blueprintResult.getResultTree();
+ assertNull(blueprintTree.getProperty("isCollection"));
+ assertEquals(1, blueprintTree.getChildren().size());
+
+ TreeNode<Resource> blueprintNode = blueprintTree.getChildren().iterator().next();
+ assertEquals(0, blueprintNode.getChildren().size());
+ Resource blueprintResource = blueprintNode.getObject();
+ Map<String, Map<String, Object>> properties = blueprintResource.getPropertiesMap();
+
+ assertEquals("HDP", properties.get("Blueprints").get("stack_name"));
+ assertEquals("1.3.3", properties.get("Blueprints").get("stack_version"));
+
+ Collection<Map<String, Object>> host_groups = (Collection<Map<String, Object>>) properties.get("").get("host_groups");
+ assertEquals(2, host_groups.size());
+
+ for (Map<String, Object> hostGroupProperties : host_groups) {
+ String host_group_name = (String) hostGroupProperties.get("name");
+ if (host_group_name.equals("host_group_1")) {
+ assertEquals("1", hostGroupProperties.get("cardinality"));
+
+ Collection<Map<String, String>> components = (Collection<Map<String, String>>) hostGroupProperties.get("components");
+ // 4 specified components and ambari server
+ assertEquals(5, components.size());
+
+ Set<String> expectedValues = new HashSet<String>(
+ Arrays.asList("JOBTRACKER", "TASKTRACKER", "NAMENODE", "DATANODE", "AMBARI_SERVER"));
+
+ Set<String> actualValues = new HashSet<String>();
+
+
+ for (Map<String, String> componentProperties : components) {
+ assertEquals(1, componentProperties.size());
+ actualValues.add(componentProperties.get("name"));
+ }
+ assertEquals(expectedValues, actualValues);
+ } else if (host_group_name.equals("host_group_2")) {
+ // cardinality is 2 because 2 hosts share same topology
+ assertEquals("2", hostGroupProperties.get("cardinality"));
+
+ Collection<Map<String, String>> components = (Collection<Map<String, String>>) hostGroupProperties.get("components");
+ assertEquals(2, components.size());
+
+ Set<String> expectedValues = new HashSet<String>(
+ Arrays.asList("TASKTRACKER", "DATANODE"));
+
+ Set<String> actualValues = new HashSet<String>();
+
+
+ for (Map<String, String> componentProperties : components) {
+ assertEquals(1, componentProperties.size());
+ actualValues.add(componentProperties.get("name"));
+ }
+ assertEquals(expectedValues, actualValues);
+ }
+ }
+
+ List<Map<String, Map<String, Map<String, ?>>>> configurationsResult =
+ (List<Map<String, Map<String, Map<String, ?>>>>)blueprintResource.getPropertyValue("configurations");
+
+ assertEquals("Incorrect number of config maps added",
+ 1, configurationsResult.size());
+
+ Map<String, Map<String, ?>> configMap =
+ configurationsResult.iterator().next().get("test-type-one");
+
+ assertNotNull("Expected config map was not included", configMap);
+
+ assertEquals("Incorrect number of maps added under expected type",
+ 2, configMap.size());
+
+ assertTrue("Expected properties map was not found",
+ configMap.containsKey("properties"));
+ assertTrue("Expected properties_attributes map was not found",
+ configMap.containsKey("properties_attributes"));
+
+ Map<String, ?> propertiesResult =
+ configMap.get("properties");
+ assertEquals("Incorrect number of config properties found",
+ 1, propertiesResult.size());
+
+ Map<String, ?> attributesResult =
+ configMap.get("properties_attributes");
+ assertEquals("Incorrect number of config attributes found",
+ 1, attributesResult.size());
+
+ // verify the correct properties were added to the exported Blueprint
+ assertEquals("Incorrect property value included",
+ "valueOne", propertiesResult.get("propertyOne"));
+
+ // verify that the expected attributes were added to the exported Blueprint
+ assertNotNull("Expected attribute not found in exported Blueprint",
+ attributesResult.get("final"));
+
+ assertTrue("Attribute type map was not included",
+ attributesResult.get("final") instanceof Map);
+
+ Map<String, ?> finalMap =
+ (Map<String, ?>)attributesResult.get("final");
+
+ assertEquals("Attribute value is not correct",
+ "true", finalMap.get("propertyOne"));
+
+ verify(controller, stackInfo);
+ }
+
//todo: collection resource
- private void createClusterResultTree(TreeNode<Resource> resultTree) throws Exception{
+ private void createClusterResultTree(TreeNode<Resource> resultTree) throws Exception {
+ createClusterResultTree(resultTree, null);
+ }
+
+ private void createClusterResultTree(TreeNode<Resource> resultTree, final Map<String, Object> desiredConfig) throws Exception{
Resource clusterResource = new ResourceImpl(Resource.Type.Cluster) {
@Override
public Map<String, Map<String, Object>> getPropertiesMap() {
Map<String, Map<String, Object>> originalMap =
super.getPropertiesMap();
- // override the properties map for simpler testing
- originalMap.put("Clusters/desired_configs", Collections.<String, Object>emptyMap());
+ if (desiredConfig == null) {
+ // override the properties map for simpler testing
+ originalMap.put("Clusters/desired_configs", Collections.<String, Object>emptyMap());
+ } else {
+ // allow for unit tests to customize this, needed for attributes export testing
+ originalMap.put("Clusters/desired_configs", desiredConfig);
+ }
+
return originalMap;
};
@@ -230,14 +375,16 @@ public class ClusterBlueprintRendererTest {
Map<String, Map<String, Object>> originalMap =
super.getPropertiesMap();
- // return test properties, to simluate valid configuration entry
+ // return test properties, to simulate valid configuration entry
originalMap.put("properties", Collections.<String, Object>singletonMap("propertyOne", "valueOne"));
+ originalMap.put("properties_attributes", Collections.<String, Object>singletonMap("final", Collections.singletonMap("propertyOne", "true")));
return originalMap;
}
};
resourceTwo.setProperty("type", "test-type-one");
+ resourceTwo.setProperty("tag", "test-tag-one");
configurations.addChild(resourceTwo, "resourceTwo");
http://git-wip-us.apache.org/repos/asf/ambari/blob/bc6f2c78/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index c8d1214..9f47163 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -924,6 +924,29 @@ public class BlueprintConfigurationProcessorTest {
}
@Test
+ public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPorts_singleHostValue() {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> yarnSiteConfig = new HashMap<String, String>();
+
+ yarnSiteConfig.put("hadoop.registry.zk.quorum", "%HOSTGROUP::host_group_1%:2181");
+ properties.put("yarn-site", yarnSiteConfig);
+
+ Collection<String> hgComponents = new HashSet<String>();
+ hgComponents.add("NAMENODE");
+ hgComponents.add("SECONDARY_NAMENODE");
+ hgComponents.add("ZOOKEEPER_SERVER");
+ HostGroup group1 = new TestHostGroup("host_group_1", Collections.singleton("testhost"), hgComponents);
+
+ Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>();
+ hostGroups.put(group1.getName(), group1);
+
+ BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties);
+ Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null);
+ assertEquals("Multi-host property with single host value was not correctly updated for cluster create.",
+ "testhost:2181", updatedProperties.get("yarn-site").get("hadoop.registry.zk.quorum"));
+ }
+
+ @Test
public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___YAML() {
Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
Map<String, String> typeProps = new HashMap<String, String>();