You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/05/06 20:04:57 UTC
[1/2] git commit: AMBARI-5674. Supervisor goes into stopped state
after deploying cluster from blueprint. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-1.6.0 4f0aa6477 -> 686722423
AMBARI-5674. Supervisor goes into stopped state after deploying cluster from blueprint. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/48cfc899
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/48cfc899
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/48cfc899
Branch: refs/heads/branch-1.6.0
Commit: 48cfc89940ce9e3539e1b274971ab987e223bdbd
Parents: 4f0aa64
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Mon May 5 21:58:02 2014 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue May 6 10:59:58 2014 -0700
----------------------------------------------------------------------
.../internal/ClusterResourceProvider.java | 117 +++++++++++++++---
.../internal/ClusterResourceProviderTest.java | 123 ++++++++++++++++---
2 files changed, 207 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/48cfc899/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index 19bca09..b7b20a2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -830,6 +830,12 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
propertyUpdaters.put("hbase.zookeeper.quorum", new MultipleHostPropertyUpdater("ZOOKEEPER_SERVER"));
propertyUpdaters.put("templeton.zookeeper.hosts", new MultipleHostPropertyUpdater("ZOOKEEPER_SERVER"));
+ // STORM
+ propertyUpdaters.put("nimbus.host", new SingleHostPropertyUpdater("NIMBUS"));
+ propertyUpdaters.put("worker.childopts", new SingleHostPropertyUpdater("GANGLIA_SERVER"));
+ propertyUpdaters.put("storm.zookeeper.servers",
+ new YamlMultiValuePropertyDecorator(new MultipleHostPropertyUpdater("ZOOKEEPER_SERVER")));
+
// properties which need "m' appended. Required due to AMBARI-4933
propertyUpdaters.put("namenode_heapsize", new MPropertyUpdater());
propertyUpdaters.put("namenode_opt_newsize", new MPropertyUpdater());
@@ -1138,7 +1144,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
/**
* Host group representation.
*/
- private class HostGroup {
+ protected class HostGroup {
/**
* Host group entity
*/
@@ -1339,6 +1345,11 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
private String component;
/**
+ * Separator for multiple property values
+ */
+ private Character separator = ',';
+
+ /**
* Constructor.
*
* @param component component name associated with the property
@@ -1347,6 +1358,11 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
this.component = component;
}
+ public MultipleHostPropertyUpdater(String component, Character separator) {
+ this.component = component;
+ this.separator = separator;
+ }
+
//todo: specific to default values of EXACTLY 'localhost' or 'localhost:port'.
//todo: when blueprint contains source configurations, these props will contain actual host names, not localhost.
//todo: currently assuming that all hosts will share the same port
@@ -1360,31 +1376,30 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
* @return updated property value with old host names replaced by new host names
*/
public String update(Map<String, HostGroup> hostGroups, String origValue) {
- String newValue;
Collection<HostGroup> matchingGroups = getHostGroupsForComponent(component, hostGroups.values());
boolean containsPort = origValue.contains(":");
-
+ String port = null;
if (containsPort) {
- String port = origValue.substring(origValue.indexOf(":") + 1);
- StringBuilder sb = new StringBuilder();
- boolean firstHost = true;
- for (HostGroup group : matchingGroups) {
- for (String host : group.getHostInfo()) {
- if (! firstHost) {
- sb.append(',');
- } else {
- firstHost = false;
- }
- sb.append(host);
+ port = origValue.substring(origValue.indexOf(":") + 1);
+ }
+ StringBuilder sb = new StringBuilder();
+ boolean firstHost = true;
+ for (HostGroup group : matchingGroups) {
+ for (String host : group.getHostInfo()) {
+ if (!firstHost) {
+ sb.append(separator);
+ } else {
+ firstHost = false;
+ }
+ sb.append(host);
+ if (containsPort) {
sb.append(":");
sb.append(port);
}
}
- newValue = sb.toString();
- } else {
- newValue = matchingGroups.iterator().next().getHostInfo().iterator().next();
}
- return newValue;
+
+ return sb.toString();
}
}
@@ -1405,5 +1420,71 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
return origValue.endsWith("m") ? origValue : origValue + 'm';
}
}
+
+ /**
+ * Class to facilitate special formatting needs of property values.
+ */
+ private abstract class AbstractPropertyValueDecorator implements PropertyUpdater {
+ PropertyUpdater propertyUpdater;
+
+ public AbstractPropertyValueDecorator(PropertyUpdater propertyUpdater) {
+ this.propertyUpdater = propertyUpdater;
+ }
+
+ /**
+ * Return decorated form of the updated input property value.
+ * @param hostGroupMap Map of host group name to HostGroup
+ * @param origValue original value of property
+ *
+ * @return Formatted output string
+ */
+ @Override
+ public String update(Map<String, HostGroup> hostGroupMap, String origValue) {
+ return doFormat(propertyUpdater.update(hostGroupMap, origValue));
+ }
+
+ /**
+ * Transform input string to required output format.
+ * @param originalValue Original value of property
+ * @return Formatted output string
+ */
+ public abstract String doFormat(String originalValue);
+ }
+
+ /**
+ * Return properties of the form ['value']
+ */
+ private class YamlMultiValuePropertyDecorator extends AbstractPropertyValueDecorator {
+
+ public YamlMultiValuePropertyDecorator(PropertyUpdater propertyUpdater) {
+ super(propertyUpdater);
+ }
+
+ /**
+ * Format input String of the form, str1,str2 to ['str1','str2']
+ * @param origValue Input string
+ * @return Formatted string
+ */
+ @Override
+ public String doFormat(String origValue) {
+ StringBuilder sb = new StringBuilder();
+ if (origValue != null) {
+ sb.append("[");
+ boolean isFirst = true;
+ for (String value : origValue.split(",")) {
+ if (!isFirst) {
+ sb.append(",");
+ } else {
+ isFirst = false;
+ }
+ sb.append("'");
+ sb.append(value);
+ sb.append("'");
+ }
+ sb.append("]");
+ }
+ return sb.toString();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/48cfc899/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
index 5a48b49..fe5734a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java
@@ -18,21 +18,6 @@
package org.apache.ambari.server.controller.internal;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.*;
-
import com.google.gson.Gson;
import org.apache.ambari.server.api.services.PersistKeyValueImpl;
import org.apache.ambari.server.api.services.PersistKeyValueService;
@@ -61,10 +46,37 @@ import org.apache.ambari.server.orm.entities.BlueprintEntity;
import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
+import org.apache.commons.collections.CollectionUtils;
import org.easymock.Capture;
import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Test;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import org.apache.ambari.server.controller.internal.ClusterResourceProvider.HostGroup;
+import org.apache.ambari.server.controller.internal.ClusterResourceProvider.PropertyUpdater;
/**
* ClusterResourceProvider tests.
@@ -701,6 +713,87 @@ public class ClusterResourceProviderTest {
}
}
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testBlueprintPropertyUpdaters() throws Exception {
+ final Map<String, String> singleHostProperty1 =
+ Collections.singletonMap("dfs.http.address", "localhost:50070");
+
+ final Map<String, String> singleHostProperty2 =
+ Collections.singletonMap("hive.metastore.uris", "prefix.localhost.suffix");
+
+ final Map<String, String> multiHostProperty1 =
+ Collections.singletonMap("hbase.zookeeper.quorum", "localhost");
+
+ final Map<String, String> multiHostProperty2 =
+ Collections.singletonMap("storm.zookeeper.servers", "['localhost']");
+
+ final Map<String, String> mProperty =
+ Collections.singletonMap("namenode_heapsize", "1025");
+
+ final HostGroup hostGroup1 = createNiceMock(HostGroup.class);
+ final HostGroup hostGroup2 = createNiceMock(HostGroup.class);
+
+ expect(hostGroup1.getComponents()).andReturn(new ArrayList<String>() {{
+ add("NAMENODE");
+ add("HBASE_MASTER");
+ add("HIVE_SERVER");
+ add("ZOOKEEPER_SERVER");
+ }}).anyTimes();
+ expect(hostGroup1.getHostInfo()).andReturn(Collections.singletonList("h1")).anyTimes();
+
+ expect(hostGroup2.getComponents()).andReturn(Collections.singletonList("ZOOKEEPER_SERVER")).anyTimes();
+ expect(hostGroup2.getHostInfo()).andReturn(Collections.singletonList("h2")).anyTimes();
+
+ Map<String, HostGroup> hostGroups = new
+ HashMap<String, HostGroup>() {{
+ put("host_group_1", hostGroup1);
+ put("host_group_2", hostGroup2);
+ }};
+
+ AmbariManagementController managementController = createNiceMock(AmbariManagementController.class);
+
+ ClusterResourceProvider resourceProvider =
+ createMockBuilder(ClusterResourceProvider.class)
+ .withConstructor(Set.class, Map.class, AmbariManagementController.class)
+ .withArgs(new HashSet<String>(), new HashMap<Resource.Type, String>(), managementController)
+ .createMock();
+
+ replay(managementController, resourceProvider, hostGroup1, hostGroup2);
+
+ Map<String, PropertyUpdater> propertyUpdaterMap;
+ Field f = ClusterResourceProvider.class.getDeclaredField("propertyUpdaters");
+ f.setAccessible(true);
+ propertyUpdaterMap = (Map<String, PropertyUpdater>) f.get(resourceProvider);
+
+ Assert.assertNotNull(propertyUpdaterMap);
+
+ String newValue;
+
+ Map.Entry<String, String> entry = singleHostProperty1.entrySet().iterator().next();
+ newValue = propertyUpdaterMap.get(entry.getKey()).update(hostGroups, entry.getValue());
+ Assert.assertEquals("h1:50070", newValue);
+
+ entry = singleHostProperty2.entrySet().iterator().next();
+ newValue = propertyUpdaterMap.get(entry.getKey()).update(hostGroups, entry.getValue());
+ Assert.assertEquals("prefix.h1.suffix", newValue);
+
+ entry = multiHostProperty1.entrySet().iterator().next();
+ newValue = propertyUpdaterMap.get(entry.getKey()).update(hostGroups, entry.getValue());
+ Assert.assertTrue(CollectionUtils.isEqualCollection(
+ Arrays.asList("h1,h2".split(",")), Arrays.asList(newValue.split(","))
+ ));
+
+ entry = multiHostProperty2.entrySet().iterator().next();
+ newValue = propertyUpdaterMap.get(entry.getKey()).update(hostGroups, entry.getValue());
+ Assert.assertEquals("['h1','h2']", newValue);
+
+ entry = mProperty.entrySet().iterator().next();
+ newValue = propertyUpdaterMap.get(entry.getKey()).update(hostGroups, entry.getValue());
+ Assert.assertEquals("1025m", newValue);
+
+ verify(managementController, resourceProvider, hostGroup1, hostGroup2);
+ }
@Test
public void testGetResources() throws Exception{
[2/2] git commit: AMBARI-5674. Supervisor goes into stopped state
after deploying cluster from blueprint. Missing doc. (swagle)
Posted by sw...@apache.org.
AMBARI-5674. Supervisor goes into stopped state after deploying cluster from blueprint. Missing doc. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/68672242
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/68672242
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/68672242
Branch: refs/heads/branch-1.6.0
Commit: 686722423e4a1d9331d14e00a3dcb4c3ff13a73a
Parents: 48cfc89
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Tue May 6 11:04:04 2014 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue May 6 11:04:04 2014 -0700
----------------------------------------------------------------------
.../server/controller/internal/ClusterResourceProvider.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/68672242/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
index b7b20a2..e884e97 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java
@@ -1358,6 +1358,11 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
this.component = component;
}
+ /**
+ * Constructor with customized separator.
+ * @param component Component name
+ * @param separator separator character
+ */
public MultipleHostPropertyUpdater(String component, Character separator) {
this.component = component;
this.separator = separator;