You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2014/02/14 22:12:05 UTC
git commit: AMBARI-4685. API doesn't return NN or RM metrics with
customized port (ncole)
Updated Branches:
refs/heads/trunk b59b258d2 -> de4fe5f14
AMBARI-4685. API doesn't return NN or RM metrics with customized port (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/de4fe5f1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/de4fe5f1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/de4fe5f1
Branch: refs/heads/trunk
Commit: de4fe5f14350e4bbdd35adfd351ca357fb8ab757
Parents: b59b258
Author: Nate Cole <nc...@hortonworks.com>
Authored: Fri Feb 14 14:27:49 2014 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Fri Feb 14 15:51:04 2014 -0500
----------------------------------------------------------------------
.../internal/AbstractProviderModule.java | 73 ++++++++-----
.../org/apache/ambari/server/state/Service.java | 3 +-
.../internal/JMXHostProviderTest.java | 107 +++++++++++++------
3 files changed, 126 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/de4fe5f1/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
index 3053998..23eafcb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
@@ -50,6 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+import java.util.Map.Entry;
/**
* An abstract provider module implementation.
@@ -68,12 +69,12 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
private static final Map<Service.Type, String> serviceConfigVersions =
Collections.synchronizedMap(new HashMap<Service.Type, String>());
private static final Map<Service.Type, String> serviceConfigTypes = new HashMap<Service.Type, String>();
- private static final Map<Service.Type, Map<String, String>> serviceDesiredProperties = new
- HashMap<Service.Type, Map<String, String>>();
+ private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new
+ HashMap<Service.Type, Map<String, String[]>>();
private static final Map<String, Service.Type> componentServiceMap = new
HashMap<String, Service.Type>();
- private static final Map<String, Map<String, String>> jmxDesiredProperties = new HashMap<String, Map<String,String>>();
+ private static final Map<String, Map<String, String[]>> jmxDesiredProperties = new HashMap<String, Map<String,String[]>>();
private volatile Map<String, String> clusterCoreSiteConfigVersionMap = new HashMap<String, String>();
private volatile Map<String, String> clusterJmxProtocolMap = new HashMap<String, String>();
@@ -81,27 +82,35 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
serviceConfigTypes.put(Service.Type.HDFS, "hdfs-site");
serviceConfigTypes.put(Service.Type.MAPREDUCE, "mapred-site");
serviceConfigTypes.put(Service.Type.HBASE, "hbase-site");
+ serviceConfigTypes.put(Service.Type.YARN, "yarn-site");
componentServiceMap.put("NAMENODE", Service.Type.HDFS);
componentServiceMap.put("DATANODE", Service.Type.HDFS);
componentServiceMap.put("JOBTRACKER", Service.Type.MAPREDUCE);
componentServiceMap.put("TASKTRACKER", Service.Type.MAPREDUCE);
componentServiceMap.put("HBASE_MASTER", Service.Type.HBASE);
+ componentServiceMap.put("RESOURCEMANAGER", Service.Type.YARN);
- Map<String, String> initPropMap = new HashMap<String, String>();
- initPropMap.put("NAMENODE", "dfs.http.address");
- initPropMap.put("DATANODE", "dfs.datanode.http.address");
+ Map<String, String[]> initPropMap = new HashMap<String, String[]>();
+ initPropMap.put("NAMENODE", new String[] {"dfs.http.address", "dfs.namenode.http-address"});
+ initPropMap.put("DATANODE", new String[] {"dfs.datanode.http.address"});
serviceDesiredProperties.put(Service.Type.HDFS, initPropMap);
- initPropMap = new HashMap<String, String>();
- initPropMap.put("JOBTRACKER", "mapred.job.tracker.http.address");
- initPropMap.put("TASKTRACKER", "mapred.task.tracker.http.address");
+
+ initPropMap = new HashMap<String, String[]>();
+ initPropMap.put("JOBTRACKER", new String[] {"mapred.job.tracker.http.address"});
+ initPropMap.put("TASKTRACKER", new String[] {"mapred.task.tracker.http.address"});
serviceDesiredProperties.put(Service.Type.MAPREDUCE, initPropMap);
- initPropMap = new HashMap<String, String>();
- initPropMap.put("HBASE_MASTER", "hbase.master.info.port");
+
+ initPropMap = new HashMap<String, String[]>();
+ initPropMap.put("HBASE_MASTER", new String[] {"hbase.master.info.port"});
serviceDesiredProperties.put(Service.Type.HBASE, initPropMap);
+
+ initPropMap = new HashMap<String, String[]>();
+ initPropMap.put("RESOURCEMANAGER", new String[] {"yarn.resourcemanager.webapp.address"});
+ serviceDesiredProperties.put(Service.Type.YARN, initPropMap);
- initPropMap = new HashMap<String, String>();
- initPropMap.put("NAMENODE", "hadoop.ssl.enabled");
+ initPropMap = new HashMap<String, String[]>();
+ initPropMap.put("NAMENODE", new String[] {"hadoop.ssl.enabled"});
jmxDesiredProperties.put("NAMENODE", initPropMap);
}
@@ -208,6 +217,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
}
}
Service.Type service = componentServiceMap.get(componentName);
+
if (service != null) {
try {
String currVersion = getDesiredConfigVersion(clusterName,
@@ -216,9 +226,11 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
String oldVersion = serviceConfigVersions.get(service);
if (!currVersion.equals(oldVersion)) {
serviceConfigVersions.put(service, currVersion);
+
Map<String, String> portMap = getDesiredConfigMap(clusterName,
currVersion, serviceConfigTypes.get(service),
serviceDesiredProperties.get(service));
+
for (String compName : portMap.keySet()) {
clusterJmxPorts.put(compName, getPortString(portMap.get
(compName)));
@@ -594,7 +606,7 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
}
private Map<String, String> getDesiredConfigMap(String clusterName,
- String versionTag, String configType, Map<String, String> keys) throws
+ String versionTag, String configType, Map<String, String[]> keys) throws
NoSuchParentResourceException, UnsupportedPropertyException,
SystemException {
// Get desired configs based on the tag
@@ -613,20 +625,30 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
LOG.info("Resource for the desired config not found. " + e);
return Collections.emptyMap();
}
+
Map<String, String> mConfigs = new HashMap<String, String>();
if (configResources != null) {
for (Resource res : configResources) {
Map<String, String> evalutedProperties = null;
- for (String key : keys.keySet()) {
- String value = (String) res.getPropertyValue
- (PropertyHelper.getPropertyId(PROPERTIES_CATEGORY, keys.get(key)));
- if (value != null && value.contains("${"))
+ for (Entry<String,String[]> entry : keys.entrySet()) {
+ String propName = null;
+ String value = null;
+
+ for (String pname : entry.getValue()) {
+ propName = pname;
+ value = (String) res.getPropertyValue(PropertyHelper.getPropertyId(
+ PROPERTIES_CATEGORY, pname));
+ if (null != value)
+ break;
+ }
+
+ if (value != null && value.contains("${")) {
if (evalutedProperties == null){
evalutedProperties = new HashMap<String, String>();
Map<String, Object> properties = res.getPropertiesMap().get(PROPERTIES_CATEGORY);
- for (Map.Entry<String, Object> entry : properties.entrySet()) {
- String keyString = entry.getKey();
- Object object = entry.getValue();
+ for (Map.Entry<String, Object> subentry : properties.entrySet()) {
+ String keyString = subentry.getKey();
+ Object object = subentry.getValue();
String valueString;
if (object != null && object instanceof String){
valueString = (String)object;
@@ -635,10 +657,11 @@ public abstract class AbstractProviderModule implements ProviderModule, Resource
}
}
}
- value = postProcessPropertyValue(keys.get(key), value, evalutedProperties, null);
- LOG.debug("PROPERTY -> key: " + keys.get(key) + ", " +
- "value: " + value);
- mConfigs.put(key, value);
+ }
+ value = postProcessPropertyValue(propName, value, evalutedProperties, null);
+ LOG.debug("PROPERTY -> key: " + propName + ", " + "value: " + value);
+
+ mConfigs.put(entry.getKey(), value);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/de4fe5f1/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
index b4238e0..d3d8fd2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
@@ -114,6 +114,7 @@ public interface Service {
ZOOKEEPER,
PIG,
HCATALOG,
- FLUME
+ FLUME,
+ YARN
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/de4fe5f1/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java
index 0b5f660..ec82e55 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/JMXHostProviderTest.java
@@ -17,13 +17,27 @@
*/
package org.apache.ambari.server.controller.internal;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.persist.PersistService;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.*;
-import org.apache.ambari.server.controller.spi.*;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
+import org.apache.ambari.server.controller.ServiceComponentHostRequest;
+import org.apache.ambari.server.controller.ServiceComponentRequest;
+import org.apache.ambari.server.controller.ServiceRequest;
+import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
+import org.apache.ambari.server.controller.spi.NoSuchResourceException;
+import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException;
+import org.apache.ambari.server.controller.spi.ResourceProvider;
+import org.apache.ambari.server.controller.spi.SystemException;
+import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
import org.apache.ambari.server.controller.utilities.PropertyHelper;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
@@ -36,17 +50,16 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.persist.PersistService;
public class JMXHostProviderTest {
private Injector injector;
private Clusters clusters;
static AmbariManagementController controller;
- private static final String NAMENODE_PORT = "dfs.http.address";
+ private static final String NAMENODE_PORT_V1 = "dfs.http.address";
+ private static final String NAMENODE_PORT_V2 = "dfs.namenode.http-address";
private static final String DATANODE_PORT = "dfs.datanode.http.address";
@Before
@@ -106,7 +119,7 @@ public class JMXHostProviderTest {
controller.createHostComponents(requests);
}
- private void createHDFSServiceConfigs() throws AmbariException {
+ private void createHDFSServiceConfigs(boolean version1) throws AmbariException {
String clusterName = "c1";
ClusterRequest r = new ClusterRequest(null, clusterName, "HDP-0.1", null);
controller.createCluster(r);
@@ -148,21 +161,32 @@ public class JMXHostProviderTest {
host1, null);
createServiceComponentHost(clusterName, serviceName, componentName3,
host2, null);
-
+
// Create configs
- Map<String, String> configs = new HashMap<String, String>();
- configs.put(NAMENODE_PORT, "localhost:${ambari.dfs.datanode.http.port}");
- configs.put(DATANODE_PORT, "localhost:70075");
- configs.put("ambari.dfs.datanode.http.port", "70070");
- ConfigurationRequest cr = new ConfigurationRequest(clusterName,
- "hdfs-site", "version1", configs);
- controller.createConfiguration(cr);
-
- Map<String, String> configVersions = new HashMap<String, String>();
- Set<ServiceRequest> sReqs = new HashSet<ServiceRequest>();
- configVersions.put("hdfs-site", "version1");
- sReqs.add(new ServiceRequest(clusterName, serviceName, null));
- ServiceResourceProviderTest.updateServices(controller, sReqs, mapRequestProps, true, false);
+ if (version1) {
+ Map<String, String> configs = new HashMap<String, String>();
+ configs.put(NAMENODE_PORT_V1, "localhost:${ambari.dfs.datanode.http.port}");
+ configs.put(DATANODE_PORT, "localhost:70075");
+ configs.put("ambari.dfs.datanode.http.port", "70070");
+
+ ConfigurationRequest cr = new ConfigurationRequest(clusterName,
+ "hdfs-site", "version1", configs);
+ ClusterRequest crequest = new ClusterRequest(null, clusterName, null, null);
+ crequest.setDesiredConfig(cr);
+ controller.updateClusters(Collections.singleton(crequest), new HashMap<String,String>());
+
+ } else {
+ Map<String, String> configs = new HashMap<String, String>();
+ configs.put(NAMENODE_PORT_V2, "localhost:70071");
+ configs.put(DATANODE_PORT, "localhost:70075");
+
+ ConfigurationRequest cr = new ConfigurationRequest(clusterName,
+ "hdfs-site", "version2", configs);
+
+ ClusterRequest crequest = new ClusterRequest(null, clusterName, null, null);
+ crequest.setDesiredConfig(cr);
+ controller.updateClusters(Collections.singleton(crequest), new HashMap<String,String>());
+ }
}
private void createConfigs() throws AmbariException {
@@ -207,8 +231,9 @@ public class JMXHostProviderTest {
// Create configs
Map<String, String> configs = new HashMap<String, String>();
- configs.put(NAMENODE_PORT, "localhost:70070");
+ configs.put(NAMENODE_PORT_V1, "localhost:${ambari.dfs.datanode.http.port}");
configs.put(DATANODE_PORT, "localhost:70075");
+ configs.put("ambari.dfs.datanode.http.port", "70070");
ConfigurationRequest cr1 = new ConfigurationRequest(clusterName,
"hdfs-site", "versionN", configs);
@@ -219,18 +244,18 @@ public class JMXHostProviderTest {
Cluster cluster = clusters.getCluster(clusterName);
Assert.assertEquals("versionN", cluster.getDesiredConfigByType("hdfs-site")
.getVersionTag());
- Assert.assertEquals("localhost:70070", cluster.getDesiredConfigByType
- ("hdfs-site").getProperties().get(NAMENODE_PORT));
+ Assert.assertEquals("localhost:${ambari.dfs.datanode.http.port}", cluster.getDesiredConfigByType
+ ("hdfs-site").getProperties().get(NAMENODE_PORT_V1));
}
@Test
- public void testJMXPortMapInitAtServiceLevel() throws
+ public void testJMXPortMapInitAtServiceLevelVersion1() throws
NoSuchParentResourceException,
ResourceAlreadyExistsException, UnsupportedPropertyException,
SystemException, AmbariException, NoSuchResourceException {
- createHDFSServiceConfigs();
+ createHDFSServiceConfigs(true);
JMXHostProviderModule providerModule = new JMXHostProviderModule();
providerModule.registerResourceProvider(Resource.Type.Service);
@@ -243,6 +268,26 @@ public class JMXHostProviderTest {
Assert.assertEquals(null, providerModule.getPort("c1", "TASKTRACKER"));
Assert.assertEquals(null, providerModule.getPort("c1", "HBASE_MASTER"));
}
+
+ @Test
+ public void testJMXPortMapInitAtServiceLevelVersion2() throws
+ NoSuchParentResourceException,
+ ResourceAlreadyExistsException, UnsupportedPropertyException,
+ SystemException, AmbariException, NoSuchResourceException {
+
+ createHDFSServiceConfigs(false);
+
+ JMXHostProviderModule providerModule = new JMXHostProviderModule();
+ providerModule.registerResourceProvider(Resource.Type.Service);
+ providerModule.registerResourceProvider(Resource.Type.Configuration);
+ // Non default port addresses
+ Assert.assertEquals("70071", providerModule.getPort("c1", "NAMENODE"));
+ Assert.assertEquals("70075", providerModule.getPort("c1", "DATANODE"));
+ // Default port addresses
+ Assert.assertEquals(null, providerModule.getPort("c1", "JOBTRACKER"));
+ Assert.assertEquals(null, providerModule.getPort("c1", "TASKTRACKER"));
+ Assert.assertEquals(null, providerModule.getPort("c1", "HBASE_MASTER"));
+ }
@Test
public void testJMXPortMapInitAtClusterLevel() throws