You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2013/04/25 01:19:46 UTC
svn commit: r1471764 - in /incubator/ambari/trunk: ./
ambari-server/src/main/java/org/apache/ambari/server/controller/internal/
ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/
ambari-server/src/test/java/org/apache/ambari/server/co...
Author: tbeerbower
Date: Wed Apr 24 23:19:46 2013
New Revision: 1471764
URL: http://svn.apache.org/r1471764
Log:
AMBARI-2022 - Service Component metric collection API takes over a minute on large cluster
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java
incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Wed Apr 24 23:19:46 2013
@@ -269,6 +269,9 @@ Trunk (unreleased changes):
IMPROVEMENTS
+ AMBARI-2022. Service Component metric collection API takes over a minute
+ on large cluster. (tbeerbower)
+
AMBARI-2005. When adding a component to a host (after cluster deployment),
UI should warn that nagios server need to be restarted. (yusaku)
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java Wed Apr 24 23:19:46 2013
@@ -42,6 +42,9 @@ import java.util.*;
*/
public abstract class AbstractProviderModule implements ProviderModule, ResourceProviderObserver, JMXHostProvider, GangliaHostProvider {
+ private static final int PROPERTY_REQUEST_CONNECT_TIMEOUT = 5000;
+ private static final int PROPERTY_REQUEST_READ_TIMEOUT = 10000;
+
private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters", "cluster_name");
private static final String HOST_COMPONENT_CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
private static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
@@ -106,7 +109,7 @@ public abstract class AbstractProviderMo
/**
* JMX ports read from the configs
*/
- private Map<String, Map<String, String>> jmxPortMap = Collections
+ private final Map<String, Map<String, String>> jmxPortMap = Collections
.synchronizedMap(new HashMap<String, Map<String, String>>());
private volatile boolean initialized = false;
@@ -175,6 +178,7 @@ public abstract class AbstractProviderMo
Map<String,String> clusterJmxPorts = jmxPortMap.get(clusterName);
if (clusterJmxPorts == null) {
synchronized (jmxPortMap) {
+ clusterJmxPorts = jmxPortMap.get(clusterName);
if (clusterJmxPorts == null) {
clusterJmxPorts = new HashMap<String, String>();
jmxPortMap.put(clusterName, clusterJmxPorts);
@@ -242,7 +246,8 @@ public abstract class AbstractProviderMo
List<PropertyProvider> providers = new LinkedList<PropertyProvider>();
- URLStreamProvider streamProvider = new URLStreamProvider();
+ URLStreamProvider streamProvider = new URLStreamProvider(
+ PROPERTY_REQUEST_CONNECT_TIMEOUT, PROPERTY_REQUEST_READ_TIMEOUT);
switch (type){
case Cluster :
@@ -268,7 +273,9 @@ public abstract class AbstractProviderMo
this,
PropertyHelper.getPropertyId("ServiceComponentInfo", "cluster_name"),
null,
- PropertyHelper.getPropertyId("ServiceComponentInfo", "component_name")));
+ PropertyHelper.getPropertyId("ServiceComponentInfo", "component_name"),
+ PropertyHelper.getPropertyId("ServiceComponentInfo", "state"),
+ Collections.singleton("STARTED")));
providers.add(new GangliaComponentPropertyProvider(
PropertyHelper.getGangliaPropertyIds(type),
@@ -284,7 +291,9 @@ public abstract class AbstractProviderMo
this,
PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
PropertyHelper.getPropertyId("HostRoles", "host_name"),
- PropertyHelper.getPropertyId("HostRoles", "component_name")));
+ PropertyHelper.getPropertyId("HostRoles", "component_name"),
+ PropertyHelper.getPropertyId("HostRoles", "state"),
+ Collections.singleton("STARTED")));
providers.add(new GangliaHostComponentPropertyProvider(
PropertyHelper.getGangliaPropertyIds(type),
@@ -325,7 +334,7 @@ public abstract class AbstractProviderMo
Request request = PropertyHelper.getReadRequest(CLUSTER_NAME_PROPERTY_ID);
try {
- jmxPortMap = new HashMap<String, Map<String, String>>();
+ jmxPortMap.clear();
Set<Resource> clusters = provider.getResources(request, null);
clusterHostComponentMap = new HashMap<String, Map<String, String>>();
@@ -411,8 +420,8 @@ public abstract class AbstractProviderMo
String versionTag = "version1";
if (serviceResource != null) {
for (Resource res : serviceResource) {
- Map<String, String> configs = (Map<String,
- String>) res.getPropertyValue(ServiceResourceProvider.SERVICE_DESIRED_CONFIGS_PROPERTY_ID);
+ Map<String, String> configs = (Map<String, String>)
+ res.getPropertyValue(ServiceResourceProvider.SERVICE_DESIRED_CONFIGS_PROPERTY_ID);
if (configs != null) {
versionTag = configs.get(configType);
}
@@ -431,7 +440,7 @@ public abstract class AbstractProviderMo
(ConfigurationResourceProvider.CONFIGURATION_CLUSTER_NAME_PROPERTY_ID).equals(clusterName).and()
.property(ConfigurationResourceProvider.CONFIGURATION_CONFIG_TYPE_PROPERTY_ID).equals(configType).and()
.property(ConfigurationResourceProvider.CONFIGURATION_CONFIG_TAG_PROPERTY_ID).equals(versionTag).toPredicate();
- Set<Resource> configResources = null;
+ Set<Resource> configResources;
try {
configResources = configResourceProvider.getResources
(PropertyHelper.getReadRequest(ConfigurationResourceProvider.CONFIGURATION_CLUSTER_NAME_PROPERTY_ID,
@@ -439,7 +448,7 @@ public abstract class AbstractProviderMo
ConfigurationResourceProvider.CONFIGURATION_CONFIG_TAG_PROPERTY_ID), configPredicate);
} catch (NoSuchResourceException e) {
LOG.info("Resource for the desired config not found. " + e);
- return Collections.EMPTY_MAP;
+ return Collections.emptyMap();
}
Map<String, String> mConfigs = new HashMap<String, String>();
if (configResources != null) {
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java Wed Apr 24 23:19:46 2013
@@ -68,17 +68,20 @@ class HostComponentResourceProvider exte
//Component name mappings
private static final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>();
-
- private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_TIMEOUT = 1500;
-
+
+ private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT = 1500;
+ private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT = 10000;
+
static {
HOST_COMPONENT_PROPERTIES_PROVIDER.put(
"NAGIOS_SERVER",
- new HttpProxyPropertyProvider(new URLStreamProvider(
- HOST_COMPONENT_HTTP_PROPERTY_REQUEST_TIMEOUT), PropertyHelper
- .getPropertyId("HostRoles", "cluster_name"), PropertyHelper
- .getPropertyId("HostRoles", "host_name"), PropertyHelper
- .getPropertyId("HostRoles", "component_name")));
+ new HttpProxyPropertyProvider(
+ new URLStreamProvider(
+ HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT,
+ HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT),
+ PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+ PropertyHelper.getPropertyId("HostRoles", "host_name"),
+ PropertyHelper.getPropertyId("HostRoles", "component_name")));
}
//Parameters from the predicate
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/URLStreamProvider.java Wed Apr 24 23:19:46 2013
@@ -29,28 +29,29 @@ import org.apache.ambari.server.controll
* URL based implementation of a stream provider.
*/
public class URLStreamProvider implements StreamProvider {
-
- private int connTimeout = -1;
-
- public URLStreamProvider() {
- }
-
+
+ private final int connTimeout;
+ private final int readTimeout;
+
/**
* Provide the connection timeout for the underlying connection.
- *
+ *
* @param connectionTimeout time, in milliseconds, to attempt a connection
+ * @param readTimeout
*/
- public URLStreamProvider(int connectionTimeout) {
- connTimeout = connectionTimeout;
+ public URLStreamProvider(int connectionTimeout, int readTimeout) {
+ this.connTimeout = connectionTimeout;
+ this.readTimeout = readTimeout;
}
@Override
public InputStream readFrom(String spec) throws IOException {
URLConnection connection = new URL(spec).openConnection();
- if (connTimeout > 0) {
- connection.setConnectTimeout(connTimeout);
- }
+
+ connection.setConnectTimeout(connTimeout);
+ connection.setReadTimeout(readTimeout);
connection.setDoOutput(true);
+
return connection.getInputStream();
}
}
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/jmx/JMXPropertyProvider.java Wed Apr 24 23:19:46 2013
@@ -57,6 +57,10 @@ public class JMXPropertyProvider extends
private final String componentNamePropertyId;
+ private final String statePropertyId;
+
+ private final Set<String> healthyStates;
+
private final static ObjectReader objectReader;
@@ -88,13 +92,17 @@ public class JMXPropertyProvider extends
* @param clusterNamePropertyId the cluster name property id
* @param hostNamePropertyId the host name property id
* @param componentNamePropertyId the component name property id
+ * @param statePropertyId the state property id
+ * @param healthyStates the set of healthy state values
*/
public JMXPropertyProvider(Map<String, Map<String, PropertyInfo>> componentMetrics,
StreamProvider streamProvider,
JMXHostProvider jmxHostProvider,
String clusterNamePropertyId,
String hostNamePropertyId,
- String componentNamePropertyId) {
+ String componentNamePropertyId,
+ String statePropertyId,
+ Set<String> healthyStates) {
super(componentMetrics);
@@ -103,6 +111,8 @@ public class JMXPropertyProvider extends
this.clusterNamePropertyId = clusterNamePropertyId;
this.hostNamePropertyId = hostNamePropertyId;
this.componentNamePropertyId = componentNamePropertyId;
+ this.statePropertyId = statePropertyId;
+ this.healthyStates = healthyStates;
}
@@ -144,16 +154,25 @@ public class JMXPropertyProvider extends
* @param request the request
* @param predicate the predicate
*
- * @return true if the resource was successfully populated with the requested properties
+ * @return true if the resource should be part of the result set for the given predicate
*/
private boolean populateResource(Resource resource, Request request, Predicate predicate)
throws SystemException {
Set<String> ids = getRequestPropertyIds(request, predicate);
if (ids.isEmpty()) {
+ // no properties requested
return true;
}
+ // Don't attempt to get the JMX properties if the resource is in an unhealthy state
+ if (statePropertyId != null) {
+ String state = (String) resource.getPropertyValue(statePropertyId);
+ if (state != null && !healthyStates.contains(state)) {
+ return true;
+ }
+ }
+
String componentName = (String) resource.getPropertyValue(componentNamePropertyId);
if (getComponentMetrics().get(componentName) == null) {
Modified: incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java?rev=1471764&r1=1471763&r2=1471764&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java (original)
+++ incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/jmx/JMXPropertyProviderTest.java Wed Apr 24 23:19:46 2013
@@ -37,22 +37,29 @@ import java.util.Set;
public class JMXPropertyProviderTest {
protected static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
protected static final String HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name");
+ protected static final String HOST_COMPONENT_STATE_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "state");
@Test
- public void testGetResources() throws Exception {
+ public void testPopulateResources() throws Exception {
TestStreamProvider streamProvider = new TestStreamProvider();
TestJMXHostProvider hostProvider = new TestJMXHostProvider(false);
JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
streamProvider,
- hostProvider, PropertyHelper.getPropertyId("HostRoles", "cluster_name"), PropertyHelper.getPropertyId("HostRoles", "host_name"), PropertyHelper.getPropertyId("HostRoles", "component_name"));
+ hostProvider,
+ PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+ PropertyHelper.getPropertyId("HostRoles", "host_name"),
+ PropertyHelper.getPropertyId("HostRoles", "component_name"),
+ PropertyHelper.getPropertyId("HostRoles", "state"),
+ Collections.singleton("STARTED"));
// namenode
Resource resource = new ResourceImpl(Resource.Type.HostComponent);
resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal");
resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE");
+ resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED");
// request with an empty set should get all supported properties
Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
@@ -194,6 +201,7 @@ public class JMXPropertyProviderTest {
resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-14-ee-b3.compute-1.internal");
resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "HBASE_MASTER");
+ resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "STARTED");
// only ask for specific properties
properties = new HashSet<String>();
@@ -208,7 +216,7 @@ public class JMXPropertyProviderTest {
Assert.assertEquals(propertyProvider.getSpec("domu-12-31-39-14-ee-b3.compute-1.internal", "60010"), streamProvider.getLastSpec());
- Assert.assertEquals(7, PropertyHelper.getProperties(resource).size());
+ Assert.assertEquals(8, PropertyHelper.getProperties(resource).size());
Assert.assertEquals(1069416448, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryMax")));
Assert.assertEquals(4806976, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "HeapMemoryUsed")));
Assert.assertEquals(136314880, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryMax")));
@@ -219,14 +227,19 @@ public class JMXPropertyProviderTest {
}
@Test
- public void testGetResourcesWithUnknownPort() throws Exception {
+ public void testPopulateResourcesWithUnknownPort() throws Exception {
TestStreamProvider streamProvider = new TestStreamProvider();
TestJMXHostProvider hostProvider = new TestJMXHostProvider(true);
JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
streamProvider,
- hostProvider, PropertyHelper.getPropertyId("HostRoles", "cluster_name"), PropertyHelper.getPropertyId("HostRoles", "host_name"), PropertyHelper.getPropertyId("HostRoles", "component_name"));
+ hostProvider,
+ PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+ PropertyHelper.getPropertyId("HostRoles", "host_name"),
+ PropertyHelper.getPropertyId("HostRoles", "component_name"),
+ PropertyHelper.getPropertyId("HostRoles", "state"),
+ Collections.singleton("STARTED"));
// namenode
Resource resource = new ResourceImpl(Resource.Type.HostComponent);
@@ -250,6 +263,37 @@ public class JMXPropertyProviderTest {
Assert.assertEquals(23634400, resource.getPropertyValue(PropertyHelper.getPropertyId("metrics/jvm", "NonHeapMemoryUsed")));
}
+ @Test
+ public void testPopulateResourcesUnhealthyResource() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider();
+ TestJMXHostProvider hostProvider = new TestJMXHostProvider(true);
+
+ JMXPropertyProvider propertyProvider = new JMXPropertyProvider(
+ PropertyHelper.getJMXPropertyIds(Resource.Type.HostComponent),
+ streamProvider,
+ hostProvider,
+ PropertyHelper.getPropertyId("HostRoles", "cluster_name"),
+ PropertyHelper.getPropertyId("HostRoles", "host_name"),
+ PropertyHelper.getPropertyId("HostRoles", "component_name"),
+ PropertyHelper.getPropertyId("HostRoles", "state"),
+ Collections.singleton("STARTED"));
+
+ // namenode
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_COMPONENT_HOST_NAME_PROPERTY_ID, "domu-12-31-39-0e-34-e1.compute-1.internal");
+ resource.setProperty(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID, "NAMENODE");
+ resource.setProperty(HOST_COMPONENT_STATE_PROPERTY_ID, "INSTALLED");
+
+ // request with an empty set should get all supported properties
+ Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet());
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ // Assert that the stream provider was never called.
+ Assert.assertNull(streamProvider.getLastSpec());
+ }
+
private static class TestJMXHostProvider implements JMXHostProvider {
private final boolean unknownPort;