You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2013/08/15 17:33:22 UTC
[2/2] git commit: AMBARI-2910. Add various YARN time-series metrics.
(ncole)
AMBARI-2910. Add various YARN time-series metrics. (ncole)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/83c4991e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/83c4991e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/83c4991e
Branch: refs/heads/trunk
Commit: 83c4991ec867ce816a3f68b9d1be4fe18a6e2f8b
Parents: 0800104
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Aug 15 09:35:11 2013 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu Aug 15 11:33:06 2013 -0400
----------------------------------------------------------------------
.../hadoop-metrics2.properties-GANGLIA.erb | 3 +
.../templates/hadoop-metrics2.properties.erb | 3 +
.../main/resources/ganglia_properties_2.json | 61 +
.../ganglia/GangliaPropertyProviderTest.java | 42 +-
.../temporal_ganglia_data_yarn_queues.txt | 15720 +++++++++++++++++
5 files changed, 15828 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
index 3ab9875..206c64c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties-GANGLIA.erb
@@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host"
resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664
nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660
historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=*
+resourcemanager.sink.ganglia.tagsForPrefix.rpc=*
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
index 3ab9875..206c64c 100644
--- a/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
+++ b/ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-metrics2.properties.erb
@@ -37,3 +37,6 @@ reducetask.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host"
resourcemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8664
nodemanager.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8660
historyserver.sink.ganglia.servers=<%=scope.function_hdp_host("ganglia_server_host")%>:8666
+
+resourcemanager.sink.ganglia.tagsForPrefix.yarn=*
+resourcemanager.sink.ganglia.tagsForPrefix.rpc=*
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/main/resources/ganglia_properties_2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/ganglia_properties_2.json b/ambari-server/src/main/resources/ganglia_properties_2.json
index 7dc2df1..5d4284c 100644
--- a/ambari-server/src/main/resources/ganglia_properties_2.json
+++ b/ambari-server/src/main/resources/ganglia_properties_2.json
@@ -6722,7 +6722,38 @@
"metric":"ugi.ugi.LoginFailureAvgTime",
"pointInTime":false,
"temporal":true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers",
+ "pointInTime" : false,
+ "temporal" : true
}
+
},
"NODEMANAGER":{
"metrics/boottime":{
@@ -16877,6 +16908,36 @@
"metric":"ugi.ugi.LoginFailureAvgTime",
"pointInTime":false,
"temporal":true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AvailableMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AvailableMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).PendingMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedMB":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedMB",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/AllocatedContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).AllocatedContainers",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/ReservedContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).ReservedContainers",
+ "pointInTime" : false,
+ "temporal" : true
+ },
+ "metrics/yarn/Queue/$1.replaceAll(\"([.])\",\"/\")/PendingContainers":{
+ "metric":"yarn.QueueMetrics.Queue=(.+).PendingContainers",
+ "pointInTime" : false,
+ "temporal" : true
}
},
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/83c4991e/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
index d476f04..1fc1e02 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/ganglia/GangliaPropertyProviderTest.java
@@ -56,7 +56,10 @@ public class GangliaPropertyProviderTest {
private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
private static final String HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
private static final String COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name");
-
+
+ private static final String RM_CATEGORY_1 = "metrics/yarn/Queue/root/default";
+ private static final String RM_AVAILABLE_MEMORY_PROPERTY = PropertyHelper.getPropertyId(RM_CATEGORY_1, "AvailableMB");
+
private ComponentSSLConfiguration configuration;
@Parameterized.Parameters
@@ -534,6 +537,43 @@ public class GangliaPropertyProviderTest {
Assert.assertEquals(11, PropertyHelper.getProperties(resource).size());
Assert.assertNotNull(resource.getPropertyValue(FLUME_CHANNEL_CAPACITY_PROPERTY));
}
+
+ @Test
+ public void testPopulateResources_params_category5() throws Exception {
+ TestStreamProvider streamProvider = new TestStreamProvider("temporal_ganglia_data_yarn_queues.txt");
+ TestGangliaHostProvider hostProvider = new TestGangliaHostProvider();
+
+ GangliaPropertyProvider propertyProvider = new GangliaHostComponentPropertyProvider(
+ PropertyHelper.getGangliaPropertyIds(Resource.Type.HostComponent, MetricsVersion.HDP2),
+ streamProvider,
+ configuration,
+ hostProvider,
+ CLUSTER_NAME_PROPERTY_ID,
+ HOST_NAME_PROPERTY_ID,
+ COMPONENT_NAME_PROPERTY_ID);
+
+ Resource resource = new ResourceImpl(Resource.Type.HostComponent);
+
+ resource.setProperty(HOST_NAME_PROPERTY_ID, "dev01.ambari.apache.org");
+ resource.setProperty(COMPONENT_NAME_PROPERTY_ID, "RESOURCEMANAGER");
+
+ // only ask for one property
+ Map<String, TemporalInfo> temporalInfoMap = new HashMap<String, TemporalInfo>();
+ temporalInfoMap.put(RM_CATEGORY_1, new TemporalInfoImpl(10L, 20L, 1L));
+
+ Request request = PropertyHelper.getReadRequest(Collections.singleton(RM_CATEGORY_1), temporalInfoMap);
+
+ Assert.assertEquals(1, propertyProvider.populateResources(Collections.singleton(resource), request, null).size());
+
+ String expected = (configuration.isGangliaSSL() ? "https" : "http") +
+ "://domU-12-31-39-0E-34-E1.compute-1.internal/cgi-bin/rrd.py?c=HDPResourceManager&h=dev01.ambari.apache.org&s=10&e=20&r=1";
+ Assert.assertEquals(expected, streamProvider.getLastSpec());
+
+ System.out.println(PropertyHelper.getProperties(resource));
+
+ Assert.assertEquals(14, PropertyHelper.getProperties(resource).size());
+ Assert.assertNotNull(resource.getPropertyValue(RM_AVAILABLE_MEMORY_PROPERTY));
+ }
private static class TestGangliaHostProvider implements GangliaHostProvider {