You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2011/09/08 00:58:59 UTC
svn commit: r1166459 - in
/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./
src/main/conf/ src/main/java/org/apache/hadoop/metrics/ganglia/
src/main/java/org/apache/hadoop/metrics2/sink/ganglia/
src/test/java/org/apache/hadoop...
Author: todd
Date: Wed Sep 7 22:58:58 2011
New Revision: 1166459
URL: http://svn.apache.org/viewvc?rev=1166459&view=rev
Log:
HADOOP-7507. Allow ganglia metrics to include the metrics system tags in the gmetric names. Contributed by Alejandro Abdelnur.
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1166459&r1=1166458&r2=1166459&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Sep 7 22:58:58 2011
@@ -361,6 +361,9 @@ Release 0.23.0 - Unreleased
HADOOP-7580. Add a version of getLocalPathForWrite to LocalDirAllocator
which doesn't create dirs. (Chris Douglas & Siddharth Seth via acmurthy)
+ HADOOP-7507. Allow ganglia metrics to include the metrics system tags
+ in the gmetric names. (Alejandro Abdelnur via todd)
+
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties?rev=1166459&r1=1166458&r2=1166459&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties Wed Sep 7 22:58:58 2011
@@ -43,6 +43,16 @@
#*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
#*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
+# Tag values to use for the ganglia prefix. If not defined no tags are used.
+# If '*' all tags are used. If specifiying multiple tags separate them with
+# commas. Note that the last segment of the property name is the context name.
+#
+#*.sink.ganglia.tagsForPrefix.jvm=ProcesName
+#*.sink.ganglia.tagsForPrefix.dfs=
+#*.sink.ganglia.tagsForPrefix.rpc=
+#*.sink.ganglia.tagsForPrefix.mapred=
+#*.sink.ganglia.tagsForPrefix.fairscheduler=
+
#namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#datanode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java?rev=1166459&r1=1166458&r2=1166459&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java Wed Sep 7 22:58:58 2011
@@ -132,6 +132,12 @@ public class GangliaContext extends Abst
StringBuilder sb = new StringBuilder();
sb.append(contextName);
sb.append('.');
+
+ if (contextName.equals("jvm") && outRec.getTag("processName") != null) {
+ sb.append(outRec.getTag("processName"));
+ sb.append('.');
+ }
+
sb.append(recordName);
sb.append('.');
int sbBaseLen = sb.length();
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java?rev=1166459&r1=1166458&r2=1166459&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java Wed Sep 7 22:58:58 2011
@@ -20,13 +20,21 @@ package org.apache.hadoop.metrics2.sink.
import java.io.IOException;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
+import org.apache.commons.configuration.SubsetConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.hadoop.metrics2.impl.MsInfo;
import org.apache.hadoop.metrics2.util.MetricsCache;
import org.apache.hadoop.metrics2.util.MetricsCache.Record;
@@ -38,8 +46,67 @@ public class GangliaSink30 extends Abstr
public final Log LOG = LogFactory.getLog(this.getClass());
+ private static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix.";
+
private MetricsCache metricsCache = new MetricsCache();
+ // a key with a NULL value means ALL
+ private Map<String,Set<String>> useTagsMap = new HashMap<String,Set<String>>();
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(SubsetConfiguration conf) {
+ super.init(conf);
+
+ conf.setListDelimiter(',');
+ Iterator<String> it = (Iterator<String>) conf.getKeys();
+ while (it.hasNext()) {
+ String propertyName = it.next();
+ if (propertyName.startsWith(TAGS_FOR_PREFIX_PROPERTY_PREFIX)) {
+ String contextName = propertyName.substring(TAGS_FOR_PREFIX_PROPERTY_PREFIX.length());
+ String[] tags = conf.getStringArray(propertyName);
+ boolean useAllTags = false;
+ Set<String> set = null;
+ if (tags.length > 0) {
+ set = new HashSet<String>();
+ for (String tag : tags) {
+ tag = tag.trim();
+ useAllTags |= tag.equals("*");
+ if (tag.length() > 0) {
+ set.add(tag);
+ }
+ }
+ if (useAllTags) {
+ set = null;
+ }
+ }
+ useTagsMap.put(contextName, set);
+ }
+ }
+ }
+
+ @InterfaceAudience.Private
+ public void appendPrefix(MetricsRecord record, StringBuilder sb) {
+ String contextName = record.context();
+ Collection<MetricsTag> tags = record.tags();
+ if (useTagsMap.containsKey(contextName)) {
+ Set<String> useTags = useTagsMap.get(contextName);
+ for (MetricsTag t : tags) {
+ if (useTags == null || useTags.contains(t.name())) {
+
+ // the context is always skipped here because it is always added
+
+ // the hostname is always skipped to avoid case-mismatches
+ // from different DNSes.
+
+ if (t.info() != MsInfo.Context && t.info() != MsInfo.Hostname && t.value() != null) {
+ sb.append('.').append(t.name()).append('=').append(t.value());
+ }
+ }
+ }
+ }
+ }
+
@Override
public void putMetrics(MetricsRecord record) {
// The method handles both cases whether Ganglia support dense publish
@@ -53,6 +120,8 @@ public class GangliaSink30 extends Abstr
sb.append('.');
sb.append(recordName);
+ appendPrefix(record, sb);
+
String groupName = sb.toString();
sb.append('.');
int sbBaseLen = sb.length();
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java?rev=1166459&r1=1166458&r2=1166459&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java Wed Sep 7 22:58:58 2011
@@ -26,12 +26,17 @@ import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.SocketException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
@@ -54,6 +59,44 @@ public class TestGangliaMetrics {
"test.s1rec.S1NumOps",
"test.s1rec.S1AvgTime" };
+ @Test
+ public void testTagsForPrefix() throws Exception {
+ ConfigBuilder cb = new ConfigBuilder()
+ .add("test.sink.ganglia.tagsForPrefix.all", "*")
+ .add("test.sink.ganglia.tagsForPrefix.some", "NumActiveSinks, NumActiveSources")
+ .add("test.sink.ganglia.tagsForPrefix.none", "");
+ GangliaSink30 sink = new GangliaSink30();
+ sink.init(cb.subset("test.sink.ganglia"));
+
+ List<MetricsTag> tags = new ArrayList<MetricsTag>();
+ tags.add(new MetricsTag(MsInfo.Context, "all"));
+ tags.add(new MetricsTag(MsInfo.NumActiveSources, "foo"));
+ tags.add(new MetricsTag(MsInfo.NumActiveSinks, "bar"));
+ tags.add(new MetricsTag(MsInfo.NumAllSinks, "haa"));
+ tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+ Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
+ MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 1, tags, metrics);
+
+ StringBuilder sb = new StringBuilder();
+ sink.appendPrefix(record, sb);
+ assertEquals(".NumActiveSources=foo.NumActiveSinks=bar.NumAllSinks=haa", sb.toString());
+
+ tags.set(0, new MetricsTag(MsInfo.Context, "some"));
+ sb = new StringBuilder();
+ sink.appendPrefix(record, sb);
+ assertEquals(".NumActiveSources=foo.NumActiveSinks=bar", sb.toString());
+
+ tags.set(0, new MetricsTag(MsInfo.Context, "none"));
+ sb = new StringBuilder();
+ sink.appendPrefix(record, sb);
+ assertEquals("", sb.toString());
+
+ tags.set(0, new MetricsTag(MsInfo.Context, "nada"));
+ sb = new StringBuilder();
+ sink.appendPrefix(record, sb);
+ assertEquals("", sb.toString());
+ }
+
@Test public void testGangliaMetrics2() throws Exception {
ConfigBuilder cb = new ConfigBuilder().add("default.period", 10)
.add("test.sink.gsink30.context", "test") // filter out only "test"