You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/05/22 21:46:17 UTC
svn commit: r540720 - in /lucene/hadoop/trunk: ./
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/metrics/
src/java/org/apache/hadoop/metrics/spi/
Author: cutting
Date: Tue May 22 12:46:16 2007
New Revision: 540720
URL: http://svn.apache.org/viewvc?view=rev&rev=540720
Log:
HADOOP-1406. Plug a leak in MapReduce's use of metrics. Contributed by David Bowen.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue May 22 12:46:16 2007
@@ -51,6 +51,9 @@
number of block locations per request from the namenode.
(Konstantin Shvachko via cutting)
+ 17. HADOOP-1406. Plug a leak in MapReduce's use of metrics.
+ (David Bowen via cutting)
+
Branch 0.13 (unreleased changes)
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue May 22 12:46:16 2007
@@ -152,6 +152,7 @@
this.jobMetrics = metricsContext.createRecord("job");
this.jobMetrics.setTag("user", conf.getUser());
this.jobMetrics.setTag("jobName", conf.getJobName());
+ this.jobMetrics.setTag("jobId", jobid);
}
/**
@@ -172,6 +173,21 @@
}
}
}
+
+
+ /**
+ * Called when the job is complete
+ */
+ public void cleanUpMetrics() {
+ // Deletes all metric data for this job (in internal table in metrics package).
+ // This frees up RAM and possibly saves network bandwidth, since otherwise
+ // the metrics package implementation might continue to send these job metrics
+ // after the job has finished.
+ jobMetrics.removeTag("group");
+ jobMetrics.removeTag("counter");
+ jobMetrics.remove();
+ }
+
/**
* Construct the splits, etc. This is invoked from an async
@@ -1035,6 +1051,8 @@
} catch (IOException e) {
LOG.warn("Error cleaning up "+profile.getJobId()+": "+e);
}
+
+ cleanUpMetrics();
}
/**
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java Tue May 22 12:46:16 2007
@@ -111,6 +111,13 @@
public abstract void setTag(String tagName, byte tagValue);
/**
+ * Removes any tag of the specified name.
+ *
+ * @param tagName name of a tag
+ */
+ public abstract void removeTag(String tagName);
+
+ /**
* Sets the named metric to the specified value.
*
* @param metricName name of the metric
@@ -198,8 +205,11 @@
public abstract void update();
/**
- * Removes, from the buffered data table, the row (if it exists) having tags
- * that equal the tags that have been set on this record.
+ * Removes, from the buffered data table, all rows having tags
+ * that equal the tags that have been set on this record. For example,
+ * if there are no tags on this record, all rows for this record name
+ * would be removed. Or, if there is a single tag on this record, then
+ * just rows containing a tag with the same name and value would be removed.
*/
public abstract void remove();
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java Tue May 22 12:46:16 2007
@@ -25,6 +25,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
@@ -65,7 +66,21 @@
TagMap(TagMap orig) {
super(orig);
}
+ /**
+ * Returns true if this tagmap contains every tag in other.
+ */
+ public boolean containsAll(TagMap other) {
+ for (Map.Entry<String,Object> entry : other.entrySet()) {
+ Object value = get(entry.getKey());
+ if (value == null || !value.equals(entry.getValue())) {
+ // either key does not exist here, or the value is different
+ return false;
+ }
+ }
+ return true;
+ }
}
+
static class MetricMap extends TreeMap<String,Number> {
private static final long serialVersionUID = -7495051861141631609L;
}
@@ -367,9 +382,10 @@
}
/**
- * Called by MetricsRecordImpl.remove(). Removes any matching row in
+ * Called by MetricsRecordImpl.remove(). Removes all matching rows in
* the internal table of metric data. A row matches if it has the same
- * tag names and tag values.
+ * tag names and values as record, but it may also have additional
+ * tags.
*/
protected void remove(MetricsRecordImpl record) {
String recordName = record.getRecordName();
@@ -377,7 +393,13 @@
RecordMap recordMap = getRecordMap(recordName);
synchronized (recordMap) {
- recordMap.remove(tagTable);
+ Iterator<TagMap> it = recordMap.keySet().iterator();
+ while (it.hasNext()) {
+ TagMap rowTags = it.next();
+ if (rowTags.containsAll(tagTable)) {
+ it.remove();
+ }
+ }
}
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java Tue May 22 12:46:16 2007
@@ -103,6 +103,13 @@
}
/**
+ * Removes any tag of the specified name.
+ */
+ public void removeTag(String tagName) {
+ tagTable.remove(tagName);
+ }
+
+ /**
* Sets the named metric to the specified value.
*
* @param metricName name of the metric