You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2010/04/26 06:15:07 UTC
svn commit: r937924 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java
src/java/org/apache/hadoop/mapred/JobInProgress.java
src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
Author: sharad
Date: Mon Apr 26 04:15:07 2010
New Revision: 937924
URL: http://svn.apache.org/viewvc?rev=937924&view=rev
Log:
MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease undue load on jobtracker. Contributed by Sreekanth Ramakrishnan.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=937924&r1=937923&r2=937924&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Apr 26 04:15:07 2010
@@ -576,6 +576,9 @@ Trunk (unreleased changes)
MAPREDUCE-1618. Add missing javadoc to JobStatus::*JobAcls. (Amareshwari
Sriramadasu via cdouglas)
+ MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease
+ undue load on jobtracker. (Sreekanth Ramakrishnan via sharad)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java?rev=937924&r1=937923&r2=937924&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java Mon Apr 26 04:15:07 2010
@@ -80,12 +80,6 @@ public class SimulatorJobInProgress exte
this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
- MetricsContext metricsContext = MetricsUtil.getContext("mapred");
- this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
- this.jobMetrics.setTag("user", conf.getUser());
- this.jobMetrics.setTag("sessionId", conf.getSessionId());
- this.jobMetrics.setTag("jobName", conf.getJobName());
- this.jobMetrics.setTag("jobId", jobid.toString());
this.maxLevel = jobtracker.getNumTaskCacheLevels();
this.anyCacheLevel = this.maxLevel + 1;
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=937924&r1=937923&r2=937924&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Mon Apr 26 04:15:07 2010
@@ -239,8 +239,6 @@ public class JobInProgress {
Counters jobCounters = new Counters();
- MetricsRecord jobMetrics;
-
// Maximum no. of fetch-failure notifications after which map task is killed
private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
@@ -435,12 +433,6 @@ public class JobInProgress {
this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
- MetricsContext metricsContext = MetricsUtil.getContext("mapred");
- this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
- this.jobMetrics.setTag("user", conf.getUser());
- this.jobMetrics.setTag("sessionId", conf.getSessionId());
- this.jobMetrics.setTag("jobName", conf.getJobName());
- this.jobMetrics.setTag("jobId", this.jobId.toString());
hasSpeculativeMaps = conf.getMapSpeculativeExecution();
hasSpeculativeReduces = conf.getReduceSpeculativeExecution();
this.maxLevel = jobtracker.getNumTaskCacheLevels();
@@ -467,35 +459,6 @@ public class JobInProgress {
DelegationTokenRenewal.registerDelegationTokensForRenewal(
jobInfo.getJobID(), ts, this.conf);
}
-
- /**
- * Called periodically by JobTrackerMetrics to update the metrics for
- * this job.
- */
- public void updateMetrics() {
- Counters counters = getCounters();
- for (Counters.Group group : counters) {
- jobMetrics.setTag("group", group.getDisplayName());
- for (Counters.Counter counter : group) {
- jobMetrics.setTag("counter", counter.getDisplayName());
- jobMetrics.setMetric("value", (float) counter.getCounter());
- jobMetrics.update();
- }
- }
- }
-
- /**
- * Called when the job is complete
- */
- public void cleanUpMetrics() {
- // Deletes all metric data for this job (in internal table in metrics package).
- // This frees up RAM and possibly saves network bandwidth, since otherwise
- // the metrics package implementation might continue to send these job metrics
- // after the job has finished.
- jobMetrics.removeTag("group");
- jobMetrics.removeTag("counter");
- jobMetrics.remove();
- }
private void printCache (Map<Node, List<TaskInProgress>> cache) {
LOG.info("The taskcache info:");
@@ -3295,7 +3258,6 @@ public class JobInProgress {
LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
}
- cleanUpMetrics();
// free up the memory used by the data structures
this.nonRunningMapCache = null;
this.runningMapCache = null;
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java?rev=937924&r1=937923&r2=937924&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Mon Apr 26 04:15:07 2010
@@ -154,12 +154,6 @@ class JobTrackerMetricsInst extends JobT
numTrackersBlackListed = 0;
}
metricsRecord.update();
-
- if (tracker != null) {
- for (JobInProgress jip : tracker.getRunningJobs()) {
- jip.updateMetrics();
- }
- }
}
@Override