You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/03/23 21:07:58 UTC
svn commit: r388254 - in /lucene/hadoop/trunk/src:
java/org/apache/hadoop/mapred/JobTracker.java webapps/mapred/jobtracker.jsp
Author: cutting
Date: Thu Mar 23 12:07:56 2006
New Revision: 388254
URL: http://svn.apache.org/viewcvs?rev=388254&view=rev
Log:
Fix for HADOOP-98. Keep more accurate task counts. Contributed by Owen O'Malley.
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=388254&r1=388253&r2=388254&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Mar 23 12:07:56 2006
@@ -101,6 +101,7 @@
// Remove profile from head of queue
trackerExpiryQueue.remove(leastRecent);
+ String trackerName = leastRecent.getTrackerName();
// Figure out if last-seen time should be updated, or if tracker is dead
TaskTrackerStatus newProfile = (TaskTrackerStatus) taskTrackers.get(leastRecent.getTrackerName());
@@ -110,12 +111,7 @@
if (newProfile != null) {
if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
// Remove completely
-
- TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(leastRecent.getTrackerName());
- if (oldStatus != null) {
- totalMaps -= oldStatus.countMapTasks();
- totalReduces -= oldStatus.countReduceTasks();
- }
+ updateTaskTrackerStatus(trackerName, null);
lostTaskTracker(leastRecent.getTrackerName());
} else {
// Update time by inserting latest profile
@@ -478,16 +474,42 @@
// InterTrackerProtocol
////////////////////////////////////////////////////
public void initialize(String taskTrackerName) {
- if (taskTrackers.get(taskTrackerName) != null) {
- TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(taskTrackerName);
- totalMaps -= oldStatus.countMapTasks();
- totalReduces -= oldStatus.countReduceTasks();
-
- lostTaskTracker(taskTrackerName);
+ synchronized (taskTrackers) {
+ boolean seenBefore = updateTaskTrackerStatus(taskTrackerName, null);
+ if (seenBefore) {
+ lostTaskTracker(taskTrackerName);
}
+ }
}
/**
+ * Update the last recorded status for the given task tracker.
+ * It assumes that the taskTrackers are locked on entry.
+ * @author Owen O'Malley
+ * @param trackerName The name of the tracker
+ * @param status The new status for the task tracker
+ * @return Was an old status found?
+ */
+ private boolean updateTaskTrackerStatus(String trackerName,
+ TaskTrackerStatus status) {
+ TaskTrackerStatus oldStatus =
+ (TaskTrackerStatus) taskTrackers.get(trackerName);
+ if (oldStatus != null) {
+ totalMaps -= oldStatus.countMapTasks();
+ totalReduces -= oldStatus.countReduceTasks();
+ if (status == null) {
+ taskTrackers.remove(trackerName);
+ }
+ }
+ if (status != null) {
+ totalMaps += status.countMapTasks();
+ totalReduces += status.countReduceTasks();
+ taskTrackers.put(trackerName, status);
+ }
+ return oldStatus != null;
+ }
+
+ /**
* Process incoming heartbeat messages from the task trackers.
*/
public synchronized int emitHeartbeat(TaskTrackerStatus trackerStatus, boolean initialContact) {
@@ -496,26 +518,20 @@
synchronized (taskTrackers) {
synchronized (trackerExpiryQueue) {
+ boolean seenBefore = updateTaskTrackerStatus(trackerName,
+ trackerStatus);
if (initialContact) {
// If it's first contact, then clear out any state hanging around
- if (taskTrackers.get(trackerName) != null) {
- TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(trackerName);
- totalMaps -= oldStatus.countMapTasks();
- totalReduces -= oldStatus.countReduceTasks();
+ if (seenBefore) {
lostTaskTracker(trackerName);
}
} else {
// If not first contact, there should be some record of the tracker
- if (taskTrackers.get(trackerName) == null) {
+ if (!seenBefore) {
return InterTrackerProtocol.UNKNOWN_TASKTRACKER;
}
}
- // Store latest state. If first contact, then save current
- // state in expiry queue
- totalMaps += trackerStatus.countMapTasks();
- totalReduces += trackerStatus.countReduceTasks();
- taskTrackers.put(trackerName, trackerStatus);
if (initialContact) {
trackerExpiryQueue.add(trackerStatus);
}
Modified: lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp?rev=388254&r1=388253&r2=388254&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/mapred/jobtracker.jsp Thu Mar 23 12:07:56 2006
@@ -78,6 +78,17 @@
out.print("</table>\n");
out.print("</center>\n");
}
+
+ public void generateSummaryTable(JspWriter out) throws IOException {
+ ClusterStatus status = tracker.getClusterStatus();
+ out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n"+
+ "<tr><th>Maps</th><th>Reduces</th>" +
+ "<th>Capacity</th><th>Nodes</th></tr>\n");
+ out.print("<tr><td>" + status.getMapTasks() + "</td><td>" +
+ status.getReduceTasks() + "</td><td>" +
+ status.getMaxTasks() + "</td><td>" +
+ status.getTaskTrackers() + "</td></tr></table>\n");
+ }
%>
<html>
@@ -88,6 +99,13 @@
<h1>JobTracker '<%=trackerLabel%>'</h1>
This JobTracker has been up since <%= new Date(tracker.getStartTime())%>.<br>
+<hr>
+<h2>Cluster Summary</h2>
+<center>
+<%
+ generateSummaryTable(out);
+%>
+</center>
<hr>