You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/06/26 20:37:39 UTC
svn commit: r417274 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/TaskTracker.java
src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
Author: cutting
Date: Mon Jun 26 11:37:38 2006
New Revision: 417274
URL: http://svn.apache.org/viewvc?rev=417274&view=rev
Log:
HADOOP-305. Fix tasktracker to ask for new tasks as soon as a task is finished. Contributed by Mahadev.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 26 11:37:38 2006
@@ -52,8 +52,13 @@
error. (omalley via cutting)
13. HADOOP-304. Improve error message for
- UnregisterdDatanodeException, adding expected node name.
+ UnregisterdDatanodeException to include expected node name.
(Konstantin Shvachko via cutting)
+
+14. HADOOP-305. Fix TaskTracker to ask for new tasks as soon as a
+ task is finished, rather than waiting for the next heartbeat.
+ This improves performance when tasks are short.
+ (Mahadev Konar via cutting)
Release 0.3.2 - 2006-06-09
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Mon Jun 26 11:37:38 2006
@@ -79,7 +79,7 @@
private int maxCurrentTasks;
private int failures;
-
+ private int finishedCount[] = new int[1];
/**
* A list of tips that should be cleaned up.
*/
@@ -284,11 +284,16 @@
long waitTime = HEARTBEAT_INTERVAL - (now - lastHeartbeat);
if (waitTime > 0) {
try {
- Thread.sleep(waitTime);
+ // sleeps for the wait time, wakes up if a task is finished.
+ synchronized(finishedCount) {
+ if (finishedCount[0] == 0) {
+ finishedCount.wait(waitTime);
+ }
+ finishedCount[0] = 0;
+ }
} catch (InterruptedException ie) {
- }
- continue;
- }
+ }
+ }
lastHeartbeat = now;
//
@@ -856,6 +861,10 @@
}
if (tip != null) {
tip.taskFinished();
+ synchronized(finishedCount) {
+ finishedCount[0]++;
+ finishedCount.notifyAll();
+ }
} else {
LOG.warn("Unknown child task finshed: "+taskid+". Ignored.");
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=417274&r1=417273&r2=417274&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java Mon Jun 26 11:37:38 2006
@@ -114,10 +114,12 @@
int mapCount = 0;
for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
TaskStatus ts = (TaskStatus) it.next();
- if (ts.getIsMap()) {
- mapCount++;
- }
- }
+ if (ts.getIsMap() &&
+ ((ts.getRunState() == TaskStatus.RUNNING)
+ || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
+ mapCount++;
+ }
+ }
return mapCount;
}
@@ -125,7 +127,16 @@
* Return the current ReduceTask count
*/
public int countReduceTasks() {
- return taskReports.size() - countMapTasks();
+ int reduceCount = 0;
+ for (Iterator it = taskReports.iterator(); it.hasNext(); ) {
+ TaskStatus ts = (TaskStatus) it.next();
+ if ((!ts.getIsMap()) &&
+ ((ts.getRunState() == TaskStatus.RUNNING)
+ || (ts.getRunState() == TaskStatus.UNASSIGNED))) {
+ reduceCount++;
+ }
+ }
+ return reduceCount;
}
/**