You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by mc...@apache.org on 2006/02/02 18:58:44 UTC

svn commit: r374443 [1/2] - in /lucene/nutch/trunk/src: java/org/apache/nutch/fs/ java/org/apache/nutch/mapred/ test/org/apache/nutch/mapred/ webapps/jobtracker/

Author: mc
Date: Thu Feb  2 09:58:38 2006
New Revision: 374443

URL: http://svn.apache.org/viewcvs?rev=374443&view=rev
Log:

  New MapReduce JobTracker internals.  This fixes
several bugs, as listed in JIRA NUTCH-183.



Added:
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskInProgress.java
    lucene/nutch/trunk/src/test/org/apache/nutch/mapred/MapredLoadTest.java
Modified:
    lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskRunner.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskStatus.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskTracker.java
    lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskTrackerStatus.java
    lucene/nutch/trunk/src/webapps/jobtracker/jobdetails.jsp
    lucene/nutch/trunk/src/webapps/jobtracker/jobtracker.jsp

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java Thu Feb  2 09:58:38 2006
@@ -57,7 +57,7 @@
       } catch (FileNotFoundException e) {         // quietly ignore
         stopSumming();
       } catch (IOException e) {                   // loudly ignore
-        LOG.warning("Problem opening checksum file: "+e+". Ignoring.");
+        LOG.warning("Problem opening checksum file: "+ file + ".  Ignoring with exception " + e + ".");
         stopSumming();
       }
     }

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java Thu Feb  2 09:58:38 2006
@@ -40,10 +40,11 @@
   /** Called to get new tasks from from the job tracker for this tracker.*/
   Task pollForNewTask(String trackerName);
 
-  /** Called to find which tasks that have been run by this tracker are now
-   * closed, i.e., their job is complete.  This is used to, e.g., notify a map
-   * task that its output is no longer needed and may be removed. */
-  String pollForClosedTask(String trackerName);
+  /** Called to find which tasks that have been run by this tracker should now
+   * be closed because their job is complete.  This is used to, e.g., 
+   * notify a map task that its output is no longer needed and may 
+   * be removed. */
+  String pollForTaskWithClosedJob(String trackerName);
 
   /** Called by a reduce task to find which map tasks are completed.
    *
@@ -51,7 +52,7 @@
    * @param mapTasksNeeded an array of UTF8 naming map task ids whose output is needed.
    * @return an array of MapOutputLocation
    */
-  MapOutputLocation[] locateMapOutputs(String taskId, String[] mapTasksNeeded);
+  MapOutputLocation[] locateMapOutputs(String taskId, String[][] mapTasksNeeded);
 
   /**
    * The task tracker calls this once, to discern where it can find

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java Thu Feb  2 09:58:38 2006
@@ -296,11 +296,7 @@
           } catch (InterruptedException e) {}
           running = jc.getJob(jobId);
           String report = null;
-          if (running.reduceProgress() == 0.0)
-            report=" map "+Math.round(running.mapProgress()*100)+"%";
-          else {
-            report=" reduce "+Math.round(running.reduceProgress()*100)+"%";
-          }
+          report = " map "+Math.round(running.mapProgress()*100)+"%  reduce " + Math.round(running.reduceProgress()*100)+"%";
           if (!report.equals(lastReport)) {
             LOG.info(report);
             lastReport = report;

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java Thu Feb  2 09:58:38 2006
@@ -16,6 +16,7 @@
 
 package org.apache.nutch.mapred;
 
+
 import java.io.IOException;
 import java.io.File;
 import java.io.InputStream;
@@ -112,18 +113,34 @@
 
   /** Constructs a local file name.  Files are distributed among configured
    * local directories.*/
-  public File getLocalFile(String subdir, String name)
-    throws IOException {
+  public File getLocalFile(String subdir, String name) throws IOException {
+      String param[] = new String[1];
+      param[0] = name;
+      return getLocalFile(subdir, param, "", false);
+  }
+  // REMIND - mjc - rename this!  getLocalFile() is not quite the same.
+  public File getLocalFile(String subdir, String names[], String ending) throws IOException {
+      return getLocalFile(subdir, names, ending, true);
+  }
+  File getLocalFile(String subdir, String names[], String ending, boolean existingFileTest) throws IOException {
     String[] localDirs = getLocalDirs();
-    String path = subdir + File.separator + name;
-    int hashCode = path.hashCode();
-    for (int i = 0; i < localDirs.length; i++) {  // try each local dir
-      int index = (hashCode+i & Integer.MAX_VALUE) % localDirs.length;
-      File file = new File(localDirs[index], path);
-      File dir = file.getParentFile();
-      if (dir.exists() || dir.mkdirs()) {
-        return file;
-      }
+    for (int k = 0; k < names.length; k++) {
+        String path = subdir + File.separator + names[k] + ending;
+        int hashCode = path.hashCode();
+        for (int i = 0; i < localDirs.length; i++) {  // try each local dir
+            int index = (hashCode+i & Integer.MAX_VALUE) % localDirs.length;
+            File file = new File(localDirs[index], path);
+            File dir = file.getParentFile();
+            if (existingFileTest) {
+                if (file.exists()) {
+                    return file;
+                }
+            } else {
+                if (dir.exists() || dir.mkdirs()) {
+                    return file;
+                }
+            }
+        }
     }
     throw new IOException("No valid local directories.");
   }

Added: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java?rev=374443&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java Thu Feb  2 09:58:38 2006
@@ -0,0 +1,436 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.mapred;
+
+import org.apache.nutch.io.*;
+import org.apache.nutch.fs.*;
+import org.apache.nutch.ipc.*;
+import org.apache.nutch.util.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+///////////////////////////////////////////////////////
+// JobInProgress maintains all the info for keeping
+// a Job on the straight and narrow.  It keeps its JobProfile
+// and its latest JobStatus, plus a set of tables for 
+// doing bookkeeping of its Tasks.
+///////////////////////////////////////////////////////
+public class JobInProgress {
+    public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.mapred.JobInProgress");
+
+    JobProfile profile;
+    JobStatus status;
+    File localJobFile = null;
+
+    TaskInProgress maps[] = null;
+    TaskInProgress reduces[] = null;
+    int numMapTasks = 0;
+    int numReduceTasks = 0;
+
+    JobTracker jobtracker = null;
+
+    long startTime;
+    long finishTime;
+    String deleteUponCompletion = null;
+
+    NutchConf nutchConf;
+    boolean tasksInited = false;
+
+    /**
+     * Create a JobInProgress with the given job file, plus a handle
+     * to the tracker.
+     */
+    public JobInProgress(String jobFile, JobTracker jobtracker, NutchConf nutchConf) throws IOException {
+        String jobid = "job_" + jobtracker.createUniqueId();
+        String url = "http://" + jobtracker.getJobTrackerMachine() + ":" + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
+        this.nutchConf = nutchConf;
+        this.jobtracker = jobtracker;
+        this.profile = new JobProfile(jobid, jobFile, url);
+        this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.RUNNING);
+        this.startTime = System.currentTimeMillis();
+
+        this.localJobFile = new JobConf(nutchConf).getLocalFile(JobTracker.SUBDIR, jobid + ".xml");
+        NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+        fs.copyToLocalFile(new File(jobFile), localJobFile);
+
+        JobConf jd = new JobConf(localJobFile);
+        this.numMapTasks = jd.getNumMapTasks();
+        this.numReduceTasks = jd.getNumReduceTasks();
+
+        //
+        // If a jobFile is in the systemDir, we can delete it (and
+        // its JAR) upon completion
+        //
+        File systemDir = jd.getSystemDir();
+        if (jobFile.startsWith(systemDir.getPath())) {
+            this.deleteUponCompletion = jobFile;
+        }
+    }
+
+    /**
+     * Construct the splits, etc
+     */
+    void initTasks() throws IOException {
+        if (tasksInited) {
+            return;
+        }
+
+        //
+        // construct input splits
+        //
+        String jobid = profile.getJobId();
+        String jobFile = profile.getJobFile();
+
+        JobConf jd = new JobConf(localJobFile);
+        NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+        FileSplit[] splits =
+            jd.getInputFormat().getSplits(fs, jd, numMapTasks);
+
+        //
+        // sort splits by decreasing length, to reduce job's tail
+        //
+        Arrays.sort(splits, new Comparator() {
+            public int compare(Object a, Object b) {
+                long diff =
+                    ((FileSplit)b).getLength() - ((FileSplit)a).getLength();
+                return diff==0 ? 0 : (diff > 0 ? 1 : -1);
+            }
+        });
+
+        //
+        // adjust number of map tasks to actual number of splits
+        //
+        this.numMapTasks = splits.length;
+        // create a map task for each split
+        this.maps = new TaskInProgress[numMapTasks];
+        for (int i = 0; i < numMapTasks; i++) {
+            maps[i] = new TaskInProgress(jobFile, splits[i], jobtracker, nutchConf, this);
+        }
+
+        //
+        // Create reduce tasks
+        //
+        this.reduces = new TaskInProgress[numReduceTasks];
+        for (int i = 0; i < numReduceTasks; i++) {
+            reduces[i] = new TaskInProgress(jobFile, maps, i, jobtracker, nutchConf, this);
+        }
+
+        tasksInited = true;
+    }
+
+    /////////////////////////////////////////////////////
+    // Accessors for the JobInProgress
+    /////////////////////////////////////////////////////
+    public JobProfile getProfile() {
+        return profile;
+    }
+    public JobStatus getStatus() {
+        return status;
+    }
+    public long getStartTime() {
+        return startTime;
+    }
+    public long getFinishTime() {
+        return finishTime;
+    }
+    public int desiredMaps() {
+        return numMapTasks;
+    }
+    public int finishedMaps() {
+        int finishedCount = 0;
+        for (int i = 0; i < maps.length; i++) {
+            if (maps[i].isComplete()) {
+                finishedCount++;
+            }
+        }
+        return finishedCount;
+    }
+    public int desiredReduces() {
+        return numReduceTasks;
+    }
+    public int finishedReduces() {
+        int finishedCount = 0;
+        for (int i = 0; i < reduces.length; i++) {
+            if (reduces[i].isComplete()) {
+                finishedCount++;
+            }
+        }
+        return finishedCount;
+    }
+
+    /**
+     * Return a treeset of completed TaskInProgress objects
+     */
+    public Vector reportTasksInProgress(boolean shouldBeMap, boolean shouldBeComplete) {
+        Vector results = new Vector();
+        TaskInProgress tips[] = null;
+        if (shouldBeMap) {
+            tips = maps;
+        } else {
+            tips = reduces;
+        }
+        for (int i = 0; i < tips.length; i++) {
+            if (tips[i].isComplete() == shouldBeComplete) {
+                results.add(tips[i]);
+            }
+        }
+        return results;
+    }
+
+    ////////////////////////////////////////////////////
+    // Status update methods
+    ////////////////////////////////////////////////////
+    public void updateTaskStatus(TaskInProgress tip, TaskStatus status) {
+        tip.updateStatus(status);
+
+        //
+        // Update JobInProgress status
+        //
+        if (maps.length == 0) {
+            this.status.setMapProgress(1.0f);
+        } else {
+            double reportedProgress = 0;
+            for (int i = 0; i < maps.length; i++) {
+                reportedProgress += maps[i].getProgress();
+            }
+            this.status.setMapProgress((float) (reportedProgress / maps.length));
+        }
+        if (reduces.length == 0) {
+            this.status.setReduceProgress(1.0f);
+        } else {
+            double reportedProgress = 0;
+            for (int i = 0; i < reduces.length; i++) {
+                reportedProgress += reduces[i].getProgress();
+            }
+            this.status.setReduceProgress((float) (reportedProgress / reduces.length));
+        }
+    }
+
+    /////////////////////////////////////////////////////
+    // Create/manage tasks
+    /////////////////////////////////////////////////////
+    /**
+     * Return a MapTask, if appropriate, to run on the given tasktracker
+     */
+    public Task obtainNewMapTask(String taskTracker, TaskTrackerStatus tts) {
+        if (! tasksInited) {
+            try {
+                initTasks();
+            } catch (IOException ie) {
+                ie.printStackTrace();
+                LOG.info("Cannot create task split for " + profile.getJobId());
+            }
+        }
+
+        Task t = null;
+        int cacheTarget = -1;
+        int stdTarget = -1;
+        int specTarget = -1;
+        double totalProgress = 0;
+
+        //
+        // We end up creating two tasks for the same bucket, because
+        // we call obtainNewMapTask() really fast, twice in a row.
+        // There's not enough time for the "recentTasks"
+        //
+        for (int i = 0; i < maps.length; i++) {
+            if (maps[i].hasTaskWithCacheHit(taskTracker, tts)) {
+                if (cacheTarget < 0) {
+                    cacheTarget = i;
+                }
+            } else if (maps[i].hasTask()) {
+                if (stdTarget < 0) {
+                    stdTarget = i;
+                }
+            }
+            totalProgress += maps[i].getProgress();
+        }
+        double avgProgress = totalProgress / maps.length;
+
+        for (int i = 0; i < maps.length; i++) {        
+            if (maps[i].hasSpeculativeTask(avgProgress)) {
+                if (specTarget < 0) {
+                    specTarget = i;
+                }
+            }
+        }
+        
+        if (cacheTarget >= 0) {
+            t = maps[cacheTarget].getTaskToRun(taskTracker, tts, avgProgress);
+        } else if (stdTarget >= 0) {
+            t = maps[stdTarget].getTaskToRun(taskTracker, tts, avgProgress);
+        } else if (specTarget >= 0) {
+            t = maps[specTarget].getTaskToRun(taskTracker, tts, avgProgress);
+        }
+        return t;
+    }
+
+    /**
+     * Return a ReduceTask, if appropriate, to run on the given tasktracker.
+     * We don't have cache-sensitivity for reduce tasks, as they
+     *  work on temporary MapRed files.  
+     */
+    public Task obtainNewReduceTask(String taskTracker, TaskTrackerStatus tts) {
+        if (! tasksInited) {
+            try {
+                initTasks();
+            } catch (IOException ie) {
+                ie.printStackTrace();
+                LOG.info("Cannot create task split for " + profile.getJobId());
+            }
+        }
+
+        Task t = null;
+        int stdTarget = -1;
+        int specTarget = -1;
+        int totalProgress = 0;
+        for (int i = 0; i < reduces.length; i++) {
+            totalProgress += reduces[i].getProgress();
+        }
+        double avgProgress = (1.0 * totalProgress) / reduces.length;
+
+        for (int i = 0; i < reduces.length; i++) {
+            if (reduces[i].hasTask()) {
+                if (stdTarget < 0) {
+                    stdTarget = i;
+                }
+            } else if (reduces[i].hasSpeculativeTask(avgProgress)) {
+                if (specTarget < 0) {
+                    specTarget = i;
+                }
+            }
+        }
+        
+        if (stdTarget >= 0) {
+            t = reduces[stdTarget].getTaskToRun(taskTracker, tts, avgProgress);
+        } else if (specTarget >= 0) {
+            t = reduces[specTarget].getTaskToRun(taskTracker, tts, avgProgress);
+        }
+        return t;
+    }
+
+    /**
+     * A taskid assigned to this JobInProgress has reported in successfully.
+     */
+    public synchronized void completedTask(TaskInProgress tip, String taskid) {
+        LOG.info("Taskid '" + taskid + "' has finished successfully.");
+        tip.completed(taskid);
+
+        //
+        // Figure out whether the Job is done
+        //
+        boolean allDone = true;
+        for (int i = 0; i < maps.length; i++) {
+            if (! maps[i].isComplete()) {
+                allDone = false;
+                break;
+            }
+        }
+        if (allDone) {
+            for (int i = 0; i < reduces.length; i++) {
+                if (! reduces[i].isComplete()) {
+                    allDone = false;
+                    break;
+                }
+            }
+        }
+
+        //
+        // If all tasks are complete, then the job is done!
+        //
+        if (status.getRunState() == JobStatus.RUNNING && allDone) {
+            this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.SUCCEEDED);
+            this.finishTime = System.currentTimeMillis();
+        }
+    }
+
+    /**
+     * Kill the job and all its component tasks.
+     */
+    public synchronized void kill() {
+        if (status.getRunState() != JobStatus.FAILED) {
+            this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.FAILED);
+            this.finishTime = System.currentTimeMillis();
+
+            //
+            // kill all TIPs.
+            //
+            for (int i = 0; i < maps.length; i++) {
+                maps[i].kill();
+            }
+            for (int i = 0; i < reduces.length; i++) {
+                reduces[i].kill();
+            }
+        }
+    }
+
+    /**
+     * A task assigned to this JobInProgress has reported in as failed.
+     * Most of the time, we'll just reschedule execution.  However, after
+     * many repeated failures we may instead decide to allow the entire 
+     * job to fail.
+     *
+     * Even if a task has reported as completed in the past, it might later
+     * be reported as failed.  That's because the TaskTracker that hosts a map
+     * task might die before the entire job can complete.  If that happens,
+     * we need to schedule reexecution so that downstream reduce tasks can 
+     * obtain the map task's output.
+     */
+    public void failedTask(TaskInProgress tip, String taskid, String trackerName) {
+        tip.failedSubTask(taskid, trackerName);
+            
+        //
+        // Check if we need to kill the job because of too many failures
+        //
+        if (tip.isFailed()) {
+            LOG.info("Aborting job " + profile.getJobId());
+            kill();
+        }
+    }
+
+    /**
+     * The job is dead.  We're now GC'ing it, getting rid of the job
+     * from all tables.  Be sure to remove all of this job's tasks
+     * from the various tables.
+     */
+    public synchronized void garbageCollect() throws IOException {
+        //
+        // Remove this job from all tables
+        //
+
+        // Definitely remove the local-disk copy of the job file
+        if (localJobFile != null) {
+            localJobFile.delete();
+            localJobFile = null;
+        }
+
+        //
+        // If the job file was in the temporary system directory,
+        // we should delete it upon garbage collect.
+        //
+        if (deleteUponCompletion != null) {
+            JobConf jd = new JobConf(deleteUponCompletion);
+            NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+            fs.delete(new File(jd.getJar()));
+            fs.delete(new File(deleteUponCompletion));
+            deleteUponCompletion = null;
+        }
+    }
+}
+

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java Thu Feb  2 09:58:38 2006
@@ -15,6 +15,7 @@
  */
 package org.apache.nutch.mapred;
 
+
 import org.apache.nutch.io.*;
 import org.apache.nutch.fs.*;
 import org.apache.nutch.ipc.*;
@@ -32,7 +33,11 @@
  * @author Mike Cafarella
  *******************************************************/
 public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmissionProtocol {
-    static final int MAX_TASK_FAILURES = 4;
+    static long RETIRE_JOB_INTERVAL;
+    static long RETIRE_JOB_CHECK_INTERVAL;
+    static float TASK_ALLOC_EPSILON;
+    static float PAD_FRACTION;
+    static float MIN_SLOTS_FOR_PADDING;
 
     public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.mapred.JobTracker");
 
@@ -104,7 +109,12 @@
                             if (newProfile != null) {
                                 if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
                                     // Remove completely
-                                    taskTrackers.remove(leastRecent.getTrackerName());
+
+                                    TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(leastRecent.getTrackerName());
+                                    if (oldStatus != null) {
+                                        totalMaps -= oldStatus.countMapTasks();
+                                        totalReduces -= oldStatus.countReduceTasks();
+                                    }
                                     lostTaskTracker(leastRecent.getTrackerName());
                                 } else {
                                     // Update time by inserting latest profile
@@ -125,6 +135,45 @@
         }
     }
 
+    ///////////////////////////////////////////////////////
+    // Used to remove old finished Jobs that have been around for too long
+    ///////////////////////////////////////////////////////
+    class RetireJobs implements Runnable {
+        boolean shouldRun = true;
+        public RetireJobs() {
+        }
+
+        /**
+         * The run method lives for the life of the JobTracker,
+         * and removes Jobs that are not still running, but which
+         * finished a long time ago.
+         */
+        public void run() {
+            while (shouldRun) {
+                try {
+                    Thread.sleep(RETIRE_JOB_CHECK_INTERVAL);
+                } catch (InterruptedException ie) {
+                }
+                
+                synchronized (jobs) {
+                    for (Iterator it = jobs.keySet().iterator(); it.hasNext(); ) {
+                        String jobid = (String) it.next();
+                        JobInProgress job = (JobInProgress) jobs.get(jobid);
+
+                        if (job.getStatus().getRunState() != JobStatus.RUNNING &&
+                            (job.getFinishTime() + RETIRE_JOB_INTERVAL < System.currentTimeMillis())) {
+                            it.remove();
+                            jobsByArrival.remove(job);
+                        }
+                    }
+                }
+            }
+        }
+        public void stopRetirer() {
+            shouldRun = false;
+        }
+    }
+
     /////////////////////////////////////////////////////////////////
     // The real JobTracker
     ////////////////////////////////////////////////////////////////
@@ -134,6 +183,8 @@
     int totalSubmissions = 0;
     Random r = new Random();
 
+    private int maxCurrentTasks;
+
     //
     // Properties to maintain while running Jobs and Tasks:
     //
@@ -152,28 +203,26 @@
 
     // All the known jobs.  (jobid->JobInProgress)
     TreeMap jobs = new TreeMap();
+    Vector jobsByArrival = new Vector();
 
-    // (taskId->jobid that contains the task)
-    TreeMap taskToJobMap = new TreeMap();
+    // All the known TaskInProgress items, mapped to by taskids (taskid->TIP)
+    TreeMap taskidToTIPMap = new TreeMap();
 
-    // (taskId->trackerID) locates a given task
-    TreeMap taskToTrackerMap = new TreeMap();
+    // (taskid --> trackerID) 
+    TreeMap taskidToTrackerMap = new TreeMap();
 
     // (trackerID->TreeSet of taskids running at that tracker)
     TreeMap trackerToTaskMap = new TreeMap();
 
-    // a list of taskIds waiting for a task tracker to run on
-    Vector unassignedTasks = new Vector();
-
-    // Tasks that we've closed
-    TreeSet tasksReportedClosed = new TreeSet();
-
     //
     // Watch and expire TaskTracker objects using these structures.
     // We can map from Name->TaskTrackerStatus, or we can expire by time.
     //
+    int totalMaps = 0;
+    int totalReduces = 0;
     TreeMap taskTrackers = new TreeMap();
     ExpireTrackers expireTrackers = new ExpireTrackers();
+    RetireJobs retireJobs = new RetireJobs();
 
     /**
      * It might seem like a bug to maintain a TreeSet of status objects,
@@ -209,18 +258,27 @@
     static final String SUBDIR = "jobTracker";
     NutchFileSystem fs;
     File systemDir;
-
     private NutchConf nutchConf;
 
     /**
      * Start the JobTracker process, listen on the indicated port
      */
     JobTracker(NutchConf conf) throws IOException {
+        //
+        // Grab some static constants
+        //
+        maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
+        RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
+        RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
+        TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
+        PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
+        MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
+
         // This is a directory of temporary submission files.  We delete it
         // on startup, and can delete any files that we're done with
+        this.nutchConf = conf;
         JobConf jobConf = new JobConf(conf);
         this.systemDir = jobConf.getSystemDir();
-        this.nutchConf = conf;
         this.fs = NutchFileSystem.get(conf);
         FileUtil.fullyDelete(fs, systemDir);
         fs.mkdirs(systemDir);
@@ -248,6 +306,7 @@
         this.startTime = System.currentTimeMillis();
 
         new Thread(this.expireTrackers).start();
+        new Thread(this.retireJobs).start();
     }
 
     public static InetSocketAddress getAddress(NutchConf conf) {
@@ -274,6 +333,41 @@
     }
 
     ///////////////////////////////////////////////////////
+    // Maintain lookup tables; called by JobInProgress
+    // and TaskInProgress
+    ///////////////////////////////////////////////////////
+    void createTaskEntry(String taskid, String taskTracker, TaskInProgress tip) {
+        LOG.info("Adding task '" + taskid + "' to tip " + tip.getTIPId() + ", for tracker '" + taskTracker + "'");
+
+        // taskid --> tracker
+        taskidToTrackerMap.put(taskid, taskTracker);
+
+        // tracker --> taskid
+        TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
+        if (taskset == null) {
+            taskset = new TreeSet();
+            trackerToTaskMap.put(taskTracker, taskset);
+        }
+        taskset.add(taskid);
+
+        // taskid --> TIP
+        taskidToTIPMap.put(taskid, tip);
+    }
+    void removeTaskEntry(String taskid) {
+        // taskid --> tracker
+        String tracker = (String) taskidToTrackerMap.remove(taskid);
+
+        // tracker --> taskid
+        TreeSet trackerSet = (TreeSet) trackerToTaskMap.get(tracker);
+        if (trackerSet != null) {
+            trackerSet.remove(taskid);
+        }
+
+        // taskid --> TIP
+        taskidToTIPMap.remove(taskid);
+    }
+
+    ///////////////////////////////////////////////////////
     // Accessors for objects that want info on jobs, tasks,
     // trackers, etc.
     ///////////////////////////////////////////////////////
@@ -286,6 +380,9 @@
     public int getTrackerPort() {
         return port;
     }
+    public int getInfoPort() {
+        return infoPort;
+    }
     public long getStartTime() {
         return startTime;
     }
@@ -334,7 +431,10 @@
     ////////////////////////////////////////////////////
     public void initialize(String taskTrackerName) {
         if (taskTrackers.get(taskTrackerName) != null) {
-            taskTrackers.remove(taskTrackerName);
+            TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(taskTrackerName);
+            totalMaps -= oldStatus.countMapTasks();
+            totalReduces -= oldStatus.countReduceTasks();
+
             lostTaskTracker(taskTrackerName);
         }
     }
@@ -351,7 +451,9 @@
                 if (initialContact) {
                     // If it's first contact, then clear out any state hanging around
                     if (taskTrackers.get(trackerName) != null) {
-                        taskTrackers.remove(trackerName);
+                        TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(trackerName);
+                        totalMaps -= oldStatus.countMapTasks();
+                        totalReduces -= oldStatus.countReduceTasks();
                         lostTaskTracker(trackerName);
                     }
                 } else {
@@ -363,6 +465,8 @@
 
                 // Store latest state.  If first contact, then save current
                 // state in expiry queue
+                totalMaps += trackerStatus.countMapTasks();
+                totalReduces += trackerStatus.countReduceTasks();
                 taskTrackers.put(trackerName, trackerStatus);
                 if (initialContact) {
                     trackerExpiryQueue.add(trackerStatus);
@@ -376,38 +480,129 @@
     }
 
     /**
-     * A tracker wants to know if there's a Task to run
+     * A tracker wants to know if there's a Task to run.  Returns
+     * a task we'd like the TaskTracker to execute right now.
+     *
+     * Eventually this function should compute load on the various TaskTrackers,
+     * and incorporate knowledge of NDFS file placement.  But for right now, it
+     * just grabs a single item out of the pending task list and hands it back.
      */
-    public synchronized Task pollForNewTask(String trackerName) {
-        //LOG.info("Unassigned tasks: " + unassignedTasks.size());
+    public synchronized Task pollForNewTask(String taskTracker) {
+        //
+        // Compute average map and reduce task numbers across pool
+        //
+        int avgMaps = 0;
+        int avgReduces = 0;
+        if (taskTrackers.size() > 0) {
+            avgMaps = totalMaps / taskTrackers.size();
+            avgReduces = totalReduces / taskTrackers.size();
+        }
+        int totalCapacity = taskTrackers.size() * maxCurrentTasks;
+
+        //
+        // Get map + reduce counts for the current tracker.
+        //
+        TaskTrackerStatus tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
+        int numMaps = tts.countMapTasks();
+        int numReduces = tts.countReduceTasks();
+
+        //
+        // In the below steps, we allocate first a map task (if appropriate),
+        // and then a reduce task if appropriate.  We go through all jobs
+        // in order of job arrival; jobs only get serviced if their 
+        // predecessors are serviced, too.
+        //
+
+        //
+        // We hand a task to the current taskTracker if the given machine 
+        // has a workload that's equal to or less than the averageMaps 
+        // +/- TASK_ALLOC_EPSILON.  (That epsilon is in place in case
+        // there is an odd machine that is failing for some reason but 
+        // has not yet been removed from the pool, making capacity seem
+        // larger than it really is.)
+        //
+        if ((numMaps < maxCurrentTasks) &&
+            (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
+
+            int totalNeededMaps = 0;
+            for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
+                JobInProgress job = (JobInProgress) it.next();
+                if (job.getStatus().getRunState() != JobStatus.RUNNING) {
+                    continue;
+                }
+
+                Task t = job.obtainNewMapTask(taskTracker, tts);
+                if (t != null) {
+                    return t;
+                }
+
+                //
+                // Beyond the highest-priority task, reserve a little 
+                // room for failures and speculative executions; don't 
+                // schedule tasks to the hilt.
+                //
+                totalNeededMaps += job.desiredMaps();
+                double padding = 0;
+                if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
+                    padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
+                }
+                if (totalNeededMaps + padding >= totalCapacity) {
+                    break;
+                }
+            }
+        }
+
+        //
+        // Same thing, but for reduce tasks
+        //
+        if ((numReduces < maxCurrentTasks) &&
+            (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
+
+            int totalNeededReduces = 0;
+            for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
+                JobInProgress job = (JobInProgress) it.next();
+                if (job.getStatus().getRunState() != JobStatus.RUNNING) {
+                    continue;
+                }
+
+                Task t = job.obtainNewReduceTask(taskTracker, tts);
+                if (t != null) {
+                    return t;
+                }
 
-        // Allocate a pending task to this TaskTracker
-        return getTaskAssignment(trackerName);
+                //
+                // Beyond the highest-priority task, reserve a little 
+                // room for failures and speculative executions; don't 
+                // schedule tasks to the hilt.
+                //
+                totalNeededReduces += job.desiredReduces();
+                double padding = 0;
+                if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
+                    padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
+                }
+                if (totalNeededReduces + padding >= totalCapacity) {
+                    break;
+                }
+            }
+        }
+        return null;
     }
 
     /**
      * A tracker wants to know if any of its Tasks have been
      * closed (because the job completed, whether successfully or not)
      */
-    public String pollForClosedTask(String trackerName) {
-        TreeSet taskIds = (TreeSet) trackerToTaskMap.get(trackerName);
+    public String pollForTaskWithClosedJob(String taskTracker) {
+        TreeSet taskIds = (TreeSet) trackerToTaskMap.get(taskTracker);
         if (taskIds != null) {
             for (Iterator it = taskIds.iterator(); it.hasNext(); ) {
                 String taskId = (String) it.next();
-                String jobId = (String) taskToJobMap.get(taskId);
-                JobInProgress job = (JobInProgress) jobs.get(jobId);
-                int runState = job.getStatus().getRunState();
-
-                if (! tasksReportedClosed.contains(taskId) &&
-                    (runState == JobStatus.SUCCEEDED ||
-                     runState == JobStatus.FAILED)) {
-                    tasksReportedClosed.add(taskId);
-
-                    //
-                    // REMIND - mjc - what happens when a job dies, but no one
-                    // arrives to claim the tasks?  I suppose when the job is
-                    // eventually GC'ed, we make sure all its tasks are flushed
-                    // from lookup tables.
+                TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
+                if (tip.shouldCloseForClosedJob(taskId)) {
+                    // 
+                    // This is how the JobTracker ends a task at the TaskTracker.
+                    // It may be successfully completed, or may be killed in
+                    // mid-execution.
                     //
                     return taskId;
                 }
@@ -417,15 +612,27 @@
     }
 
     /**
-     * A tracker wants to know the physical locations of completed, but not
+     * A TaskTracker wants to know the physical locations of completed, but not
      * yet closed, tasks.  This exists so the reduce task thread can locate
      * map task outputs.
-     * 
      */
-    public synchronized MapOutputLocation[] locateMapOutputs(String taskId, String[] mapTasksNeeded) {
-        JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskId));
-        MapOutputLocation outputs[] = job.locateTasks(mapTasksNeeded);
-        return outputs;
+    public synchronized MapOutputLocation[] locateMapOutputs(String taskId, String[][] mapTasksNeeded) {
+        ArrayList v = new ArrayList();
+        for (int i = 0; i < mapTasksNeeded.length; i++) {
+            for (int j = 0; j < mapTasksNeeded[i].length; j++) {
+                TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(mapTasksNeeded[i][j]);
+                if (tip != null && tip.isComplete(mapTasksNeeded[i][j])) {
+                    String trackerId = (String) taskidToTrackerMap.get(mapTasksNeeded[i][j]);
+                    TaskTrackerStatus tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
+                    v.add(new MapOutputLocation(mapTasksNeeded[i][j], tracker.getHost(), tracker.getPort()));
+                    break;
+                }
+            }
+        }
+        // randomly shuffle results to load-balance map output requests
+        Collections.shuffle(v);
+
+        return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
     }
 
     /**
@@ -471,15 +678,15 @@
             return new Vector[0];
         } else {
             Vector reports = new Vector();
-            TreeMap completeMapTasks = job.getCompleteMapTasks();
-            for (Iterator it = completeMapTasks.keySet().iterator(); it.hasNext(); ) {
-                String taskid = (String) it.next();
-                reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
-            }
-            TreeMap incompleteMapTasks = job.getIncompleteMapTasks();
-            for (Iterator it = incompleteMapTasks.keySet().iterator(); it.hasNext(); ) {
-                String taskid = (String) it.next();
-                reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
+            Vector completeMapTasks = job.reportTasksInProgress(true, true);
+            for (Iterator it = completeMapTasks.iterator(); it.hasNext(); ) {
+                TaskInProgress tip = (TaskInProgress) it.next();
+                reports.add(tip.generateSingleReport());
+            }
+            Vector incompleteMapTasks = job.reportTasksInProgress(true, false);
+            for (Iterator it = incompleteMapTasks.iterator(); it.hasNext(); ) {
+                TaskInProgress tip = (TaskInProgress) it.next();
+                reports.add(tip.generateSingleReport());
             }
             return (Vector[]) reports.toArray(new Vector[reports.size()]);
         }
@@ -491,39 +698,32 @@
             return new Vector[0];
         } else {
             Vector reports = new Vector();
-            TreeMap completeReduceTasks = job.getCompleteReduceTasks();
-            for (Iterator it = completeReduceTasks.keySet().iterator(); it.hasNext(); ) {
-                String taskid = (String) it.next();
-                reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
-            }
-            TreeMap incompleteReduceTasks = job.getIncompleteReduceTasks();
-            for (Iterator it = incompleteReduceTasks.keySet().iterator(); it.hasNext(); ) {
-                String taskid = (String) it.next();
-                reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
+            Vector completeReduceTasks = job.reportTasksInProgress(false, true);
+            for (Iterator it = completeReduceTasks.iterator(); it.hasNext(); ) {
+                TaskInProgress tip = (TaskInProgress) it.next();
+                reports.add(tip.generateSingleReport());
+            }
+            Vector incompleteReduceTasks = job.reportTasksInProgress(false, false);
+            for (Iterator it = incompleteReduceTasks.iterator(); it.hasNext(); ) {
+                TaskInProgress tip = (TaskInProgress) it.next();
+                reports.add(tip.generateSingleReport());
             }
             return (Vector[]) reports.toArray(new Vector[reports.size()]);
         }
     }
 
-    //////////////////////////////////////////////////////////////
-    //  (See InterTrackerProtocol section for getFilesystemName())
-    //////////////////////////////////////////////////////////////
-
-    Vector generateSingleReport(String taskid, TaskStatus status, Vector diagInfo, String stateString) {
-        Vector report = new Vector();
-        report.add(taskid);
-        report.add("" + status.getProgress());
-        report.add(stateString);
-        report.addAll(diagInfo);
-        return report;
-    }
-
     ///////////////////////////////////////////////////////////////
     // JobTracker methods
     ///////////////////////////////////////////////////////////////
-    public JobTracker.JobInProgress getJob(String jobid) {
+    public JobInProgress getJob(String jobid) {
         return (JobInProgress) jobs.get(jobid);
     }
+    /**
+     * Grab random num for task id
+     */
+    String createUniqueId() {
+        return "" + Integer.toString(Math.abs(r.nextInt()),36);
+    }
 
     /**
      * JobProfile createJob() kicks off a new job.  
@@ -531,439 +731,19 @@
      * tasks.  The tasks are added to the unassignedTasks structure.  
      * (The precise structure will change as we get more sophisticated about 
      * task allocation.)
+     *
+     * Create a 'JobInProgress' object, which contains both JobProfile
+     * and JobStatus.  Those two sub-objects are sometimes shipped outside
+     * of the JobTracker.  But JobInProgress adds info that's useful for
+     * the JobTracker alone.
      */
     JobInProgress createJob(String jobFile) throws IOException {
-        JobInProgress job = new JobInProgress(jobFile, this.nutchConf);
+        JobInProgress job = new JobInProgress(jobFile, this, this.nutchConf);
         jobs.put(job.getProfile().getJobId(), job);
-
-        boolean error = true;
-        try {
-          job.launch();
-          error = false;
-        } finally {
-          if (error) {
-            job.kill();
-          }
-        }
-
+        jobsByArrival.add(job);
         return job;
     }
 
-    ///////////////////////////////////////////////////////
-    // JobInProgress maintains all the info for keeping
-    // a Job on the straight and narrow.  It keeps its JobProfile
-    // and its latest JobStatus, plus a set of tables for 
-    // doing bookkeeping of its Tasks.
-    ///////////////////////////////////////////////////////
-    public class JobInProgress {
-        JobProfile profile;
-        JobStatus status;
-        Vector reducesToLaunch = new Vector();
-        File localJobFile = null;
-
-        TreeMap taskStatus = new TreeMap();
-        TreeMap incompleteMapTasks = new TreeMap();
-        TreeMap completeMapTasks = new TreeMap();
-        TreeMap incompleteReduceTasks = new TreeMap();
-        TreeMap completeReduceTasks = new TreeMap();
-        TreeMap taskFailures = new TreeMap();
-        TreeMap taskDiagnosticData = new TreeMap();
-        TreeMap taskStateStrings = new TreeMap();
-
-        // Info for user; useless for JobTracker
-        int numMapTasks = 0;
-        int numReduceTasks = 0;
-        float totalReportedMapProgress = 0.0f;
-        float totalReportedReduceProgress = 0.0f;
-        int attemptedMapExecutions = 0;
-        int attemptedReduceExecutions = 0;
-        long startTime;
-        long finishTime;
-        String deleteUponCompletion = null;
-        private NutchConf nutchConf;
-
-        /**
-         * Create a 'JobInProgress' object, which contains both JobProfile
-         * and JobStatus.  Those two sub-objects are sometimes shipped outside
-         * of the JobTracker.  But JobInProgress adds info that's useful for
-         * the JobTracker alone.
-         */
-        public JobInProgress(String jobFile, NutchConf nutchConf) throws IOException {
-            String jobid = createJobId();
-            String url = "http://" + localMachine + ":" + infoPort + "/jobdetails.jsp?jobid=" + jobid;
-            this.profile = new JobProfile(jobid, jobFile, url);
-            this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.RUNNING);
-
-            this.localJobFile = new JobConf(nutchConf).getLocalFile(SUBDIR, jobid + ".xml");
-            fs.copyToLocalFile(new File(jobFile), localJobFile);
-
-            JobConf jd = new JobConf(localJobFile);
-            this.numMapTasks = jd.getNumMapTasks();
-            this.numReduceTasks = jd.getNumReduceTasks();
-            this.startTime = System.currentTimeMillis();
-
-            // If a jobFile is in the systemDir, we can delete it (and
-            // its JAR) upon completion
-            if (jobFile.startsWith(systemDir.getPath())) {
-                this.deleteUponCompletion = jobFile;
-            }
-            this.nutchConf = nutchConf;
-        }
-
-        /**
-         * Start up the tasks
-         */
-        public void launch() throws IOException {
-            String jobid = profile.getJobId();
-            String jobFile = profile.getJobFile();
-
-            // construct input splits
-            JobConf jd = new JobConf(localJobFile);
-            NutchFileSystem fs = NutchFileSystem.get(nutchConf);
-            FileSplit[] splits =
-              jd.getInputFormat().getSplits(fs, jd, numMapTasks);
-
-            // sort splits by decreasing length, to reduce job's tail
-            Arrays.sort(splits, new Comparator() {
-                public int compare(Object a, Object b) {
-                  long diff =
-                    ((FileSplit)b).getLength() - ((FileSplit)a).getLength();
-                  return diff==0 ? 0 : (diff > 0 ? 1 : -1);
-                }
-              });
-
-            // adjust number of map tasks to actual number of splits
-            numMapTasks = splits.length;
-
-            // create a map task for each split
-            String mapIds[] = new String[numMapTasks];
-            for (int i = 0; i < numMapTasks; i++) {
-                mapIds[i] = createMapTaskId();
-                Task t = new MapTask(jobFile, mapIds[i], splits[i]);
-                t.setConf(this.nutchConf);
-
-                incompleteMapTasks.put(mapIds[i], t);
-                taskToJobMap.put(mapIds[i], jobid);
-            }
-
-            // Create reduce tasks
-            for (int i = 0; i < numReduceTasks; i++) {
-                String taskid = createReduceTaskId();
-                Task t = new ReduceTask(jobFile, taskid, mapIds, i);
-                t.setConf(this.nutchConf);
-                reducesToLaunch.add(t);
-                taskToJobMap.put(taskid, jobid);
-            }
-
-            // Launch the map tasks
-            for (int i = 0; i < mapIds.length; i++) {
-                executeTask(mapIds[i]);
-            }
-        }
-
-        /**
-         * Kill the job and all its component tasks.
-         */
-        public synchronized void kill() {
-            if (status.getRunState() != JobStatus.FAILED) {
-                this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.FAILED);
-
-                //
-                // Kill all the pending tasks
-                //
-                synchronized (unassignedTasks) {
-                    for (Iterator it = unassignedTasks.iterator(); it.hasNext(); ) {
-                        String taskid = (String) it.next();
-                        if ((incompleteMapTasks.get(taskid) != null) ||
-                            (incompleteReduceTasks.get(taskid) != null)) {
-                            it.remove();
-                        }
-                    }
-                }
-
-                this.finishTime = System.currentTimeMillis();
-            }
-        }
-
-        /**
-         * The job is dead.  We're now GC'ing it, getting rid of the job
-         * from all tables.  Be sure to remove all of this job's tasks
-         * from the various tables.
-         */
-        public synchronized void garbageCollect() throws IOException {
-            //
-            // Remove this job from all tables
-            //
-            
-
-            // Definitely remove the local-disk copy of the job file
-            if (localJobFile != null) {
-                localJobFile.delete();
-                localJobFile = null;
-            }
-
-            //
-            // If the job file was in the temporary system directory,
-            // we should delete it upon garbage collect.
-            //
-            if (deleteUponCompletion != null) {
-                JobConf jd = new JobConf(deleteUponCompletion);
-                fs.delete(new File(jd.getJar()));
-                fs.delete(new File(deleteUponCompletion));
-                deleteUponCompletion = null;
-            }
-        }
-
-        /**
-         * A task assigned to this JobInProgress has reported in successfully.
-         * 
-         * This might prompt us to launch more tasks, or it might even indicate
-         * the job is now complete.
-         */
-        public synchronized void completedTask(String taskid) {
-            LOG.info("Task '" + taskid + "' has finished successfully.");
-
-            Task t = null;
-            if ((t = (Task) incompleteMapTasks.get(taskid)) != null) {
-                incompleteMapTasks.remove(taskid);
-                completeMapTasks.put(taskid, t);
-            } else if ((t = (Task) incompleteReduceTasks.get(taskid)) != null) {
-                incompleteReduceTasks.remove(taskid);
-                completeReduceTasks.put(taskid, t);
-            } else {
-                // Impossible situation; taskid is not 'incomplete' yet is reported
-                // as just finishing
-                throw new IllegalArgumentException("Impossible state: task " + taskid + " reported as complete, but was not known as incomplete");
-            }
-
-            //
-            // We may have Reduce tasks that are still waiting to be scheduled for execution;
-            // we don't want to launch them all while still waiting for most Map tasks to
-            // finish.  Nor do we want to wait for all the maps to finish, as the reduce task
-            // could be downloading files while other map tasks complete.
-            //
-            // WELL, maybe we do want to wait.  It's not obvious what the right launch rate
-            // would be.  For the moment, we wait until ALL map tasks have finished
-            // before launching even a single reduce task.
-            //
-            while (status.getRunState() == JobStatus.RUNNING &&
-                   reducesToLaunch.size() > 0 && 
-                   completeMapTasks.size() == numMapTasks) {
-                t = (Task) reducesToLaunch.elementAt(0);
-                reducesToLaunch.removeElement(t);
-                incompleteReduceTasks.put(t.getTaskId(), t);
-                executeTask(t.getTaskId());
-            }
-
-            //
-            // If all tasks are complete, then the job is done!
-            //
-            if (status.getRunState() == JobStatus.RUNNING &&
-                completeReduceTasks.size() == numReduceTasks) {
-                this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.SUCCEEDED);
-                this.finishTime = System.currentTimeMillis();
-            }
-        }
-
-        /**
-         * A task assigned to this JobInProgress has reported in as failed.
-         * Most of the time, we'll just reschedule execution.  However, after
-         * many repeated failures we may instead decide to allow the entire 
-         * job to fail.
-         *
-         * Even if a task has reported as completed in the past, it might later
-         * be reported as failed.  That's because the TaskTracker that hosts a map
-         * task might die before the entire job can complete.  If that happens,
-         * we need to schedule reexecution so that downstream reduce tasks can 
-         * obtain the map task's output.
-         */
-        public void failedTask(String taskid) {
-            LOG.info("Task '" + taskid + "' has been lost.");
-
-            Task t = null;
-            if ((t = (Task) completeMapTasks.get(taskid)) != null) {
-                completeMapTasks.remove(taskid);
-                incompleteMapTasks.put(taskid, t);
-            } else if ((t = (Task) completeReduceTasks.get(taskid)) != null) {
-                completeReduceTasks.remove(taskid);
-                incompleteReduceTasks.put(taskid, t);
-            }
-            
-            // Check if we need to kill the job because of excess failures
-            Integer failures = (Integer) taskFailures.get(taskid);
-            int numFailures = ((failures == null) ? 0 : failures.intValue()) + 1;
-            taskFailures.put(taskid, new Integer(numFailures));
-            if (numFailures >= MAX_TASK_FAILURES) {
-                LOG.info("Task " + taskid + " has failed " + numFailures + " times.  Aborting owning job " + profile.getJobId());
-                kill();
-            }
-
-            if (status.getRunState() == JobStatus.RUNNING) {
-                executeTask(taskid);
-            }
-        }
-
-        /**
-         * Return locations for all the indicated taskIds.  If the task
-         * is not complete, don't return anything.
-         */
-        public MapOutputLocation[] locateTasks(String taskIds[]) {
-            ArrayList v = new ArrayList();
-            for (int i = 0; i < taskIds.length; i++) {
-                if (completeMapTasks.get(taskIds[i]) != null) {
-                    String trackerId = (String) taskToTrackerMap.get(taskIds[i]);
-                    TaskTrackerStatus tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
-                    v.add(new MapOutputLocation(taskIds[i], tracker.getHost(), tracker.getPort()));
-                }
-            }
-            // randomly shuffle results to load-balance map output requests
-            Collections.shuffle(v);
-
-            return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
-        }
-
-        //////////////////////////
-        // A number of accessors
-        //////////////////////////
-        TreeMap getCompleteMapTasks() {
-            return completeMapTasks;
-        }
-        TreeMap getIncompleteMapTasks() {
-            return incompleteMapTasks;
-        }
-        TreeMap getCompleteReduceTasks() {
-            return completeReduceTasks;
-        }
-        TreeMap getIncompleteReduceTasks() {
-            return incompleteReduceTasks;
-        }
-        public JobProfile getProfile() {
-            return profile;
-        }
-        public JobStatus getStatus() {
-            return status;
-        }
-        public Task getTask(String taskid) {
-            Task t = null;
-            synchronized (incompleteMapTasks) {
-                if ((t = (Task) incompleteMapTasks.get(taskid)) != null) {
-                    return t;
-                }
-                synchronized (completeMapTasks) {
-                    if ((t = (Task) completeMapTasks.get(taskid)) != null) {
-                        return t;
-                    }
-                    synchronized (incompleteReduceTasks) {
-                        if ((t = (Task) incompleteReduceTasks.get(taskid)) != null) {
-                            return t;
-                        }
-                        synchronized (completeReduceTasks) {
-                            t = (Task) completeReduceTasks.get(taskid);
-                        }
-                    }
-                }
-            }
-            return t;
-        }
-        public Vector getTaskDiagnosticInfo(String taskid) {
-            Vector v = (Vector) taskDiagnosticData.get(taskid);
-            if (v == null) {
-                return new Vector();
-            } else {
-                return v;
-            }
-        }
-        public String getTaskStateString(String taskid) {
-            return (String) taskStateStrings.get(taskid);
-        }
-        public float completedRatio() {
-            return (0.5f * status.mapProgress()) + (0.5f * status.reduceProgress());
-        }
-        public long getStartTime() {
-            return startTime;
-        }
-        public long getFinishTime() {
-            return finishTime;
-        }
-        public int desiredMaps() {
-            return numMapTasks;
-        }
-        public int attemptedMaps() {
-            return attemptedMapExecutions;
-        }
-        public int completedMaps() {
-            return completeMapTasks.size();
-        }
-        public int desiredReduces() {
-            return numReduceTasks;
-        }
-        public int attemptedReduces() {
-            return attemptedReduceExecutions;
-        }
-        public int completedReduces() {
-            return completeReduceTasks.size();
-        }
-        public void updateTaskStatus(String taskid, TaskStatus status) {
-            TaskStatus oldStatus = (TaskStatus) taskStatus.put(taskid, status);
-            
-            float delta = 0.0f;
-            if (oldStatus != null) {
-                delta -= oldStatus.getProgress();
-            }
-            delta += status.getProgress();
-
-            String diagInfo = status.getDiagnosticInfo();
-            if (diagInfo != null && diagInfo.length() > 0) {
-                Vector v = (Vector) taskDiagnosticData.get(taskid);
-                if (v == null) {
-                    v = new Vector();
-                    taskDiagnosticData.put(taskid, v);
-                }
-                v.add(diagInfo);
-            }
-
-            taskStateStrings.put(taskid, status.getStateString());
-
-            if (incompleteMapTasks.get(taskid) != null || 
-                completeMapTasks.get(taskid) != null) {
-                totalReportedMapProgress += delta;
-                if (numMapTasks == 0) {
-                    this.status.setMapProgress(1.0f);
-                } else {
-                    this.status.setMapProgress((float) (totalReportedMapProgress / numMapTasks));
-                }
-            } else if (incompleteReduceTasks.get(taskid) != null ||
-                       completeReduceTasks.get(taskid) != null) {
-                totalReportedReduceProgress += delta;
-                if (numReduceTasks == 0) {
-                    this.status.setReduceProgress(1.0f);
-                } else {
-                    this.status.setReduceProgress((float) (totalReportedReduceProgress / numReduceTasks));
-                }
-            } else {
-                LOG.info("Serious problem.  While updating status, cannot find taskid " + taskid);
-            }
-        }
-        public TaskStatus getTaskStatus(String taskid) {
-            return (TaskStatus) taskStatus.get(taskid);
-        }
-
-        /////////////////////////////
-        // Some private util methods
-        /////////////////////////////
-        void executeTask(String taskid) {
-            if (incompleteMapTasks.get(taskid) != null) {
-                attemptedMapExecutions++;
-            } else if (incompleteReduceTasks.get(taskid) != null) {
-                attemptedReduceExecutions++;
-            }
-            updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.UNASSIGNED, "", ""));
-            synchronized (unassignedTasks) {
-                unassignedTasks.add(taskid);
-            }
-        }
-    }
-
     ////////////////////////////////////////////////////
     // Methods to track all the TaskTrackers
     ////////////////////////////////////////////////////
@@ -977,16 +757,19 @@
     void updateTaskStatuses(TaskTrackerStatus status) {
         for (Iterator it = status.taskReports(); it.hasNext(); ) {
             TaskStatus report = (TaskStatus) it.next();
-            JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(report.getTaskId()));
-            job.updateTaskStatus(report.getTaskId(), report);
+            TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(report.getTaskId());
+            if (tip == null) {
+                LOG.info("Serious problem.  While updating status, cannot find taskid " + report.getTaskId());
+            } else {
+                JobInProgress job = tip.getJob();
+                job.updateTaskStatus(tip, report);
 
-            if (report.getRunState() == TaskStatus.SUCCEEDED) {
-                job.completedTask(report.getTaskId());
-            } else if (report.getRunState() == TaskStatus.FAILED) {
-                TreeSet taskset = (TreeSet) trackerToTaskMap.get(status.getTrackerName());
-                taskset.remove(report.getTaskId());
-                taskToTrackerMap.remove(report.getTaskId());
-                job.failedTask(report.getTaskId());
+                if (report.getRunState() == TaskStatus.SUCCEEDED) {
+                    job.completedTask(tip, report.getTaskId());
+                } else if (report.getRunState() == TaskStatus.FAILED) {
+                    // Tell the job to fail the relevant task
+                    job.failedTask(tip, report.getTaskId(), status.getTrackerName());
+                }
             }
         }
     }
@@ -1004,61 +787,13 @@
         if (lostTasks != null) {
             for (Iterator it = lostTasks.iterator(); it.hasNext(); ) {
                 String taskId = (String) it.next();
-                taskToTrackerMap.remove(taskId);
-                JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskId));
-                job.failedTask(taskId);
-            }
-        }
-    }
+                TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
 
-    /**
-     * Task getTaskAssignment() returns
-     * a task we'd like the taskTracker to execute right now.
-     *
-     * Eventually this function should compute load on the various TaskTrackers,
-     * and incorporate knowledge of NDFS file placement.  But for right now, it
-     * just grabs a single item out of the pending task list and hands it back.
-     */
-    Task getTaskAssignment(String taskTracker) {
-        synchronized (unassignedTasks) {
-            if (unassignedTasks.size() > 0) {
-                String taskid = (String) unassignedTasks.elementAt(0);
-                unassignedTasks.remove(taskid);
-
-                // Move task status to RUNNING
-                JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskid));
-                job.updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.RUNNING, "", ""));
-
-                // Remember where we are running it
-                TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
-                if (taskset == null) {
-                    taskset = new TreeSet();
-                    trackerToTaskMap.put(taskTracker, taskset);
-                }
-                LOG.info("Adding task '" + taskid + "' to set for tracker '" + taskTracker + "'");
-                taskset.add(taskid);
-
-                taskToTrackerMap.put(taskid, taskTracker);
-
-                return job.getTask(taskid);
-            } else {
-                return null;
+                // Tell the job to fail the relevant task
+                JobInProgress job = tip.getJob();
+                job.failedTask(tip, taskId, trackerName);
             }
         }
-    }
-
-    /**
-     * Grab random num for task id
-     */
-    String createMapTaskId() {
-        return "task_m_" + Integer.toString(Math.abs(r.nextInt()),36);
-    }
-    String createReduceTaskId() {
-        return "task_r_" + Integer.toString(Math.abs(r.nextInt()),36);
-    }
-
-    String createJobId() {
-        return "job_" + Integer.toString(Math.abs(r.nextInt()),36);
     }
 
     ////////////////////////////////////////////////////////////

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java Thu Feb  2 09:58:38 2006
@@ -89,9 +89,13 @@
         }
 
         // run a single reduce task
+        String mapDependencies[][] = new String[mapIds.size()][1];
+        for (int i = 0; i < mapIds.size(); i++) {
+            mapDependencies[i][0] = (String) mapIds.get(i);
+        }
         ReduceTask reduce =
           new ReduceTask(file, reduceId,
-                         (String[])mapIds.toArray(new String[0]),
+                         mapDependencies,
                          0);
         reduce.setConf(job);
         reduce.run(job, this);

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java Thu Feb  2 09:58:38 2006
@@ -25,7 +25,7 @@
     // Timeouts, constants
     //
     public static final long HEARTBEAT_INTERVAL = 3 * 1000;
-    public static final long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000;
+    public static final long TASKTRACKER_EXPIRY_INTERVAL = 60 * 1000;
 
     //
     // Result codes

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java Thu Feb  2 09:58:38 2006
@@ -58,6 +58,10 @@
     throws IOException {
     return this.jobConf.getLocalFile(reduceTaskId, mapTaskId+".out");
   }
+  public File getInputFile(String mapTaskId[], String reduceTaskId)
+    throws IOException {
+    return this.jobConf.getLocalFile(reduceTaskId, mapTaskId, ".out");
+  }
 
   /** Removes all of the files related to a task. */
   public void removeAll(String taskId) throws IOException {

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java Thu Feb  2 09:58:38 2006
@@ -37,6 +37,10 @@
     this.split = split;
   }
 
+  public boolean isMapTask() {
+      return true;
+  }
+
   public TaskRunner createRunner(TaskTracker tracker) {
     return new MapTaskRunner(this, tracker, this.nutchConf);
   }

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java Thu Feb  2 09:58:38 2006
@@ -35,8 +35,9 @@
   }
   
   /** Delete any temporary files from previous failed attempts. */
-  public void prepare() throws IOException {
+  public boolean prepare() throws IOException {
     this.mapOutputFile.removeAll(getTask().getTaskId());
+    return true;
   }
 
   /** Delete all of the temporary map output files. */

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java Thu Feb  2 09:58:38 2006
@@ -27,7 +27,7 @@
 
 /** A Reduce task. */
 public class ReduceTask extends Task {
-  private String[] mapTaskIds;
+  private String[][] mapTaskIds;
   private int partition;
   private boolean sortComplete;
 
@@ -43,7 +43,7 @@
   public ReduceTask() {}
 
   public ReduceTask(String jobFile, String taskId,
-                    String[] mapTaskIds, int partition) {
+                    String[][] mapTaskIds, int partition) {
     super(jobFile, taskId);
     this.mapTaskIds = mapTaskIds;
     this.partition = partition;
@@ -53,7 +53,11 @@
     return new ReduceTaskRunner(this, tracker, this.nutchConf);
   }
 
-  public String[] getMapTaskIds() { return mapTaskIds; }
+  public boolean isMapTask() {
+      return false;
+  }
+
+  public String[][] getMapTaskIds() { return mapTaskIds; }
   public int getPartition() { return partition; }
 
   public void write(DataOutput out) throws IOException {
@@ -61,7 +65,10 @@
 
     out.writeInt(mapTaskIds.length);              // write mapTaskIds
     for (int i = 0; i < mapTaskIds.length; i++) {
-      UTF8.writeString(out, mapTaskIds[i]);
+        out.writeInt(mapTaskIds[i].length);
+        for (int j = 0; j < mapTaskIds[i].length; j++) {
+            UTF8.writeString(out, mapTaskIds[i][j]);
+        }
     }
 
     out.writeInt(partition);                      // write partition
@@ -70,9 +77,12 @@
   public void readFields(DataInput in) throws IOException {
     super.readFields(in);
 
-    mapTaskIds = new String[in.readInt()];        // read mapTaskIds
+    mapTaskIds = new String[in.readInt()][];        // read mapTaskIds
     for (int i = 0; i < mapTaskIds.length; i++) {
-      mapTaskIds[i] = UTF8.readString(in);
+        mapTaskIds[i] = new String[in.readInt()];
+        for (int j = 0; j < mapTaskIds[i].length; j++) {
+            mapTaskIds[i][j] = UTF8.readString(in);
+        }
     }
 
     this.partition = in.readInt();                // read partition
@@ -182,6 +192,7 @@
         float progPerByte = 1.0f / lfs.getLength(partFile);
         Progress phase = appendPhase.phase();
         phase.setStatus(partFile.toString());
+
         SequenceFile.Reader in =
           new SequenceFile.Reader(lfs, partFile.toString(), job);
         try {

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java Thu Feb  2 09:58:38 2006
@@ -37,14 +37,14 @@
   }
 
   /** Assemble all of the map output files. */
-  public void prepare() throws IOException {
+  public boolean prepare() throws IOException {
     ReduceTask task = ((ReduceTask)getTask());
     this.mapOutputFile.removeAll(task.getTaskId());    // cleanup from failures
-    String[] mapTaskIds = task.getMapTaskIds();
+    String[][] mapTaskIds = task.getMapTaskIds();
     final Progress copyPhase = getTask().getProgress().phase();
 
     // we need input from every map task
-    HashSet needed = new HashSet();
+    Vector needed = new Vector();
     for (int i = 0; i < mapTaskIds.length; i++) {
       needed.add(mapTaskIds[i]);
       copyPhase.addPhase();                       // add sub-phase per file
@@ -52,17 +52,21 @@
 
     InterTrackerProtocol jobClient = getTracker().getJobClient();
     while (needed.size() > 0) {
-
       getTask().reportProgress(getTracker());
 
       // get list of available map output locations from job tracker
-      String[] neededStrings =
-        (String[])needed.toArray(new String[needed.size()]);
+      String[][] neededStrings = new String[needed.size()][];
+      for (int i = 0; i < needed.size(); i++) {
+          neededStrings[i] = (String[]) needed.elementAt(i);
+      }
       MapOutputLocation[] locs =
         jobClient.locateMapOutputs(task.getTaskId(), neededStrings);
 
       if (locs.length == 0) {
         try {
+          if (killed) {
+            return false;
+          }
           Thread.sleep(1000);
         } catch (InterruptedException e) {
         }
@@ -96,9 +100,19 @@
           
           client.getFile(loc.getMapTaskId(), task.getTaskId(),
                          new IntWritable(task.getPartition()));
-          
-          needed.remove(loc.getMapTaskId());     // success: remove from needed
-          
+
+          // Success: remove from 'needed'
+          boolean foundit = false;
+          for (Iterator it = needed.iterator(); it.hasNext() && !foundit; ) {
+              String idsForSingleMap[] = (String[]) it.next();
+              for (int j = 0; j < idsForSingleMap.length; j++) {
+                  if (idsForSingleMap[j].equals(loc.getMapTaskId())) {
+                      it.remove();
+                      foundit = true;
+                      break;
+                  }
+              }
+          }
           copyPhase.startNextPhase();
           
         } catch (IOException e) {                 // failed: try again later
@@ -109,11 +123,10 @@
         } finally {
           this.mapOutputFile.setProgressReporter(null);
         }
-        
       }
-      
     }
     getTask().reportProgress(getTracker());
+    return true;
   }
 
   /** Delete all of the temporary map output files. */

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java Thu Feb  2 09:58:38 2006
@@ -82,6 +82,8 @@
   private transient long nextProgressTime =
     System.currentTimeMillis() + PROGRESS_INTERVAL;
 
+  public abstract boolean isMapTask();
+
   public Progress getProgress() { return taskProgress; }
 
   public Reporter getReporter(final TaskUmbilicalProtocol umbilical,