You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by mc...@apache.org on 2006/02/02 18:58:44 UTC
svn commit: r374443 [1/2] - in /lucene/nutch/trunk/src:
java/org/apache/nutch/fs/ java/org/apache/nutch/mapred/
test/org/apache/nutch/mapred/ webapps/jobtracker/
Author: mc
Date: Thu Feb 2 09:58:38 2006
New Revision: 374443
URL: http://svn.apache.org/viewcvs?rev=374443&view=rev
Log:
New MapReduce JobTracker internals. This fixes
several bugs, as listed in JIRA NUTCH-183.
Added:
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskInProgress.java
lucene/nutch/trunk/src/test/org/apache/nutch/mapred/MapredLoadTest.java
Modified:
lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskRunner.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskStatus.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskTracker.java
lucene/nutch/trunk/src/java/org/apache/nutch/mapred/TaskTrackerStatus.java
lucene/nutch/trunk/src/webapps/jobtracker/jobdetails.jsp
lucene/nutch/trunk/src/webapps/jobtracker/jobtracker.jsp
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fs/NFSDataInputStream.java Thu Feb 2 09:58:38 2006
@@ -57,7 +57,7 @@
} catch (FileNotFoundException e) { // quietly ignore
stopSumming();
} catch (IOException e) { // loudly ignore
- LOG.warning("Problem opening checksum file: "+e+". Ignoring.");
+ LOG.warning("Problem opening checksum file: "+ file + ". Ignoring with exception " + e + ".");
stopSumming();
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/InterTrackerProtocol.java Thu Feb 2 09:58:38 2006
@@ -40,10 +40,11 @@
/** Called to get new tasks from from the job tracker for this tracker.*/
Task pollForNewTask(String trackerName);
- /** Called to find which tasks that have been run by this tracker are now
- * closed, i.e., their job is complete. This is used to, e.g., notify a map
- * task that its output is no longer needed and may be removed. */
- String pollForClosedTask(String trackerName);
+ /** Called to find which tasks that have been run by this tracker should now
+ * be closed because their job is complete. This is used to, e.g.,
+ * notify a map task that its output is no longer needed and may
+ * be removed. */
+ String pollForTaskWithClosedJob(String trackerName);
/** Called by a reduce task to find which map tasks are completed.
*
@@ -51,7 +52,7 @@
* @param mapTasksNeeded an array of UTF8 naming map task ids whose output is needed.
* @return an array of MapOutputLocation
*/
- MapOutputLocation[] locateMapOutputs(String taskId, String[] mapTasksNeeded);
+ MapOutputLocation[] locateMapOutputs(String taskId, String[][] mapTasksNeeded);
/**
* The task tracker calls this once, to discern where it can find
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobClient.java Thu Feb 2 09:58:38 2006
@@ -296,11 +296,7 @@
} catch (InterruptedException e) {}
running = jc.getJob(jobId);
String report = null;
- if (running.reduceProgress() == 0.0)
- report=" map "+Math.round(running.mapProgress()*100)+"%";
- else {
- report=" reduce "+Math.round(running.reduceProgress()*100)+"%";
- }
+ report = " map "+Math.round(running.mapProgress()*100)+"% reduce " + Math.round(running.reduceProgress()*100)+"%";
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobConf.java Thu Feb 2 09:58:38 2006
@@ -16,6 +16,7 @@
package org.apache.nutch.mapred;
+
import java.io.IOException;
import java.io.File;
import java.io.InputStream;
@@ -112,18 +113,34 @@
/** Constructs a local file name. Files are distributed among configured
* local directories.*/
- public File getLocalFile(String subdir, String name)
- throws IOException {
+ public File getLocalFile(String subdir, String name) throws IOException {
+ String param[] = new String[1];
+ param[0] = name;
+ return getLocalFile(subdir, param, "", false);
+ }
+ // REMIND - mjc - rename this! getLocalFile() is not quite the same.
+ public File getLocalFile(String subdir, String names[], String ending) throws IOException {
+ return getLocalFile(subdir, names, ending, true);
+ }
+ File getLocalFile(String subdir, String names[], String ending, boolean existingFileTest) throws IOException {
String[] localDirs = getLocalDirs();
- String path = subdir + File.separator + name;
- int hashCode = path.hashCode();
- for (int i = 0; i < localDirs.length; i++) { // try each local dir
- int index = (hashCode+i & Integer.MAX_VALUE) % localDirs.length;
- File file = new File(localDirs[index], path);
- File dir = file.getParentFile();
- if (dir.exists() || dir.mkdirs()) {
- return file;
- }
+ for (int k = 0; k < names.length; k++) {
+ String path = subdir + File.separator + names[k] + ending;
+ int hashCode = path.hashCode();
+ for (int i = 0; i < localDirs.length; i++) { // try each local dir
+ int index = (hashCode+i & Integer.MAX_VALUE) % localDirs.length;
+ File file = new File(localDirs[index], path);
+ File dir = file.getParentFile();
+ if (existingFileTest) {
+ if (file.exists()) {
+ return file;
+ }
+ } else {
+ if (dir.exists() || dir.mkdirs()) {
+ return file;
+ }
+ }
+ }
}
throw new IOException("No valid local directories.");
}
Added: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java?rev=374443&view=auto
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java (added)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobInProgress.java Thu Feb 2 09:58:38 2006
@@ -0,0 +1,436 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.mapred;
+
+import org.apache.nutch.io.*;
+import org.apache.nutch.fs.*;
+import org.apache.nutch.ipc.*;
+import org.apache.nutch.util.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+///////////////////////////////////////////////////////
+// JobInProgress maintains all the info for keeping
+// a Job on the straight and narrow. It keeps its JobProfile
+// and its latest JobStatus, plus a set of tables for
+// doing bookkeeping of its Tasks.
+///////////////////////////////////////////////////////
+public class JobInProgress {
+ public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.mapred.JobInProgress");
+
+ JobProfile profile;
+ JobStatus status;
+ File localJobFile = null;
+
+ TaskInProgress maps[] = null;
+ TaskInProgress reduces[] = null;
+ int numMapTasks = 0;
+ int numReduceTasks = 0;
+
+ JobTracker jobtracker = null;
+
+ long startTime;
+ long finishTime;
+ String deleteUponCompletion = null;
+
+ NutchConf nutchConf;
+ boolean tasksInited = false;
+
+ /**
+ * Create a JobInProgress with the given job file, plus a handle
+ * to the tracker.
+ */
+ public JobInProgress(String jobFile, JobTracker jobtracker, NutchConf nutchConf) throws IOException {
+ String jobid = "job_" + jobtracker.createUniqueId();
+ String url = "http://" + jobtracker.getJobTrackerMachine() + ":" + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
+ this.nutchConf = nutchConf;
+ this.jobtracker = jobtracker;
+ this.profile = new JobProfile(jobid, jobFile, url);
+ this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.RUNNING);
+ this.startTime = System.currentTimeMillis();
+
+ this.localJobFile = new JobConf(nutchConf).getLocalFile(JobTracker.SUBDIR, jobid + ".xml");
+ NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+ fs.copyToLocalFile(new File(jobFile), localJobFile);
+
+ JobConf jd = new JobConf(localJobFile);
+ this.numMapTasks = jd.getNumMapTasks();
+ this.numReduceTasks = jd.getNumReduceTasks();
+
+ //
+ // If a jobFile is in the systemDir, we can delete it (and
+ // its JAR) upon completion
+ //
+ File systemDir = jd.getSystemDir();
+ if (jobFile.startsWith(systemDir.getPath())) {
+ this.deleteUponCompletion = jobFile;
+ }
+ }
+
+ /**
+ * Construct the splits, etc
+ */
+ void initTasks() throws IOException {
+ if (tasksInited) {
+ return;
+ }
+
+ //
+ // construct input splits
+ //
+ String jobid = profile.getJobId();
+ String jobFile = profile.getJobFile();
+
+ JobConf jd = new JobConf(localJobFile);
+ NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+ FileSplit[] splits =
+ jd.getInputFormat().getSplits(fs, jd, numMapTasks);
+
+ //
+ // sort splits by decreasing length, to reduce job's tail
+ //
+ Arrays.sort(splits, new Comparator() {
+ public int compare(Object a, Object b) {
+ long diff =
+ ((FileSplit)b).getLength() - ((FileSplit)a).getLength();
+ return diff==0 ? 0 : (diff > 0 ? 1 : -1);
+ }
+ });
+
+ //
+ // adjust number of map tasks to actual number of splits
+ //
+ this.numMapTasks = splits.length;
+ // create a map task for each split
+ this.maps = new TaskInProgress[numMapTasks];
+ for (int i = 0; i < numMapTasks; i++) {
+ maps[i] = new TaskInProgress(jobFile, splits[i], jobtracker, nutchConf, this);
+ }
+
+ //
+ // Create reduce tasks
+ //
+ this.reduces = new TaskInProgress[numReduceTasks];
+ for (int i = 0; i < numReduceTasks; i++) {
+ reduces[i] = new TaskInProgress(jobFile, maps, i, jobtracker, nutchConf, this);
+ }
+
+ tasksInited = true;
+ }
+
+ /////////////////////////////////////////////////////
+ // Accessors for the JobInProgress
+ /////////////////////////////////////////////////////
+ public JobProfile getProfile() {
+ return profile;
+ }
+ public JobStatus getStatus() {
+ return status;
+ }
+ public long getStartTime() {
+ return startTime;
+ }
+ public long getFinishTime() {
+ return finishTime;
+ }
+ public int desiredMaps() {
+ return numMapTasks;
+ }
+ public int finishedMaps() {
+ int finishedCount = 0;
+ for (int i = 0; i < maps.length; i++) {
+ if (maps[i].isComplete()) {
+ finishedCount++;
+ }
+ }
+ return finishedCount;
+ }
+ public int desiredReduces() {
+ return numReduceTasks;
+ }
+ public int finishedReduces() {
+ int finishedCount = 0;
+ for (int i = 0; i < reduces.length; i++) {
+ if (reduces[i].isComplete()) {
+ finishedCount++;
+ }
+ }
+ return finishedCount;
+ }
+
+ /**
+ * Return a treeset of completed TaskInProgress objects
+ */
+ public Vector reportTasksInProgress(boolean shouldBeMap, boolean shouldBeComplete) {
+ Vector results = new Vector();
+ TaskInProgress tips[] = null;
+ if (shouldBeMap) {
+ tips = maps;
+ } else {
+ tips = reduces;
+ }
+ for (int i = 0; i < tips.length; i++) {
+ if (tips[i].isComplete() == shouldBeComplete) {
+ results.add(tips[i]);
+ }
+ }
+ return results;
+ }
+
+ ////////////////////////////////////////////////////
+ // Status update methods
+ ////////////////////////////////////////////////////
+ public void updateTaskStatus(TaskInProgress tip, TaskStatus status) {
+ tip.updateStatus(status);
+
+ //
+ // Update JobInProgress status
+ //
+ if (maps.length == 0) {
+ this.status.setMapProgress(1.0f);
+ } else {
+ double reportedProgress = 0;
+ for (int i = 0; i < maps.length; i++) {
+ reportedProgress += maps[i].getProgress();
+ }
+ this.status.setMapProgress((float) (reportedProgress / maps.length));
+ }
+ if (reduces.length == 0) {
+ this.status.setReduceProgress(1.0f);
+ } else {
+ double reportedProgress = 0;
+ for (int i = 0; i < reduces.length; i++) {
+ reportedProgress += reduces[i].getProgress();
+ }
+ this.status.setReduceProgress((float) (reportedProgress / reduces.length));
+ }
+ }
+
+ /////////////////////////////////////////////////////
+ // Create/manage tasks
+ /////////////////////////////////////////////////////
+ /**
+ * Return a MapTask, if appropriate, to run on the given tasktracker
+ */
+ public Task obtainNewMapTask(String taskTracker, TaskTrackerStatus tts) {
+ if (! tasksInited) {
+ try {
+ initTasks();
+ } catch (IOException ie) {
+ ie.printStackTrace();
+ LOG.info("Cannot create task split for " + profile.getJobId());
+ }
+ }
+
+ Task t = null;
+ int cacheTarget = -1;
+ int stdTarget = -1;
+ int specTarget = -1;
+ double totalProgress = 0;
+
+ //
+ // We end up creating two tasks for the same bucket, because
+ // we call obtainNewMapTask() really fast, twice in a row.
+ // There's not enough time for the "recentTasks"
+ //
+ for (int i = 0; i < maps.length; i++) {
+ if (maps[i].hasTaskWithCacheHit(taskTracker, tts)) {
+ if (cacheTarget < 0) {
+ cacheTarget = i;
+ }
+ } else if (maps[i].hasTask()) {
+ if (stdTarget < 0) {
+ stdTarget = i;
+ }
+ }
+ totalProgress += maps[i].getProgress();
+ }
+ double avgProgress = totalProgress / maps.length;
+
+ for (int i = 0; i < maps.length; i++) {
+ if (maps[i].hasSpeculativeTask(avgProgress)) {
+ if (specTarget < 0) {
+ specTarget = i;
+ }
+ }
+ }
+
+ if (cacheTarget >= 0) {
+ t = maps[cacheTarget].getTaskToRun(taskTracker, tts, avgProgress);
+ } else if (stdTarget >= 0) {
+ t = maps[stdTarget].getTaskToRun(taskTracker, tts, avgProgress);
+ } else if (specTarget >= 0) {
+ t = maps[specTarget].getTaskToRun(taskTracker, tts, avgProgress);
+ }
+ return t;
+ }
+
+ /**
+ * Return a ReduceTask, if appropriate, to run on the given tasktracker.
+ * We don't have cache-sensitivity for reduce tasks, as they
+ * work on temporary MapRed files.
+ */
+ public Task obtainNewReduceTask(String taskTracker, TaskTrackerStatus tts) {
+ if (! tasksInited) {
+ try {
+ initTasks();
+ } catch (IOException ie) {
+ ie.printStackTrace();
+ LOG.info("Cannot create task split for " + profile.getJobId());
+ }
+ }
+
+ Task t = null;
+ int stdTarget = -1;
+ int specTarget = -1;
+ int totalProgress = 0;
+ for (int i = 0; i < reduces.length; i++) {
+ totalProgress += reduces[i].getProgress();
+ }
+ double avgProgress = (1.0 * totalProgress) / reduces.length;
+
+ for (int i = 0; i < reduces.length; i++) {
+ if (reduces[i].hasTask()) {
+ if (stdTarget < 0) {
+ stdTarget = i;
+ }
+ } else if (reduces[i].hasSpeculativeTask(avgProgress)) {
+ if (specTarget < 0) {
+ specTarget = i;
+ }
+ }
+ }
+
+ if (stdTarget >= 0) {
+ t = reduces[stdTarget].getTaskToRun(taskTracker, tts, avgProgress);
+ } else if (specTarget >= 0) {
+ t = reduces[specTarget].getTaskToRun(taskTracker, tts, avgProgress);
+ }
+ return t;
+ }
+
+ /**
+ * A taskid assigned to this JobInProgress has reported in successfully.
+ */
+ public synchronized void completedTask(TaskInProgress tip, String taskid) {
+ LOG.info("Taskid '" + taskid + "' has finished successfully.");
+ tip.completed(taskid);
+
+ //
+ // Figure out whether the Job is done
+ //
+ boolean allDone = true;
+ for (int i = 0; i < maps.length; i++) {
+ if (! maps[i].isComplete()) {
+ allDone = false;
+ break;
+ }
+ }
+ if (allDone) {
+ for (int i = 0; i < reduces.length; i++) {
+ if (! reduces[i].isComplete()) {
+ allDone = false;
+ break;
+ }
+ }
+ }
+
+ //
+ // If all tasks are complete, then the job is done!
+ //
+ if (status.getRunState() == JobStatus.RUNNING && allDone) {
+ this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.SUCCEEDED);
+ this.finishTime = System.currentTimeMillis();
+ }
+ }
+
+ /**
+ * Kill the job and all its component tasks.
+ */
+ public synchronized void kill() {
+ if (status.getRunState() != JobStatus.FAILED) {
+ this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.FAILED);
+ this.finishTime = System.currentTimeMillis();
+
+ //
+ // kill all TIPs.
+ //
+ for (int i = 0; i < maps.length; i++) {
+ maps[i].kill();
+ }
+ for (int i = 0; i < reduces.length; i++) {
+ reduces[i].kill();
+ }
+ }
+ }
+
+ /**
+ * A task assigned to this JobInProgress has reported in as failed.
+ * Most of the time, we'll just reschedule execution. However, after
+ * many repeated failures we may instead decide to allow the entire
+ * job to fail.
+ *
+ * Even if a task has reported as completed in the past, it might later
+ * be reported as failed. That's because the TaskTracker that hosts a map
+ * task might die before the entire job can complete. If that happens,
+ * we need to schedule reexecution so that downstream reduce tasks can
+ * obtain the map task's output.
+ */
+ public void failedTask(TaskInProgress tip, String taskid, String trackerName) {
+ tip.failedSubTask(taskid, trackerName);
+
+ //
+ // Check if we need to kill the job because of too many failures
+ //
+ if (tip.isFailed()) {
+ LOG.info("Aborting job " + profile.getJobId());
+ kill();
+ }
+ }
+
+ /**
+ * The job is dead. We're now GC'ing it, getting rid of the job
+ * from all tables. Be sure to remove all of this job's tasks
+ * from the various tables.
+ */
+ public synchronized void garbageCollect() throws IOException {
+ //
+ // Remove this job from all tables
+ //
+
+ // Definitely remove the local-disk copy of the job file
+ if (localJobFile != null) {
+ localJobFile.delete();
+ localJobFile = null;
+ }
+
+ //
+ // If the job file was in the temporary system directory,
+ // we should delete it upon garbage collect.
+ //
+ if (deleteUponCompletion != null) {
+ JobConf jd = new JobConf(deleteUponCompletion);
+ NutchFileSystem fs = NutchFileSystem.get(nutchConf);
+ fs.delete(new File(jd.getJar()));
+ fs.delete(new File(deleteUponCompletion));
+ deleteUponCompletion = null;
+ }
+ }
+}
+
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/JobTracker.java Thu Feb 2 09:58:38 2006
@@ -15,6 +15,7 @@
*/
package org.apache.nutch.mapred;
+
import org.apache.nutch.io.*;
import org.apache.nutch.fs.*;
import org.apache.nutch.ipc.*;
@@ -32,7 +33,11 @@
* @author Mike Cafarella
*******************************************************/
public class JobTracker implements MRConstants, InterTrackerProtocol, JobSubmissionProtocol {
- static final int MAX_TASK_FAILURES = 4;
+ static long RETIRE_JOB_INTERVAL;
+ static long RETIRE_JOB_CHECK_INTERVAL;
+ static float TASK_ALLOC_EPSILON;
+ static float PAD_FRACTION;
+ static float MIN_SLOTS_FOR_PADDING;
public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.mapred.JobTracker");
@@ -104,7 +109,12 @@
if (newProfile != null) {
if (now - newProfile.getLastSeen() > TASKTRACKER_EXPIRY_INTERVAL) {
// Remove completely
- taskTrackers.remove(leastRecent.getTrackerName());
+
+ TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(leastRecent.getTrackerName());
+ if (oldStatus != null) {
+ totalMaps -= oldStatus.countMapTasks();
+ totalReduces -= oldStatus.countReduceTasks();
+ }
lostTaskTracker(leastRecent.getTrackerName());
} else {
// Update time by inserting latest profile
@@ -125,6 +135,45 @@
}
}
+ ///////////////////////////////////////////////////////
+ // Used to remove old finished Jobs that have been around for too long
+ ///////////////////////////////////////////////////////
+ class RetireJobs implements Runnable {
+ boolean shouldRun = true;
+ public RetireJobs() {
+ }
+
+ /**
+ * The run method lives for the life of the JobTracker,
+ * and removes Jobs that are not still running, but which
+ * finished a long time ago.
+ */
+ public void run() {
+ while (shouldRun) {
+ try {
+ Thread.sleep(RETIRE_JOB_CHECK_INTERVAL);
+ } catch (InterruptedException ie) {
+ }
+
+ synchronized (jobs) {
+ for (Iterator it = jobs.keySet().iterator(); it.hasNext(); ) {
+ String jobid = (String) it.next();
+ JobInProgress job = (JobInProgress) jobs.get(jobid);
+
+ if (job.getStatus().getRunState() != JobStatus.RUNNING &&
+ (job.getFinishTime() + RETIRE_JOB_INTERVAL < System.currentTimeMillis())) {
+ it.remove();
+ jobsByArrival.remove(job);
+ }
+ }
+ }
+ }
+ }
+ public void stopRetirer() {
+ shouldRun = false;
+ }
+ }
+
/////////////////////////////////////////////////////////////////
// The real JobTracker
////////////////////////////////////////////////////////////////
@@ -134,6 +183,8 @@
int totalSubmissions = 0;
Random r = new Random();
+ private int maxCurrentTasks;
+
//
// Properties to maintain while running Jobs and Tasks:
//
@@ -152,28 +203,26 @@
// All the known jobs. (jobid->JobInProgress)
TreeMap jobs = new TreeMap();
+ Vector jobsByArrival = new Vector();
- // (taskId->jobid that contains the task)
- TreeMap taskToJobMap = new TreeMap();
+ // All the known TaskInProgress items, mapped to by taskids (taskid->TIP)
+ TreeMap taskidToTIPMap = new TreeMap();
- // (taskId->trackerID) locates a given task
- TreeMap taskToTrackerMap = new TreeMap();
+ // (taskid --> trackerID)
+ TreeMap taskidToTrackerMap = new TreeMap();
// (trackerID->TreeSet of taskids running at that tracker)
TreeMap trackerToTaskMap = new TreeMap();
- // a list of taskIds waiting for a task tracker to run on
- Vector unassignedTasks = new Vector();
-
- // Tasks that we've closed
- TreeSet tasksReportedClosed = new TreeSet();
-
//
// Watch and expire TaskTracker objects using these structures.
// We can map from Name->TaskTrackerStatus, or we can expire by time.
//
+ int totalMaps = 0;
+ int totalReduces = 0;
TreeMap taskTrackers = new TreeMap();
ExpireTrackers expireTrackers = new ExpireTrackers();
+ RetireJobs retireJobs = new RetireJobs();
/**
* It might seem like a bug to maintain a TreeSet of status objects,
@@ -209,18 +258,27 @@
static final String SUBDIR = "jobTracker";
NutchFileSystem fs;
File systemDir;
-
private NutchConf nutchConf;
/**
* Start the JobTracker process, listen on the indicated port
*/
JobTracker(NutchConf conf) throws IOException {
+ //
+ // Grab some static constants
+ //
+ maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
+ RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
+ RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
+ TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
+ PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
+ MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
+
// This is a directory of temporary submission files. We delete it
// on startup, and can delete any files that we're done with
+ this.nutchConf = conf;
JobConf jobConf = new JobConf(conf);
this.systemDir = jobConf.getSystemDir();
- this.nutchConf = conf;
this.fs = NutchFileSystem.get(conf);
FileUtil.fullyDelete(fs, systemDir);
fs.mkdirs(systemDir);
@@ -248,6 +306,7 @@
this.startTime = System.currentTimeMillis();
new Thread(this.expireTrackers).start();
+ new Thread(this.retireJobs).start();
}
public static InetSocketAddress getAddress(NutchConf conf) {
@@ -274,6 +333,41 @@
}
///////////////////////////////////////////////////////
+ // Maintain lookup tables; called by JobInProgress
+ // and TaskInProgress
+ ///////////////////////////////////////////////////////
+ void createTaskEntry(String taskid, String taskTracker, TaskInProgress tip) {
+ LOG.info("Adding task '" + taskid + "' to tip " + tip.getTIPId() + ", for tracker '" + taskTracker + "'");
+
+ // taskid --> tracker
+ taskidToTrackerMap.put(taskid, taskTracker);
+
+ // tracker --> taskid
+ TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
+ if (taskset == null) {
+ taskset = new TreeSet();
+ trackerToTaskMap.put(taskTracker, taskset);
+ }
+ taskset.add(taskid);
+
+ // taskid --> TIP
+ taskidToTIPMap.put(taskid, tip);
+ }
+ void removeTaskEntry(String taskid) {
+ // taskid --> tracker
+ String tracker = (String) taskidToTrackerMap.remove(taskid);
+
+ // tracker --> taskid
+ TreeSet trackerSet = (TreeSet) trackerToTaskMap.get(tracker);
+ if (trackerSet != null) {
+ trackerSet.remove(taskid);
+ }
+
+ // taskid --> TIP
+ taskidToTIPMap.remove(taskid);
+ }
+
+ ///////////////////////////////////////////////////////
// Accessors for objects that want info on jobs, tasks,
// trackers, etc.
///////////////////////////////////////////////////////
@@ -286,6 +380,9 @@
public int getTrackerPort() {
return port;
}
+ public int getInfoPort() {
+ return infoPort;
+ }
public long getStartTime() {
return startTime;
}
@@ -334,7 +431,10 @@
////////////////////////////////////////////////////
public void initialize(String taskTrackerName) {
if (taskTrackers.get(taskTrackerName) != null) {
- taskTrackers.remove(taskTrackerName);
+ TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(taskTrackerName);
+ totalMaps -= oldStatus.countMapTasks();
+ totalReduces -= oldStatus.countReduceTasks();
+
lostTaskTracker(taskTrackerName);
}
}
@@ -351,7 +451,9 @@
if (initialContact) {
// If it's first contact, then clear out any state hanging around
if (taskTrackers.get(trackerName) != null) {
- taskTrackers.remove(trackerName);
+ TaskTrackerStatus oldStatus = (TaskTrackerStatus) taskTrackers.remove(trackerName);
+ totalMaps -= oldStatus.countMapTasks();
+ totalReduces -= oldStatus.countReduceTasks();
lostTaskTracker(trackerName);
}
} else {
@@ -363,6 +465,8 @@
// Store latest state. If first contact, then save current
// state in expiry queue
+ totalMaps += trackerStatus.countMapTasks();
+ totalReduces += trackerStatus.countReduceTasks();
taskTrackers.put(trackerName, trackerStatus);
if (initialContact) {
trackerExpiryQueue.add(trackerStatus);
@@ -376,38 +480,129 @@
}
/**
- * A tracker wants to know if there's a Task to run
+ * A tracker wants to know if there's a Task to run. Returns
+ * a task we'd like the TaskTracker to execute right now.
+ *
+ * Eventually this function should compute load on the various TaskTrackers,
+ * and incorporate knowledge of NDFS file placement. But for right now, it
+ * just grabs a single item out of the pending task list and hands it back.
*/
- public synchronized Task pollForNewTask(String trackerName) {
- //LOG.info("Unassigned tasks: " + unassignedTasks.size());
+ public synchronized Task pollForNewTask(String taskTracker) {
+ //
+ // Compute average map and reduce task numbers across pool
+ //
+ int avgMaps = 0;
+ int avgReduces = 0;
+ if (taskTrackers.size() > 0) {
+ avgMaps = totalMaps / taskTrackers.size();
+ avgReduces = totalReduces / taskTrackers.size();
+ }
+ int totalCapacity = taskTrackers.size() * maxCurrentTasks;
+
+ //
+ // Get map + reduce counts for the current tracker.
+ //
+ TaskTrackerStatus tts = (TaskTrackerStatus) taskTrackers.get(taskTracker);
+ int numMaps = tts.countMapTasks();
+ int numReduces = tts.countReduceTasks();
+
+ //
+ // In the below steps, we allocate first a map task (if appropriate),
+ // and then a reduce task if appropriate. We go through all jobs
+ // in order of job arrival; jobs only get serviced if their
+ // predecessors are serviced, too.
+ //
+
+ //
+ // We hand a task to the current taskTracker if the given machine
+ // has a workload that's equal to or less than the averageMaps
+ // +/- TASK_ALLOC_EPSILON. (That epsilon is in place in case
+ // there is an odd machine that is failing for some reason but
+ // has not yet been removed from the pool, making capacity seem
+ // larger than it really is.)
+ //
+ if ((numMaps < maxCurrentTasks) &&
+ (numMaps <= (avgMaps + TASK_ALLOC_EPSILON))) {
+
+ int totalNeededMaps = 0;
+ for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
+ JobInProgress job = (JobInProgress) it.next();
+ if (job.getStatus().getRunState() != JobStatus.RUNNING) {
+ continue;
+ }
+
+ Task t = job.obtainNewMapTask(taskTracker, tts);
+ if (t != null) {
+ return t;
+ }
+
+ //
+ // Beyond the highest-priority task, reserve a little
+ // room for failures and speculative executions; don't
+ // schedule tasks to the hilt.
+ //
+ totalNeededMaps += job.desiredMaps();
+ double padding = 0;
+ if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
+ padding = Math.min(maxCurrentTasks, totalNeededMaps * PAD_FRACTION);
+ }
+ if (totalNeededMaps + padding >= totalCapacity) {
+ break;
+ }
+ }
+ }
+
+ //
+ // Same thing, but for reduce tasks
+ //
+ if ((numReduces < maxCurrentTasks) &&
+ (numReduces <= (avgReduces + TASK_ALLOC_EPSILON))) {
+
+ int totalNeededReduces = 0;
+ for (Iterator it = jobsByArrival.iterator(); it.hasNext(); ) {
+ JobInProgress job = (JobInProgress) it.next();
+ if (job.getStatus().getRunState() != JobStatus.RUNNING) {
+ continue;
+ }
+
+ Task t = job.obtainNewReduceTask(taskTracker, tts);
+ if (t != null) {
+ return t;
+ }
- // Allocate a pending task to this TaskTracker
- return getTaskAssignment(trackerName);
+ //
+ // Beyond the highest-priority task, reserve a little
+ // room for failures and speculative executions; don't
+ // schedule tasks to the hilt.
+ //
+ totalNeededReduces += job.desiredReduces();
+ double padding = 0;
+ if (totalCapacity > MIN_SLOTS_FOR_PADDING) {
+ padding = Math.min(maxCurrentTasks, totalNeededReduces * PAD_FRACTION);
+ }
+ if (totalNeededReduces + padding >= totalCapacity) {
+ break;
+ }
+ }
+ }
+ return null;
}
/**
* A tracker wants to know if any of its Tasks have been
* closed (because the job completed, whether successfully or not)
*/
- public String pollForClosedTask(String trackerName) {
- TreeSet taskIds = (TreeSet) trackerToTaskMap.get(trackerName);
+ public String pollForTaskWithClosedJob(String taskTracker) {
+ TreeSet taskIds = (TreeSet) trackerToTaskMap.get(taskTracker);
if (taskIds != null) {
for (Iterator it = taskIds.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
- String jobId = (String) taskToJobMap.get(taskId);
- JobInProgress job = (JobInProgress) jobs.get(jobId);
- int runState = job.getStatus().getRunState();
-
- if (! tasksReportedClosed.contains(taskId) &&
- (runState == JobStatus.SUCCEEDED ||
- runState == JobStatus.FAILED)) {
- tasksReportedClosed.add(taskId);
-
- //
- // REMIND - mjc - what happens when a job dies, but no one
- // arrives to claim the tasks? I suppose when the job is
- // eventually GC'ed, we make sure all its tasks are flushed
- // from lookup tables.
+ TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
+ if (tip.shouldCloseForClosedJob(taskId)) {
+ //
+ // This is how the JobTracker ends a task at the TaskTracker.
+ // It may be successfully completed, or may be killed in
+ // mid-execution.
//
return taskId;
}
@@ -417,15 +612,27 @@
}
/**
- * A tracker wants to know the physical locations of completed, but not
+ * A TaskTracker wants to know the physical locations of completed, but not
* yet closed, tasks. This exists so the reduce task thread can locate
* map task outputs.
- *
*/
- public synchronized MapOutputLocation[] locateMapOutputs(String taskId, String[] mapTasksNeeded) {
- JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskId));
- MapOutputLocation outputs[] = job.locateTasks(mapTasksNeeded);
- return outputs;
+ public synchronized MapOutputLocation[] locateMapOutputs(String taskId, String[][] mapTasksNeeded) {
+ ArrayList v = new ArrayList();
+ for (int i = 0; i < mapTasksNeeded.length; i++) {
+ for (int j = 0; j < mapTasksNeeded[i].length; j++) {
+ TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(mapTasksNeeded[i][j]);
+ if (tip != null && tip.isComplete(mapTasksNeeded[i][j])) {
+ String trackerId = (String) taskidToTrackerMap.get(mapTasksNeeded[i][j]);
+ TaskTrackerStatus tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
+ v.add(new MapOutputLocation(mapTasksNeeded[i][j], tracker.getHost(), tracker.getPort()));
+ break;
+ }
+ }
+ }
+ // randomly shuffle results to load-balance map output requests
+ Collections.shuffle(v);
+
+ return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
}
/**
@@ -471,15 +678,15 @@
return new Vector[0];
} else {
Vector reports = new Vector();
- TreeMap completeMapTasks = job.getCompleteMapTasks();
- for (Iterator it = completeMapTasks.keySet().iterator(); it.hasNext(); ) {
- String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
- }
- TreeMap incompleteMapTasks = job.getIncompleteMapTasks();
- for (Iterator it = incompleteMapTasks.keySet().iterator(); it.hasNext(); ) {
- String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
+ Vector completeMapTasks = job.reportTasksInProgress(true, true);
+ for (Iterator it = completeMapTasks.iterator(); it.hasNext(); ) {
+ TaskInProgress tip = (TaskInProgress) it.next();
+ reports.add(tip.generateSingleReport());
+ }
+ Vector incompleteMapTasks = job.reportTasksInProgress(true, false);
+ for (Iterator it = incompleteMapTasks.iterator(); it.hasNext(); ) {
+ TaskInProgress tip = (TaskInProgress) it.next();
+ reports.add(tip.generateSingleReport());
}
return (Vector[]) reports.toArray(new Vector[reports.size()]);
}
@@ -491,39 +698,32 @@
return new Vector[0];
} else {
Vector reports = new Vector();
- TreeMap completeReduceTasks = job.getCompleteReduceTasks();
- for (Iterator it = completeReduceTasks.keySet().iterator(); it.hasNext(); ) {
- String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
- }
- TreeMap incompleteReduceTasks = job.getIncompleteReduceTasks();
- for (Iterator it = incompleteReduceTasks.keySet().iterator(); it.hasNext(); ) {
- String taskid = (String) it.next();
- reports.add(generateSingleReport(taskid, job.getTaskStatus(taskid), job.getTaskDiagnosticInfo(taskid), job.getTaskStateString(taskid)));
+ Vector completeReduceTasks = job.reportTasksInProgress(false, true);
+ for (Iterator it = completeReduceTasks.iterator(); it.hasNext(); ) {
+ TaskInProgress tip = (TaskInProgress) it.next();
+ reports.add(tip.generateSingleReport());
+ }
+ Vector incompleteReduceTasks = job.reportTasksInProgress(false, false);
+ for (Iterator it = incompleteReduceTasks.iterator(); it.hasNext(); ) {
+ TaskInProgress tip = (TaskInProgress) it.next();
+ reports.add(tip.generateSingleReport());
}
return (Vector[]) reports.toArray(new Vector[reports.size()]);
}
}
- //////////////////////////////////////////////////////////////
- // (See InterTrackerProtocol section for getFilesystemName())
- //////////////////////////////////////////////////////////////
-
- Vector generateSingleReport(String taskid, TaskStatus status, Vector diagInfo, String stateString) {
- Vector report = new Vector();
- report.add(taskid);
- report.add("" + status.getProgress());
- report.add(stateString);
- report.addAll(diagInfo);
- return report;
- }
-
///////////////////////////////////////////////////////////////
// JobTracker methods
///////////////////////////////////////////////////////////////
- public JobTracker.JobInProgress getJob(String jobid) {
+ public JobInProgress getJob(String jobid) {
return (JobInProgress) jobs.get(jobid);
}
+ /**
+ * Grab random num for task id
+ */
+ String createUniqueId() {
+ return "" + Integer.toString(Math.abs(r.nextInt()),36);
+ }
/**
* JobProfile createJob() kicks off a new job.
@@ -531,439 +731,19 @@
* tasks. The tasks are added to the unassignedTasks structure.
* (The precise structure will change as we get more sophisticated about
* task allocation.)
+ *
+ * Create a 'JobInProgress' object, which contains both JobProfile
+ * and JobStatus. Those two sub-objects are sometimes shipped outside
+ * of the JobTracker. But JobInProgress adds info that's useful for
+ * the JobTracker alone.
*/
JobInProgress createJob(String jobFile) throws IOException {
- JobInProgress job = new JobInProgress(jobFile, this.nutchConf);
+ JobInProgress job = new JobInProgress(jobFile, this, this.nutchConf);
jobs.put(job.getProfile().getJobId(), job);
-
- boolean error = true;
- try {
- job.launch();
- error = false;
- } finally {
- if (error) {
- job.kill();
- }
- }
-
+ jobsByArrival.add(job);
return job;
}
- ///////////////////////////////////////////////////////
- // JobInProgress maintains all the info for keeping
- // a Job on the straight and narrow. It keeps its JobProfile
- // and its latest JobStatus, plus a set of tables for
- // doing bookkeeping of its Tasks.
- ///////////////////////////////////////////////////////
- public class JobInProgress {
- JobProfile profile;
- JobStatus status;
- Vector reducesToLaunch = new Vector();
- File localJobFile = null;
-
- TreeMap taskStatus = new TreeMap();
- TreeMap incompleteMapTasks = new TreeMap();
- TreeMap completeMapTasks = new TreeMap();
- TreeMap incompleteReduceTasks = new TreeMap();
- TreeMap completeReduceTasks = new TreeMap();
- TreeMap taskFailures = new TreeMap();
- TreeMap taskDiagnosticData = new TreeMap();
- TreeMap taskStateStrings = new TreeMap();
-
- // Info for user; useless for JobTracker
- int numMapTasks = 0;
- int numReduceTasks = 0;
- float totalReportedMapProgress = 0.0f;
- float totalReportedReduceProgress = 0.0f;
- int attemptedMapExecutions = 0;
- int attemptedReduceExecutions = 0;
- long startTime;
- long finishTime;
- String deleteUponCompletion = null;
- private NutchConf nutchConf;
-
- /**
- * Create a 'JobInProgress' object, which contains both JobProfile
- * and JobStatus. Those two sub-objects are sometimes shipped outside
- * of the JobTracker. But JobInProgress adds info that's useful for
- * the JobTracker alone.
- */
- public JobInProgress(String jobFile, NutchConf nutchConf) throws IOException {
- String jobid = createJobId();
- String url = "http://" + localMachine + ":" + infoPort + "/jobdetails.jsp?jobid=" + jobid;
- this.profile = new JobProfile(jobid, jobFile, url);
- this.status = new JobStatus(jobid, 0.0f, 0.0f, JobStatus.RUNNING);
-
- this.localJobFile = new JobConf(nutchConf).getLocalFile(SUBDIR, jobid + ".xml");
- fs.copyToLocalFile(new File(jobFile), localJobFile);
-
- JobConf jd = new JobConf(localJobFile);
- this.numMapTasks = jd.getNumMapTasks();
- this.numReduceTasks = jd.getNumReduceTasks();
- this.startTime = System.currentTimeMillis();
-
- // If a jobFile is in the systemDir, we can delete it (and
- // its JAR) upon completion
- if (jobFile.startsWith(systemDir.getPath())) {
- this.deleteUponCompletion = jobFile;
- }
- this.nutchConf = nutchConf;
- }
-
- /**
- * Start up the tasks
- */
- public void launch() throws IOException {
- String jobid = profile.getJobId();
- String jobFile = profile.getJobFile();
-
- // construct input splits
- JobConf jd = new JobConf(localJobFile);
- NutchFileSystem fs = NutchFileSystem.get(nutchConf);
- FileSplit[] splits =
- jd.getInputFormat().getSplits(fs, jd, numMapTasks);
-
- // sort splits by decreasing length, to reduce job's tail
- Arrays.sort(splits, new Comparator() {
- public int compare(Object a, Object b) {
- long diff =
- ((FileSplit)b).getLength() - ((FileSplit)a).getLength();
- return diff==0 ? 0 : (diff > 0 ? 1 : -1);
- }
- });
-
- // adjust number of map tasks to actual number of splits
- numMapTasks = splits.length;
-
- // create a map task for each split
- String mapIds[] = new String[numMapTasks];
- for (int i = 0; i < numMapTasks; i++) {
- mapIds[i] = createMapTaskId();
- Task t = new MapTask(jobFile, mapIds[i], splits[i]);
- t.setConf(this.nutchConf);
-
- incompleteMapTasks.put(mapIds[i], t);
- taskToJobMap.put(mapIds[i], jobid);
- }
-
- // Create reduce tasks
- for (int i = 0; i < numReduceTasks; i++) {
- String taskid = createReduceTaskId();
- Task t = new ReduceTask(jobFile, taskid, mapIds, i);
- t.setConf(this.nutchConf);
- reducesToLaunch.add(t);
- taskToJobMap.put(taskid, jobid);
- }
-
- // Launch the map tasks
- for (int i = 0; i < mapIds.length; i++) {
- executeTask(mapIds[i]);
- }
- }
-
- /**
- * Kill the job and all its component tasks.
- */
- public synchronized void kill() {
- if (status.getRunState() != JobStatus.FAILED) {
- this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.FAILED);
-
- //
- // Kill all the pending tasks
- //
- synchronized (unassignedTasks) {
- for (Iterator it = unassignedTasks.iterator(); it.hasNext(); ) {
- String taskid = (String) it.next();
- if ((incompleteMapTasks.get(taskid) != null) ||
- (incompleteReduceTasks.get(taskid) != null)) {
- it.remove();
- }
- }
- }
-
- this.finishTime = System.currentTimeMillis();
- }
- }
-
- /**
- * The job is dead. We're now GC'ing it, getting rid of the job
- * from all tables. Be sure to remove all of this job's tasks
- * from the various tables.
- */
- public synchronized void garbageCollect() throws IOException {
- //
- // Remove this job from all tables
- //
-
-
- // Definitely remove the local-disk copy of the job file
- if (localJobFile != null) {
- localJobFile.delete();
- localJobFile = null;
- }
-
- //
- // If the job file was in the temporary system directory,
- // we should delete it upon garbage collect.
- //
- if (deleteUponCompletion != null) {
- JobConf jd = new JobConf(deleteUponCompletion);
- fs.delete(new File(jd.getJar()));
- fs.delete(new File(deleteUponCompletion));
- deleteUponCompletion = null;
- }
- }
-
- /**
- * A task assigned to this JobInProgress has reported in successfully.
- *
- * This might prompt us to launch more tasks, or it might even indicate
- * the job is now complete.
- */
- public synchronized void completedTask(String taskid) {
- LOG.info("Task '" + taskid + "' has finished successfully.");
-
- Task t = null;
- if ((t = (Task) incompleteMapTasks.get(taskid)) != null) {
- incompleteMapTasks.remove(taskid);
- completeMapTasks.put(taskid, t);
- } else if ((t = (Task) incompleteReduceTasks.get(taskid)) != null) {
- incompleteReduceTasks.remove(taskid);
- completeReduceTasks.put(taskid, t);
- } else {
- // Impossible situation; taskid is not 'incomplete' yet is reported
- // as just finishing
- throw new IllegalArgumentException("Impossible state: task " + taskid + " reported as complete, but was not known as incomplete");
- }
-
- //
- // We may have Reduce tasks that are still waiting to be scheduled for execution;
- // we don't want to launch them all while still waiting for most Map tasks to
- // finish. Nor do we want to wait for all the maps to finish, as the reduce task
- // could be downloading files while other map tasks complete.
- //
- // WELL, maybe we do want to wait. It's not obvious what the right launch rate
- // would be. For the moment, we wait until ALL map tasks have finished
- // before launching even a single reduce task.
- //
- while (status.getRunState() == JobStatus.RUNNING &&
- reducesToLaunch.size() > 0 &&
- completeMapTasks.size() == numMapTasks) {
- t = (Task) reducesToLaunch.elementAt(0);
- reducesToLaunch.removeElement(t);
- incompleteReduceTasks.put(t.getTaskId(), t);
- executeTask(t.getTaskId());
- }
-
- //
- // If all tasks are complete, then the job is done!
- //
- if (status.getRunState() == JobStatus.RUNNING &&
- completeReduceTasks.size() == numReduceTasks) {
- this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.SUCCEEDED);
- this.finishTime = System.currentTimeMillis();
- }
- }
-
- /**
- * A task assigned to this JobInProgress has reported in as failed.
- * Most of the time, we'll just reschedule execution. However, after
- * many repeated failures we may instead decide to allow the entire
- * job to fail.
- *
- * Even if a task has reported as completed in the past, it might later
- * be reported as failed. That's because the TaskTracker that hosts a map
- * task might die before the entire job can complete. If that happens,
- * we need to schedule reexecution so that downstream reduce tasks can
- * obtain the map task's output.
- */
- public void failedTask(String taskid) {
- LOG.info("Task '" + taskid + "' has been lost.");
-
- Task t = null;
- if ((t = (Task) completeMapTasks.get(taskid)) != null) {
- completeMapTasks.remove(taskid);
- incompleteMapTasks.put(taskid, t);
- } else if ((t = (Task) completeReduceTasks.get(taskid)) != null) {
- completeReduceTasks.remove(taskid);
- incompleteReduceTasks.put(taskid, t);
- }
-
- // Check if we need to kill the job because of excess failures
- Integer failures = (Integer) taskFailures.get(taskid);
- int numFailures = ((failures == null) ? 0 : failures.intValue()) + 1;
- taskFailures.put(taskid, new Integer(numFailures));
- if (numFailures >= MAX_TASK_FAILURES) {
- LOG.info("Task " + taskid + " has failed " + numFailures + " times. Aborting owning job " + profile.getJobId());
- kill();
- }
-
- if (status.getRunState() == JobStatus.RUNNING) {
- executeTask(taskid);
- }
- }
-
- /**
- * Return locations for all the indicated taskIds. If the task
- * is not complete, don't return anything.
- */
- public MapOutputLocation[] locateTasks(String taskIds[]) {
- ArrayList v = new ArrayList();
- for (int i = 0; i < taskIds.length; i++) {
- if (completeMapTasks.get(taskIds[i]) != null) {
- String trackerId = (String) taskToTrackerMap.get(taskIds[i]);
- TaskTrackerStatus tracker = (TaskTrackerStatus) taskTrackers.get(trackerId);
- v.add(new MapOutputLocation(taskIds[i], tracker.getHost(), tracker.getPort()));
- }
- }
- // randomly shuffle results to load-balance map output requests
- Collections.shuffle(v);
-
- return (MapOutputLocation[]) v.toArray(new MapOutputLocation[v.size()]);
- }
-
- //////////////////////////
- // A number of accessors
- //////////////////////////
- TreeMap getCompleteMapTasks() {
- return completeMapTasks;
- }
- TreeMap getIncompleteMapTasks() {
- return incompleteMapTasks;
- }
- TreeMap getCompleteReduceTasks() {
- return completeReduceTasks;
- }
- TreeMap getIncompleteReduceTasks() {
- return incompleteReduceTasks;
- }
- public JobProfile getProfile() {
- return profile;
- }
- public JobStatus getStatus() {
- return status;
- }
- public Task getTask(String taskid) {
- Task t = null;
- synchronized (incompleteMapTasks) {
- if ((t = (Task) incompleteMapTasks.get(taskid)) != null) {
- return t;
- }
- synchronized (completeMapTasks) {
- if ((t = (Task) completeMapTasks.get(taskid)) != null) {
- return t;
- }
- synchronized (incompleteReduceTasks) {
- if ((t = (Task) incompleteReduceTasks.get(taskid)) != null) {
- return t;
- }
- synchronized (completeReduceTasks) {
- t = (Task) completeReduceTasks.get(taskid);
- }
- }
- }
- }
- return t;
- }
- public Vector getTaskDiagnosticInfo(String taskid) {
- Vector v = (Vector) taskDiagnosticData.get(taskid);
- if (v == null) {
- return new Vector();
- } else {
- return v;
- }
- }
- public String getTaskStateString(String taskid) {
- return (String) taskStateStrings.get(taskid);
- }
- public float completedRatio() {
- return (0.5f * status.mapProgress()) + (0.5f * status.reduceProgress());
- }
- public long getStartTime() {
- return startTime;
- }
- public long getFinishTime() {
- return finishTime;
- }
- public int desiredMaps() {
- return numMapTasks;
- }
- public int attemptedMaps() {
- return attemptedMapExecutions;
- }
- public int completedMaps() {
- return completeMapTasks.size();
- }
- public int desiredReduces() {
- return numReduceTasks;
- }
- public int attemptedReduces() {
- return attemptedReduceExecutions;
- }
- public int completedReduces() {
- return completeReduceTasks.size();
- }
- public void updateTaskStatus(String taskid, TaskStatus status) {
- TaskStatus oldStatus = (TaskStatus) taskStatus.put(taskid, status);
-
- float delta = 0.0f;
- if (oldStatus != null) {
- delta -= oldStatus.getProgress();
- }
- delta += status.getProgress();
-
- String diagInfo = status.getDiagnosticInfo();
- if (diagInfo != null && diagInfo.length() > 0) {
- Vector v = (Vector) taskDiagnosticData.get(taskid);
- if (v == null) {
- v = new Vector();
- taskDiagnosticData.put(taskid, v);
- }
- v.add(diagInfo);
- }
-
- taskStateStrings.put(taskid, status.getStateString());
-
- if (incompleteMapTasks.get(taskid) != null ||
- completeMapTasks.get(taskid) != null) {
- totalReportedMapProgress += delta;
- if (numMapTasks == 0) {
- this.status.setMapProgress(1.0f);
- } else {
- this.status.setMapProgress((float) (totalReportedMapProgress / numMapTasks));
- }
- } else if (incompleteReduceTasks.get(taskid) != null ||
- completeReduceTasks.get(taskid) != null) {
- totalReportedReduceProgress += delta;
- if (numReduceTasks == 0) {
- this.status.setReduceProgress(1.0f);
- } else {
- this.status.setReduceProgress((float) (totalReportedReduceProgress / numReduceTasks));
- }
- } else {
- LOG.info("Serious problem. While updating status, cannot find taskid " + taskid);
- }
- }
- public TaskStatus getTaskStatus(String taskid) {
- return (TaskStatus) taskStatus.get(taskid);
- }
-
- /////////////////////////////
- // Some private util methods
- /////////////////////////////
- void executeTask(String taskid) {
- if (incompleteMapTasks.get(taskid) != null) {
- attemptedMapExecutions++;
- } else if (incompleteReduceTasks.get(taskid) != null) {
- attemptedReduceExecutions++;
- }
- updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.UNASSIGNED, "", ""));
- synchronized (unassignedTasks) {
- unassignedTasks.add(taskid);
- }
- }
- }
-
////////////////////////////////////////////////////
// Methods to track all the TaskTrackers
////////////////////////////////////////////////////
@@ -977,16 +757,19 @@
void updateTaskStatuses(TaskTrackerStatus status) {
for (Iterator it = status.taskReports(); it.hasNext(); ) {
TaskStatus report = (TaskStatus) it.next();
- JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(report.getTaskId()));
- job.updateTaskStatus(report.getTaskId(), report);
+ TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(report.getTaskId());
+ if (tip == null) {
+ LOG.info("Serious problem. While updating status, cannot find taskid " + report.getTaskId());
+ } else {
+ JobInProgress job = tip.getJob();
+ job.updateTaskStatus(tip, report);
- if (report.getRunState() == TaskStatus.SUCCEEDED) {
- job.completedTask(report.getTaskId());
- } else if (report.getRunState() == TaskStatus.FAILED) {
- TreeSet taskset = (TreeSet) trackerToTaskMap.get(status.getTrackerName());
- taskset.remove(report.getTaskId());
- taskToTrackerMap.remove(report.getTaskId());
- job.failedTask(report.getTaskId());
+ if (report.getRunState() == TaskStatus.SUCCEEDED) {
+ job.completedTask(tip, report.getTaskId());
+ } else if (report.getRunState() == TaskStatus.FAILED) {
+ // Tell the job to fail the relevant task
+ job.failedTask(tip, report.getTaskId(), status.getTrackerName());
+ }
}
}
}
@@ -1004,61 +787,13 @@
if (lostTasks != null) {
for (Iterator it = lostTasks.iterator(); it.hasNext(); ) {
String taskId = (String) it.next();
- taskToTrackerMap.remove(taskId);
- JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskId));
- job.failedTask(taskId);
- }
- }
- }
+ TaskInProgress tip = (TaskInProgress) taskidToTIPMap.get(taskId);
- /**
- * Task getTaskAssignment() returns
- * a task we'd like the taskTracker to execute right now.
- *
- * Eventually this function should compute load on the various TaskTrackers,
- * and incorporate knowledge of NDFS file placement. But for right now, it
- * just grabs a single item out of the pending task list and hands it back.
- */
- Task getTaskAssignment(String taskTracker) {
- synchronized (unassignedTasks) {
- if (unassignedTasks.size() > 0) {
- String taskid = (String) unassignedTasks.elementAt(0);
- unassignedTasks.remove(taskid);
-
- // Move task status to RUNNING
- JobInProgress job = (JobInProgress) jobs.get((String) taskToJobMap.get(taskid));
- job.updateTaskStatus(taskid, new TaskStatus(taskid, 0.0f, TaskStatus.RUNNING, "", ""));
-
- // Remember where we are running it
- TreeSet taskset = (TreeSet) trackerToTaskMap.get(taskTracker);
- if (taskset == null) {
- taskset = new TreeSet();
- trackerToTaskMap.put(taskTracker, taskset);
- }
- LOG.info("Adding task '" + taskid + "' to set for tracker '" + taskTracker + "'");
- taskset.add(taskid);
-
- taskToTrackerMap.put(taskid, taskTracker);
-
- return job.getTask(taskid);
- } else {
- return null;
+ // Tell the job to fail the relevant task
+ JobInProgress job = tip.getJob();
+ job.failedTask(tip, taskId, trackerName);
}
}
- }
-
- /**
- * Grab random num for task id
- */
- String createMapTaskId() {
- return "task_m_" + Integer.toString(Math.abs(r.nextInt()),36);
- }
- String createReduceTaskId() {
- return "task_r_" + Integer.toString(Math.abs(r.nextInt()),36);
- }
-
- String createJobId() {
- return "job_" + Integer.toString(Math.abs(r.nextInt()),36);
}
////////////////////////////////////////////////////////////
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/LocalJobRunner.java Thu Feb 2 09:58:38 2006
@@ -89,9 +89,13 @@
}
// run a single reduce task
+ String mapDependencies[][] = new String[mapIds.size()][1];
+ for (int i = 0; i < mapIds.size(); i++) {
+ mapDependencies[i][0] = (String) mapIds.get(i);
+ }
ReduceTask reduce =
new ReduceTask(file, reduceId,
- (String[])mapIds.toArray(new String[0]),
+ mapDependencies,
0);
reduce.setConf(job);
reduce.run(job, this);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MRConstants.java Thu Feb 2 09:58:38 2006
@@ -25,7 +25,7 @@
// Timeouts, constants
//
public static final long HEARTBEAT_INTERVAL = 3 * 1000;
- public static final long TASKTRACKER_EXPIRY_INTERVAL = 10 * 60 * 1000;
+ public static final long TASKTRACKER_EXPIRY_INTERVAL = 60 * 1000;
//
// Result codes
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapOutputFile.java Thu Feb 2 09:58:38 2006
@@ -58,6 +58,10 @@
throws IOException {
return this.jobConf.getLocalFile(reduceTaskId, mapTaskId+".out");
}
+ public File getInputFile(String mapTaskId[], String reduceTaskId)
+ throws IOException {
+ return this.jobConf.getLocalFile(reduceTaskId, mapTaskId, ".out");
+ }
/** Removes all of the files related to a task. */
public void removeAll(String taskId) throws IOException {
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTask.java Thu Feb 2 09:58:38 2006
@@ -37,6 +37,10 @@
this.split = split;
}
+ public boolean isMapTask() {
+ return true;
+ }
+
public TaskRunner createRunner(TaskTracker tracker) {
return new MapTaskRunner(this, tracker, this.nutchConf);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/MapTaskRunner.java Thu Feb 2 09:58:38 2006
@@ -35,8 +35,9 @@
}
/** Delete any temporary files from previous failed attempts. */
- public void prepare() throws IOException {
+ public boolean prepare() throws IOException {
this.mapOutputFile.removeAll(getTask().getTaskId());
+ return true;
}
/** Delete all of the temporary map output files. */
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTask.java Thu Feb 2 09:58:38 2006
@@ -27,7 +27,7 @@
/** A Reduce task. */
public class ReduceTask extends Task {
- private String[] mapTaskIds;
+ private String[][] mapTaskIds;
private int partition;
private boolean sortComplete;
@@ -43,7 +43,7 @@
public ReduceTask() {}
public ReduceTask(String jobFile, String taskId,
- String[] mapTaskIds, int partition) {
+ String[][] mapTaskIds, int partition) {
super(jobFile, taskId);
this.mapTaskIds = mapTaskIds;
this.partition = partition;
@@ -53,7 +53,11 @@
return new ReduceTaskRunner(this, tracker, this.nutchConf);
}
- public String[] getMapTaskIds() { return mapTaskIds; }
+ public boolean isMapTask() {
+ return false;
+ }
+
+ public String[][] getMapTaskIds() { return mapTaskIds; }
public int getPartition() { return partition; }
public void write(DataOutput out) throws IOException {
@@ -61,7 +65,10 @@
out.writeInt(mapTaskIds.length); // write mapTaskIds
for (int i = 0; i < mapTaskIds.length; i++) {
- UTF8.writeString(out, mapTaskIds[i]);
+ out.writeInt(mapTaskIds[i].length);
+ for (int j = 0; j < mapTaskIds[i].length; j++) {
+ UTF8.writeString(out, mapTaskIds[i][j]);
+ }
}
out.writeInt(partition); // write partition
@@ -70,9 +77,12 @@
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- mapTaskIds = new String[in.readInt()]; // read mapTaskIds
+ mapTaskIds = new String[in.readInt()][]; // read mapTaskIds
for (int i = 0; i < mapTaskIds.length; i++) {
- mapTaskIds[i] = UTF8.readString(in);
+ mapTaskIds[i] = new String[in.readInt()];
+ for (int j = 0; j < mapTaskIds[i].length; j++) {
+ mapTaskIds[i][j] = UTF8.readString(in);
+ }
}
this.partition = in.readInt(); // read partition
@@ -182,6 +192,7 @@
float progPerByte = 1.0f / lfs.getLength(partFile);
Progress phase = appendPhase.phase();
phase.setStatus(partFile.toString());
+
SequenceFile.Reader in =
new SequenceFile.Reader(lfs, partFile.toString(), job);
try {
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/ReduceTaskRunner.java Thu Feb 2 09:58:38 2006
@@ -37,14 +37,14 @@
}
/** Assemble all of the map output files. */
- public void prepare() throws IOException {
+ public boolean prepare() throws IOException {
ReduceTask task = ((ReduceTask)getTask());
this.mapOutputFile.removeAll(task.getTaskId()); // cleanup from failures
- String[] mapTaskIds = task.getMapTaskIds();
+ String[][] mapTaskIds = task.getMapTaskIds();
final Progress copyPhase = getTask().getProgress().phase();
// we need input from every map task
- HashSet needed = new HashSet();
+ Vector needed = new Vector();
for (int i = 0; i < mapTaskIds.length; i++) {
needed.add(mapTaskIds[i]);
copyPhase.addPhase(); // add sub-phase per file
@@ -52,17 +52,21 @@
InterTrackerProtocol jobClient = getTracker().getJobClient();
while (needed.size() > 0) {
-
getTask().reportProgress(getTracker());
// get list of available map output locations from job tracker
- String[] neededStrings =
- (String[])needed.toArray(new String[needed.size()]);
+ String[][] neededStrings = new String[needed.size()][];
+ for (int i = 0; i < needed.size(); i++) {
+ neededStrings[i] = (String[]) needed.elementAt(i);
+ }
MapOutputLocation[] locs =
jobClient.locateMapOutputs(task.getTaskId(), neededStrings);
if (locs.length == 0) {
try {
+ if (killed) {
+ return false;
+ }
Thread.sleep(1000);
} catch (InterruptedException e) {
}
@@ -96,9 +100,19 @@
client.getFile(loc.getMapTaskId(), task.getTaskId(),
new IntWritable(task.getPartition()));
-
- needed.remove(loc.getMapTaskId()); // success: remove from needed
-
+
+ // Success: remove from 'needed'
+ boolean foundit = false;
+ for (Iterator it = needed.iterator(); it.hasNext() && !foundit; ) {
+ String idsForSingleMap[] = (String[]) it.next();
+ for (int j = 0; j < idsForSingleMap.length; j++) {
+ if (idsForSingleMap[j].equals(loc.getMapTaskId())) {
+ it.remove();
+ foundit = true;
+ break;
+ }
+ }
+ }
copyPhase.startNextPhase();
} catch (IOException e) { // failed: try again later
@@ -109,11 +123,10 @@
} finally {
this.mapOutputFile.setProgressReporter(null);
}
-
}
-
}
getTask().reportProgress(getTracker());
+ return true;
}
/** Delete all of the temporary map output files. */
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java?rev=374443&r1=374442&r2=374443&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/mapred/Task.java Thu Feb 2 09:58:38 2006
@@ -82,6 +82,8 @@
private transient long nextProgressTime =
System.currentTimeMillis() + PROGRESS_INTERVAL;
+ public abstract boolean isMapTask();
+
public Progress getProgress() { return taskProgress; }
public Reporter getReporter(final TaskUmbilicalProtocol umbilical,