You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2008/10/10 01:52:23 UTC
svn commit: r703294 - in /hadoop/core/branches/branch-0.19: ./ conf/
src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/
Author: dhruba
Date: Thu Oct 9 16:52:23 2008
New Revision: 703294
URL: http://svn.apache.org/viewvc?rev=703294&view=rev
Log:
HADOOP-4018. The number of tasks for a single job cannot exceed a
pre-configured maximum value. (dhruba)
Added:
hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTaskLimits.java
- copied unchanged from r703292, hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskLimits.java
Modified:
hadoop/core/branches/branch-0.19/CHANGES.txt
hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java
Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Thu Oct 9 16:52:23 2008
@@ -819,6 +819,9 @@
HADOOP-4267. Occasional exceptions during shutting down HSQLDB is logged
but not rethrown. (enis)
+ HADOOP-4018. The number of tasks for a single job cannot exceed a
+ pre-configured maximum value. (dhruba)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/conf/hadoop-default.xml?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/conf/hadoop-default.xml (original)
+++ hadoop/core/branches/branch-0.19/conf/hadoop-default.xml Thu Oct 9 16:52:23 2008
@@ -968,6 +968,13 @@
</property>
<property>
+ <name>mapred.jobtracker.maxtasks.per.job</name>
+ <value>-1</value>
+ <description>The maximum number of tasks for a single job.
+ A value of -1 indicates that there is no maximum. </description>
+</property>
+
+<property>
<name>mapred.submit.replication</name>
<value>10</value>
<description>The replication level for submitted job files. This
Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Thu Oct 9 16:52:23 2008
@@ -52,7 +52,7 @@
* ***********************************************************
*/
class JobInProgress {
- private static final Log LOG = LogFactory.getLog(JobInProgress.class);
+ static final Log LOG = LogFactory.getLog(JobInProgress.class);
JobProfile profile;
JobStatus status;
@@ -363,6 +363,18 @@
splitFile.close();
}
numMapTasks = splits.length;
+
+
+ // if the number of splits is larger than a configured value
+ // then fail the job.
+ int maxTasks = jobtracker.getMaxTasksPerJob();
+ if (maxTasks > 0 && numMapTasks + numReduceTasks > maxTasks) {
+ throw new IOException(
+ "The number of tasks for this job " +
+ (numMapTasks + numReduceTasks) +
+ " exceeds the configured limit " + maxTasks);
+ }
+
maps = new TaskInProgress[numMapTasks];
for(int i=0; i < numMapTasks; ++i) {
inputLength += splits[i].getDataLength();
Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu Oct 9 16:52:23 2008
@@ -2721,5 +2721,12 @@
return (JobStatus[]) jobStatusList.toArray(
new JobStatus[jobStatusList.size()]);
}
+
+ /**
+ * Returns the confgiured maximum number of tasks for a single job
+ */
+ int getMaxTasksPerJob() {
+ return conf.getInt("mapred.jobtracker.maxtasks.per.job", -1);
+ }
}