You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/08/31 16:29:13 UTC
svn commit: r1379467 - in /hadoop/common/branches/branch-1: ./ src/mapred/
src/mapred/org/apache/hadoop/mapred/ src/mapred/org/apache/hadoop/mapreduce/
src/test/org/apache/hadoop/mapreduce/
Author: tomwhite
Date: Fri Aug 31 14:29:13 2012
New Revision: 1379467
URL: http://svn.apache.org/viewvc?rev=1379467&view=rev
Log:
Revert MAPREDUCE-4488 and MAPREDUCE-4567.
Removed:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestNoJobSetupCleanup.java
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobContext.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1379467&r1=1379466&r2=1379467&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Aug 31 14:29:13 2012
@@ -27,9 +27,6 @@ Release 1.2.0 - unreleased
module external to HDFS to specify how HDFS blocks should be placed.
(Sumadhur Reddy Bolli via szetszwo)
- MAPREDUCE-4488. Port MAPREDUCE-463 (The job setup and cleanup tasks
- should be optional) to branch-1. (tomwhite)
-
HADOOP-7754. Expose file descriptors from Hadoop-wrapped local
FileSystems (todd and ahmed via tucu)
@@ -206,8 +203,6 @@ Release 1.2.0 - unreleased
HADOOP-8611. Allow fall-back to the shell-based implementation when
JNI-based users-group mapping fails (Robert Parker via bobby)
- MAPREDUCE-4567. Fix failing TestJobKillAndFail in branch-1. (tomwhite)
-
MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson
via atm)
Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1379467&r1=1379466&r2=1379467&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Fri Aug 31 14:29:13 2012
@@ -35,14 +35,6 @@
</description>
</property>
-<property>
- <name>mapred.committer.job.setup.cleanup.needed</name>
- <value>true</value>
- <description> true, if job needs job-setup and job-cleanup.
- false, otherwise
- </description>
-</property>
-
<!-- i/o properties -->
<property>
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1379467&r1=1379466&r2=1379467&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Fri Aug 31 14:29:13 2012
@@ -131,7 +131,6 @@ public class JobInProgress {
private volatile boolean launchedSetup = false;
private volatile boolean jobKilled = false;
private volatile boolean jobFailed = false;
- private boolean jobSetupCleanupNeeded = true;
JobPriority priority = JobPriority.NORMAL;
final JobTracker jobtracker;
@@ -362,8 +361,6 @@ public class JobInProgress {
this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>
(numMapTasks + numReduceTasks + 10);
- JobContext jobContext = new JobContext(conf, jobId);
- this.jobSetupCleanupNeeded = jobContext.getJobSetupCleanupNeeded();
try {
this.userUGI = UserGroupInformation.getCurrentUser();
} catch (IOException ie){
@@ -452,9 +449,6 @@ public class JobInProgress {
this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>
(numMapTasks + numReduceTasks + 10);
-
- JobContext jobContext = new JobContext(conf, jobId);
- this.jobSetupCleanupNeeded = jobContext.getJobSetupCleanupNeeded();
// Construct the jobACLs
status.setJobACLs(jobtracker.getJobACLsManager().constructJobACLs(conf));
@@ -763,35 +757,7 @@ public class JobInProgress {
// ... use the same for estimating the total output of all maps
resourceEstimator.setThreshhold(completedMapsForReduceSlowstart);
-
- initSetupCleanupTasks(jobFile);
-
- synchronized(jobInitKillStatus){
- jobInitKillStatus.initDone = true;
-
- // set this before the throw to make sure cleanup works properly
- tasksInited = true;
-
- if(jobInitKillStatus.killed) {
- throw new KillInterruptedException("Job " + jobId + " killed in init");
- }
- }
-
- JobHistory.JobInfo.logInited(profile.getJobID(), this.launchTime,
- numMapTasks, numReduceTasks);
- // if setup is not needed, mark it complete
- if (!jobSetupCleanupNeeded) {
- setupComplete();
- }
- }
-
- private void initSetupCleanupTasks(String jobFile) {
- if (!jobSetupCleanupNeeded) {
- // nothing to initialize
- return;
- }
-
// create cleanup two cleanup tips, one map and one reduce.
cleanup = new TaskInProgress[2];
@@ -821,23 +787,25 @@ public class JobInProgress {
numReduceTasks + 1, jobtracker, conf, this, 1);
setup[1].setJobSetupTask();
+ synchronized(jobInitKillStatus){
+ jobInitKillStatus.initDone = true;
+
+ // set this before the throw to make sure cleanup works properly
+ tasksInited = true;
+
+ if(jobInitKillStatus.killed) {
+ throw new KillInterruptedException("Job " + jobId + " killed in init");
+ }
+ }
+
+ JobHistory.JobInfo.logInited(profile.getJobID(), this.launchTime,
+ numMapTasks, numReduceTasks);
+
// Log the number of map and reduce tasks
LOG.info("Job " + jobId + " initialized successfully with " + numMapTasks
+ " map tasks and " + numReduceTasks + " reduce tasks.");
}
- private void setupComplete() {
- status.setSetupProgress(1.0f);
- if (maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded) {
- jobComplete();
- return;
- }
- if (this.status.getRunState() == JobStatus.PREP) {
- changeStateTo(JobStatus.RUNNING);
- JobHistory.JobInfo.logStarted(profile.getJobID());
- }
- }
-
TaskSplitMetaInfo[] createSplits(org.apache.hadoop.mapreduce.JobID jobId)
throws IOException {
TaskSplitMetaInfo[] allTaskSplitMetaInfo =
@@ -2652,7 +2620,13 @@ public class JobInProgress {
if (tip.isJobSetupTask()) {
// setup task has finished. kill the extra setup tip
killSetupTip(!tip.isMapTask());
- setupComplete();
+ // Job can start running now.
+ this.status.setSetupProgress(1.0f);
+ // move the job to running state if the job is in prep state
+ if (this.status.getRunState() == JobStatus.PREP) {
+ changeStateTo(JobStatus.RUNNING);
+ JobHistory.JobInfo.logStarted(profile.getJobID());
+ }
} else if (tip.isJobCleanupTask()) {
// cleanup task has finished. Kill the extra cleanup tip
if (tip.isMapTask()) {
@@ -2711,9 +2685,6 @@ public class JobInProgress {
}
}
}
- if (!jobSetupCleanupNeeded && canLaunchJobCleanupTask()) {
- jobComplete();
- }
return true;
}
@@ -2774,8 +2745,7 @@ public class JobInProgress {
//
// All tasks are complete, then the job is done!
//
- if (this.status.getRunState() == JobStatus.RUNNING ||
- this.status.getRunState() == JobStatus.PREP) {
+ if (this.status.getRunState() == JobStatus.RUNNING ) {
changeStateTo(JobStatus.SUCCEEDED);
this.status.setCleanupProgress(1.0f);
if (maps.length == 0) {
@@ -2909,9 +2879,6 @@ public class JobInProgress {
for (int i = 0; i < reduces.length; i++) {
reduces[i].kill();
}
- if (!jobSetupCleanupNeeded) {
- terminateJob(jobTerminationState);
- }
}
}
@@ -3200,9 +3167,7 @@ public class JobInProgress {
}
boolean isSetupFinished() {
- // if there is no setup to be launched, consider setup is finished.
- if ((tasksInited && setup.length == 0) ||
- setup[0].isComplete() || setup[0].isFailed() || setup[1].isComplete()
+ if (setup[0].isComplete() || setup[0].isFailed() || setup[1].isComplete()
|| setup[1].isFailed()) {
return true;
}
@@ -3316,12 +3281,10 @@ public class JobInProgress {
*/
public synchronized TaskInProgress getTaskInProgress(TaskID tipid) {
if (tipid.isMap()) {
- // cleanup map tip
- if (cleanup.length > 0 && tipid.equals(cleanup[0].getTIPId())) {
+ if (tipid.equals(cleanup[0].getTIPId())) { // cleanup map tip
return cleanup[0];
}
- // setup map tip
- if (setup.length > 0 && tipid.equals(setup[0].getTIPId())) {
+ if (tipid.equals(setup[0].getTIPId())) { //setup map tip
return setup[0];
}
for (int i = 0; i < maps.length; i++) {
@@ -3330,12 +3293,10 @@ public class JobInProgress {
}
}
} else {
- // cleanup reduce tip
- if (cleanup.length > 0 && tipid.equals(cleanup[1].getTIPId())) {
+ if (tipid.equals(cleanup[1].getTIPId())) { // cleanup reduce tip
return cleanup[1];
}
- // setup reduce tip
- if (setup.length > 0 && tipid.equals(setup[1].getTIPId())) {
+ if (tipid.equals(setup[1].getTIPId())) { //setup reduce tip
return setup[1];
}
for (int i = 0; i < reduces.length; i++) {
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/Job.java?rev=1379467&r1=1379466&r2=1379467&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/Job.java Fri Aug 31 14:29:13 2012
@@ -349,18 +349,6 @@ public class Job extends JobContext {
}
/**
- * Specify whether job-setup and job-cleanup is needed for the job
- *
- * @param needed If <code>true</code>, job-setup and job-cleanup will be
- * considered from {@link OutputCommitter}
- * else ignored.
- */
- public void setJobSetupCleanupNeeded(boolean needed) {
- ensureState(JobState.DEFINE);
- conf.setBoolean("mapred.committer.job.setup.cleanup.needed", needed);
- }
-
- /**
* Get the URL where some job progress information will be displayed.
*
* @return the URL where some job progress information will be displayed.
Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobContext.java?rev=1379467&r1=1379466&r2=1379467&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobContext.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobContext.java Fri Aug 31 14:29:13 2012
@@ -275,13 +275,4 @@ public class JobContext {
public RawComparator<?> getGroupingComparator() {
return conf.getOutputValueGroupingComparator();
}
-
- /**
- * Get whether job-setup and job-cleanup is needed for the job
- *
- * @return boolean
- */
- public boolean getJobSetupCleanupNeeded() {
- return conf.getBoolean("mapred.committer.job.setup.cleanup.needed", true);
- }
}