You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/03/19 09:29:15 UTC
svn commit: r638746 - in /hadoop/core/trunk: ./
src/java/org/apache/hadoop/mapred/
Author: ddas
Date: Wed Mar 19 01:29:13 2008
New Revision: 638746
URL: http://svn.apache.org/viewvc?rev=638746&view=rev
Log:
HADOOP-2985. Fixes LocalJobRunner to tolerate null job output path. Also makes the _temporary a constant in MRConstants.java. Contributed by Amareshwari Sriramadasu.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/Task.java
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 19 01:29:13 2008
@@ -294,6 +294,10 @@
HADOOP-2978. Fixes the JobHistory log format for counters.
(Runping Qi via ddas)
+ HADOOP-2985. Fixes LocalJobRunner to tolerate null job output path.
+ Also makes the _temporary a constant in MRConstants.java.
+ (Amareshwari Sriramadasu via ddas)
+
Release 0.16.1 - 2008-03-13
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Mar 19 01:29:13 2008
@@ -336,13 +336,11 @@
// create job specific temporary directory in output path
Path outputPath = conf.getOutputPath();
if (outputPath != null) {
- Path tmpDir = new Path(outputPath, "_temporary");
+ Path tmpDir = new Path(outputPath, MRConstants.TEMP_DIR_NAME);
FileSystem fileSys = tmpDir.getFileSystem(conf);
if (!fileSys.mkdirs(tmpDir)) {
LOG.error("Mkdirs failed to create " + tmpDir.toString());
}
- } else {
- LOG.error("Null Output path");
}
this.status = new JobStatus(status.getJobId(), 0.0f, 0.0f, JobStatus.RUNNING);
@@ -1276,7 +1274,7 @@
// delete the temporary directory in output directory
Path outputPath = conf.getOutputPath();
if (outputPath != null) {
- Path tmpDir = new Path(outputPath, "_temporary");
+ Path tmpDir = new Path(outputPath, MRConstants.TEMP_DIR_NAME);
FileSystem fileSys = tmpDir.getFileSystem(conf);
if (fileSys.exists(tmpDir)) {
FileUtil.fullyDelete(fileSys, tmpDir);
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Wed Mar 19 01:29:13 2008
@@ -114,10 +114,15 @@
job.setNumReduceTasks(1);
}
// create job specific temp directory in output path
- Path tmpDir = new Path(job.getOutputPath(), "_temporary");
- FileSystem fileSys = tmpDir.getFileSystem(job);
- if (!fileSys.mkdirs(tmpDir)) {
- LOG.error("Mkdirs failed to create " + tmpDir.toString());
+ Path outputPath = job.getOutputPath();
+ FileSystem outputFs = null;
+ Path tmpDir = null;
+ if (outputPath != null) {
+ tmpDir = new Path(outputPath, MRConstants.TEMP_DIR_NAME);
+ outputFs = tmpDir.getFileSystem(job);
+ if (!outputFs.mkdirs(tmpDir)) {
+ LOG.error("Mkdirs failed to create " + tmpDir.toString());
+ }
}
DataOutputBuffer buffer = new DataOutputBuffer();
@@ -133,15 +138,17 @@
splits[i].getClass().getName(),
split);
JobConf localConf = new JobConf(job);
- if (fileSys.exists(tmpDir)) {
- Path taskTmpDir = new Path(tmpDir, "_" + mapId);
- if (!fileSys.mkdirs(taskTmpDir)) {
- throw new IOException("Mkdirs failed to create "
- + taskTmpDir.toString());
- }
- } else {
- throw new IOException("The directory " + tmpDir.toString()
+ if (outputFs != null) {
+ if (outputFs.exists(tmpDir)) {
+ Path taskTmpDir = new Path(tmpDir, "_" + mapId);
+ if (!outputFs.mkdirs(taskTmpDir)) {
+ throw new IOException("Mkdirs failed to create "
+ + taskTmpDir.toString());
+ }
+ } else {
+ throw new IOException("The directory " + tmpDir.toString()
+ " doesnt exist " );
+ }
}
map.localizeConfiguration(localConf);
map.setConf(localConf);
@@ -175,15 +182,17 @@
"tip_r_0001",
reduceId, 0, mapIds.size());
JobConf localConf = new JobConf(job);
- if (fileSys.exists(tmpDir)) {
- Path taskTmpDir = new Path(tmpDir, "_" + reduceId);
- if (!fileSys.mkdirs(taskTmpDir)) {
- throw new IOException("Mkdirs failed to create "
- + taskTmpDir.toString());
+ if (outputFs != null) {
+ if (outputFs.exists(tmpDir)) {
+ Path taskTmpDir = new Path(tmpDir, "_" + reduceId);
+ if (!outputFs.mkdirs(taskTmpDir)) {
+ throw new IOException("Mkdirs failed to create "
+ + taskTmpDir.toString());
+ }
+ } else {
+ throw new IOException("The directory " + tmpDir.toString()
+ + " doesnt exist ");
}
- } else {
- throw new IOException("The directory " + tmpDir.toString()
- + " doesnt exist ");
}
reduce.localizeConfiguration(localConf);
reduce.setConf(localConf);
@@ -207,8 +216,10 @@
}
// delete the temporary directory in output directory
try {
- if (fileSys.exists(tmpDir)) {
- FileUtil.fullyDelete(fileSys, tmpDir);
+ if (outputFs != null) {
+ if (outputFs.exists(tmpDir)) {
+ FileUtil.fullyDelete(outputFs, tmpDir);
+ }
}
} catch (IOException e) {
LOG.error("Exception in deleting " + tmpDir.toString());
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MRConstants.java Wed Mar 19 01:29:13 2008
@@ -54,4 +54,9 @@
* The custom http header used for the map output length.
*/
public static final String MAP_OUTPUT_LENGTH = "Map-Output-Length";
+
+ /**
+ * Temporary directory name
+ */
+ public static final String TEMP_DIR_NAME = "_temporary";
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/Task.java?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/Task.java Wed Mar 19 01:29:13 2008
@@ -191,8 +191,8 @@
public String toString() { return taskId; }
private Path getTaskOutputPath(JobConf conf) {
- Path p = new Path(conf.getOutputPath(), ("_temporary"
- + Path.SEPARATOR + "_" + taskId));
+ Path p = new Path(conf.getOutputPath(),
+ (MRConstants.TEMP_DIR_NAME + Path.SEPARATOR + "_" + taskId));
try {
FileSystem fs = p.getFileSystem(conf);
return p.makeQualified(fs);
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=638746&r1=638745&r2=638746&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Wed Mar 19 01:29:13 2008
@@ -1398,7 +1398,7 @@
// create _taskid directory in output path temporary directory.
Path outputPath = localJobConf.getOutputPath();
if (outputPath != null) {
- Path jobTmpDir = new Path(outputPath, "_temporary");
+ Path jobTmpDir = new Path(outputPath, MRConstants.TEMP_DIR_NAME);
FileSystem fs = jobTmpDir.getFileSystem(localJobConf);
if (fs.exists(jobTmpDir)) {
Path taskTmpDir = new Path(jobTmpDir, "_" + task.getTaskId());