You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2008/12/04 23:00:00 UTC
svn commit: r723463 - in /hadoop/core/branches/branch-0.19: ./ CHANGES.txt
src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java
src/mapred/org/apache/hadoop/mapred/JobClient.java
Author: hairong
Date: Thu Dec 4 14:00:00 2008
New Revision: 723463
URL: http://svn.apache.org/viewvc?rev=723463&view=rev
Log:
Merge -r 723459:723460 from main to move the change log of HADOOP-4746 to branch 0.19
Modified:
hadoop/core/branches/branch-0.19/ (props changed)
hadoop/core/branches/branch-0.19/CHANGES.txt (contents, props changed)
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java
hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobClient.java
Propchange: hadoop/core/branches/branch-0.19/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Dec 4 14:00:00 2008
@@ -1 +1 @@
-/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013
+/hadoop/core/trunk:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,709040,709303,712881,713888,720602,723013,723460
Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=723463&r1=723462&r2=723463&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Thu Dec 4 14:00:00 2008
@@ -1054,6 +1054,8 @@
HADOOP-4679. Datanode prints tons of log messages: waiting for threadgroup
to exit, active threads is XX. (hairong)
+ HADOOP-4746. Job output directory should be normalized. (hairong)
+
Release 0.18.2 - 2008-11-03
BUG FIXES
Propchange: hadoop/core/branches/branch-0.19/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Dec 4 14:00:00 2008
@@ -1 +1 @@
-/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013
+/hadoop/core/trunk/CHANGES.txt:697306,698176,699056,699098,699415,699424,699444,699490,699517,700163,700628,700923,701273,701398,703923,704203,704261,704701,704703,704707,704712,704732,704748,704989,705391,705420,705430,705762,706350,706707,706719,706796,706802,707258,707262,708623,708641,708710,708723,709040,709303,711717,712881,713888,720602,723013,723460
Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java?rev=723463&r1=723462&r2=723463&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/FileOutputFormat.java Thu Dec 4 14:00:00 2008
@@ -101,9 +101,16 @@
if (outDir == null && job.getNumReduceTasks() != 0) {
throw new InvalidJobConfException("Output directory not set in JobConf.");
}
- if (outDir != null && outDir.getFileSystem(job).exists(outDir)) {
- throw new FileAlreadyExistsException("Output directory " + outDir +
- " already exists");
+ if (outDir != null) {
+ FileSystem fs = outDir.getFileSystem(job);
+ // normalize the output directory
+ outDir = fs.makeQualified(outDir);
+ setOutputPath(job, outDir);
+ // check its existence
+ if (fs.exists(outDir)) {
+ throw new FileAlreadyExistsException("Output directory " + outDir +
+ " already exists");
+ }
}
}
Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=723463&r1=723462&r2=723463&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobClient.java Thu Dec 4 14:00:00 2008
@@ -748,7 +748,21 @@
// job submission directory is world readable/writable/executable
final static FsPermission JOB_DIR_PERMISSION =
FsPermission.createImmutable((short) 0777); // rwx-rwx-rwx
-
+
+ /** Normalize the output path defined in job configuration.
+ * Store the normalized path back into job configuration.
+ *
+ * @param job job configuration
+ * @throws IOException if any error occurs during normalization
+ */
+ private static void normalizeOutputPath(JobConf job) throws IOException {
+ Path out = FileOutputFormat.getOutputPath(job);
+ if (out!=null) {
+ FileOutputFormat.setOutputPath(job,
+ out.getFileSystem(job).makeQualified(out));
+ }
+ }
+
/**
* Submit a job to the MR system.
* This returns a handle to the {@link RunningJob} which can be used to track
@@ -774,7 +788,8 @@
configureCommandLineOptions(job, submitJobDir, submitJarFile);
Path submitJobFile = new Path(submitJobDir, "job.xml");
- // Check the output specification
+ // Normalize and check the output specification
+ normalizeOutputPath(job);
job.getOutputFormat().checkOutputSpecs(fs, job);
// Create the splits for the job