svn commit: r584276 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/util/CopyFiles.java

[ni...@apache.org]

Author: nigel
Date: Fri Oct 12 14:25:29 2007
New Revision: 584276

URL: http://svn.apache.org/viewvc?rev=584276&view=rev
Log:
HADOOP-2028. Fix distcp so that the log dir does not need to be specified and the destination does not need to exist. Contributed by Chris.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=584276&r1=584275&r2=584276&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Oct 12 14:25:29 2007
@@ -294,6 +294,10 @@
     HADOOP-1771. Fix a NullPointerException in streaming caused by an 
     IOException in MROutputThread. (lohit vijayarenu via nigel)
 
+    HADOOP-2028. Fix distcp so that the log dir does not need to be 
+    specified and the destination does not need to exist.
+    (Chris Douglas via nigel)
+
   IMPROVEMENTS
 
     HADOOP-1908. Restructure data node code so that block sending and 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=584276&r1=584275&r2=584276&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Fri Oct 12 14:25:29 2007
@@ -238,7 +238,8 @@
      * @param dstpath dst path
      * @param reporter
      */
-    private void copy(FileStatus srcstat, Path dstpath, Reporter reporter)
+    private void copy(FileStatus srcstat, Path dstpath,
+        OutputCollector<WritableComparable, Text> outc, Reporter reporter)
         throws IOException {
 
       int totfiles = job.getInt("distcp.file.count", -1);
@@ -273,6 +274,7 @@
         if (destFileSys.exists(dstpath)
            && (!overwrite && !(update
                && needsUpdate(srcstat, destFileSys.getFileStatus(dstpath))))) {
+          outc.collect(null, new Text("SKIP: " + srcstat.getPath()));
           reporter.setStatus("Skipped " + srcstat.getPath());
           return;
         }
@@ -345,7 +347,7 @@
       FileStatus srcstat = value.input;
       Path dstpath = value.output;
       try {
-        copy(srcstat, dstpath, reporter);
+        copy(srcstat, dstpath, out, reporter);
       } catch (IOException except) {
         out.collect(null, new Text("Failed to copy " + srcstat.getPath() +
               " : " + StringUtils.stringifyException(except)));
@@ -593,8 +595,8 @@
    * @param logPath : Log output directory
    * @param flags : Command-line flags
    */
-  private static void setup(Configuration conf, JobConf jobConf, 
-                            List<Path> srcPaths, Path destPath, 
+  private static void setup(Configuration conf, JobConf jobConf,
+                            List<Path> srcPaths, Path destPath,
                             Path logPath, EnumSet<cpOpts> flags)
       throws IOException {
     boolean update;
@@ -632,10 +634,11 @@
     // default logPath
     FileSystem dstfs = destPath.getFileSystem(conf);
     if (logPath == null) {
-      FileStatus stat = dstfs.getFileStatus(destPath);
       String filename = "_distcp_logs_" + randomId;
-      if (!stat.isDir()) {
-        logPath = new Path(destPath.getParent(), filename);        
+      if (!dstfs.exists(destPath) || !dstfs.getFileStatus(destPath).isDir()) {
+        Path parent = destPath.getParent();
+        dstfs.mkdirs(parent);
+        logPath = new Path(parent, filename);
       } else {
         logPath = new Path(destPath, filename);
       }