svn commit: r413984 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/util/CopyFiles.java src/java/org/apache/hadoop/util/StringUtils.java

[cu...@apache.org]

Author: cutting
Date: Tue Jun 13 14:56:36 2006
New Revision: 413984

URL: http://svn.apache.org/viewvc?rev=413984&view=rev
Log:
HADOOP-298.  Improved progress reports for CopyFiles, the distributed file copier.  Contributed by Owen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=413984&r1=413983&r2=413984&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Jun 13 14:56:36 2006
@@ -7,6 +7,9 @@
     selector, and a thread per connection is no longer required.  This
     should permit larger clusters.  (Devaraj Das via cutting)
 
+ 2. HADOOP-298.  Improved progress reports for CopyFiles utility, the
+    distributed file copier.  (omalley via cutting)
+
 
 Release 0.3.2 - 2006-06-09
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=413984&r1=413983&r2=413984&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Tue Jun 13 14:56:36 2006
@@ -19,9 +19,9 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
 import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.StringTokenizer;
@@ -75,11 +75,13 @@
     private static final long reportInterval = 1L << 25;
     private long bytesSinceLastReport = 0L;
     private long totalBytesCopied = 0L;
+    private static DecimalFormat percentFormat = new DecimalFormat("0.00");
     
     private void copy(String src, Reporter reporter) throws IOException {
       // open source file
       Path srcFile = new Path(srcPath, src);
       FSDataInputStream in = srcFileSys.open(srcFile);
+      long totalBytes = srcFileSys.getLength(srcFile);
       
       // create directories to hold destination file and create destFile
       Path destFile = new Path(destPath, src);
@@ -96,7 +98,12 @@
         if (bytesSinceLastReport > reportInterval) {
             totalBytesCopied += bytesSinceLastReport;
             bytesSinceLastReport = 0L;
-            reporter.setStatus("Total bytes copied: "+totalBytesCopied);
+            reporter.setStatus("Copy "+ src + ": " + 
+                               percentFormat.format(100.0 * totalBytesCopied / 
+                                                    totalBytes) +
+                               "% and " +
+                               StringUtils.humanReadableInt(totalBytesCopied) +
+                               " bytes");
         }
       }
       
@@ -105,7 +112,8 @@
       // report at least once for each file
       totalBytesCopied += bytesSinceLastReport;
       bytesSinceLastReport = 0L;
-      reporter.setStatus("Total bytes copied: "+totalBytesCopied);
+      reporter.setStatus("Finished. Bytes copied: " + 
+                         StringUtils.humanReadableInt(totalBytesCopied));
     }
     
     /** Mapper configuration.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java?rev=413984&r1=413983&r2=413984&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java Tue Jun 13 14:56:36 2006
@@ -18,6 +18,7 @@
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.text.DecimalFormat;
 
 /**
  * General string utils
@@ -51,4 +52,31 @@
     return fullHostname;
   }
 
+  private static DecimalFormat numFormat = new DecimalFormat("0.0");
+  
+  /**
+   * Given an integer, return a string that is in an approximate, but human 
+   * readable format. 
+   * It uses the bases 'k', 'm', and 'g' for 1024, 1024**2, and 1024**3.
+   * @param number the number to format
+   * @return a human readable form of the integer
+   */
+  public static String humanReadableInt(long number) {
+    long absNumber = Math.abs(number);
+    double result = number;
+    String suffix = "";
+    if (absNumber < 1024) {
+      // nothing
+    } else if (absNumber < 1024 * 1024) {
+      result = number / 1024.0;
+      suffix = "k";
+    } else if (absNumber < 1024 * 1024 * 1024) {
+      result = number / (1024.0 * 1024);
+      suffix = "m";
+    } else {
+      result = number / (1024.0 * 1024 * 1024);
+      suffix = "g";
+    }
+    return numFormat.format(result) + suffix;
+  }
 }