svn commit: r614192 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java src/java/org/apache/hadoop/mapred/MergeSorter.java

[dd...@apache.org]

Author: ddas
Date: Tue Jan 22 04:32:30 2008
New Revision: 614192

URL: http://svn.apache.org/viewvc?rev=614192&view=rev
Log:
HADOOP-2284. Reduce the number of progress updates during the sorting in the map task. Contributed by Amar Kamat.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MergeSorter.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=614192&r1=614191&r2=614192&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Jan 22 04:32:30 2008
@@ -289,6 +289,9 @@
     HADOOP-2208. Counter update frequency (from TaskTracker to JobTracker) is 
     capped at 1 minute.  (Amareshwari Sri Ramadasu via ddas)
 
+    HADOOP-2284. Reduce the number of progress updates during the sorting in 
+    the map task. (Amar Kamat via ddas)
+
   BUG FIXES
 
     HADOOP-2583.  Fixes a bug in the Eclipse plug-in UI to edit locations.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java?rev=614192&r1=614191&r2=614192&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/BasicTypeSorterBase.java Tue Jan 22 04:32:30 2008
@@ -60,7 +60,7 @@
   private int maxValLength = 0;
 
   //Reference to the Progressable object for sending KeepAlive
-  private Progressable reporter;
+  protected Progressable reporter;
 
   //Implementation of methods of the SorterBase interface
   //
@@ -124,16 +124,6 @@
     //release the large key-value buffer so that the GC, if necessary,
     //can collect it away
     keyValBuffer = null;
-  }
-  //A compare method that references the keyValBuffer through the indirect
-  //pointers
-  protected int compare(int i, int j) {
-    // indicate we're making progress
-    reporter.progress();
-    return comparator.compare(keyValBuffer.getData(), startOffsets[i],
-                              keyLengths[i],
-                              keyValBuffer.getData(), startOffsets[j], 
-                              keyLengths[j]);
   }
   
   private void grow() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MergeSorter.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MergeSorter.java?rev=614192&r1=614191&r2=614192&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MergeSorter.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MergeSorter.java Tue Jan 22 04:32:30 2008
@@ -34,6 +34,8 @@
  */
 class MergeSorter extends BasicTypeSorterBase 
 implements Comparator<IntWritable> {
+  private static int progressUpdateFrequency = 10000;
+  private int progressCalls = 0;
   
   /** The sort method derived from BasicTypeSorterBase and overridden here*/
   public RawKeyValueIterator sort() {
@@ -47,8 +49,7 @@
     return new MRSortResultIterator(super.keyValBuffer, pointersCopy, 
                                     super.startOffsets, super.keyLengths, super.valueLengths);
   }
-  /** The implementation of the compare method from Comparator. This basically
-   * forwards the call to the super class's compare. Note that
+  /** The implementation of the compare method from Comparator. Note that
    * Comparator.compare takes objects as inputs and so the int values are
    * wrapped in (reusable) IntWritables from the class util.MergeSort
    * @param i
@@ -56,7 +57,17 @@
    * @return int as per the specification of Comparator.compare
    */
   public int compare (IntWritable i, IntWritable j) {
-    return super.compare(i.get(), j.get());
+    // indicate we're making progress but do a batch update
+    if (progressCalls < progressUpdateFrequency) {
+      progressCalls++;
+    } else {
+      progressCalls = 0;
+      reporter.progress();
+    }  
+    return comparator.compare(keyValBuffer.getData(), startOffsets[i.get()],
+                              keyLengths[i.get()],
+                              keyValBuffer.getData(), startOffsets[j.get()], 
+                              keyLengths[j.get()]);
   }
   
   /** Add the extra memory that will be utilized by the sort method */