svn commit: r728605 - in /hadoop/core/branches/branch-0.20: CHANGES.txt src/mapred/org/apache/hadoop/mapred/ReduceTask.java src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java

[dd...@apache.org]

Author: ddas
Date: Sun Dec 21 20:42:02 2008
New Revision: 728605

URL: http://svn.apache.org/viewvc?rev=728605&view=rev
Log:
Merge -r 728603:728604 from trunk onto 0.20 branch. Fixes HADOOP-4716.

Modified:
    hadoop/core/branches/branch-0.20/CHANGES.txt
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=728605&r1=728604&r2=728605&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Sun Dec 21 20:42:02 2008
@@ -480,6 +480,9 @@
     restart apart from the initialContact flag that there was earlier.
     (Amareshwari Sriramadasu via ddas)
 
+    HADOOP-4716. Fixes ReduceTask.java to clear out the mapping between
+    hosts and MapOutputLocation upon a JT restart (Amar Kamat via ddas)
+
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=728605&r1=728604&r2=728605&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Sun Dec 21 20:42:02 2008
@@ -1803,10 +1803,18 @@
             MapOutputLocation loc = locItr.next(); 
             List<MapOutputLocation> locList = 
               mapLocations.get(loc.getHost());
-            //Add to the beginning of the list so that this map is 
-            //tried again before the others and we can hasten the 
-            //re-execution of this map should there be a problem
-            locList.add(0, loc);
+            
+            // Check if the list exists. Map output location mapping is cleared 
+            // once the jobtracker restarts and is rebuilt from scratch.
+            // Note that map-output-location mapping will be recreated and hence
+            // we continue with the hope that we might find some locations
+            // from the rebuild map.
+            if (locList != null) {
+              // Add to the beginning of the list so that this map is 
+              //tried again before the others and we can hasten the 
+              //re-execution of this map should there be a problem
+              locList.add(0, loc);
+            }
           }
 
           if (retryFetches.size() > 0) {
@@ -1839,7 +1847,13 @@
               List<MapOutputLocation> knownOutputsByLoc = 
                 mapLocations.get(host);
 
-              if (knownOutputsByLoc.size() == 0) {
+              // Check if the list exists. Map output location mapping is 
+              // cleared once the jobtracker restarts and is rebuilt from 
+              // scratch.
+              // Note that map-output-location mapping will be recreated and 
+              // hence we continue with the hope that we might find some 
+              // locations from the rebuild map and add then for fetching.
+              if (knownOutputsByLoc == null || knownOutputsByLoc.size() == 0) {
                 continue;
               }
               
@@ -2598,6 +2612,7 @@
         if (update.shouldReset()) {
           fromEventId.set(0);
           obsoleteMapIds.clear(); // clear the obsolete map
+          mapLocations.clear(); // clear the map locations mapping
         }
         
         // Update the last seen event ID

Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java?rev=728605&r1=728604&r2=728605&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java (original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java Sun Dec 21 20:42:02 2008
@@ -148,6 +148,7 @@
       jtConf.set("mapred.jobtracker.job.history.buffer.size", "1024");
       jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
       jtConf.setLong("mapred.tasktracker.expiry.interval", 25 * 1000);
+      jtConf.setInt("mapred.reduce.copy.backoff", 4);
       
       mr = new MiniMRCluster(2, namenode, 1, null, null, jtConf);