svn commit: r1067308 - in /hadoop/mapreduce/branches/branch-0.22: CHANGES.txt src/java/org/apache/hadoop/mapred/IndexCache.java src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java

[sh...@apache.org]

Author: shv
Date: Fri Feb  4 22:04:16 2011
New Revision: 1067308

URL: http://svn.apache.org/viewvc?rev=1067308&view=rev
Log:
MAPREDUCE-1915. Merge r1067305 from trunk to branch 0.22.

Modified:
    hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/IndexCache.java
    hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java

Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1067308&r1=1067307&r2=1067308&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Fri Feb  4 22:04:16 2011
@@ -466,6 +466,9 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2188. The new API MultithreadedMapper doesn't initialize
     RecordReader. (Owen O'Malley via shv)
 
+    MAPREDUCE-1915. Fix IndexOutOfBoundsException in IndexCache.
+    (Priyo Mustafi via shv)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/IndexCache.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/IndexCache.java?rev=1067308&r1=1067307&r2=1067308&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/IndexCache.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/IndexCache.java Fri Feb  4 22:04:16 2011
@@ -80,7 +80,7 @@ class IndexCache {
     }
 
     if (info.mapSpillRecord.size() == 0 ||
-        info.mapSpillRecord.size() < reduce) {
+        info.mapSpillRecord.size() <= reduce) {
       throw new IOException("Invalid request " +
         " Map Id = " + mapId + " Reducer = " + reduce +
         " Index Info Length = " + info.mapSpillRecord.size());

Modified: hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java?rev=1067308&r1=1067307&r2=1067308&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/test/mapred/org/apache/hadoop/mapred/TestIndexCache.java Fri Feb  4 22:04:16 2011
@@ -149,6 +149,50 @@ public class TestIndexCache extends Test
     }
   }
 
+  public void testInvalidReduceNumberOrLength() throws Exception {
+    JobConf conf = new JobConf();
+    FileSystem fs = FileSystem.getLocal(conf).getRaw();
+    Path p = new Path(System.getProperty("test.build.data", "/tmp"),
+                      "cache").makeQualified(fs);
+    fs.delete(p, true);
+    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    final int partsPerMap = 1000;
+    final int bytesPerFile = partsPerMap * 24;
+    IndexCache cache = new IndexCache(conf);
+
+    // fill cache
+    Path feq = new Path(p, "invalidReduceOrPartsPerMap");
+    writeFile(fs, feq, bytesPerFile, partsPerMap);
+
+    // Number of reducers should always be less than partsPerMap as reducer
+    // numbers start from 0 and there cannot be more reducer than parts
+
+    try {
+      // Number of reducers equal to partsPerMap
+      cache.getIndexInformation("reduceEqualPartsPerMap", 
+               partsPerMap, // reduce number == partsPerMap
+               feq, UserGroupInformation.getCurrentUser().getShortUserName());
+      fail("Number of reducers equal to partsPerMap did not fail");
+    } catch (Exception e) {
+      if (!(e instanceof IOException)) {
+        throw e;
+      }
+    }
+
+    try {
+      // Number of reducers more than partsPerMap
+      cache.getIndexInformation(
+      "reduceMorePartsPerMap", 
+      partsPerMap + 1, // reduce number > partsPerMap
+      feq, UserGroupInformation.getCurrentUser().getShortUserName());
+      fail("Number of reducers more than partsPerMap did not fail");
+    } catch (Exception e) {
+      if (!(e instanceof IOException)) {
+        throw e;
+      }
+    }
+  }
+
   private static void checkRecord(IndexRecord rec, long fill) {
     assertEquals(fill, rec.startOffset);
     assertEquals(fill, rec.rawLength);