svn commit: r724459 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java

[cd...@apache.org]

Author: cdouglas
Date: Mon Dec  8 11:43:13 2008
New Revision: 724459

URL: http://svn.apache.org/viewvc?rev=724459&view=rev
Log:
HADOOP-4688. Modify the MiniMRDFSSort unit test to spill multiple times,
exercising the map-side merge code.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=724459&r1=724458&r2=724459&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Dec  8 11:43:13 2008
@@ -201,6 +201,9 @@
     Restarting the TaskTracker moves it out of the blacklist.
     (Amareshwari Sriramadasu via ddas)
 
+    HADOOP-4688. Modify the MiniMRDFSSort unit test to spill multiple times,
+    exercising the map-side merge code. (cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java?rev=724459&r1=724458&r2=724459&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRDFSSort.java Mon Dec  8 11:43:13 2008
@@ -39,13 +39,13 @@
 
   // Knobs to control randomwriter; and hence sort
   private static final int NUM_HADOOP_SLAVES = 3;
-  private static final int RW_BYTES_PER_MAP = 50000;
-  private static final int RW_MAPS_PER_HOST = 5;
-  
+  private static final int RW_BYTES_PER_MAP = 2 * 1024 * 1024;
+  private static final int RW_MAPS_PER_HOST = 2;
+
   private static void runRandomWriter(JobConf job, Path sortInput) 
   throws Exception {
     // Scale down the default settings for RandomWriter for the test-case
-    // Generates NUM_HADOOP_SLAVES * RW_MAPS_PER_HOST * RW_BYTES_PER_MAP -> 1MB
+    // Generates NUM_HADOOP_SLAVES * RW_MAPS_PER_HOST * RW_BYTES_PER_MAP
     job.setInt("test.randomwrite.bytes_per_map", RW_BYTES_PER_MAP);
     job.setInt("test.randomwriter.maps_per_host", RW_MAPS_PER_HOST);
     String[] rwArgs = {sortInput.toString()};
@@ -56,6 +56,9 @@
   
   private static void runSort(JobConf job, Path sortInput, Path sortOutput) 
   throws Exception {
+
+    job.setInt("io.sort.mb", 1);
+    job.setLong("mapred.min.split.size", Long.MAX_VALUE);
     // Setup command-line arguments to 'sort'
     String[] sortArgs = {sortInput.toString(), sortOutput.toString()};
     
@@ -78,9 +81,6 @@
     MiniMRCluster mr = null;
     FileSystem fileSys = null;
     try {
-      // set io.sort.mb and fsinmemory.size.mb to lower value in test
-      conf.setInt("io.sort.mb", 5);
-      conf.setInt("fs.inmemory.size.mb", 20);
 
       // Start the mini-MR and mini-DFS clusters
       dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null);