svn commit: r537933 - in /lucene/hadoop/trunk/src/test/org/apache/hadoop: mapred/BigMapOutput.java test/AllTestDriver.java

[cu...@apache.org]

Author: cutting
Date: Mon May 14 11:22:33 2007
New Revision: 537933

URL: http://svn.apache.org/viewvc?view=rev&rev=537933
Log:
HADOOP-1350.  Add test code for big map outputs.  Contributed by Devaraj.

Added:
    lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/BigMapOutput.java
Modified:
    lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/BigMapOutput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/BigMapOutput.java?view=auto&rev=537933
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/BigMapOutput.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/BigMapOutput.java Mon May 14 11:22:33 2007
@@ -0,0 +1,61 @@
+package org.apache.hadoop.mapred;
+
+import java.io.*;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import org.apache.hadoop.mapred.SortValidator.RecordStatsChecker.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.mapred.lib.*;
+import org.apache.hadoop.fs.*;
+
+public class BigMapOutput {
+
+  public static void main(String[] args) throws IOException {
+    if (args.length != 4) { //input-dir should contain a huge file ( > 2GB)
+      System.err.println("BigMapOutput " +
+                       "-input <input-dir> -output <output-dir>");
+      System.exit(1);
+    } 
+    Path bigMapInput = null;
+    Path outputPath = null;
+    for(int i=0; i < args.length; ++i) {
+      if ("-input".equals(args[i])){
+        bigMapInput = new Path(args[++i]);
+      } else if ("-output".equals(args[i])){
+        outputPath = new Path(args[++i]);
+      }
+    }
+    Configuration defaults = new Configuration();
+    FileSystem fs = FileSystem.get(defaults);
+    
+    JobConf jobConf = new JobConf(defaults, BigMapOutput.class);
+
+    jobConf.setJobName("BigMapOutput");
+    jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
+    jobConf.setOutputFormat(SequenceFileOutputFormat.class);
+    jobConf.setInputPath(bigMapInput);
+    if (fs.exists(outputPath)) {
+      fs.delete(outputPath);
+    }
+    jobConf.setOutputPath(outputPath);
+    jobConf.setMapperClass(IdentityMapper.class);
+    jobConf.setReducerClass(IdentityReducer.class);
+    jobConf.setOutputKeyClass(BytesWritable.class);
+    jobConf.setOutputValueClass(BytesWritable.class);
+
+    Date startTime = new Date();
+    System.out.println("Job started: " + startTime);
+    JobClient.runJob(jobConf);
+    Date end_time = new Date();
+    System.out.println("Job ended: " + end_time);
+      
+  }
+}

Modified: lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java?view=diff&rev=537933&r1=537932&r2=537933
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java (original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/test/AllTestDriver.java Mon May 14 11:22:33 2007
@@ -22,6 +22,7 @@
 import org.apache.hadoop.mapred.MRBench;
 import org.apache.hadoop.mapred.SortValidator;
 import org.apache.hadoop.mapred.TestMapRed;
+import org.apache.hadoop.mapred.BigMapOutput;
 import org.apache.hadoop.mapred.TestTextInputFormat;
 import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
 import org.apache.hadoop.dfs.ClusterTestDFS;
@@ -61,6 +62,9 @@
       pgd.addClass("DistributedFSCheck", DistributedFSCheck.class, "Distributed checkup of the file system consistency.");
       pgd.addClass("testmapredsort", SortValidator.class, 
                    "A map/reduce program that validates the map-reduce framework's sort.");
+      pgd.addClass("testbigmapoutput", BigMapOutput.class, 
+                   "A map/reduce program that works on a very big " + 
+                   "non-splittable file and does identity map/reduce");
       pgd.driver(argv);
     } catch(Throwable e) {
       e.printStackTrace();