svn commit: r470209 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobConf.java src/java/org/apache/hadoop/mapred/MapTask.java

[cu...@apache.org]

Author: cutting
Date: Wed Nov  1 18:55:20 2006
New Revision: 470209

URL: http://svn.apache.org/viewvc?view=rev&rev=470209
Log:
HADOOP-647.  Permit map outputs to use a different compression type than job output.  Contributed by Owen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=470209&r1=470208&r2=470209
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Nov  1 18:55:20 2006
@@ -134,6 +134,9 @@
     types, replace Vector with ArrayList, etc.
     (Konstantin Shvachko via cutting)
 
+38. HADOOP-647.  Permit map outputs to use a different compression
+    type than the job output.  (omalley via cutting)
+
 
 Release 0.7.2 - 2006-10-18
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?view=diff&rev=470209&r1=470208&r2=470209
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Wed Nov  1 18:55:20 2006
@@ -34,11 +34,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.conf.Configuration;
 
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.compress.CompressionCodec;
 
 import org.apache.hadoop.mapred.lib.IdentityMapper;
@@ -338,6 +334,28 @@
     return getBoolean("mapred.compress.map.output", false);
   }
 
+  /**
+   * Set the compression type for the map outputs.
+   * @param style NONE, RECORD, or BLOCK to control how the map outputs are 
+   *        compressed
+   */
+  public void setMapOutputCompressionType(SequenceFile.CompressionType style) {
+    set("map.output.compression.type", style.toString());
+  }
+  
+  /**
+   * Get the compression type for the map outputs.
+   * @return the compression type, defaulting to job output compression type
+   */
+  public SequenceFile.CompressionType getMapOutputCompressionType() {
+    String val = get("map.output.compression.type");
+    if (val == null) {
+      return SequenceFile.getCompressionType(this);
+    } else {
+      return SequenceFile.CompressionType.valueOf(val);
+    }
+  }
+  
   /**
    * Set the given class as the  compression codec for the map outputs.
    * @param codecClass the CompressionCodec class that will compress the 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?view=diff&rev=470209&r1=470208&r2=470209
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Wed Nov  1 18:55:20 2006
@@ -137,7 +137,7 @@
       CompressionType compressionType = CompressionType.NONE;
       if (job.getCompressMapOutput()) {
         // find the kind of compression to do, defaulting to record
-        compressionType = SequenceFile.getCompressionType(job);
+        compressionType = job.getMapOutputCompressionType();
 
         // find the right codec
         Class codecClass =