svn commit: r940989 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java src/test/core/org/apache/hadoop/io/compress/TestCodec.java

[cd...@apache.org]

Author: cdouglas
Date: Tue May  4 18:21:48 2010
New Revision: 940989

URL: http://svn.apache.org/viewvc?rev=940989&view=rev
Log:
HADOOP-6669. Respect compression configuration when creating DefaultCodec
instances. Contributed by Koji Noguchi

Modified:
    hadoop/common/trunk/CHANGES.txt
    hadoop/common/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
    hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java

Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=940989&r1=940988&r2=940989&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue May  4 18:21:48 2010
@@ -7,6 +7,9 @@ Trunk (unreleased changes)
     HADOOP-6730. Bug in FileContext#copy and provide base class for FileContext 
     tests. (Ravi Phulari via jghoman)
 
+    HADOOP-6669. Respect compression configuration when creating DefaultCodec
+    instances. (Koji Noguchi via cdouglas)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java?rev=940989&r1=940988&r2=940989&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java Tue May  4 18:21:48 2010
@@ -86,7 +86,8 @@ public class ZlibFactory {
    */
   public static Compressor getZlibCompressor(Configuration conf) {
     return (isNativeZlibLoaded(conf)) ? 
-      new ZlibCompressor() : new BuiltInZlibDeflater(); 
+      new ZlibCompressor(conf) :
+      new BuiltInZlibDeflater(ZlibFactory.getCompressionLevel(conf).compressionLevel());
   }
 
   /**

Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java?rev=940989&r1=940988&r2=940989&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java Tue May  4 18:21:48 2010
@@ -166,6 +166,8 @@ public class TestCodec {
       RandomDatum v2 = new RandomDatum();
       k2.readFields(inflateIn);
       v2.readFields(inflateIn);
+      assertTrue("original and compressed-then-decompressed-output not equal",
+                 k1.equals(k2) && v1.equals(v2));
     }
     LOG.info("SUCCESS! Completed checking " + count + " records");
   }
@@ -322,6 +324,61 @@ public class TestCodec {
                outbytes.length >= b.length);
   }
 
+  private static void codecTestWithNOCompression (Configuration conf,
+                      String codecClass) throws IOException {
+    // Create a compressor with NO_COMPRESSION and make sure that
+    // output is not compressed by comparing the size with the
+    // original input
+
+    CompressionCodec codec = null;
+    ZlibFactory.setCompressionLevel(conf, CompressionLevel.NO_COMPRESSION);
+    try {
+      codec = (CompressionCodec)
+        ReflectionUtils.newInstance(conf.getClassByName(codecClass), conf);
+    } catch (ClassNotFoundException cnfe) {
+      throw new IOException("Illegal codec!");
+    }
+    Compressor c = codec.createCompressor();
+    // ensure same compressor placed earlier
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    CompressionOutputStream cos = null;
+    // write trivially compressable data
+    byte[] b = new byte[1 << 15];
+    Arrays.fill(b, (byte) 43);
+    try {
+      cos = codec.createOutputStream(bos, c);
+      cos.write(b);
+    } finally {
+      if (cos != null) {
+        cos.close();
+      }
+    }
+    byte[] outbytes = bos.toByteArray();
+    // verify data were not compressed
+    assertTrue("Compressed bytes contrary to configuration(NO_COMPRESSION)",
+               outbytes.length >= b.length);
+  }
+
+  @Test
+  public void testCodecInitWithCompressionLevel() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean("io.native.lib.available", true);
+    if (ZlibFactory.isNativeZlibLoaded(conf)) {
+      LOG.info("testCodecInitWithCompressionLevel with native");
+      codecTestWithNOCompression(conf,
+                            "org.apache.hadoop.io.compress.GzipCodec");
+      codecTestWithNOCompression(conf,
+                         "org.apache.hadoop.io.compress.DefaultCodec");
+    } else {
+      LOG.warn("testCodecInitWithCompressionLevel for native skipped"
+               + ": native libs not loaded");
+    }
+    conf = new Configuration();
+    conf.setBoolean("io.native.lib.available", false);
+    codecTestWithNOCompression( conf,
+                         "org.apache.hadoop.io.compress.DefaultCodec");
+  }
+
   @Test
   public void testCodecPoolCompressorReinit() throws Exception {
     Configuration conf = new Configuration();