svn commit: r1362293 - in /hadoop/common/branches/branch-1: CHANGES.txt src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java src/test/org/apache/hadoop/io/compress/TestCodec.java

[to...@apache.org]

Author: todd
Date: Mon Jul 16 22:35:17 2012
New Revision: 1362293

URL: http://svn.apache.org/viewvc?rev=1362293&view=rev
Log:
HADOOP-8423. MapFile.Reader.get() crashes jvm or throws EOFException on Snappy or LZO block-compressed data. Contributed by Todd Lipcon and Harsh J Chouraria.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/compress/TestCodec.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1362293&r1=1362292&r2=1362293&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jul 16 22:35:17 2012
@@ -98,6 +98,9 @@ Release 1.2.0 - unreleased
  
     HDFS-3654. TestJspHelper#testGetUgi fails with NPE. (eli)
 
+    HADOOP-8423. MapFile.Reader.get() crashes jvm or throws EOFException on
+    Snappy or LZO block-compressed data (todd and harsh)
+
 Release 1.1.0 - unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java?rev=1362293&r1=1362292&r2=1362293&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/BlockDecompressorStream.java Mon Jul 16 22:35:17 2012
@@ -114,6 +114,8 @@ public class BlockDecompressorStream ext
   }
 
   public void resetState() throws IOException {
+    originalBlockSize = 0;
+    noUncompressedBytes = 0;
     super.resetState();
   }
 

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/compress/TestCodec.java?rev=1362293&r1=1362292&r2=1362293&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/io/compress/TestCodec.java Mon Jul 16 22:35:17 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MapFile;
 import org.apache.hadoop.io.RandomDatum;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
@@ -622,4 +623,56 @@ public class TestCodec extends TestCase 
 
     verifyGzipFile(fileName, msg);
   }
+
+  /**
+   * Regression test for HADOOP-8423: seeking in a block-compressed
+   * stream would not properly reset the block decompressor state.
+   */
+  public void testSnappyMapFile() throws Exception {
+    if (SnappyCodec.isNativeSnappyLoaded(conf)) {
+      codecTestMapFile(SnappyCodec.class, CompressionType.BLOCK, 100);
+    } else {
+      System.err.println(
+          "Could not find the snappy codec to test MapFiles with!");
+    }
+  }
+
+  private void codecTestMapFile(Class<? extends CompressionCodec> clazz,
+      CompressionType type, int records) throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    LOG.info("Creating MapFiles with " + records  +
+            " records using codec " + clazz.getSimpleName());
+    Path path = new Path(new Path(
+        System.getProperty("test.build.data", "/tmp")),
+      clazz.getSimpleName() + "-" + type + "-" + records);
+
+    LOG.info("Writing " + path);
+    createMapFile(conf, fs, path, clazz.newInstance(), type, records);
+    MapFile.Reader reader = new MapFile.Reader(fs, path.toString(), conf);
+    Text key1 = new Text("002");
+    assertNotNull(reader.get(key1, new Text()));
+    Text key2 = new Text("004");
+    assertNotNull(reader.get(key2, new Text()));
+  }
+
+  private static void createMapFile(Configuration conf, FileSystem fs,
+      Path path, CompressionCodec codec, CompressionType type, int records)
+          throws IOException {
+    MapFile.Writer writer =
+        new MapFile.Writer(
+              conf,
+              fs,
+              path.toString(),
+              Text.class,
+              Text.class,
+              type,
+              codec,
+              null);
+    Text key = new Text();
+    for (int j = 0; j < records; j++) {
+        key.set(String.format("%03d", j));
+        writer.append(key, key);
+    }
+    writer.close();
+  }
 }