You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/01/10 14:08:37 UTC

[1/2] commons-compress git commit: COMPRESS-246 support for framed Snappy format

Repository: commons-compress
Updated Branches:
  refs/heads/master 7b9fdb6e9 -> 4282bc34d


COMPRESS-246 support for framed Snappy format


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/6ac58a59
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/6ac58a59
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/6ac58a59

Branch: refs/heads/master
Commit: 6ac58a5915cb471ae58c13cf667ad81f574dde34
Parents: 7b9fdb6
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 10 15:07:26 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 10 15:07:26 2017 +0100

----------------------------------------------------------------------
 src/changes/changes.xml                         |   3 +
 .../compressors/CompressorStreamFactory.java    |   8 +-
 .../FramedSnappyCompressorInputStream.java      |   5 +-
 .../FramedSnappyCompressorOutputStream.java     | 134 +++++++++++++++++++
 src/site/xdoc/index.xml                         |   9 +-
 src/site/xdoc/limitations.xml                   |   3 +-
 .../compressors/FramedSnappyTestCase.java       |  58 ++++++++
 7 files changed, 214 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 25b3c7f..9e36923 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -44,6 +44,9 @@ The <action> type attribute can be add,update,fix,remove.
   <body>
     <release version="1.14" date="not released, yet"
              description="Release 1.14">
+      <action issue="COMPRESS-246" type="update" date="2017-01-10">
+        Added write support for Snappy.
+      </action>
     </release>
     <release version="1.13" date="2016-12-29"
              description="Release 1.13 - API compatible to 1.12 but requires Java 7 at runtime.">

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 6a46a06..6f09924 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -43,6 +43,7 @@ import org.apache.commons.compress.compressors.lzma.LZMAUtils;
 import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
 import org.apache.commons.compress.compressors.pack200.Pack200CompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
+import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
 import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
@@ -497,7 +498,8 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
      * 
      * @param name
      *            the compressor name, i.e. {@value #GZIP}, {@value #BZIP2},
-     *            {@value #XZ}, {@value #PACK200} or {@value #DEFLATE}
+     *            {@value #XZ}, {@value #PACK200}, {@value SNAPPY_FRAMED}
+     *            or {@value #DEFLATE}
      * @param out
      *            the output stream
      * @return the compressor output stream
@@ -539,6 +541,10 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
                 return new DeflateCompressorOutputStream(out);
             }
 
+            if (SNAPPY_FRAMED.equalsIgnoreCase(name)) {
+                return new FramedSnappyCompressorOutputStream(out);
+            }
+
         } catch (final IOException e) {
             throw new CompressorException("Could not create CompressorOutputStream", e);
         }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
index d7f4209..245d495 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
@@ -43,14 +43,15 @@ public class FramedSnappyCompressorInputStream extends CompressorInputStream {
     static final long MASK_OFFSET = 0xa282ead8L;
 
     private static final int STREAM_IDENTIFIER_TYPE = 0xff;
-    private static final int COMPRESSED_CHUNK_TYPE = 0;
+    static final int COMPRESSED_CHUNK_TYPE = 0;
     private static final int UNCOMPRESSED_CHUNK_TYPE = 1;
     private static final int PADDING_CHUNK_TYPE = 0xfe;
     private static final int MIN_UNSKIPPABLE_TYPE = 2;
     private static final int MAX_UNSKIPPABLE_TYPE = 0x7f;
     private static final int MAX_SKIPPABLE_TYPE = 0xfd;
 
-    private static final byte[] SZ_SIGNATURE = new byte[] {
+    // used by FramedSnappyCompressorOutputStream as well
+    static final byte[] SZ_SIGNATURE = new byte[] {
         (byte) STREAM_IDENTIFIER_TYPE, // tag
         6, 0, 0, // length
         's', 'N', 'a', 'P', 'p', 'Y'

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
new file mode 100644
index 0000000..2d9f2be
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.snappy;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.commons.compress.compressors.CompressorOutputStream;
+
+/**
+ * CompressorOutputStream for the framing Snappy format.
+ *
+ * <p>Based on the "spec" in the version "Last revised: 2013-10-25"</p>
+ *
+ * @see <a href="https://github.com/google/snappy/blob/master/framing_format.txt">Snappy framing format description</a>
+ * @since 1.14
+ * @NotThreadSafe
+ */
+public class FramedSnappyCompressorOutputStream extends CompressorOutputStream {
+    // see spec:
+    // > However, we place an additional restriction that the uncompressed data
+    // > in a chunk must be no longer than 65536 bytes. This allows consumers to
+    // > easily use small fixed-size buffers.
+    private static final int MAX_COMPRESSED_BUFFER_SIZE = 1 << 16;
+
+    private final OutputStream out;
+    private final PureJavaCrc32C checksum = new PureJavaCrc32C();
+    // used in one-arg write method
+    private final byte[] oneByte = new byte[1];
+    private final byte[] buffer = new byte[MAX_COMPRESSED_BUFFER_SIZE];
+    private int currentIndex = 0;
+
+    /**
+     * Constructs a new output stream that compresses
+     * snappy-framed-compressed data to the specified output stream.
+     * @param out the OutputStream to which to write the compressed data
+     * @throws IOException if writing the signature fails
+     */
+    public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException {
+        this.out = out;
+        out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE);
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+        oneByte[0] = (byte) (b & 0xff);
+        write(oneByte);
+    }
+
+    @Override
+    public void write(byte[] data, int off, int len) throws IOException {
+        if (currentIndex + len > MAX_COMPRESSED_BUFFER_SIZE) {
+            flushBuffer();
+            while (len > MAX_COMPRESSED_BUFFER_SIZE) {
+                System.arraycopy(data, off, buffer, 0, MAX_COMPRESSED_BUFFER_SIZE);
+                off += MAX_COMPRESSED_BUFFER_SIZE;
+                len -= MAX_COMPRESSED_BUFFER_SIZE;
+                currentIndex = MAX_COMPRESSED_BUFFER_SIZE;
+                flushBuffer();
+            }
+        }
+        System.arraycopy(data, off, buffer, currentIndex, len);
+        currentIndex += len;
+    }
+
+    @Override
+    public void close() throws IOException {
+        finish();
+        out.close();
+    }
+
+    /**
+     * Compresses all remaining data and writes it to the stream,
+     * doesn't close the underlying stream.
+     * @throws IOException if an error occurs
+     */
+    public void finish() throws IOException {
+        if (currentIndex > 0) {
+            flushBuffer();
+        }
+    }
+
+    private void flushBuffer() throws IOException {
+        out.write(FramedSnappyCompressorInputStream.COMPRESSED_CHUNK_TYPE);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex)) {
+            o.write(buffer, 0, currentIndex);
+        }
+        byte[] b = baos.toByteArray();
+        writeLittleEndian(3, b.length + 4 /* CRC */);
+        writeCrc();
+        out.write(b);
+        currentIndex = 0;
+    }
+
+    private void writeLittleEndian(final int numBytes, long num) throws IOException {
+        for (int i = 0; i < numBytes; i++) {
+            out.write((int) (num & 0xff));
+            num >>= 8;
+        }
+    }
+
+    private void writeCrc() throws IOException {
+        checksum.update(buffer, 0, currentIndex);
+        writeLittleEndian(4, mask(checksum.getValue()));
+        checksum.reset();
+    }
+
+    static long mask(long x) {
+        // ugly, maybe we should just have used ints and deal with the
+        // overflow
+        x = ((x >> 15) | (x << 17));
+        x += FramedSnappyCompressorInputStream.MASK_OFFSET;
+        x &= 0xffffFFFFL;
+        return x;
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/site/xdoc/index.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index 6aa203a..60c0184 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -57,6 +57,11 @@
           of changes see the <a href="changes-report.html">Changes
           Report</a>.</p>
 
+          <subsection name="What's coming in 1.14?">
+            <ul>
+              <li>Added support for writing the Snappy format</li>
+            </ul>
+          </subsection>
           <subsection name="What's new in 1.13?">
             <ul>
               <li>The 7z package as well as
@@ -93,8 +98,8 @@
             by the <code>java.util.jar</code> package of the Java
             class library.  XZ and lzma support is provided by the public
             domain <a href="http://tukaani.org/xz/java.html">XZ for
-            Java</a> library.  As of Commons Compress 1.13 support for
-            the Z and Snappy formats is read-only.</p>
+            Java</a> library.  As of Commons Compress 1.14 support for
+            the Z format is read-only.</p>
 
           <p>The ar, arj, cpio, dump, tar, 7z and zip formats are supported as
             archivers where the <a href="zip.html">zip</a>

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/site/xdoc/limitations.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/limitations.xml b/src/site/xdoc/limitations.xml
index c5c2be4..45c3a27 100644
--- a/src/site/xdoc/limitations.xml
+++ b/src/site/xdoc/limitations.xml
@@ -120,7 +120,8 @@
      </section>
      <section name="SNAPPY">
        <ul>
-         <li>read-only support</li>
+         <li>Commons Compress 1.13 and earlier only support reading
+         the format</li>
        </ul>
      </section>
      <section name="TAR">

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java b/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
index 9af2518..b85db74 100644
--- a/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
@@ -26,6 +26,8 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Random;
 
 import org.apache.commons.compress.AbstractTestCase;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
@@ -95,4 +97,60 @@ public final class FramedSnappyTestCase
         }
     }
 
+    @Test
+    public void testRoundtrip() throws Exception {
+        testRoundtrip(getFile("test.txt"));
+        testRoundtrip(getFile("bla.tar"));
+        testRoundtrip(getFile("COMPRESS-256.7z"));
+    }
+
+    @Test
+    public void testRoundtripWithOneBigWrite() throws Exception {
+        Random r = new Random();
+        File input = new File(dir, "bigChunkTest");
+        try (FileOutputStream fs = new FileOutputStream(input)) {
+            for (int i = 0 ; i < 1 << 17; i++) {
+                fs.write(r.nextInt(256));
+            }
+        }
+        long start = System.currentTimeMillis();
+        final File outputSz = new File(dir, input.getName() + ".sz");
+        try (FileInputStream is = new FileInputStream(input);
+             FileOutputStream os = new FileOutputStream(outputSz);
+             CompressorOutputStream sos = new CompressorStreamFactory()
+                 .createCompressorOutputStream("snappy-framed", os)) {
+            byte[] b = IOUtils.toByteArray(is);
+            sos.write(b[0]);
+            sos.write(b, 1, b.length - 1); // must be split into multiple compressed chunks
+        }
+        System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+            + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+        try (FileInputStream is = new FileInputStream(input);
+             CompressorInputStream sis = new CompressorStreamFactory()
+                 .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) {
+            byte[] expected = IOUtils.toByteArray(is);
+            byte[] actual = IOUtils.toByteArray(sis);
+            assertArrayEquals(expected, actual);
+        }
+    }
+
+    private void testRoundtrip(File input)  throws Exception {
+        long start = System.currentTimeMillis();
+        final File outputSz = new File(dir, input.getName() + ".sz");
+        try (FileInputStream is = new FileInputStream(input);
+             FileOutputStream os = new FileOutputStream(outputSz);
+             CompressorOutputStream sos = new CompressorStreamFactory()
+                 .createCompressorOutputStream("snappy-framed", os)) {
+            IOUtils.copy(is, sos);
+        }
+        System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+            + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+        try (FileInputStream is = new FileInputStream(input);
+             CompressorInputStream sis = new CompressorStreamFactory()
+                 .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) {
+            byte[] expected = IOUtils.toByteArray(is);
+            byte[] actual = IOUtils.toByteArray(sis);
+            assertArrayEquals(expected, actual);
+        }
+    }
 }


[2/2] commons-compress git commit: COMPRESS-378 record fixes to SnappyCompressorInputStream

Posted by bo...@apache.org.
COMPRESS-378 record fixes to SnappyCompressorInputStream


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4282bc34
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4282bc34
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4282bc34

Branch: refs/heads/master
Commit: 4282bc34d6b64fee1a9f4f55fb14fc6d3fd89521
Parents: 6ac58a5
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 10 15:07:52 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 10 15:07:52 2017 +0100

----------------------------------------------------------------------
 src/changes/changes.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4282bc34/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 9e36923..6e6ee4a 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -44,7 +44,11 @@ The <action> type attribute can be add,update,fix,remove.
   <body>
     <release version="1.14" date="not released, yet"
              description="Release 1.14">
-      <action issue="COMPRESS-246" type="update" date="2017-01-10">
+      <action issue="COMPRESS-378" type="fix" date="2017-01-09">
+        SnappyCompressorInputStream slides the window too early
+        leading to ArrayIndexOutOfBoundsExceptions for some streams.
+      </action>
+      <action issue="COMPRESS-246" type="add" date="2017-01-10">
         Added write support for Snappy.
       </action>
     </release>