You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/01/10 14:08:37 UTC
[1/2] commons-compress git commit: COMPRESS-246 support for framed
Snappy format
Repository: commons-compress
Updated Branches:
refs/heads/master 7b9fdb6e9 -> 4282bc34d
COMPRESS-246 support for framed Snappy format
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/6ac58a59
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/6ac58a59
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/6ac58a59
Branch: refs/heads/master
Commit: 6ac58a5915cb471ae58c13cf667ad81f574dde34
Parents: 7b9fdb6
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 10 15:07:26 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 10 15:07:26 2017 +0100
----------------------------------------------------------------------
src/changes/changes.xml | 3 +
.../compressors/CompressorStreamFactory.java | 8 +-
.../FramedSnappyCompressorInputStream.java | 5 +-
.../FramedSnappyCompressorOutputStream.java | 134 +++++++++++++++++++
src/site/xdoc/index.xml | 9 +-
src/site/xdoc/limitations.xml | 3 +-
.../compressors/FramedSnappyTestCase.java | 58 ++++++++
7 files changed, 214 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 25b3c7f..9e36923 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -44,6 +44,9 @@ The <action> type attribute can be add,update,fix,remove.
<body>
<release version="1.14" date="not released, yet"
description="Release 1.14">
+ <action issue="COMPRESS-246" type="update" date="2017-01-10">
+ Added write support for Snappy.
+ </action>
</release>
<release version="1.13" date="2016-12-29"
description="Release 1.13 - API compatible to 1.12 but requires Java 7 at runtime.">
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 6a46a06..6f09924 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -43,6 +43,7 @@ import org.apache.commons.compress.compressors.lzma.LZMAUtils;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorOutputStream;
import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
+import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorOutputStream;
import org.apache.commons.compress.compressors.snappy.SnappyCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
@@ -497,7 +498,8 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
*
* @param name
* the compressor name, i.e. {@value #GZIP}, {@value #BZIP2},
- * {@value #XZ}, {@value #PACK200} or {@value #DEFLATE}
+ * {@value #XZ}, {@value #PACK200}, {@value SNAPPY_FRAMED}
+ * or {@value #DEFLATE}
* @param out
* the output stream
* @return the compressor output stream
@@ -539,6 +541,10 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return new DeflateCompressorOutputStream(out);
}
+ if (SNAPPY_FRAMED.equalsIgnoreCase(name)) {
+ return new FramedSnappyCompressorOutputStream(out);
+ }
+
} catch (final IOException e) {
throw new CompressorException("Could not create CompressorOutputStream", e);
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
index d7f4209..245d495 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
@@ -43,14 +43,15 @@ public class FramedSnappyCompressorInputStream extends CompressorInputStream {
static final long MASK_OFFSET = 0xa282ead8L;
private static final int STREAM_IDENTIFIER_TYPE = 0xff;
- private static final int COMPRESSED_CHUNK_TYPE = 0;
+ static final int COMPRESSED_CHUNK_TYPE = 0;
private static final int UNCOMPRESSED_CHUNK_TYPE = 1;
private static final int PADDING_CHUNK_TYPE = 0xfe;
private static final int MIN_UNSKIPPABLE_TYPE = 2;
private static final int MAX_UNSKIPPABLE_TYPE = 0x7f;
private static final int MAX_SKIPPABLE_TYPE = 0xfd;
- private static final byte[] SZ_SIGNATURE = new byte[] {
+ // used by FramedSnappyCompressorOutputStream as well
+ static final byte[] SZ_SIGNATURE = new byte[] {
(byte) STREAM_IDENTIFIER_TYPE, // tag
6, 0, 0, // length
's', 'N', 'a', 'P', 'p', 'Y'
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
new file mode 100644
index 0000000..2d9f2be
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.snappy;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.commons.compress.compressors.CompressorOutputStream;
+
+/**
+ * CompressorOutputStream for the framing Snappy format.
+ *
+ * <p>Based on the "spec" in the version "Last revised: 2013-10-25"</p>
+ *
+ * @see <a href="https://github.com/google/snappy/blob/master/framing_format.txt">Snappy framing format description</a>
+ * @since 1.14
+ * @NotThreadSafe
+ */
+public class FramedSnappyCompressorOutputStream extends CompressorOutputStream {
+ // see spec:
+ // > However, we place an additional restriction that the uncompressed data
+ // > in a chunk must be no longer than 65536 bytes. This allows consumers to
+ // > easily use small fixed-size buffers.
+ private static final int MAX_COMPRESSED_BUFFER_SIZE = 1 << 16;
+
+ private final OutputStream out;
+ private final PureJavaCrc32C checksum = new PureJavaCrc32C();
+ // used in one-arg write method
+ private final byte[] oneByte = new byte[1];
+ private final byte[] buffer = new byte[MAX_COMPRESSED_BUFFER_SIZE];
+ private int currentIndex = 0;
+
+ /**
+ * Constructs a new output stream that compresses
+ * snappy-framed-compressed data to the specified output stream.
+ * @param out the OutputStream to which to write the compressed data
+ * @throws IOException if writing the signature fails
+ */
+ public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException {
+ this.out = out;
+ out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ oneByte[0] = (byte) (b & 0xff);
+ write(oneByte);
+ }
+
+ @Override
+ public void write(byte[] data, int off, int len) throws IOException {
+ if (currentIndex + len > MAX_COMPRESSED_BUFFER_SIZE) {
+ flushBuffer();
+ while (len > MAX_COMPRESSED_BUFFER_SIZE) {
+ System.arraycopy(data, off, buffer, 0, MAX_COMPRESSED_BUFFER_SIZE);
+ off += MAX_COMPRESSED_BUFFER_SIZE;
+ len -= MAX_COMPRESSED_BUFFER_SIZE;
+ currentIndex = MAX_COMPRESSED_BUFFER_SIZE;
+ flushBuffer();
+ }
+ }
+ System.arraycopy(data, off, buffer, currentIndex, len);
+ currentIndex += len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ finish();
+ out.close();
+ }
+
+ /**
+ * Compresses all remaining data and writes it to the stream,
+ * doesn't close the underlying stream.
+ * @throws IOException if an error occurs
+ */
+ public void finish() throws IOException {
+ if (currentIndex > 0) {
+ flushBuffer();
+ }
+ }
+
+ private void flushBuffer() throws IOException {
+ out.write(FramedSnappyCompressorInputStream.COMPRESSED_CHUNK_TYPE);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex)) {
+ o.write(buffer, 0, currentIndex);
+ }
+ byte[] b = baos.toByteArray();
+ writeLittleEndian(3, b.length + 4 /* CRC */);
+ writeCrc();
+ out.write(b);
+ currentIndex = 0;
+ }
+
+ private void writeLittleEndian(final int numBytes, long num) throws IOException {
+ for (int i = 0; i < numBytes; i++) {
+ out.write((int) (num & 0xff));
+ num >>= 8;
+ }
+ }
+
+ private void writeCrc() throws IOException {
+ checksum.update(buffer, 0, currentIndex);
+ writeLittleEndian(4, mask(checksum.getValue()));
+ checksum.reset();
+ }
+
+ static long mask(long x) {
+ // ugly, maybe we should just have used ints and deal with the
+ // overflow
+ x = ((x >> 15) | (x << 17));
+ x += FramedSnappyCompressorInputStream.MASK_OFFSET;
+ x &= 0xffffFFFFL;
+ return x;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/site/xdoc/index.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index 6aa203a..60c0184 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -57,6 +57,11 @@
of changes see the <a href="changes-report.html">Changes
Report</a>.</p>
+ <subsection name="What's coming in 1.14?">
+ <ul>
+ <li>Added support for writing the Snappy format</li>
+ </ul>
+ </subsection>
<subsection name="What's new in 1.13?">
<ul>
<li>The 7z package as well as
@@ -93,8 +98,8 @@
by the <code>java.util.jar</code> package of the Java
class library. XZ and lzma support is provided by the public
domain <a href="http://tukaani.org/xz/java.html">XZ for
- Java</a> library. As of Commons Compress 1.13 support for
- the Z and Snappy formats is read-only.</p>
+ Java</a> library. As of Commons Compress 1.14 support for
+ the Z format is read-only.</p>
<p>The ar, arj, cpio, dump, tar, 7z and zip formats are supported as
archivers where the <a href="zip.html">zip</a>
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/site/xdoc/limitations.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/limitations.xml b/src/site/xdoc/limitations.xml
index c5c2be4..45c3a27 100644
--- a/src/site/xdoc/limitations.xml
+++ b/src/site/xdoc/limitations.xml
@@ -120,7 +120,8 @@
</section>
<section name="SNAPPY">
<ul>
- <li>read-only support</li>
+ <li>Commons Compress 1.13 and earlier only support reading
+ the format</li>
</ul>
</section>
<section name="TAR">
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/6ac58a59/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java b/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
index 9af2518..b85db74 100644
--- a/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
@@ -26,6 +26,8 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Random;
import org.apache.commons.compress.AbstractTestCase;
import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
@@ -95,4 +97,60 @@ public final class FramedSnappyTestCase
}
}
+ @Test
+ public void testRoundtrip() throws Exception {
+ testRoundtrip(getFile("test.txt"));
+ testRoundtrip(getFile("bla.tar"));
+ testRoundtrip(getFile("COMPRESS-256.7z"));
+ }
+
+ @Test
+ public void testRoundtripWithOneBigWrite() throws Exception {
+ Random r = new Random();
+ File input = new File(dir, "bigChunkTest");
+ try (FileOutputStream fs = new FileOutputStream(input)) {
+ for (int i = 0 ; i < 1 << 17; i++) {
+ fs.write(r.nextInt(256));
+ }
+ }
+ long start = System.currentTimeMillis();
+ final File outputSz = new File(dir, input.getName() + ".sz");
+ try (FileInputStream is = new FileInputStream(input);
+ FileOutputStream os = new FileOutputStream(outputSz);
+ CompressorOutputStream sos = new CompressorStreamFactory()
+ .createCompressorOutputStream("snappy-framed", os)) {
+ byte[] b = IOUtils.toByteArray(is);
+ sos.write(b[0]);
+ sos.write(b, 1, b.length - 1); // must be split into multiple compressed chunks
+ }
+ System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+ + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+ try (FileInputStream is = new FileInputStream(input);
+ CompressorInputStream sis = new CompressorStreamFactory()
+ .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) {
+ byte[] expected = IOUtils.toByteArray(is);
+ byte[] actual = IOUtils.toByteArray(sis);
+ assertArrayEquals(expected, actual);
+ }
+ }
+
+ private void testRoundtrip(File input) throws Exception {
+ long start = System.currentTimeMillis();
+ final File outputSz = new File(dir, input.getName() + ".sz");
+ try (FileInputStream is = new FileInputStream(input);
+ FileOutputStream os = new FileOutputStream(outputSz);
+ CompressorOutputStream sos = new CompressorStreamFactory()
+ .createCompressorOutputStream("snappy-framed", os)) {
+ IOUtils.copy(is, sos);
+ }
+ System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+ + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+ try (FileInputStream is = new FileInputStream(input);
+ CompressorInputStream sis = new CompressorStreamFactory()
+ .createCompressorInputStream("snappy-framed", new FileInputStream(outputSz))) {
+ byte[] expected = IOUtils.toByteArray(is);
+ byte[] actual = IOUtils.toByteArray(sis);
+ assertArrayEquals(expected, actual);
+ }
+ }
}
[2/2] commons-compress git commit: COMPRESS-378 record fixes to
SnappyCompressorInputStream
Posted by bo...@apache.org.
COMPRESS-378 record fixes to SnappyCompressorInputStream
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4282bc34
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4282bc34
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4282bc34
Branch: refs/heads/master
Commit: 4282bc34d6b64fee1a9f4f55fb14fc6d3fd89521
Parents: 6ac58a5
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 10 15:07:52 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 10 15:07:52 2017 +0100
----------------------------------------------------------------------
src/changes/changes.xml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4282bc34/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 9e36923..6e6ee4a 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -44,7 +44,11 @@ The <action> type attribute can be add,update,fix,remove.
<body>
<release version="1.14" date="not released, yet"
description="Release 1.14">
- <action issue="COMPRESS-246" type="update" date="2017-01-10">
+ <action issue="COMPRESS-378" type="fix" date="2017-01-09">
+ SnappyCompressorInputStream slides the window too early
+ leading to ArrayIndexOutOfBoundsExceptions for some streams.
+ </action>
+ <action issue="COMPRESS-246" type="add" date="2017-01-10">
Added write support for Snappy.
</action>
</release>