You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/01/24 07:04:07 UTC
[2/2] commons-compress git commit: COMPRESS-271 incomplete output
stream for LZ4 frame format
COMPRESS-271 incomplete output stream for LZ4 frame format
the resulting output will get rejected by other implementations as the
header checksum is incorrect. xxhash32 is up next.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/2820db32
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/2820db32
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/2820db32
Branch: refs/heads/master
Commit: 2820db32e3fdeff47fbf6cb4be39eeebc1ed287e
Parents: 48d7def
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 24 08:02:26 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 24 08:02:26 2017 +0100
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 9 +-
.../lz4/FramedLZ4CompressorOutputStream.java | 162 +++++++++++++++++++
.../FramedLZ4CompressorOutputStreamTest.java | 32 ++++
.../lz4/FramedLZ4CompressorRoundtripTest.java | 102 ++++++++++++
4 files changed, 303 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2820db32/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 64cac10..29ba9fd 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -40,6 +40,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream;
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
+import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream;
import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream;
import org.apache.commons.compress.compressors.lzma.LZMAUtils;
@@ -538,7 +539,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
* @param name
* the compressor name, i.e. {@value #GZIP}, {@value #BZIP2},
* {@value #XZ}, {@value #PACK200}, {@value SNAPPY_FRAMED},
- * {@value LZ4_BLOCK}
+ * {@value LZ4_BLOCK}, {@value LZ4_FRAMED}
* or {@value #DEFLATE}
* @param out
* the output stream
@@ -589,6 +590,10 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return new BlockLZ4CompressorOutputStream(out);
}
+ if (LZ4_FRAMED.equalsIgnoreCase(name)) {
+ return new FramedLZ4CompressorOutputStream(out);
+ }
+
} catch (final IOException e) {
throw new CompressorException("Could not create CompressorOutputStream", e);
}
@@ -632,7 +637,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
@Override
public Set<String> getOutputStreamCompressorNames() {
- return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, LZ4_BLOCK);
+ return Sets.newHashSet(GZIP, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_FRAMED, LZ4_BLOCK, LZ4_FRAMED);
}
/**
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2820db32/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
new file mode 100644
index 0000000..f240e0b
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.lz4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.compress.compressors.CompressorOutputStream;
+import org.apache.commons.compress.utils.ByteUtils;
+
+/**
+ * CompressorOutputStream for the LZ4 frame format.
+ *
+ * <p>Based on the "spec" in the version "1.5.1 (31/03/2015)"</p>
+ *
+ * @see <a href="http://lz4.github.io/lz4/lz4_Frame_format.html">LZ4 Frame Format Description</a>
+ * @since 1.14
+ * @NotThreadSafe
+ */
+public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
+ /*
+ * TODO before releasing 1.14:
+ *
+ * + xxhash32 checksum creation for headers, content, blocks
+ * + block dependence
+ */
+
+ private static final int DEFAULT_BLOCK_SIZE = 4096 * 1024;
+ private static final List<Integer> BLOCK_SIZES = Arrays.asList(64 * 1024, 256 * 1024, 1024 * 1024,
+ DEFAULT_BLOCK_SIZE);
+ private static final byte[] END_MARK = new byte[4];
+
+ // used in one-arg write method
+ private final byte[] oneByte = new byte[1];
+
+ private final byte[] blockData;
+ private final OutputStream out;
+ private boolean finished = false;
+ private int currentIndex = 0;
+
+ /**
+ * Constructs a new output stream that compresses data using the
+ * LZ4 frame format using the default block size of 4MB.
+ * @param out the OutputStream to which to write the compressed data
+ * @throws IOException if writing the signature fails
+ */
+ public FramedLZ4CompressorOutputStream(OutputStream out) throws IOException {
+ this(out, DEFAULT_BLOCK_SIZE);
+ }
+
+ /**
+ * Constructs a new output stream that compresses data using the
+ * LZ4 frame format using the given block size.
+ * @param out the OutputStream to which to write the compressed data
+ * @param blockSize block size, one of 64 KB, 256 KB, 1 MB or 4 MB.
+ * @throws IOException if writing the signature fails
+ * @throws IllegalArgumentException if the block size is not supported
+ */
+ public FramedLZ4CompressorOutputStream(OutputStream out, int blockSize) throws IOException {
+ if (!BLOCK_SIZES.contains(blockSize)) {
+ throw new IllegalArgumentException("Unsupported block size");
+ }
+ blockData = new byte[blockSize];
+ this.out = out;
+ out.write(FramedLZ4CompressorInputStream.LZ4_SIGNATURE);
+ writeFrameDescriptor();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ oneByte[0] = (byte) (b & 0xff);
+ write(oneByte);
+ }
+
+ @Override
+ public void write(byte[] data, int off, int len) throws IOException {
+ if (currentIndex + len > blockData.length) {
+ flushBlock();
+ while (len > blockData.length) {
+ System.arraycopy(data, off, blockData, 0, blockData.length);
+ off += blockData.length;
+ len -= blockData.length;
+ currentIndex = blockData.length;
+ flushBlock();
+ }
+ }
+ System.arraycopy(data, off, blockData, currentIndex, len);
+ currentIndex += len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ finish();
+ out.close();
+ }
+
+ /**
+ * Compresses all remaining data and writes it to the stream,
+ * doesn't close the underlying stream.
+ * @throws IOException if an error occurs
+ */
+ public void finish() throws IOException {
+ if (!finished) {
+ if (currentIndex > 0) {
+ flushBlock();
+ }
+ writeTrailer();
+ finished = true;
+ }
+ }
+
+ private void writeFrameDescriptor() throws IOException {
+ out.write(FramedLZ4CompressorInputStream.SUPPORTED_VERSION
+ | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK);
+ out.write(BLOCK_SIZES.indexOf(blockData.length) << 4);
+ out.write(0); // TODO header checksum
+ }
+
+ private void flushBlock() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (OutputStream o = new BlockLZ4CompressorOutputStream(baos)) {
+ o.write(blockData, 0, currentIndex);
+ }
+ byte[] b = baos.toByteArray();
+ if (b.length > currentIndex) { // compression increased size, maybe beyond blocksize
+ ByteUtils.toLittleEndian(out, currentIndex | FramedLZ4CompressorInputStream.UNCOMPRESSED_FLAG_MASK,
+ 4);
+ out.write(blockData, 0, currentIndex);
+ } else {
+ ByteUtils.toLittleEndian(out, b.length, 4);
+ out.write(b);
+ }
+ // TODO block checksum
+ currentIndex = 0;
+ }
+
+ private void writeTrailer() throws IOException {
+ out.write(END_MARK);
+ // TODO content checksum
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2820db32/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java
new file mode 100644
index 0000000..5f2802e
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStreamTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.lz4;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+import org.junit.Test;
+
+public final class FramedLZ4CompressorOutputStreamTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void illegalBlockSize() throws IOException {
+ new FramedLZ4CompressorOutputStream(new ByteArrayOutputStream(), 32 * 1024);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2820db32/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
new file mode 100644
index 0000000..1a743e7
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.lz4;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Random;
+import org.apache.commons.compress.AbstractTestCase;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.commons.compress.compressors.lz77support.Parameters;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase {
+
+ private void roundTripTest(String testFile) throws IOException {
+ File input = getFile(testFile);
+ long start = System.currentTimeMillis();
+ final File outputSz = new File(dir, input.getName() + ".framed.lz4");
+ try (FileInputStream is = new FileInputStream(input);
+ FileOutputStream os = new FileOutputStream(outputSz);
+ FramedLZ4CompressorOutputStream los = new FramedLZ4CompressorOutputStream(os)) {
+ IOUtils.copy(is, los);
+ }
+ System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+ + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+ start = System.currentTimeMillis();
+ try (FileInputStream is = new FileInputStream(input);
+ FramedLZ4CompressorInputStream sis = new FramedLZ4CompressorInputStream(new FileInputStream(outputSz))) {
+ byte[] expected = IOUtils.toByteArray(is);
+ byte[] actual = IOUtils.toByteArray(sis);
+ Assert.assertArrayEquals(expected, actual);
+ }
+ System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms");
+ }
+
+ // should yield decent compression
+ @Test
+ public void blaTarRoundtrip() throws IOException {
+ try {
+ roundTripTest("bla.tar");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ // yields no compression at all
+ @Test
+ public void gzippedLoremIpsumRoundtrip() throws IOException {
+ roundTripTest("lorem-ipsum.txt.gz");
+ }
+
+ @Test
+ public void biggerFileRoundtrip() throws IOException {
+ roundTripTest("COMPRESS-256.7z");
+ }
+
+ @Test
+ public void roundtripViaFactory() throws Exception {
+ File input = getFile("bla.tar");
+ long start = System.currentTimeMillis();
+ final File outputSz = new File(dir, input.getName() + ".framed.lz4");
+ try (FileInputStream is = new FileInputStream(input);
+ FileOutputStream os = new FileOutputStream(outputSz);
+ OutputStream los = new CompressorStreamFactory()
+ .createCompressorOutputStream(CompressorStreamFactory.getLZ4Framed(), os)) {
+ IOUtils.copy(is, los);
+ }
+ System.err.println(input.getName() + " written, uncompressed bytes: " + input.length()
+ + ", compressed bytes: " + outputSz.length() + " after " + (System.currentTimeMillis() - start) + "ms");
+ start = System.currentTimeMillis();
+ try (FileInputStream is = new FileInputStream(input);
+ InputStream sis = new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.LZ4_FRAMED, new FileInputStream(outputSz))) {
+ byte[] expected = IOUtils.toByteArray(is);
+ byte[] actual = IOUtils.toByteArray(sis);
+ Assert.assertArrayEquals(expected, actual);
+ }
+ System.err.println(outputSz.getName() + " read after " + (System.currentTimeMillis() - start) + "ms");
+ }
+}