You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/02/05 11:55:52 UTC
commons-compress git commit: COMPRESS-271 support for
decompressConcatenated in lz4 frame input
Repository: commons-compress
Updated Branches:
refs/heads/master 3af95ce3a -> 25ecc9b17
COMPRESS-271 support for decompressConcatenated in lz4 frame input
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/25ecc9b1
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/25ecc9b1
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/25ecc9b1
Branch: refs/heads/master
Commit: 25ecc9b177ef21bcc35a82776b71cbdae4a7f708
Parents: 3af95ce
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Feb 5 12:55:12 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Feb 5 12:55:12 2017 +0100
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 2 +-
.../lz4/FramedLZ4CompressorInputStream.java | 45 ++++++--
.../lz4/FramedLZ4CompressorInputStreamTest.java | 105 +++++++++++++++++++
3 files changed, 143 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 29ba9fd..4f0685b 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -518,7 +518,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (LZ4_FRAMED.equalsIgnoreCase(name)) {
- return new FramedLZ4CompressorInputStream(in);
+ return new FramedLZ4CompressorInputStream(in, actualDecompressConcatenated);
}
} catch (final IOException e) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 418f320..e597b5a 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -40,9 +40,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
/*
* TODO before releasing 1.14:
*
- * + xxhash32 checksum validation
* + skippable frames
- * + decompressConcatenated
* + block dependence
*/
@@ -71,6 +69,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
};
private final InputStream in;
+ private final boolean decompressConcatenated;
private boolean expectBlockChecksum;
private boolean expectContentSize;
@@ -84,15 +83,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
/**
* Creates a new input stream that decompresses streams compressed
- * using the LZ4 frame format.
+ * using the LZ4 frame format and stops after decompressing the
+ * first frame.
* @param in the InputStream from which to read the compressed data
* @throws IOException if reading fails
*/
public FramedLZ4CompressorInputStream(InputStream in) throws IOException {
+ this(in, false);
+ }
+
+ /**
+ * Creates a new input stream that decompresses streams compressed
+ * using the LZ4 frame format.
+ * @param in the InputStream from which to read the compressed data
+ * @param decompressConcatenated if true, decompress until the end
+ * of the input; if false, stop after the first LZ4 frame
+ * and leave the input position to point to the next byte
+ * after the frame stream
+ * @throws IOException if reading fails
+ */
+ public FramedLZ4CompressorInputStream(InputStream in, boolean decompressConcatenated) throws IOException {
this.in = in;
- readSignature();
- readFrameDescriptor();
- nextBlock();
+ this.decompressConcatenated = decompressConcatenated;
+ init(true);
}
/** {@inheritDoc} */
@@ -130,13 +143,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
return r;
}
- private void readSignature() throws IOException {
+ private void init(boolean firstFrame) throws IOException {
+ if (readSignature(firstFrame)) {
+ readFrameDescriptor();
+ nextBlock();
+ }
+ }
+
+ private boolean readSignature(boolean firstFrame) throws IOException {
final byte[] b = new byte[4];
final int read = IOUtils.readFully(in, b);
count(read);
+ if (4 != read && !firstFrame) {
+ endReached = true;
+ return false;
+ }
if (4 != read || !matches(b, 4)) {
throw new IOException("Not a LZ4 frame stream");
}
+ return true;
}
private void readFrameDescriptor() throws IOException {
@@ -185,8 +210,12 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0;
int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK));
if (realLen == 0) {
- endReached = true;
verifyContentChecksum();
+ if (!decompressConcatenated) {
+ endReached = true;
+ } else {
+ init(false);
+ }
return;
}
InputStream capped = new BoundedInputStream(in, realLen);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 3f5c701..0363e5e 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -27,6 +27,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.commons.compress.AbstractTestCase;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
@@ -82,6 +83,86 @@ public final class FramedLZ4CompressorInputStreamTest
}
}
+ @Test
+ public void readBlaLz4WithDecompressConcatenated() throws IOException {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.lz4")), true);
+ FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+ byte[] expected = IOUtils.toByteArray(e);
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(expected, actual);
+ }
+ }
+
+ @Test
+ public void readDoubledBlaLz4WithDecompressConcatenatedTrue() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new FramedLZ4CompressorInputStream(in, true);
+ }
+ }, true);
+ }
+
+ @Test
+ public void readDoubledBlaLz4WithDecompressConcatenatedFalse() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new FramedLZ4CompressorInputStream(in, false);
+ }
+ }, false);
+ }
+
+ @Test
+ public void readDoubledBlaLz4WithoutExplicitDecompressConcatenated() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new FramedLZ4CompressorInputStream(in);
+ }
+ }, false);
+ }
+
+ @Test
+ public void readBlaLz4ViaFactoryWithDecompressConcatenated() throws Exception {
+ try (InputStream a = new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(),
+ new FileInputStream(getFile("bla.tar.lz4")),
+ true);
+ FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+ byte[] expected = IOUtils.toByteArray(e);
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(expected, actual);
+ }
+ }
+
+ @Test
+ public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedTrue() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, true);
+ }
+ }, true);
+ }
+
+ @Test
+ public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedFalse() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, false);
+ }
+ }, false);
+ }
+
+ @Test
+ public void readDoubledBlaLz4ViaFactoryWithoutExplicitDecompressConcatenated() throws Exception {
+ readDoubledBlaLz4(new StreamWrapper() {
+ public InputStream wrap(InputStream in) throws Exception {
+ return new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in);
+ }
+ }, false);
+ }
+
@Test(expected = IOException.class)
public void rejectsNonLZ4Stream() throws IOException {
try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar")))) {
@@ -280,4 +361,28 @@ public final class FramedLZ4CompressorInputStreamTest
assertThat(ex.getMessage(), containsString("content checksum mismatch"));
}
}
+
+ interface StreamWrapper {
+ InputStream wrap(InputStream in) throws Exception;
+ }
+
+ private void readDoubledBlaLz4(StreamWrapper wrapper, boolean expectDuplicateOutput) throws Exception {
+ byte[] singleInput;
+ try (InputStream i = new FileInputStream(getFile("bla.tar.lz4"))) {
+ singleInput = IOUtils.toByteArray(i);
+ }
+ byte[] input = duplicate(singleInput);
+ try (InputStream a = wrapper.wrap(new ByteArrayInputStream(input));
+ FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+ byte[] expected = IOUtils.toByteArray(e);
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(expectDuplicateOutput ? duplicate(expected) : expected, actual);
+ }
+ }
+
+ private static byte[] duplicate(byte[] from) {
+ byte[] to = Arrays.copyOf(from, 2 * from.length);
+ System.arraycopy(from, 0, to, from.length, from.length);
+ return to;
+ }
}