You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/02/05 18:09:07 UTC
[2/2] commons-compress git commit: COMPRESS-271 add read-support for
LZ4 with block dependency
COMPRESS-271 add read-support for LZ4 with block dependency
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/8c75380e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/8c75380e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/8c75380e
Branch: refs/heads/master
Commit: 8c75380e482d7c9945098a258ebba4fe2eba4264
Parents: dc45445
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Feb 5 19:08:31 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Feb 5 19:08:31 2017 +0100
----------------------------------------------------------------------
.../lz4/FramedLZ4CompressorInputStream.java | 40 ++++++++++++++++----
.../AbstractLZ77CompressorInputStream.java | 21 ++++++++++
.../lz4/FramedLZ4CompressorInputStreamTest.java | 10 +++++
3 files changed, 63 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 2929bae..9b47d7b 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -37,11 +37,6 @@ import org.apache.commons.compress.utils.IOUtils;
* @NotThreadSafe
*/
public class FramedLZ4CompressorInputStream extends CompressorInputStream {
- /*
- * TODO before releasing 1.14:
- *
- * + block dependence
- */
// used by FramedLZ4CompressorOutputStream as well
static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR
@@ -75,6 +70,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
private final boolean decompressConcatenated;
private boolean expectBlockChecksum;
+ private boolean expectBlockDependency;
private boolean expectContentSize;
private boolean expectContentChecksum;
@@ -84,6 +80,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
// used for frame header checksum and content checksum, if present
private final XXHash32 contentHash = new XXHash32();
+ // only created if the frame doesn't set the block independence flag
+ private byte[] blockDependencyBuffer;
+
/**
* Creates a new input stream that decompresses streams compressed
* using the LZ4 frame format and stops after decompressing the
@@ -140,6 +139,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
r = readOnce(b, off, len);
}
}
+ if (expectBlockDependency) {
+ appendToBlockDependencyBuffer(b, off, r);
+ }
if (expectContentChecksum && r != -1) {
contentHash.update(b, off, r);
}
@@ -186,8 +188,13 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
if ((flags & VERSION_MASK) != SUPPORTED_VERSION) {
throw new IOException("Unsupported version " + (flags >> 6));
}
- if ((flags & BLOCK_INDEPENDENCE_MASK) == 0) {
- throw new IOException("Block dependence is not supported");
+ expectBlockDependency = (flags & BLOCK_INDEPENDENCE_MASK) == 0;
+ if (expectBlockDependency) {
+ if (blockDependencyBuffer == null) {
+ blockDependencyBuffer = new byte[BlockLZ4CompressorInputStream.WINDOW_SIZE];
+ }
+ } else {
+ blockDependencyBuffer = null;
}
expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0;
expectContentSize = (flags & CONTENT_SIZE_MASK) != 0;
@@ -237,7 +244,11 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
currentBlock = capped;
} else {
inUncompressed = false;
- currentBlock = new BlockLZ4CompressorInputStream(capped);
+ BlockLZ4CompressorInputStream s = new BlockLZ4CompressorInputStream(capped);
+ if (expectBlockDependency) {
+ s.prefill(blockDependencyBuffer);
+ }
+ currentBlock = s;
}
}
@@ -329,6 +340,19 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
return read;
}
+ private void appendToBlockDependencyBuffer(final byte[] b, final int off, int len) {
+ len = Math.min(len, blockDependencyBuffer.length);
+ if (len > 0) {
+ int keep = blockDependencyBuffer.length - len;
+ if (keep > 0) {
+ // move last keep bytes towards the start of the buffer
+ System.arraycopy(blockDependencyBuffer, len, blockDependencyBuffer, 0, keep);
+ }
+ // append new data
+ System.arraycopy(b, off, blockDependencyBuffer, keep, len);
+ }
+ }
+
/**
* Checks if the signature matches what is expected for a .lz4 file.
*
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
index 49372e4..a406b85 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
@@ -120,6 +120,27 @@ public abstract class AbstractLZ77CompressorInputStream extends CompressorInputS
}
/**
+ * Adds some initial data to fill the window with.
+ *
+ * <p>This is used if the stream has been cut into blocks and
+ * back-references of one block may refer to data of the previous
+ * block(s). One such example is the LZ4 frame format using block
+ * dependency.</p>
+ *
+ * @param data the data to fill the window with.
+ * @throws IllegalStateException if the stream has already started to read data
+ */
+ public void prefill(byte[] data) {
+ if (writeIndex != 0) {
+ throw new IllegalStateException("the stream has already been read from, can't prefill anymore");
+ }
+ int len = Math.min(windowSize, data.length);
+ System.arraycopy(data, data.length - len, buf, 0, len);
+ writeIndex += len;
+ readIndex += len;
+ }
+
+ /**
* Used by subclasses to signal the next block contains the given
* amount of literal data.
* @param length the length of the block
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 9dda432..a1e28b5 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -163,6 +163,16 @@ public final class FramedLZ4CompressorInputStreamTest
}, false);
}
+ @Test
+ public void readBlaDumpLz4() throws IOException {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.dump.lz4")));
+ FileInputStream e = new FileInputStream(getFile("bla.dump"))) {
+ byte[] expected = IOUtils.toByteArray(e);
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(expected, actual);
+ }
+ }
+
@Test(expected = IOException.class)
public void rejectsNonLZ4Stream() throws IOException {
try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar")))) {