You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/02/05 14:21:12 UTC
commons-compress git commit: COMPRESS-271 support for skipping
skippable frames in lz4
Repository: commons-compress
Updated Branches:
refs/heads/master 2239893ab -> 68bc083e6
COMPRESS-271 support for skipping skippable frames in lz4
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/68bc083e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/68bc083e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/68bc083e
Branch: refs/heads/master
Commit: 68bc083e6ba002f7c997f9781f0e24a80d8a3274
Parents: 2239893
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Feb 5 15:20:37 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Feb 5 15:20:37 2017 +0100
----------------------------------------------------------------------
.../lz4/FramedLZ4CompressorInputStream.java | 56 ++++-
.../lz4/FramedLZ4CompressorInputStreamTest.java | 214 +++++++++++++++++++
2 files changed, 266 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 115a2d6..2929bae 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -40,7 +40,6 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
/*
* TODO before releasing 1.14:
*
- * + skippable frames
* + block dependence
*/
@@ -48,6 +47,10 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR
4, 0x22, 0x4d, 0x18
};
+ private static final byte[] SKIPPABLE_FRAME_TRAILER = new byte[] {
+ 0x2a, 0x4d, 0x18
+ };
+ private static final byte SKIPPABLE_FRAME_PREFIX_BYTE_MASK = 0x50;
static final int VERSION_MASK = 0xC0;
static final int SUPPORTED_VERSION = 0x40;
@@ -151,15 +154,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
}
private boolean readSignature(boolean firstFrame) throws IOException {
+ String garbageMessage = firstFrame ? "Not a LZ4 frame stream" : "LZ4 frame stream followed by garbage";
final byte[] b = new byte[4];
- final int read = IOUtils.readFully(in, b);
+ int read = IOUtils.readFully(in, b);
count(read);
- if (4 != read && !firstFrame) {
+ if (0 == read && !firstFrame) {
+ endReached = true;
+ return false;
+ }
+ if (4 != read) {
+ throw new IOException(garbageMessage);
+ }
+
+ read = skipSkippableFrame(b);
+ if (0 == read && !firstFrame) {
endReached = true;
return false;
}
if (4 != read || !matches(b, 4)) {
- throw new IOException("Not a LZ4 frame stream");
+ throw new IOException(garbageMessage);
}
return true;
}
@@ -281,6 +294,41 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
}
}
+ private static boolean isSkippableFrameSignature(byte[] b) {
+ if ((b[0] & SKIPPABLE_FRAME_PREFIX_BYTE_MASK) != SKIPPABLE_FRAME_PREFIX_BYTE_MASK) {
+ return false;
+ }
+ for (int i = 1; i < 4; i++) {
+ if (b[i] != SKIPPABLE_FRAME_TRAILER[i - 1]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Skips over the contents of a skippable frame as well as
+ * skippable frames following it.
+ *
+ * <p>It then tries to read four more bytes which are supposed to
+ * hold an LZ4 signature and returns the number of bytes read
+ * while storing the bytes in the given array.</p>
+ */
+ private int skipSkippableFrame(byte[] b) throws IOException {
+ int read = 4;
+ while (read == 4 && isSkippableFrameSignature(b)) {
+ long len = ByteUtils.fromLittleEndian(supplier, 4);
+ long skipped = IOUtils.skip(in, len);
+ count(skipped);
+ if (len != skipped) {
+ throw new IOException("Premature end of stream while skipping frame");
+ }
+ read = IOUtils.readFully(in, b);
+ count(read);
+ }
+ return read;
+ }
+
/**
* Checks if the signature matches what is expected for a .lz4 file.
*
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 0363e5e..9dda432 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -362,6 +362,220 @@ public final class FramedLZ4CompressorInputStreamTest
}
}
+ @Test
+ public void skipsOverSkippableFrames() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x5f, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, 0, // skippable frame has length 2
+ 1, 2, // content of skippable frame
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 1, 0, 0, (byte) 0x80, // 1 bytes length and uncompressed bit set
+ '!', // content
+ 0, 0, 0, 0, // empty block marker
+ };
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(new byte[] {
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!'
+ }, actual);
+ }
+ }
+
+ @Test
+ public void skipsOverTrailingSkippableFrames() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x51, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, 0, // skippable frame has length 2
+ 1, 2, // content of skippable frame
+ };
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ byte[] actual = IOUtils.toByteArray(a);
+ assertArrayEquals(new byte[] {
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!'
+ }, actual);
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameFollowedByJunk() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, 0, // skippable frame has length 2
+ 1, 2, // content of skippable frame
+ 1, 0x22, 0x4d, 0x18, // bad signature
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("garbage"));
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameFollowedByTooFewBytes() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x52, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, 0, // skippable frame has length 2
+ 1, 2, // content of skippable frame
+ 4, // too short for signature
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("garbage"));
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameWithPrematureEnd() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, 0, // skippable frame has length 2
+ 1, // content of skippable frame (should be two bytes)
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("Premature end of stream while skipping frame"));
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameWithPrematureEndInLengthBytes() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x55, 0x2a, 0x4d, 0x18, // skippable frame signature
+ 2, 0, 0, // should be four byte length
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("premature end of data"));
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameWithBadSignatureTrailer() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x51, 0x2a, 0x4d, 0x17, // broken skippable frame signature
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("garbage"));
+ }
+ }
+
+ @Test
+ public void rejectsSkippableFrameWithBadSignaturePrefix() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x60, 0x2a, 0x4d, 0x18, // broken skippable frame signature
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("garbage"));
+ }
+ }
+
+ @Test
+ public void rejectsTrailingBytesAfterValidFrame() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
+ 0x70, // block size 4MB
+ 115, // checksum
+ 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+ 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+ 0, 0, 0, 0, // empty block marker
+ 0x56, 0x2a, 0x4d, // too short for any signature
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) {
+ IOUtils.toByteArray(a);
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("garbage"));
+ }
+ }
+
interface StreamWrapper {
InputStream wrap(InputStream in) throws Exception;
}