You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/01/25 05:36:34 UTC
commons-compress git commit: COMPRESS-271 frame header checksums
Repository: commons-compress
Updated Branches:
refs/heads/master b5d6f1f62 -> 76d913a3c
COMPRESS-271 frame header checksums
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/76d913a3
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/76d913a3
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/76d913a3
Branch: refs/heads/master
Commit: 76d913a3cd2224a1e6a13af77f77709ee1b265c6
Parents: b5d6f1f
Author: Stefan Bodewig <bo...@apache.org>
Authored: Wed Jan 25 06:36:12 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Wed Jan 25 06:36:12 2017 +0100
----------------------------------------------------------------------
.../lz4/FramedLZ4CompressorInputStream.java | 20 +++++++++++++---
.../lz4/FramedLZ4CompressorOutputStream.java | 18 ++++++++++----
.../lz4/FramedLZ4CompressorInputStreamTest.java | 25 ++++++++++++++++----
3 files changed, 51 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index a7f4a4d..7cbd688 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -80,6 +80,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
private InputStream currentBlock;
private boolean endReached, inUncompressed;
+ // used for frame header checksum and content checksum, if present
+ private final XXHash32 contentHash = new XXHash32();
+
/**
* Creates a new input stream that decompresses streams compressed
* using the LZ4 frame format.
@@ -139,6 +142,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
if (flags == -1) {
throw new IOException("Premature end of stream while reading frame flags");
}
+ contentHash.update(flags);
if ((flags & VERSION_MASK) != SUPPORTED_VERSION) {
throw new IOException("Unsupported version " + (flags >> 6));
}
@@ -148,19 +152,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0;
expectContentSize = (flags & CONTENT_SIZE_MASK) != 0;
expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0;
- if (readOneByte() == -1) { // max size is irrelevant for this implementation
+ int bdByte = readOneByte();
+ if (bdByte == -1) { // max size is irrelevant for this implementation
throw new IOException("Premature end of stream while reading frame BD byte");
}
+ contentHash.update(bdByte);
if (expectContentSize) { // for now we don't care, contains the uncompressed size
- int skipped = (int) IOUtils.skip(in, 8);
+ byte[] contentSize = new byte[8];
+ int skipped = (int) IOUtils.readFully(in, contentSize);
count(skipped);
if (8 != skipped) {
throw new IOException("Premature end of stream while reading content size");
}
+ contentHash.update(contentSize, 0, contentSize.length);
}
- if (readOneByte() == -1) { // partial hash of header. not supported, yet
+ int headerHash = readOneByte();
+ if (headerHash == -1) { // partial hash of header.
throw new IOException("Premature end of stream while reading frame header checksum");
}
+ int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff);
+ contentHash.reset();
+ if (headerHash != expectedHash) {
+ throw new IOException("frame header checksum mismatch.");
+ }
}
private void nextBlock() throws IOException {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
index f240e0b..e0622e1 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -40,7 +40,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
/*
* TODO before releasing 1.14:
*
- * + xxhash32 checksum creation for headers, content, blocks
+ * + xxhash32 checksum creation for content, blocks
* + block dependence
*/
@@ -57,6 +57,9 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
private boolean finished = false;
private int currentIndex = 0;
+ // used for frame header checksum and content checksum, if present
+ private final XXHash32 contentHash = new XXHash32();
+
/**
* Constructs a new output stream that compresses data using the
* LZ4 frame format using the default block size of 4MB.
@@ -129,10 +132,15 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream {
}
private void writeFrameDescriptor() throws IOException {
- out.write(FramedLZ4CompressorInputStream.SUPPORTED_VERSION
- | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK);
- out.write(BLOCK_SIZES.indexOf(blockData.length) << 4);
- out.write(0); // TODO header checksum
+ int flags = FramedLZ4CompressorInputStream.SUPPORTED_VERSION
+ | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK;
+ out.write(flags);
+ contentHash.update(flags);
+ int bd = BLOCK_SIZES.indexOf(blockData.length) << 4;
+ out.write(bd);
+ contentHash.update(bd);
+ out.write((int) ((contentHash.getValue() >> 8) & 0xff));
+ contentHash.reset();
}
private void flushBlock() throws IOException {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 3727518..0893fbe 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -166,12 +166,29 @@ public final class FramedLZ4CompressorInputStreamTest
}
@Test
+ public void rejectsFileWithBadHeaderChecksum() throws IOException {
+ byte[] input = new byte[] {
+ 4, 0x22, 0x4d, 0x18, // signature
+ 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum
+ 0x70, // block size 4MB
+ 0,
+ };
+ try {
+ try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) {
+ fail("expected exception");
+ }
+ } catch (IOException ex) {
+ assertThat(ex.getMessage(), containsString("header checksum mismatch"));
+ }
+ }
+
+ @Test
public void readsUncompressedBlocks() throws IOException {
byte[] input = new byte[] {
4, 0x22, 0x4d, 0x18, // signature
0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
0x70, // block size 4MB
- 0x00, // checksum, revisit once it gets validated
+ 115, // checksum
13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
0, 0, 0, 0, // empty block marker
@@ -190,7 +207,7 @@ public final class FramedLZ4CompressorInputStreamTest
4, 0x22, 0x4d, 0x18, // signature
0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum
0x70, // block size 4MB
- 0x00, // checksum, revisit once it gets validated
+ 115, // checksum
13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
0, 0, 0, 0, // empty block marker
@@ -207,7 +224,7 @@ public final class FramedLZ4CompressorInputStreamTest
4, 0x22, 0x4d, 0x18, // signature
0x70, // flag - Version 01, block independent, with block checksum, no content size, no content checksum
0x70, // block size 4MB
- 0x00, // checksum, revisit once it gets validated
+ 114, // checksum
13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
};
@@ -227,7 +244,7 @@ public final class FramedLZ4CompressorInputStreamTest
4, 0x22, 0x4d, 0x18, // signature
0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum
0x70, // block size 4MB
- 0x00, // checksum, revisit once it gets validated
+ (byte) 185, // checksum
13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
0, 0, 0, 0, // empty block marker