You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/02/09 16:37:34 UTC
[1/2] commons-compress git commit: COMPRESS-271 verify block checksums
Repository: commons-compress
Updated Branches:
refs/heads/master 72fec65e1 -> e79465bbe
COMPRESS-271 verify block checksums
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/982ce0ec
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/982ce0ec
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/982ce0ec
Branch: refs/heads/master
Commit: 982ce0eca9935e65915278ae07febd3a1d52273d
Parents: 72fec65
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Feb 9 17:37:01 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Feb 9 17:37:01 2017 +0100
----------------------------------------------------------------------
.../lz4/FramedLZ4CompressorInputStream.java | 38 +++++---
.../utils/ChecksumCalculatingInputStream.java | 97 ++++++++++++++++++++
2 files changed, 120 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/982ce0ec/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 9b47d7b..8bf49ac 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.utils.BoundedInputStream;
import org.apache.commons.compress.utils.ByteUtils;
+import org.apache.commons.compress.utils.ChecksumCalculatingInputStream;
import org.apache.commons.compress.utils.IOUtils;
/**
@@ -80,6 +81,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
// used for frame header checksum and content checksum, if present
private final XXHash32 contentHash = new XXHash32();
+ // used for block checksum, if present
+ private final XXHash32 blockHash = new XXHash32();
+
// only created if the frame doesn't set the block independence flag
private byte[] blockDependencyBuffer;
@@ -239,6 +243,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
return;
}
InputStream capped = new BoundedInputStream(in, realLen);
+ if (expectBlockChecksum) {
+ capped = new ChecksumCalculatingInputStream(blockHash, capped);
+ }
if (uncompressed) {
inUncompressed = true;
currentBlock = capped;
@@ -257,31 +264,32 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
currentBlock.close();
currentBlock = null;
if (expectBlockChecksum) {
- int skipped = (int) IOUtils.skip(in, 4);
- count(skipped);
- if (4 != skipped) {
- throw new IOException("Premature end of stream while reading block checksum");
- }
+ verifyChecksum(blockHash, "block");
+ blockHash.reset();
}
}
}
private void verifyContentChecksum() throws IOException {
if (expectContentChecksum) {
- byte[] checksum = new byte[4];
- int read = IOUtils.readFully(in, checksum);
- count(read);
- if (4 != read) {
- throw new IOException("Premature end of stream while reading content checksum");
- }
- long expectedHash = contentHash.getValue();
- if (expectedHash != ByteUtils.fromLittleEndian(checksum)) {
- throw new IOException("content checksum mismatch.");
- }
+ verifyChecksum(contentHash, "content");
}
contentHash.reset();
}
+ private void verifyChecksum(XXHash32 hash, String kind) throws IOException {
+ byte[] checksum = new byte[4];
+ int read = IOUtils.readFully(in, checksum);
+ count(read);
+ if (4 != read) {
+ throw new IOException("Premature end of stream while reading " + kind + " checksum");
+ }
+ long expectedHash = hash.getValue();
+ if (expectedHash != ByteUtils.fromLittleEndian(checksum)) {
+ throw new IOException(kind + " checksum mismatch.");
+ }
+ }
+
private int readOneByte() throws IOException {
final int b = in.read();
if (b != -1) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/982ce0ec/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java b/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java
new file mode 100644
index 0000000..a698196
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.utils;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.Checksum;
+
+/**
+ * A stream that calculates the checksum of the data read.
+ * @NotThreadSafe
+ * @since 1.14
+ */
+public class ChecksumCalculatingInputStream extends InputStream {
+ private final InputStream in;
+ private final Checksum checksum;
+
+ public ChecksumCalculatingInputStream(final Checksum checksum, final InputStream in) {
+ this.checksum = checksum;
+ this.in = in;
+ }
+
+ /**
+ * Reads a single byte from the stream
+ * @throws IOException if the underlying stream throws or the
+ * stream is exhausted and the Checksum doesn't match the expected
+ * value
+ */
+ @Override
+ public int read() throws IOException {
+ final int ret = in.read();
+ if (ret >= 0) {
+ checksum.update(ret);
+ }
+ return ret;
+ }
+
+ /**
+ * Reads a byte array from the stream
+ * @throws IOException if the underlying stream throws or the
+ * stream is exhausted and the Checksum doesn't match the expected
+ * value
+ */
+ @Override
+ public int read(final byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ /**
+ * Reads from the stream into a byte array.
+ * @throws IOException if the underlying stream throws or the
+ * stream is exhausted and the Checksum doesn't match the expected
+ * value
+ */
+ @Override
+ public int read(final byte[] b, final int off, final int len) throws IOException {
+ final int ret = in.read(b, off, len);
+ if (ret >= 0) {
+ checksum.update(b, off, ret);
+ }
+ return ret;
+ }
+
+ @Override
+ public long skip(final long n) throws IOException {
+ // Can't really skip, we have to hash everything to verify the checksum
+ if (read() >= 0) {
+ return 1;
+ }
+ return 0;
+ }
+
+ /**
+ * Returns the calculated checksum.
+ * @return the calculated checksum.
+ */
+ public long getValue() {
+ return checksum.getValue();
+ }
+
+}
[2/2] commons-compress git commit: tweaks
Posted by bo...@apache.org.
tweaks
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/e79465bb
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/e79465bb
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/e79465bb
Branch: refs/heads/master
Commit: e79465bbecc6033780f91fd41ab7d66c5e9e7bae
Parents: 982ce0e
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Feb 9 17:37:17 2017 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Feb 9 17:37:17 2017 +0100
----------------------------------------------------------------------
.../lz4/FramedLZ4CompressorInputStream.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/e79465bb/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 8bf49ac..2f1eabb 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -143,11 +143,13 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
r = readOnce(b, off, len);
}
}
- if (expectBlockDependency) {
- appendToBlockDependencyBuffer(b, off, r);
- }
- if (expectContentChecksum && r != -1) {
- contentHash.update(b, off, r);
+ if (r != -1) {
+ if (expectBlockDependency) {
+ appendToBlockDependencyBuffer(b, off, r);
+ }
+ if (expectContentChecksum) {
+ contentHash.update(b, off, r);
+ }
}
return r;
}
@@ -165,6 +167,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
int read = IOUtils.readFully(in, b);
count(read);
if (0 == read && !firstFrame) {
+ // good LZ4 frame and nothing after it
endReached = true;
return false;
}
@@ -174,6 +177,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream {
read = skipSkippableFrame(b);
if (0 == read && !firstFrame) {
+ // good LZ4 frame with only some skippable frames after it
endReached = true;
return false;
}