You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2018/01/09 17:40:54 UTC
[01/28] commons-compress git commit: COMPRESS-380 initial patch by
Christian Marquez Grabia
Repository: commons-compress
Updated Branches:
refs/heads/master 7cf10d71e -> b18ef2a10
COMPRESS-380 initial patch by Christian Marquez Grabia
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/d07f04b7
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/d07f04b7
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/d07f04b7
Branch: refs/heads/master
Commit: d07f04b73c06c45b75f53f38b03b25c987b7bca5
Parents: e35049e
Author: Stefan Bodewig <bo...@apache.org>
Authored: Wed Jan 3 15:14:32 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Wed Jan 3 15:14:32 2018 +0100
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipFile.java | 4 +-
.../commons/compress/archivers/zip/ZipUtil.java | 2 +
.../compressors/zip/Deflate64InputStream.java | 71 +++
.../compressors/zip/HuffmanDecoder.java | 445 +++++++++++++++++++
.../compress/compressors/zip/HuffmanState.java | 8 +
.../zip/Deflate64InputStreamTest.java | 111 +++++
.../compressors/zip/HuffmanDecoderTest.java | 205 +++++++++
7 files changed, 845 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index 9b13cf3..b7a41a3 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -42,6 +42,7 @@ import java.util.zip.InflaterInputStream;
import java.util.zip.ZipException;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.zip.Deflate64InputStream;
import org.apache.commons.compress.utils.IOUtils;
import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD;
@@ -502,8 +503,9 @@ public class ZipFile implements Closeable {
};
case BZIP2:
return new BZip2CompressorInputStream(bis);
- case AES_ENCRYPTED:
case ENHANCED_DEFLATED:
+ return new Deflate64InputStream(bis, ze.getSize());
+ case AES_ENCRYPTED:
case EXPANDING_LEVEL_1:
case EXPANDING_LEVEL_2:
case EXPANDING_LEVEL_3:
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
index 97fd341..d22a62e 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
@@ -295,6 +295,7 @@ public abstract class ZipUtil {
}
return null;
}
+
static void copy(final byte[] from, final byte[] to, final int offset) {
if (from != null) {
System.arraycopy(from, 0, to, offset, from.length);
@@ -330,6 +331,7 @@ public abstract class ZipUtil {
|| entry.getMethod() == ZipMethod.UNSHRINKING.getCode()
|| entry.getMethod() == ZipMethod.IMPLODING.getCode()
|| entry.getMethod() == ZipEntry.DEFLATED
+ || entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode()
|| entry.getMethod() == ZipMethod.BZIP2.getCode();
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java b/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
new file mode 100644
index 0000000..aeceb2a
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
@@ -0,0 +1,71 @@
+package org.apache.commons.compress.compressors.zip;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+public class Deflate64InputStream extends InputStream {
+ private HuffmanDecoder decoder;
+ private long uncompressedSize;
+ private long totalRead = 0;
+
+ public Deflate64InputStream(InputStream in, long uncompressedSize) {
+ this(new HuffmanDecoder(in), uncompressedSize);
+ }
+
+ Deflate64InputStream(HuffmanDecoder decoder, long uncompressedSize) {
+ this.uncompressedSize = uncompressedSize;
+ this.decoder = decoder;
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] b = new byte[1];
+ while (true) {
+ int r = read(b);
+ switch (r) {
+ case 1:
+ return b[0] & 0xFF;
+ case -1:
+ return -1;
+ case 0:
+ continue;
+ default:
+ throw new IllegalStateException("Invalid return value from read: " + r);
+ }
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int read = -1;
+ if (decoder != null) {
+ read = decoder.decode(b, off, len);
+ if (read == -1) {
+ close();
+ } else {
+ totalRead += read;
+ }
+ }
+ return read;
+ }
+
+ @Override
+ public int available() throws IOException {
+ long available = 0;
+ if (decoder != null) {
+ available = uncompressedSize - totalRead;
+ if (Long.compare(available, Integer.MAX_VALUE) > 0) {
+ available = Integer.MAX_VALUE;
+ }
+ }
+ return (int) available;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeQuietly(decoder);
+ decoder = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
new file mode 100644
index 0000000..de4984b
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
@@ -0,0 +1,445 @@
+package org.apache.commons.compress.compressors.zip;
+
+import org.apache.commons.compress.utils.BitInputStream;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+
+import static org.apache.commons.compress.compressors.zip.HuffmanState.*;
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+public class HuffmanDecoder implements Closeable {
+ /**
+ * --------------------------------------------------------------------
+ * idx xtra base idx xtra base idx xtra base
+ * --------------------------------------------------------------------
+ * 257 0 3 267 1 15,16 277 4 67-82
+ * 258 0 4 268 1 17,18 278 4 83-98
+ * 259 0 5 269 2 19-22 279 4 99-114
+ * 260 0 6 270 2 23-26 280 4 115-130
+ * 261 0 7 271 2 27-30 281 5 131-162
+ * 262 0 8 272 2 31-34 282 5 163-194
+ * 263 0 9 273 3 35-42 283 5 195-226
+ * 264 0 10 274 3 43-50 284 5 227-257
+ * 265 1 11,12 275 3 51-58 285 96 3
+ * 266 1 13,14 276 3 59-66
+ * --------------------------------------------------------------------
+ * value = (base of run length) << 5 | (number of extra bits to read)
+ */
+ private static final short[] RUN_LENGTH_TABLE = {
+ 96, 128, 160, 192, 224, 256, 288, 320, 353, 417, 481, 545, 610, 738, 866,
+ 994, 1123, 1379, 1635, 1891, 2148, 2660, 3172, 3684, 4197, 5221, 6245, 7269, 112
+ };
+
+ /**
+ * --------------------------------------------------------------------
+ * idx xtra dist idx xtra dist idx xtra dist
+ * --------------------------------------------------------------------
+ * 0 0 1 10 4 33-48 20 9 1025-1536
+ * 1 0 2 11 4 49-64 21 9 1537-2048
+ * 2 0 3 12 5 65-96 22 10 2049-3072
+ * 3 0 4 13 5 97-128 23 10 3073-4096
+ * 4 1 5,6 14 6 129-192 24 11 4097-6144
+ * 5 1 7,8 15 6 193-256 25 11 6145-8192
+ * 6 2 9-12 16 7 257-384 26 12 8193-12288
+ * 7 2 13-16 17 7 385-512 27 12 12289-16384
+ * 8 3 17-24 18 8 513-768 28 13 16385-24576
+ * 9 3 25-32 19 8 769-1024 29 13 24577-32768
+ * 30 14 32769-49152
+ * 31 14 49153-65536
+ * --------------------------------------------------------------------
+ * value = (base of distance) << 4 | (number of extra bits to read)
+ */
+ private static final int[] DISTANCE_TABLE = {
+ 16, 32, 48, 64, 81, 113, 146, 210, 275, 403, // 0-9
+ 532, 788, 1045, 1557, 2070, 3094, 4119, 6167, 8216, 12312, // 10-19
+ 16409, 24601, 32794, 49178, 65563, 98331, 131100, 196636, 262173, 393245, // 20-29
+ 524318, 786462 // 30-31
+ };
+
+ /**
+ * When using dynamic huffman codes the order in which the values are stored
+ * follows the positioning below
+ */
+ private static final int[] CODE_LENGTHS_ORDER =
+ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
+
+ /**
+ * Huffman Fixed Literal / Distance tables for mode 1
+ */
+ private static final int[] FIXED_LITERALS;
+ private static final int[] FIXED_DISTANCE;
+
+ static {
+ FIXED_LITERALS = new int[288];
+ Arrays.fill(FIXED_LITERALS, 0, 144, 8);
+ Arrays.fill(FIXED_LITERALS, 144, 256, 9);
+ Arrays.fill(FIXED_LITERALS, 256, 280, 7);
+ Arrays.fill(FIXED_LITERALS, 280, 288, 8);
+
+ FIXED_DISTANCE = new int[32];
+ Arrays.fill(FIXED_DISTANCE, 5);
+ }
+
+ private boolean finalBlock = false;
+ private DecoderState state;
+ private BitInputStream reader;
+
+ private final DecodingMemory memory = new DecodingMemory();
+
+ HuffmanDecoder(InputStream in) {
+ this(new BitInputStream(in, ByteOrder.LITTLE_ENDIAN));
+ }
+
+ private HuffmanDecoder(BitInputStream reader) {
+ this.reader = reader;
+ state = new InitialState();
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(reader);
+ reader = null;
+ }
+
+ public int decode(byte[] b) throws IOException {
+ return decode(b, 0, b.length);
+ }
+
+ public int decode(byte[] b, int off, int len) throws IOException {
+ while (!finalBlock || state.hasData()) {
+ switch (state.state()) {
+ case INITIAL:
+ finalBlock = reader.readBits(1) == 1;
+ int mode = (int) reader.readBits(2);
+ switch (mode) {
+ case 0:
+ reader.readBits(Byte.SIZE - 3);
+ long bLen = reader.readBits(16);
+ long bNLen = reader.readBits(16);
+ if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
+ //noinspection DuplicateStringLiteralInspection
+ throw new IllegalStateException("Illegal LEN / NLEN values");
+ }
+ state = new UncompressedState((int) bLen);
+ break;
+ case 1:
+ state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
+ break;
+ case 2:
+ int literals = (int) (reader.readBits(5) + 257);
+ int[] literalTable = new int[literals];
+
+ int distances = (int) (reader.readBits(5) + 1);
+ int[] distanceTable = new int[distances];
+
+ populateDynamicTables(reader, literalTable, distanceTable);
+
+ state = new HuffmanCodes(DYNAMIC_CODES, literalTable, distanceTable);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported compression: " + mode);
+ }
+ break;
+ default:
+ return state.read(b, off, len);
+ }
+ }
+ return -1;
+ }
+
+
+ private static abstract class DecoderState {
+ abstract HuffmanState state();
+
+ abstract int read(byte[] b, int off, int len) throws IOException;
+
+ abstract boolean hasData();
+ }
+
+ private class UncompressedState extends DecoderState {
+ private final int blockLength;
+ private int read;
+
+ private UncompressedState(int blockLength) {
+ this.blockLength = blockLength;
+ }
+
+ @Override
+ HuffmanState state() {
+ return read < blockLength ? STORED : INITIAL;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ int max = Math.min(blockLength - read, len);
+ for (int i = 0; i < max; i++) {
+ byte next = (byte) (reader.readBits(Byte.SIZE) & 0xFF);
+ b[off + i] = memory.add(next);
+ read++;
+ }
+ return max;
+ }
+
+ @Override
+ boolean hasData() {
+ return read < blockLength;
+ }
+ }
+
+ private class InitialState extends DecoderState {
+ @Override
+ HuffmanState state() {
+ return INITIAL;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ throw new IllegalStateException("Cannot read in this state");
+ }
+
+ @Override
+ boolean hasData() {
+ return false;
+ }
+ }
+
+ private class HuffmanCodes extends DecoderState {
+ private boolean endOfBlock = false;
+ private final HuffmanState state;
+ private final BinaryTreeNode lengthTree;
+ private final BinaryTreeNode distanceTree;
+
+ private int runBufferPos = 0;
+ private byte[] runBuffer = new byte[0];
+
+ HuffmanCodes(HuffmanState state, int[] lengths, int[] distance) {
+ this.state = state;
+ lengthTree = buildTree(lengths);
+ distanceTree = buildTree(distance);
+ }
+
+ @Override
+ HuffmanState state() {
+ return endOfBlock ? INITIAL : state;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ return decodeNext(b, off, len);
+ }
+
+ private int decodeNext(byte[] b, int off, int len) throws IOException {
+ if (endOfBlock) {
+ return -1;
+ }
+ int result = copyFromRunBuffer(b, off, len);
+
+ while (result < len) {
+ int symbol = nextSymbol(reader, lengthTree);
+ if (symbol < 256) {
+ b[off + result++] = memory.add((byte) (symbol & 0xFF));
+ } else if (symbol > 256) {
+ int runMask = RUN_LENGTH_TABLE[symbol - 257];
+ int run = runMask >>> 5;
+ int runXtra = runMask & 0x1F;
+ run += reader.readBits(runXtra);
+
+ int distSym = nextSymbol(reader, distanceTree);
+
+ int distMask = DISTANCE_TABLE[distSym];
+ int dist = distMask >>> 4;
+ int distXtra = distMask & 0xF;
+ dist += reader.readBits(distXtra);
+
+ runBuffer = new byte[run];
+ runBufferPos = 0;
+ memory.recordToBuffer(dist, run, runBuffer);
+
+ result += copyFromRunBuffer(b, off + result, len - result);
+ } else {
+ endOfBlock = true;
+ return result;
+ }
+ }
+
+ return result;
+ }
+
+ private int copyFromRunBuffer(byte[] b, int off, int len) {
+ int bytesInBuffer = runBuffer.length - runBufferPos;
+ int copiedBytes = 0;
+ if (bytesInBuffer > 0) {
+ copiedBytes = Math.min(len, bytesInBuffer);
+ System.arraycopy(runBuffer, runBufferPos, b, off, copiedBytes);
+ runBufferPos += copiedBytes;
+ }
+ return copiedBytes;
+ }
+
+ @Override
+ boolean hasData() {
+ return !endOfBlock;
+ }
+ }
+
+ private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
+ BinaryTreeNode node = tree;
+ while (node != null && node.literal == -1) {
+ long bit = reader.readBits(1);
+ node = bit == 0 ? node.left : node.right;
+ }
+ return node != null ? node.literal : -1;
+ }
+
+ private static void populateDynamicTables(BitInputStream reader, int[] literals, int[] distances) throws IOException {
+ int codeLengths = (int) (reader.readBits(4) + 4);
+
+ int[] codeLengthValues = new int[19];
+ for (int cLen = 0; cLen < codeLengths; cLen++) {
+ codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) reader.readBits(3);
+ }
+
+ BinaryTreeNode codeLengthTree = buildTree(codeLengthValues);
+
+ int[] auxBuffer = new int[literals.length + distances.length];
+
+ int value = -1, length = 0;
+ for (int i = 0; i < auxBuffer.length; ) {
+ if (length > 0) {
+ auxBuffer[i++] = value;
+ length--;
+ } else {
+ int symbol = nextSymbol(reader, codeLengthTree);
+ if (symbol < 16) {
+ value = symbol;
+ auxBuffer[i++] = value;
+ } else if (symbol == 16) {
+ length = (int) (reader.readBits(2) + 3);
+ } else if (symbol == 17) {
+ value = 0;
+ length = (int) (reader.readBits(3) + 3);
+ } else if (symbol == 18) {
+ value = 0;
+ length = (int) (reader.readBits(7) + 11);
+ }
+ }
+ }
+
+ System.arraycopy(auxBuffer, 0, literals, 0, literals.length);
+ System.arraycopy(auxBuffer, literals.length, distances, 0, distances.length);
+ }
+
+ private static class BinaryTreeNode {
+ private final int bits;
+ int literal = -1;
+ BinaryTreeNode left, right;
+
+ private BinaryTreeNode(int bits) {
+ this.bits = bits;
+ }
+
+ void leaf(int symbol) {
+ literal = symbol;
+ left = null;
+ right = null;
+ }
+
+ BinaryTreeNode left() {
+ if (left == null && literal == -1) {
+ left = new BinaryTreeNode(bits + 1);
+ }
+ return left;
+ }
+
+ BinaryTreeNode right() {
+ if (right == null && literal == -1) {
+ right = new BinaryTreeNode(bits + 1);
+ }
+ return right;
+ }
+ }
+
+ private static BinaryTreeNode buildTree(int[] litTable) {
+ int[] literalCodes = getCodes(litTable);
+
+ BinaryTreeNode root = new BinaryTreeNode(0);
+
+ for (int i = 0; i < litTable.length; i++) {
+ int len = litTable[i];
+ if (len != 0) {
+ BinaryTreeNode node = root;
+ int lit = literalCodes[len - 1];
+ for (int p = len - 1; p >= 0; p--) {
+ int bit = lit & (1 << p);
+ node = bit == 0 ? node.left() : node.right();
+ }
+ node.leaf(i);
+ literalCodes[len - 1]++;
+ }
+ }
+ return root;
+ }
+
+ private static int[] getCodes(int[] litTable) {
+ int max = 0;
+ int[] blCount = new int[65];
+
+ for (int aLitTable : litTable) {
+ max = Math.max(max, aLitTable);
+ blCount[aLitTable]++;
+ }
+ blCount = Arrays.copyOf(blCount, max + 1);
+
+ int code = 0;
+ int[] nextCode = new int[max + 1];
+ for (int i = 0; i <= max; i++) {
+ code = (code + blCount[i]) << 1;
+ nextCode[i] = code;
+ }
+
+ return nextCode;
+ }
+
+ private static class DecodingMemory {
+ private final byte[] memory;
+ private final int mask;
+ private int wHead;
+
+ private DecodingMemory() {
+ this(16);
+ }
+
+ private DecodingMemory(int bits) {
+ memory = new byte[1 << bits];
+ Arrays.fill(memory, (byte) -1);
+ mask = memory.length - 1;
+ }
+
+ byte add(byte b) {
+ memory[wHead] = b;
+ wHead = incCounter(wHead);
+ return b;
+ }
+
+ void recordToBuffer(int distance, int length, byte[] buff) {
+ if (distance > memory.length) {
+ throw new IllegalStateException("Illegal distance parameter: " + distance);
+ }
+ int start = (wHead - distance) & mask;
+ if (memory[start] == -1) {
+ throw new IllegalStateException("Attempt to read beyond memory: dist=" + distance);
+ }
+ for (int i = 0, pos = start; i < length; i++, pos = incCounter(pos)) {
+ buff[i] = add(memory[pos]);
+ }
+ }
+
+ private int incCounter(int counter) {
+ return (counter + 1) & mask;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
new file mode 100644
index 0000000..c871955
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
@@ -0,0 +1,8 @@
+package org.apache.commons.compress.compressors.zip;
+
+enum HuffmanState {
+ INITIAL,
+ STORED,
+ DYNAMIC_CODES,
+ FIXED_CODES
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
new file mode 100644
index 0000000..f5f7943
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
@@ -0,0 +1,111 @@
+package org.apache.commons.compress.compressors.zip;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.times;
+
+@RunWith(MockitoJUnitRunner.class)
+public class Deflate64InputStreamTest
+{
+ private final HuffmanDecoder nullDecoder = null;
+
+ @Mock
+ private HuffmanDecoder decoder;
+
+ @Test
+ public void readWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
+ assertEquals(-1, input.read());
+ assertEquals(-1, input.read(new byte[1]));
+ assertEquals(-1, input.read(new byte[1], 0, 1));
+ }
+
+ @Test
+ public void properSizeWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
+ assertEquals(0, input.available());
+ }
+
+ @Test
+ public void properSizeWhenInRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(size, input.available());
+ }
+
+ @Test
+ public void properSizeWhenOutOfRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE + 1L;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(Integer.MAX_VALUE, input.available());
+ }
+
+ @Test
+ public void properSizeAfterReading() throws Exception
+ {
+ byte[] buf = new byte[4096];
+ int offset = 1000;
+ int length = 3096;
+
+ Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
+
+ long size = Integer.MAX_VALUE + 2047L;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(2048, input.read(buf, offset, length));
+ assertEquals(Integer.MAX_VALUE - 1, input.available());
+ }
+
+ @Test
+ public void closeCallsDecoder() throws Exception
+ {
+
+ Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void closeIsDelegatedJustOnce() throws Exception
+ {
+
+ Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
+
+ input.close();
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void uncompressedBlock() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64InputStream input = new Deflate64InputStream(new ByteArrayInputStream(data), 11);
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/d07f04b7/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
new file mode 100644
index 0000000..27d0905
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
@@ -0,0 +1,205 @@
+package org.apache.commons.compress.compressors.zip;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class HuffmanDecoderTest {
+ @Test
+ public void decodeUncompressedBlock() throws Exception {
+ byte[] data = {
+ 0b1, // end of block + no compression mode
+ 11, 0, -12, -1, // len & ~len
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(11, len);
+ assertEquals("Hello World", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeUncompressedBlockWithInvalidLenNLenValue() throws Exception {
+ byte[] data = {
+ 0b1, // end of block + no compression mode
+ 11, 0, -12, -2, // len & ~len
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ try {
+ int len = decoder.decode(result);
+ fail("Should have failed but returned " + len + " entries: " + Arrays.toString(Arrays.copyOf(result, len)));
+ } catch (IllegalStateException e) {
+ assertEquals("Illegal LEN / NLEN values", e.getMessage());
+ }
+ }
+
+ @Test
+ public void decodeSimpleFixedHuffmanBlock() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b00000000000000000000000000000001, // d + end of block
+ 0b11111111111111111111111111111100 // end of block (00) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(11, len);
+ assertEquals("Hello World", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeSimpleFixedHuffmanBlockToSmallBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b00000000000000000000000000000001, // d + end of block
+ 0b11111111111111111111111111111100 // end of block (00) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[10];
+ int len;
+ len = decoder.decode(result);
+ assertEquals(10, len);
+ assertEquals("Hello Worl", new String(result, 0, len));
+ len = decoder.decode(result);
+ assertEquals(1, len);
+ assertEquals("d", new String(result, 0, len));
+ }
+
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookup() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(48, len);
+ assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookupInSmallBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[30];
+ int len;
+
+ len = decoder.decode(result);
+ assertEquals(30, len);
+ assertEquals("Hello World\nHello World\nHello ", new String(result, 0, len));
+
+ len = decoder.decode(result);
+ assertEquals(18, len);
+ assertEquals("World\nHello World\n", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookupInExactBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[48];
+ int len;
+
+ len = decoder.decode(result);
+ assertEquals(48, len);
+ assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
+
+ len = decoder.decode(result);
+ assertEquals(0, len);
+
+ len = decoder.decode(result);
+ assertEquals(-1, len);
+ }
+}
\ No newline at end of file
[16/28] commons-compress git commit: COMPRESS-380 support data
descriptors with ENHANCED_DEFLATED
Posted by bo...@apache.org.
COMPRESS-380 support data descriptors with ENHANCED_DEFLATED
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/27b16a26
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/27b16a26
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/27b16a26
Branch: refs/heads/master
Commit: 27b16a26212d5e4b61ebe07c4bd9c36465f56702
Parents: aed74ea
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 13:31:23 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 13:31:23 2018 +0100
----------------------------------------------------------------------
.../archivers/zip/ZipArchiveInputStream.java | 39 +++++++++++++------
.../zip/ZipArchiveInputStreamTest.java | 19 +++++++++
src/test/resources/COMPRESS-380-dd.zip | Bin 0 -> 1391 bytes
3 files changed, 46 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/27b16a26/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
index ea2b34f..30ccdf3 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
@@ -313,21 +313,35 @@ public class ZipArchiveInputStream extends ArchiveInputStream {
current.entry.setDataOffset(getBytesRead());
current.entry.setStreamContiguous(true);
+ ZipMethod m = ZipMethod.getMethodByCode(current.entry.getMethod());
if (current.entry.getCompressedSize() != ArchiveEntry.SIZE_UNKNOWN) {
- // FIXME this currently leaks bis if the method is not one of the supported ones
- InputStream bis = new BoundedInputStream(in, current.entry.getCompressedSize());
- if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()) {
- current.in = new UnshrinkingInputStream(bis);
- } else if (current.entry.getMethod() == ZipMethod.IMPLODING.getCode()) {
- current.in = new ExplodingInputStream(
+ if (ZipUtil.canHandleEntryData(current.entry) && m != ZipMethod.STORED && m != ZipMethod.DEFLATED) {
+ InputStream bis = new BoundedInputStream(in, current.entry.getCompressedSize());
+ switch (m) {
+ case UNSHRINKING:
+ current.in = new UnshrinkingInputStream(bis);
+ break;
+ case IMPLODING:
+ current.in = new ExplodingInputStream(
current.entry.getGeneralPurposeBit().getSlidingDictionarySize(),
current.entry.getGeneralPurposeBit().getNumberOfShannonFanoTrees(),
bis);
- } else if (current.entry.getMethod() == ZipMethod.BZIP2.getCode()) {
- current.in = new BZip2CompressorInputStream(bis);
- } else if (current.entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode()) {
- current.in = new Deflate64CompressorInputStream(bis);
+ break;
+ case BZIP2:
+ current.in = new BZip2CompressorInputStream(bis);
+ break;
+ case ENHANCED_DEFLATED:
+ current.in = new Deflate64CompressorInputStream(bis);
+ break;
+ default:
+ // we should never get here as all supported methods have been covered
+ // will cause an error when read is invoked, don't throw an exception here so people can
+ // skip unsupported entries
+ break;
+ }
}
+ } else if (m == ZipMethod.ENHANCED_DEFLATED) {
+ current.in = new Deflate64CompressorInputStream(in);
}
entriesRead++;
@@ -776,13 +790,14 @@ public class ZipArchiveInputStream extends ArchiveInputStream {
*
* @return true if allowStoredEntriesWithDataDescriptor is true,
* the entry doesn't require any data descriptor or the method is
- * DEFLATED.
+ * DEFLATED or ENHANCED_DEFLATED.
*/
private boolean supportsDataDescriptorFor(final ZipArchiveEntry entry) {
return !entry.getGeneralPurposeBit().usesDataDescriptor()
|| (allowStoredEntriesWithDataDescriptor && entry.getMethod() == ZipEntry.STORED)
- || entry.getMethod() == ZipEntry.DEFLATED;
+ || entry.getMethod() == ZipEntry.DEFLATED
+ || entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode();
}
/**
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/27b16a26/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
index 3bb62d5..04269ad 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
@@ -221,6 +221,25 @@ public class ZipArchiveInputStreamTest {
}
}
+ @Test
+ public void readDeflate64CompressedStreamWithDataDescriptor() throws Exception {
+ // this is a copy of bla.jar with META-INF/MANIFEST.MF's method manually changed to ENHANCED_DEFLATED
+ final File archive = getFile("COMPRESS-380-dd.zip");
+ try (ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) {
+ ZipArchiveEntry e = zin.getNextZipEntry();
+ assertEquals(-1, e.getSize());
+ assertEquals(ZipMethod.ENHANCED_DEFLATED.getCode(), e.getMethod());
+ byte[] fromZip = IOUtils.toByteArray(zin);
+ byte[] expected = new byte[] {
+ 'M', 'a', 'n', 'i', 'f', 'e', 's', 't', '-', 'V', 'e', 'r', 's', 'i', 'o', 'n', ':', ' ', '1', '.', '0',
+ '\r', '\n', '\r', '\n'
+ };
+ assertArrayEquals(expected, fromZip);
+ zin.getNextZipEntry();
+ assertEquals(25, e.getSize());
+ }
+ }
+
/**
* Test case for
* <a href="https://issues.apache.org/jira/browse/COMPRESS-364"
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/27b16a26/src/test/resources/COMPRESS-380-dd.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-dd.zip b/src/test/resources/COMPRESS-380-dd.zip
new file mode 100644
index 0000000..9557996
Binary files /dev/null and b/src/test/resources/COMPRESS-380-dd.zip differ
[05/28] commons-compress git commit: COMPRESS-380 make
Defalt64InputStream a CompressorInputStream
Posted by bo...@apache.org.
COMPRESS-380 make Defalt64InputStream a CompressorInputStream
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/79e76d5f
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/79e76d5f
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/79e76d5f
Branch: refs/heads/master
Commit: 79e76d5f48fc44ae3d716ac28ebe1a76ebb90b61
Parents: a62c3d0
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Jan 4 10:48:04 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Jan 4 10:48:04 2018 +0100
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipFile.java | 4 +-
.../Deflate64CompressorInputStream.java | 101 +++++++++++++++
.../deflate64/Deflate64InputStream.java | 88 -------------
.../compressors/deflate64/HuffmanDecoder.java | 2 +-
.../compress/compressors/deflate64/package.html | 25 ++++
.../Deflate64CompressorInputStreamTest.java | 127 ++++++++++++++++++
.../deflate64/Deflate64InputStreamTest.java | 128 -------------------
7 files changed, 256 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index c10fc1a..c661eeb 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -42,7 +42,7 @@ import java.util.zip.InflaterInputStream;
import java.util.zip.ZipException;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-import org.apache.commons.compress.compressors.deflate64.Deflate64InputStream;
+import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
import org.apache.commons.compress.utils.IOUtils;
import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD;
@@ -504,7 +504,7 @@ public class ZipFile implements Closeable {
case BZIP2:
return new BZip2CompressorInputStream(bis);
case ENHANCED_DEFLATED:
- return new Deflate64InputStream(bis, ze.getSize());
+ return new Deflate64CompressorInputStream(bis, ze.getSize());
case AES_ENCRYPTED:
case EXPANDING_LEVEL_1:
case EXPANDING_LEVEL_2:
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
new file mode 100644
index 0000000..c097ab3
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+/**
+ * Deflate64 decompressor.
+ *
+ * @since 1.16
+ */
+public class Deflate64CompressorInputStream extends CompressorInputStream {
+ private HuffmanDecoder decoder;
+ private long uncompressedSize;
+ private long totalRead = 0;
+
+ /**
+ * Constructs a Deflate64CompressorInputStream.
+ *
+ * @param in the stream to read from
+ * @param uncompressedSize the uncompressed size of the data to be read from in
+ */
+ public Deflate64CompressorInputStream(InputStream in, long uncompressedSize) {
+ this(new HuffmanDecoder(in), uncompressedSize);
+ }
+
+ Deflate64CompressorInputStream(HuffmanDecoder decoder, long uncompressedSize) {
+ this.uncompressedSize = uncompressedSize;
+ this.decoder = decoder;
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] b = new byte[1];
+ while (true) {
+ int r = read(b);
+ switch (r) {
+ case 1:
+ return b[0] & 0xFF;
+ case -1:
+ return -1;
+ case 0:
+ continue;
+ default:
+ throw new IllegalStateException("Invalid return value from read: " + r);
+ }
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int read = -1;
+ if (decoder != null) {
+ read = decoder.decode(b, off, len);
+ count(read);
+ if (read == -1) {
+ close();
+ } else {
+ totalRead += read;
+ }
+ }
+ return read;
+ }
+
+ @Override
+ public int available() throws IOException {
+ long available = 0;
+ if (decoder != null) {
+ available = uncompressedSize - totalRead;
+ if (Long.compare(available, Integer.MAX_VALUE) > 0) {
+ available = Integer.MAX_VALUE;
+ }
+ }
+ return (int) available;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeQuietly(decoder);
+ decoder = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
deleted file mode 100644
index e45f87f..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.deflate64;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
-
-public class Deflate64InputStream extends InputStream {
- private HuffmanDecoder decoder;
- private long uncompressedSize;
- private long totalRead = 0;
-
- public Deflate64InputStream(InputStream in, long uncompressedSize) {
- this(new HuffmanDecoder(in), uncompressedSize);
- }
-
- Deflate64InputStream(HuffmanDecoder decoder, long uncompressedSize) {
- this.uncompressedSize = uncompressedSize;
- this.decoder = decoder;
- }
-
- @Override
- public int read() throws IOException {
- byte[] b = new byte[1];
- while (true) {
- int r = read(b);
- switch (r) {
- case 1:
- return b[0] & 0xFF;
- case -1:
- return -1;
- case 0:
- continue;
- default:
- throw new IllegalStateException("Invalid return value from read: " + r);
- }
- }
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- int read = -1;
- if (decoder != null) {
- read = decoder.decode(b, off, len);
- if (read == -1) {
- close();
- } else {
- totalRead += read;
- }
- }
- return read;
- }
-
- @Override
- public int available() throws IOException {
- long available = 0;
- if (decoder != null) {
- available = uncompressedSize - totalRead;
- if (Long.compare(available, Integer.MAX_VALUE) > 0) {
- available = Integer.MAX_VALUE;
- }
- }
- return (int) available;
- }
-
- @Override
- public void close() throws IOException {
- closeQuietly(decoder);
- decoder = null;
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index bf1c4ad..de909db 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -28,7 +28,7 @@ import java.util.Arrays;
import static org.apache.commons.compress.compressors.deflate64.HuffmanState.*;
import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
-public class HuffmanDecoder implements Closeable {
+class HuffmanDecoder implements Closeable {
/**
* --------------------------------------------------------------------
* idx xtra base idx xtra base idx xtra base
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html b/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html
new file mode 100644
index 0000000..4a0cdd0
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/package.html
@@ -0,0 +1,25 @@
+<html>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+ <body>
+ <p>Provides a stream that allows decompressing streams using the
+ DEFLATE64(tm) algorithm. DEFLATE64 is a trademark of PKWARE,
+ Inc.</p>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
new file mode 100644
index 0000000..92e7b90
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.times;
+
+@RunWith(MockitoJUnitRunner.class)
+public class Deflate64CompressorInputStreamTest {
+ private final HuffmanDecoder nullDecoder = null;
+
+ @Mock
+ private HuffmanDecoder decoder;
+
+ @Test
+ public void readWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder, size);
+ assertEquals(-1, input.read());
+ assertEquals(-1, input.read(new byte[1]));
+ assertEquals(-1, input.read(new byte[1], 0, 1));
+ }
+
+ @Test
+ public void properSizeWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder, size);
+ assertEquals(0, input.available());
+ }
+
+ @Test
+ public void properSizeWhenInRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
+ assertEquals(size, input.available());
+ }
+
+ @Test
+ public void properSizeWhenOutOfRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE + 1L;
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
+ assertEquals(Integer.MAX_VALUE, input.available());
+ }
+
+ @Test
+ public void properSizeAfterReading() throws Exception
+ {
+ byte[] buf = new byte[4096];
+ int offset = 1000;
+ int length = 3096;
+
+ Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
+
+ long size = Integer.MAX_VALUE + 2047L;
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
+ assertEquals(2048, input.read(buf, offset, length));
+ assertEquals(Integer.MAX_VALUE - 1, input.available());
+ }
+
+ @Test
+ public void closeCallsDecoder() throws Exception
+ {
+
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, 10);
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void closeIsDelegatedJustOnce() throws Exception
+ {
+
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, 10);
+
+ input.close();
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void uncompressedBlock() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data), 11);
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/79e76d5f/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
deleted file mode 100644
index f9d1f3b..0000000
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.deflate64;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.InputStreamReader;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.times;
-
-@RunWith(MockitoJUnitRunner.class)
-public class Deflate64InputStreamTest
-{
- private final HuffmanDecoder nullDecoder = null;
-
- @Mock
- private HuffmanDecoder decoder;
-
- @Test
- public void readWhenClosed() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
- assertEquals(-1, input.read());
- assertEquals(-1, input.read(new byte[1]));
- assertEquals(-1, input.read(new byte[1], 0, 1));
- }
-
- @Test
- public void properSizeWhenClosed() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
- assertEquals(0, input.available());
- }
-
- @Test
- public void properSizeWhenInRange() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(size, input.available());
- }
-
- @Test
- public void properSizeWhenOutOfRange() throws Exception
- {
- long size = Integer.MAX_VALUE + 1L;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(Integer.MAX_VALUE, input.available());
- }
-
- @Test
- public void properSizeAfterReading() throws Exception
- {
- byte[] buf = new byte[4096];
- int offset = 1000;
- int length = 3096;
-
- Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
-
- long size = Integer.MAX_VALUE + 2047L;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(2048, input.read(buf, offset, length));
- assertEquals(Integer.MAX_VALUE - 1, input.available());
- }
-
- @Test
- public void closeCallsDecoder() throws Exception
- {
-
- Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void closeIsDelegatedJustOnce() throws Exception
- {
-
- Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
-
- input.close();
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void uncompressedBlock() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- try (Deflate64InputStream input = new Deflate64InputStream(new ByteArrayInputStream(data), 11);
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- assertEquals(null, br.readLine());
- }
- }
-
-}
[15/28] commons-compress git commit: COMPRESS-380 allow underlying
stream to outlive HuffmanDecoder
Posted by bo...@apache.org.
COMPRESS-380 allow underlying stream to outlive HuffmanDecoder
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/aed74ea0
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/aed74ea0
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/aed74ea0
Branch: refs/heads/master
Commit: aed74ea03276362cd301a36787774843ebd4f70c
Parents: 5d0b71f
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 13:29:40 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 13:29:40 2018 +0100
----------------------------------------------------------------------
.../deflate64/Deflate64CompressorInputStream.java | 12 +++++++++++-
.../compress/compressors/deflate64/HuffmanDecoder.java | 3 +--
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/aed74ea0/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
index 88b2142..369fa8d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
@@ -29,6 +29,7 @@ import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
* @since 1.16
*/
public class Deflate64CompressorInputStream extends CompressorInputStream {
+ private InputStream originalStream;
private HuffmanDecoder decoder;
/**
@@ -38,6 +39,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
*/
public Deflate64CompressorInputStream(InputStream in) {
this(new HuffmanDecoder(in));
+ originalStream = in;
}
Deflate64CompressorInputStream(HuffmanDecoder decoder) {
@@ -75,7 +77,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
read = decoder.decode(b, off, len);
count(read);
if (read == -1) {
- close();
+ closeDecoder();
}
}
return read;
@@ -88,6 +90,14 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
@Override
public void close() throws IOException {
+ closeDecoder();
+ if (originalStream != null) {
+ originalStream.close();
+ originalStream = null;
+ }
+ }
+
+ private void closeDecoder() throws IOException {
closeQuietly(decoder);
decoder = null;
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/aed74ea0/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 04ec7f9..f74c7de 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -27,7 +27,6 @@ import java.nio.ByteOrder;
import java.util.Arrays;
import static org.apache.commons.compress.compressors.deflate64.HuffmanState.*;
-import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
class HuffmanDecoder implements Closeable {
/**
@@ -119,7 +118,7 @@ class HuffmanDecoder implements Closeable {
@Override
public void close() {
- closeQuietly(reader);
+ state = new InitialState();
reader = null;
}
[17/28] commons-compress git commit: COMPRESS-380 document DEFLATE64
support
Posted by bo...@apache.org.
COMPRESS-380 document DEFLATE64 support
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/87a3cfaf
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/87a3cfaf
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/87a3cfaf
Branch: refs/heads/master
Commit: 87a3cfaf42bc8bdbbde3815947e2b8b4c267beee
Parents: 27b16a2
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 14:43:02 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 14:45:01 2018 +0100
----------------------------------------------------------------------
src/changes/changes.xml | 5 +++++
src/site/xdoc/examples.xml | 25 +++++++++++++++++++++++--
src/site/xdoc/index.xml | 7 +++++--
src/site/xdoc/limitations.xml | 2 +-
src/site/xdoc/zip.xml | 36 +++++++++++++++++++++++++++++++++++-
5 files changed, 69 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/87a3cfaf/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 8d3bf93..3eaf692 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -72,6 +72,11 @@ The <action> type attribute can be add,update,fix,remove.
throw the expected IOException rather than obscure
RuntimeExceptions.
</action>
+ <action issue="COMPRESS-380" type="add" date="2018-01-03"
+ due-to="">
+ Added read-only DEFLATE64 support to ZIP archives and as
+ stand-alone CompressorInputStream.
+ </action>
</release>
<release version="1.15" date="2017-10-17"
description="Release 1.15
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/87a3cfaf/src/site/xdoc/examples.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/examples.xml b/src/site/xdoc/examples.xml
index b6997b3..d5d70c1 100644
--- a/src/site/xdoc/examples.xml
+++ b/src/site/xdoc/examples.xml
@@ -32,12 +32,12 @@
compressed) archive are archiver formats.</p>
<p>The compressor formats supported are gzip, bzip2, xz, lzma,
- Pack200, DEFLATE and Z, the archiver formats are 7z, ar, arj,
+ Pack200, DEFLATE, Brotli, DEFLATE64, ZStandard and Z, the archiver formats are 7z, ar, arj,
cpio, dump, tar and zip. Pack200 is a special case as it can
only compress JAR files.</p>
<p>We currently only provide read support for arj,
- dump and Z. arj can only read uncompressed archives, 7z can read
+ dump, Brotli, DEFLATE64 and Z. arj can only read uncompressed archives, 7z can read
archives with many compression and encryption algorithms
supported by 7z but doesn't support encryption when writing
archives.</p>
@@ -785,6 +785,27 @@ in.close();
</subsection>
+ <subsection name="DEFLATE64">
+
+ <p>Uncompressing a given DEFLATE64 compressed file (you would
+ certainly add exception handling and make sure all streams
+ get closed properly):</p>
+<source><![CDATA[
+InputStream fin = Files.newInputStream(Paths.get("some-file"));
+BufferedInputStream in = new BufferedInputStream(fin);
+OutputStream out = Files.newOutputStream(Paths.get("archive.tar"));
+Deflate64CompressorInputStream defIn = new Deflate64CompressorInputStream(in);
+final byte[] buffer = new byte[buffersize];
+int n = 0;
+while (-1 != (n = defIn.read(buffer))) {
+ out.write(buffer, 0, n);
+}
+out.close();
+defIn.close();
+]]></source>
+
+ </subsection>
+
<subsection name="Snappy">
<p>There are two different "formats" used for <a
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/87a3cfaf/src/site/xdoc/index.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index ea32924..cfadd39 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -28,7 +28,7 @@
The Apache Commons Compress library defines an API for
working with ar, cpio, Unix dump, tar, zip, gzip, XZ,
Pack200, bzip2, 7z, arj, lzma, snappy, DEFLATE, lz4,
- Brotli, Zstandard and Z files.
+ Brotli, Zstandard, DEFLATE64 and Z files.
</p>
<p>
The code in this component has many origins:
@@ -69,6 +69,9 @@
<subsection name="What's coming in 1.16?">
<ul>
<li>Support for Zstandard compression.</li>
+ <li>Read-only support for DEFLATE64 compression as
+ stand-alone CompressorInputStream and as method used in
+ ZIP archives.</li>
</ul>
</subsection>
@@ -94,7 +97,7 @@
licensed <a href="https://github.com/google/brotli">Google
Brotli decoder</a>. Zstandard support is provided by the BSD
licensed <a href="https://github.com/luben/zstd-jni">Zstd-jni</a>.
- As of Commons Compress 1.16 support for the Z and Brotli
+ As of Commons Compress 1.16 support for the DEFLATE64, Z and Brotli
formats is read-only.</p>
<p>The ar, arj, cpio, dump, tar, 7z and zip formats are supported as
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/87a3cfaf/src/site/xdoc/limitations.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/limitations.xml b/src/site/xdoc/limitations.xml
index e86b296..8645a9b 100644
--- a/src/site/xdoc/limitations.xml
+++ b/src/site/xdoc/limitations.xml
@@ -178,7 +178,7 @@
limitation of Compress' specific implementation.</li>
<li>only a subset of compression methods are supported,
including the most common STORED and DEFLATEd. IMPLODE,
- SHRINK and BZIP2 support is read-only.</li>
+ SHRINK, DEFLATE64 and BZIP2 support is read-only.</li>
<li>no support for encryption or multi-volume archives</li>
<li>In versions prior to Compress 1.6
<code>ZipArchiveEntries</code> read from an archive will
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/87a3cfaf/src/site/xdoc/zip.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/zip.xml b/src/site/xdoc/zip.xml
index 6f08ba6..b857332 100644
--- a/src/site/xdoc/zip.xml
+++ b/src/site/xdoc/zip.xml
@@ -297,7 +297,8 @@
just fine. As of version 1.7, Commons Compress can also
decompress entries compressed with the legacy SHRINK and
IMPLODE algorithms of PKZIP 1.x. Version 1.11 of Commons
- Compress adds read-only support for BZIP2.</p>
+ Compress adds read-only support for BZIP2. Version 1.16 adds
+ read-only support for DEFLATE64 - also known as "enhanced DEFLATE".</p>
<p>The ZIP specification allows for various other compression
algorithms and also supports several different ways of
@@ -312,6 +313,39 @@
mechanism. Using this method it is possible to at least
detect and skip the entries that can not be extracted.</p>
+ <table>
+ <thead>
+ <tr>
+ <th>Version of Apache Commons Compress</th>
+ <th>Supported Compression Methods</th>
+ <th>Supported Encryption Methods</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>1.0 to 1.6</td>
+ <td>STORED, DEFLATE</td>
+ <td>-</td>
+ </tr>
+ <tr>
+ <td>1.7 to 1.10</td>
+ <td>STORED, DEFLATE, SHRINK, IMPLODE</td>
+ <td>-</td>
+ </tr>
+ <tr>
+ <td>1.11 to 1.15</td>
+ <td>STORED, DEFLATE, SHRINK, IMPLODE, BZIP2</td>
+ <td>-</td>
+ </tr>
+ <tr>
+ <td>1.16 and later</td>
+ <td>STORED, DEFLATE, SHRINK, IMPLODE, BZIP2, DEFLATE64
+ (enhanced deflate)</td>
+ <td>-</td>
+ </tr>
+ </tbody>
+ </table>
+
</subsection>
<subsection name="Zip64 Support" id="zip64">
[04/28] commons-compress git commit: COMPRESS-380 add testcase for
deflat64 support
Posted by bo...@apache.org.
COMPRESS-380 add testcase for deflat64 support
based on an archive kindly provided by Dawid Weiss
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/a62c3d0d
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/a62c3d0d
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/a62c3d0d
Branch: refs/heads/master
Commit: a62c3d0d8301c450d5bd0ab1bd9bb3862aac5853
Parents: 2042aa0
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Jan 4 09:27:10 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Jan 4 09:27:10 2018 +0100
----------------------------------------------------------------------
.gitattributes | 1 +
.../compress/archivers/zip/ZipFileTest.java | 18 ++++++++++++++++++
src/test/resources/COMPRESS-380-input | Bin 0 -> 3072 bytes
src/test/resources/COMPRESS-380.zip | Bin 0 -> 2257 bytes
4 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a62c3d0d/.gitattributes
----------------------------------------------------------------------
diff --git a/.gitattributes b/.gitattributes
index 0fda71d..a02cd59 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -14,3 +14,4 @@ src/test/resources/test3.xml eol=lf
src/test/resources/test4.xml eol=lf
src/test/resources/test?with?spaces.txt eol=lf
src/test/resources/test.txt eol=lf
+src/test/resources/COMPRESS-380-input binary
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a62c3d0d/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
index 87a3ded..2903f77 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
@@ -581,6 +581,24 @@ public class ZipFileTest {
entry.setAlignment(3);
}
+ /**
+ * @see "https://issues.apache.org/jira/browse/COMPRESS-380"
+ */
+ @Test
+ public void readDeflate64CompressedStream() throws Exception {
+ final File input = getFile("COMPRESS-380-input");
+ final File archive = getFile("COMPRESS-380.zip");
+ try (FileInputStream in = new FileInputStream(input);
+ ZipFile zf = new ZipFile(archive)) {
+ byte[] orig = IOUtils.toByteArray(in);
+ ZipArchiveEntry e = zf.getEntry("input2");
+ try (InputStream s = zf.getInputStream(e)) {
+ byte[] fromZip = IOUtils.toByteArray(s);
+ assertArrayEquals(orig, fromZip);
+ }
+ }
+ }
+
private void assertAllReadMethods(byte[] expected, ZipFile zipFile, ZipArchiveEntry entry) {
// simple IOUtil read
try (InputStream stream = zf.getInputStream(entry)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a62c3d0d/src/test/resources/COMPRESS-380-input
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-input b/src/test/resources/COMPRESS-380-input
new file mode 100644
index 0000000..daf1f56
Binary files /dev/null and b/src/test/resources/COMPRESS-380-input differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a62c3d0d/src/test/resources/COMPRESS-380.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380.zip b/src/test/resources/COMPRESS-380.zip
new file mode 100644
index 0000000..d9146be
Binary files /dev/null and b/src/test/resources/COMPRESS-380.zip differ
[06/28] commons-compress git commit: COMPRESS-380
Deflate64CompressorInputStream#available promised too much
Posted by bo...@apache.org.
COMPRESS-380 Deflate64CompressorInputStream#available promised too much
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/07cc1a27
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/07cc1a27
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/07cc1a27
Branch: refs/heads/master
Commit: 07cc1a278b217d45cb090ff6cb3a7934105cb2d0
Parents: 79e76d5
Author: Stefan Bodewig <bo...@apache.org>
Authored: Thu Jan 4 16:51:20 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Thu Jan 4 16:51:20 2018 +0100
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipFile.java | 2 +-
.../Deflate64CompressorInputStream.java | 21 ++------
.../compressors/deflate64/HuffmanDecoder.java | 20 +++++++
.../commons/compress/utils/BitInputStream.java | 10 ++++
.../Deflate64CompressorInputStreamTest.java | 56 ++++++++------------
5 files changed, 58 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07cc1a27/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index c661eeb..f77339b 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -504,7 +504,7 @@ public class ZipFile implements Closeable {
case BZIP2:
return new BZip2CompressorInputStream(bis);
case ENHANCED_DEFLATED:
- return new Deflate64CompressorInputStream(bis, ze.getSize());
+ return new Deflate64CompressorInputStream(bis);
case AES_ENCRYPTED:
case EXPANDING_LEVEL_1:
case EXPANDING_LEVEL_2:
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07cc1a27/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
index c097ab3..e909f13 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
@@ -30,21 +30,17 @@ import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
*/
public class Deflate64CompressorInputStream extends CompressorInputStream {
private HuffmanDecoder decoder;
- private long uncompressedSize;
- private long totalRead = 0;
/**
* Constructs a Deflate64CompressorInputStream.
*
* @param in the stream to read from
- * @param uncompressedSize the uncompressed size of the data to be read from in
*/
- public Deflate64CompressorInputStream(InputStream in, long uncompressedSize) {
- this(new HuffmanDecoder(in), uncompressedSize);
+ public Deflate64CompressorInputStream(InputStream in) {
+ this(new HuffmanDecoder(in));
}
- Deflate64CompressorInputStream(HuffmanDecoder decoder, long uncompressedSize) {
- this.uncompressedSize = uncompressedSize;
+ Deflate64CompressorInputStream(HuffmanDecoder decoder) {
this.decoder = decoder;
}
@@ -74,8 +70,6 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
count(read);
if (read == -1) {
close();
- } else {
- totalRead += read;
}
}
return read;
@@ -83,14 +77,7 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
@Override
public int available() throws IOException {
- long available = 0;
- if (decoder != null) {
- available = uncompressedSize - totalRead;
- if (Long.compare(available, Integer.MAX_VALUE) > 0) {
- available = Integer.MAX_VALUE;
- }
- }
- return (int) available;
+ return decoder != null ? decoder.available() : 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07cc1a27/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index de909db..3a41613 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -168,6 +168,9 @@ class HuffmanDecoder implements Closeable {
return -1;
}
+ int available() throws IOException {
+ return state == null ? 0 : state.available();
+ }
private static abstract class DecoderState {
abstract HuffmanState state();
@@ -175,6 +178,8 @@ class HuffmanDecoder implements Closeable {
abstract int read(byte[] b, int off, int len) throws IOException;
abstract boolean hasData();
+
+ abstract int available() throws IOException ;
}
private class UncompressedState extends DecoderState {
@@ -205,6 +210,11 @@ class HuffmanDecoder implements Closeable {
boolean hasData() {
return read < blockLength;
}
+
+ @Override
+ int available() throws IOException {
+ return (int) Math.min(blockLength - read, reader.bitsAvailable() / Byte.SIZE);
+ }
}
private class InitialState extends DecoderState {
@@ -222,6 +232,11 @@ class HuffmanDecoder implements Closeable {
boolean hasData() {
return false;
}
+
+ @Override
+ int available() {
+ return 0;
+ }
}
private class HuffmanCodes extends DecoderState {
@@ -301,6 +316,11 @@ class HuffmanDecoder implements Closeable {
boolean hasData() {
return !endOfBlock;
}
+
+ @Override
+ int available() {
+ return runBuffer.length - runBufferPos;
+ }
}
private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07cc1a27/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
index 45d30ec..f4beaae 100644
--- a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
@@ -100,6 +100,16 @@ public class BitInputStream implements Closeable {
return bitsOut;
}
+ /**
+ * Returns an estimate of the number of bits that can be read from
+ * this input stream without blocking by the next invocation of a
+ * method for this input stream.
+ * @since 1.16
+ */
+ public long bitsAvailable() throws IOException {
+ return bitsCachedSize + 8l * in.available();
+ }
+
private long processBitsGreater57(final int count) throws IOException {
final long bitsOut;
int overflowBits = 0;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07cc1a27/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
index 92e7b90..1e32b42 100644
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
@@ -40,8 +40,7 @@ public class Deflate64CompressorInputStreamTest {
@Test
public void readWhenClosed() throws Exception
{
- long size = Integer.MAX_VALUE - 1;
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder, size);
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
assertEquals(-1, input.read());
assertEquals(-1, input.read(new byte[1]));
assertEquals(-1, input.read(new byte[1], 0, 1));
@@ -50,47 +49,24 @@ public class Deflate64CompressorInputStreamTest {
@Test
public void properSizeWhenClosed() throws Exception
{
- long size = Integer.MAX_VALUE - 1;
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder, size);
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
assertEquals(0, input.available());
}
@Test
- public void properSizeWhenInRange() throws Exception
+ public void delegatesAvailable() throws Exception
{
- long size = Integer.MAX_VALUE - 1;
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
- assertEquals(size, input.available());
- }
-
- @Test
- public void properSizeWhenOutOfRange() throws Exception
- {
- long size = Integer.MAX_VALUE + 1L;
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
- assertEquals(Integer.MAX_VALUE, input.available());
- }
+ Mockito.when(decoder.available()).thenReturn(1024);
- @Test
- public void properSizeAfterReading() throws Exception
- {
- byte[] buf = new byte[4096];
- int offset = 1000;
- int length = 3096;
-
- Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
-
- long size = Integer.MAX_VALUE + 2047L;
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, size);
- assertEquals(2048, input.read(buf, offset, length));
- assertEquals(Integer.MAX_VALUE - 1, input.available());
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
+ assertEquals(1024, input.available());
}
@Test
public void closeCallsDecoder() throws Exception
{
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, 10);
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
input.close();
Mockito.verify(decoder, times(1)).close();
@@ -100,7 +76,7 @@ public class Deflate64CompressorInputStreamTest {
public void closeIsDelegatedJustOnce() throws Exception
{
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder, 10);
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
input.close();
input.close();
@@ -116,7 +92,7 @@ public class Deflate64CompressorInputStreamTest {
'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
};
- try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data), 11);
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
BufferedReader br = new BufferedReader(new InputStreamReader(input)))
{
assertEquals("Hello World", br.readLine());
@@ -124,4 +100,18 @@ public class Deflate64CompressorInputStreamTest {
}
}
+ @Test
+ public void uncompressedBlockAvailable() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data))) {
+ assertEquals('H', input.read());
+ assertEquals(10, input.available());
+ }
+ }
+
}
[12/28] commons-compress git commit: COMPRESS-380 document
EOFException and add a test
Posted by bo...@apache.org.
COMPRESS-380 document EOFException and add a test
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/f4b75016
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/f4b75016
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/f4b75016
Branch: refs/heads/master
Commit: f4b750165bc27f6a36bc3abed650e18d474e296b
Parents: b97a02c
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 11:55:08 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 11:55:08 2018 +0100
----------------------------------------------------------------------
.../Deflate64CompressorInputStream.java | 6 ++++
.../compressors/deflate64/HuffmanDecoder.java | 2 +-
.../Deflate64CompressorInputStreamTest.java | 31 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/f4b75016/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
index e909f13..88b2142 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStream.java
@@ -44,6 +44,9 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
this.decoder = decoder;
}
+ /**
+ * @throws java.io.EOFException if the underlying stream is exhausted before the end of defalted data was reached.
+ */
@Override
public int read() throws IOException {
byte[] b = new byte[1];
@@ -62,6 +65,9 @@ public class Deflate64CompressorInputStream extends CompressorInputStream {
}
}
+ /**
+ * @throws java.io.EOFException if the underlying stream is exhausted before the end of defalted data was reached.
+ */
@Override
public int read(byte[] b, int off, int len) throws IOException {
int read = -1;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/f4b75016/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index e96959c..04ec7f9 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -488,7 +488,7 @@ class HuffmanDecoder implements Closeable {
private static long readBits(BitInputStream reader, int numBits) throws IOException {
long r = reader.readBits(numBits);
if (r == -1) {
- throw new EOFException();
+ throw new EOFException("Truncated Deflate64 Stream");
}
return r;
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/f4b75016/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
index 0232b06..b7b3cd9 100644
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
@@ -133,4 +133,35 @@ public class Deflate64CompressorInputStreamTest {
}
}
+ @Test
+ public void streamIgnoresExtraBytesAfterDeflatedInput() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', 'X'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+ @Test(expected = java.io.EOFException.class)
+ public void throwsEOFExceptionOnTruncatedStreams() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l',
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ }
+ }
+
}
[10/28] commons-compress git commit: COMPRESS-380 state is never null
Posted by bo...@apache.org.
COMPRESS-380 state is never null
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/0e74bef9
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/0e74bef9
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/0e74bef9
Branch: refs/heads/master
Commit: 0e74bef9b2bcc7f41123fb2eb6d10433ebb80a23
Parents: bca3b08
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 10:23:06 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 10:23:06 2018 +0100
----------------------------------------------------------------------
.../commons/compress/compressors/deflate64/HuffmanDecoder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e74bef9/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 3a41613..ec4a6f0 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -169,7 +169,7 @@ class HuffmanDecoder implements Closeable {
}
int available() throws IOException {
- return state == null ? 0 : state.available();
+ return state.available();
}
private static abstract class DecoderState {
[08/28] commons-compress git commit: Add DEFLATE64 support to
CompressorStreamfactory.
Posted by bo...@apache.org.
Add DEFLATE64 support to CompressorStreamfactory.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/53ec5829
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/53ec5829
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/53ec5829
Branch: refs/heads/master
Commit: 53ec5829855e8a80b2857b7cfef8bce3e63058a7
Parents: ccc3067
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 07:00:54 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 07:00:54 2018 +0100
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 24 ++++++++++++++++++--
.../Deflate64CompressorInputStreamTest.java | 19 ++++++++++++++++
2 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/53ec5829/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index e6a4400..7118744 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -37,6 +37,7 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorOutputStream;
+import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
@@ -178,6 +179,13 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
public static final String DEFLATE = "deflate";
/**
+ * Constant (value {@value}) used to identify the Deflate64 compress method.
+ *
+ * @since 1.16
+ */
+ public static final String DEFLATE64 = "deflate64";
+
+ /**
* Constant (value {@value}) used to identify the block LZ4
* compression method.
*
@@ -307,6 +315,13 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return DEFLATE;
}
+ /**
+ * @since 1.16
+ */
+ public static String getDeflate64() {
+ return DEFLATE64;
+ }
+
public static String getGzip() {
return GZIP;
}
@@ -534,7 +549,8 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
* of the compressor, i.e. {@value #GZIP}, {@value #BZIP2},
* {@value #XZ}, {@value #LZMA}, {@value #PACK200},
* {@value #SNAPPY_RAW}, {@value #SNAPPY_FRAMED}, {@value #Z},
- * {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED}, {@value #ZSTANDARD}
+ * {@value #LZ4_BLOCK}, {@value #LZ4_FRAMED}, {@value #ZSTANDARD},
+ * {@value #DEFLATE64}
* or {@value #DEFLATE}
* @param in
* the input stream
@@ -616,6 +632,10 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
return new DeflateCompressorInputStream(in);
}
+ if (DEFLATE64.equalsIgnoreCase(name)) {
+ return new Deflate64CompressorInputStream(in);
+ }
+
if (LZ4_BLOCK.equalsIgnoreCase(name)) {
return new BlockLZ4CompressorInputStream(in);
}
@@ -738,7 +758,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
@Override
public Set<String> getInputStreamCompressorNames() {
return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200, DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK,
- LZ4_FRAMED, ZSTANDARD);
+ LZ4_FRAMED, ZSTANDARD, DEFLATE64);
}
@Override
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/53ec5829/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
index 1e32b42..0232b06 100644
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.commons.compress.compressors.deflate64;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
@@ -25,6 +26,7 @@ import org.mockito.runners.MockitoJUnitRunner;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.InputStream;
import java.io.InputStreamReader;
import static org.junit.Assert.assertEquals;
@@ -101,6 +103,23 @@ public class Deflate64CompressorInputStreamTest {
}
@Test
+ public void uncompressedBlockViaFactory() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (InputStream input = new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.DEFLATE64, new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+ @Test
public void uncompressedBlockAvailable() throws Exception
{
byte[] data = {
[07/28] commons-compress git commit: COMPRESS-380 add DEFLATE64
support to ZipArchiveInputStream
Posted by bo...@apache.org.
COMPRESS-380 add DEFLATE64 support to ZipArchiveInputStream
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/ccc3067f
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/ccc3067f
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/ccc3067f
Branch: refs/heads/master
Commit: ccc3067f5216630d6c96f21f1ed2ef174aba9bd5
Parents: 07cc1a2
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 06:52:55 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 06:53:38 2018 +0100
----------------------------------------------------------------------
.../archivers/zip/ZipArchiveInputStream.java | 11 ++++++++---
.../archivers/zip/ZipArchiveInputStreamTest.java | 16 ++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ccc3067f/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
index 54b69ae..5ea9c0a 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
@@ -34,6 +34,7 @@ import java.util.zip.ZipException;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.deflate64.Deflate64CompressorInputStream;
import org.apache.commons.compress.utils.ArchiveUtils;
import org.apache.commons.compress.utils.IOUtils;
@@ -313,15 +314,18 @@ public class ZipArchiveInputStream extends ArchiveInputStream {
current.entry.setStreamContiguous(true);
if (current.entry.getCompressedSize() != ArchiveEntry.SIZE_UNKNOWN) {
+ InputStream bis = new BoundedInputStream(in, current.entry.getCompressedSize());
if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()) {
- current.in = new UnshrinkingInputStream(new BoundedInputStream(in, current.entry.getCompressedSize()));
+ current.in = new UnshrinkingInputStream(bis);
} else if (current.entry.getMethod() == ZipMethod.IMPLODING.getCode()) {
current.in = new ExplodingInputStream(
current.entry.getGeneralPurposeBit().getSlidingDictionarySize(),
current.entry.getGeneralPurposeBit().getNumberOfShannonFanoTrees(),
- new BoundedInputStream(in, current.entry.getCompressedSize()));
+ bis);
} else if (current.entry.getMethod() == ZipMethod.BZIP2.getCode()) {
- current.in = new BZip2CompressorInputStream(new BoundedInputStream(in, current.entry.getCompressedSize()));
+ current.in = new BZip2CompressorInputStream(bis);
+ } else if (current.entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode()) {
+ current.in = new Deflate64CompressorInputStream(bis);
}
}
@@ -424,6 +428,7 @@ public class ZipArchiveInputStream extends ArchiveInputStream {
read = readDeflated(buffer, offset, length);
} else if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()
|| current.entry.getMethod() == ZipMethod.IMPLODING.getCode()
+ || current.entry.getMethod() == ZipMethod.ENHANCED_DEFLATED.getCode()
|| current.entry.getMethod() == ZipMethod.BZIP2.getCode()) {
read = current.in.read(buffer, offset, length);
} else {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ccc3067f/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
index a0a5c50..3bb62d5 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
@@ -206,6 +206,22 @@ public class ZipArchiveInputStreamTest {
}
/**
+ * @see "https://issues.apache.org/jira/browse/COMPRESS-380"
+ */
+ @Test
+ public void readDeflate64CompressedStream() throws Exception {
+ final File input = getFile("COMPRESS-380-input");
+ final File archive = getFile("COMPRESS-380.zip");
+ try (FileInputStream in = new FileInputStream(input);
+ ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) {
+ byte[] orig = IOUtils.toByteArray(in);
+ ZipArchiveEntry e = zin.getNextZipEntry();
+ byte[] fromZip = IOUtils.toByteArray(zin);
+ assertArrayEquals(orig, fromZip);
+ }
+ }
+
+ /**
* Test case for
* <a href="https://issues.apache.org/jira/browse/COMPRESS-364"
* >COMPRESS-364</a>.
[14/28] commons-compress git commit: the list of supported methods
has grown by now
Posted by bo...@apache.org.
the list of supported methods has grown by now
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/5d0b71fe
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/5d0b71fe
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/5d0b71fe
Branch: refs/heads/master
Commit: 5d0b71fe9c3ab81717245f80eb71840f9a22a5aa
Parents: 334f1de
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 12:49:21 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 12:49:21 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/commons/compress/archivers/zip/ZipUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/5d0b71fe/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
index d22a62e..677ac3f 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
@@ -324,7 +324,7 @@ public abstract class ZipUtil {
* Whether this library supports the compression method used by
* the given entry.
*
- * @return true if the compression method is STORED or DEFLATED
+ * @return true if the compression method is supported
*/
private static boolean supportsMethodOf(final ZipArchiveEntry entry) {
return entry.getMethod() == ZipEntry.STORED
[25/28] commons-compress git commit: COMPRESS fix comment
Posted by bo...@apache.org.
COMPRESS fix comment
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/782b1562
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/782b1562
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/782b1562
Branch: refs/heads/master
Commit: 782b15622462d9a9bd9f483c43de711921febac3
Parents: 19ad287
Author: Stefan Bodewig <bo...@apache.org>
Authored: Mon Jan 8 13:21:04 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Mon Jan 8 13:21:04 2018 +0100
----------------------------------------------------------------------
.../commons/compress/compressors/deflate64/HuffmanDecoder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/782b1562/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index f1f1042..506b35d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -41,7 +41,7 @@ class HuffmanDecoder implements Closeable {
* 262 0 8 272 2 31-34 282 5 163-194
* 263 0 9 273 3 35-42 283 5 195-226
* 264 0 10 274 3 43-50 284 5 227-257
- * 265 1 11,12 275 3 51-58 285 96 3
+ * 265 1 11,12 275 3 51-58 285 16 3
* 266 1 13,14 276 3 59-66
* --------------------------------------------------------------------
* value = (base of run length) << 5 | (number of extra bits to read)
[22/28] commons-compress git commit: COMPRESS-380 ensure reading of
stored block stats at byte boundary
Posted by bo...@apache.org.
COMPRESS-380 ensure reading of stored block stats at byte boundary
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/2d25368d
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/2d25368d
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/2d25368d
Branch: refs/heads/master
Commit: 2d25368dd82e50dd68cf0d256084881e6a3153d9
Parents: 77a0a69
Author: Stefan Bodewig <bo...@apache.org>
Authored: Mon Jan 8 07:32:17 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Mon Jan 8 07:32:17 2018 +0100
----------------------------------------------------------------------
.../compress/compressors/deflate64/HuffmanDecoder.java | 2 +-
.../apache/commons/compress/utils/BitInputStream.java | 11 +++++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2d25368d/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 9d6585a..5ed4079 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -134,7 +134,7 @@ class HuffmanDecoder implements Closeable {
int mode = (int) readBits(2);
switch (mode) {
case 0:
- readBits(Byte.SIZE - 3);
+ reader.alignWithByteBoundary();
long bLen = readBits(16);
long bNLen = readBits(16);
if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2d25368d/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
index f4beaae..7f29ac0 100644
--- a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
@@ -110,6 +110,17 @@ public class BitInputStream implements Closeable {
return bitsCachedSize + 8l * in.available();
}
+ /**
+ * Drops bits until the next bits will be read from a byte boundary.
+ * @since 1.16
+ */
+ public void alignWithByteBoundary() throws IOException {
+ int toSkip = bitsCachedSize % 8;
+ if (toSkip > 0) {
+ readBits(toSkip);
+ }
+ }
+
private long processBitsGreater57(final int count) throws IOException {
final long bitsOut;
int overflowBits = 0;
[23/28] commons-compress git commit: COMPRESS-38ß block length is an unsigned 32bit int in C code
Posted by bo...@apache.org.
COMPRESS-38ß block length is an unsigned 32bit int in C code
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/3600a5f5
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/3600a5f5
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/3600a5f5
Branch: refs/heads/master
Commit: 3600a5f510cee298397d5ffc21c978ab6e52a160
Parents: 2d25368
Author: Stefan Bodewig <bo...@apache.org>
Authored: Mon Jan 8 08:33:32 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Mon Jan 8 08:33:32 2018 +0100
----------------------------------------------------------------------
.../compress/compressors/deflate64/HuffmanDecoder.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3600a5f5/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 5ed4079..8b006c3 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -141,7 +141,7 @@ class HuffmanDecoder implements Closeable {
//noinspection DuplicateStringLiteralInspection
throw new IllegalStateException("Illegal LEN / NLEN values");
}
- state = new UncompressedState((int) bLen);
+ state = new UncompressedState(bLen);
break;
case 1:
state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
@@ -183,10 +183,10 @@ class HuffmanDecoder implements Closeable {
}
private class UncompressedState extends DecoderState {
- private final int blockLength;
- private int read;
+ private final long blockLength;
+ private long read;
- private UncompressedState(int blockLength) {
+ private UncompressedState(long blockLength) {
this.blockLength = blockLength;
}
@@ -197,7 +197,8 @@ class HuffmanDecoder implements Closeable {
@Override
int read(byte[] b, int off, int len) throws IOException {
- int max = Math.min(blockLength - read, len);
+ // as len is an int the min must fit into an int as well
+ int max = (int) Math.min(blockLength - read, len);
for (int i = 0; i < max; i++) {
byte next = (byte) (readBits(Byte.SIZE) & 0xFF);
b[off + i] = memory.add(next);
[27/28] commons-compress git commit: fix typo, thanks @dweiss
Posted by bo...@apache.org.
fix typo, thanks @dweiss
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4b57823e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4b57823e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4b57823e
Branch: refs/heads/master
Commit: 4b57823e9675293f2055a3dc5145b77ccc0b55f2
Parents: 32d507b
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 9 13:26:22 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 9 13:26:22 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/commons/compress/utils/BitInputStream.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4b57823e/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
index d847eb9..8105075 100644
--- a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
@@ -102,7 +102,7 @@ public class BitInputStream implements Closeable {
/**
* Returns the number of bits that can be read from this input
- * stream without readong from the underlying input stream at all.
+ * stream without reading from the underlying input stream at all.
* @since 1.16
*/
public int bitsCached() {
[28/28] commons-compress git commit: Merge branch 'COMPRESS-380'
Posted by bo...@apache.org.
Merge branch 'COMPRESS-380'
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b18ef2a1
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b18ef2a1
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b18ef2a1
Branch: refs/heads/master
Commit: b18ef2a10393824bd9a79c908bda656de4d45006
Parents: 7cf10d7 4b57823
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 9 18:40:31 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 9 18:40:31 2018 +0100
----------------------------------------------------------------------
.gitattributes | 1 +
src/changes/changes.xml | 4 +
.../archivers/zip/ZipArchiveInputStream.java | 39 +-
.../commons/compress/archivers/zip/ZipFile.java | 4 +-
.../commons/compress/archivers/zip/ZipUtil.java | 4 +-
.../compressors/CompressorStreamFactory.java | 24 +-
.../Deflate64CompressorInputStream.java | 104 ++++
.../compressors/deflate64/HuffmanDecoder.java | 513 +++++++++++++++++++
.../compressors/deflate64/HuffmanState.java | 25 +
.../compress/compressors/deflate64/package.html | 25 +
.../commons/compress/utils/BitInputStream.java | 30 ++
src/site/xdoc/examples.xml | 25 +-
src/site/xdoc/index.xml | 7 +-
src/site/xdoc/limitations.xml | 2 +-
src/site/xdoc/zip.xml | 36 +-
.../zip/ZipArchiveInputStreamTest.java | 35 ++
.../compress/archivers/zip/ZipFileTest.java | 18 +
.../deflate64/Deflate64BugsTest.java | 49 ++
.../Deflate64CompressorInputStreamTest.java | 155 ++++++
.../deflate64/HuffmanDecoderTest.java | 222 ++++++++
.../resources/COMPRESS-380/COMPRESS-380-dd.zip | Bin 0 -> 1391 bytes
.../resources/COMPRESS-380/COMPRESS-380-input | Bin 0 -> 3072 bytes
.../COMPRESS-380-readbeyondmemory.zip | Bin 0 -> 15290 bytes
.../resources/COMPRESS-380/COMPRESS-380.zip | Bin 0 -> 2257 bytes
24 files changed, 1303 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/changes/changes.xml
----------------------------------------------------------------------
diff --cc src/changes/changes.xml
index d640f2a,3eaf692..de50417
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@@ -72,14 -72,10 +72,18 @@@ The <action> type attribute can be add,
throw the expected IOException rather than obscure
RuntimeExceptions.
</action>
- <action issue="COMPRESS-380" type="add" date="2018-01-03"
+ <action type="update" date="2018-01-04">
+ Updated XZ for Java dependency to 1.8 in order to pick up bug
+ fix to LZMA2InputStream's available method.
+ </action>
+ <action type="update" date="2018-01-05" issue="COMPRESS-429"
+ due-to="Damiano Albani">
+ ZipArchiveEntry now exposes how the name or comment have been
+ determined when the entry was read.
++ <action issue="COMPRESS-380" type="add" date="2018-01-09"
+ due-to="">
+ Added read-only DEFLATE64 support to ZIP archives and as
+ stand-alone CompressorInputStream.
</action>
</release>
<release version="1.15" date="2017-10-17"
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/main/java/org/apache/commons/compress/archivers/zip/ZipUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/site/xdoc/examples.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b18ef2a1/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
----------------------------------------------------------------------
diff --cc src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
index a6170b1,e29f080..8e07d7e
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
@@@ -581,23 -581,24 +581,41 @@@ public class ZipFileTest
entry.setAlignment(3);
}
+ @Test
+ public void nameSourceDefaultsToName() throws Exception {
+ nameSource("bla.zip", "test1.xml", ZipArchiveEntry.NameSource.NAME);
+ }
+
+ @Test
+ public void nameSourceIsSetToUnicodeExtraField() throws Exception {
+ nameSource("utf8-winzip-test.zip", "\u20AC_for_Dollar.txt",
+ ZipArchiveEntry.NameSource.UNICODE_EXTRA_FIELD);
+ }
+
+ @Test
+ public void nameSourceIsSetToEFS() throws Exception {
+ nameSource("utf8-7zip-test.zip", "\u20AC_for_Dollar.txt",
+ ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG);
+ }
+
+ /**
+ * @see "https://issues.apache.org/jira/browse/COMPRESS-380"
+ */
+ @Test
+ public void readDeflate64CompressedStream() throws Exception {
+ final File input = getFile("COMPRESS-380/COMPRESS-380-input");
+ final File archive = getFile("COMPRESS-380/COMPRESS-380.zip");
+ try (FileInputStream in = new FileInputStream(input);
+ ZipFile zf = new ZipFile(archive)) {
+ byte[] orig = IOUtils.toByteArray(in);
+ ZipArchiveEntry e = zf.getEntry("input2");
+ try (InputStream s = zf.getInputStream(e)) {
+ byte[] fromZip = IOUtils.toByteArray(s);
+ assertArrayEquals(orig, fromZip);
+ }
+ }
+ }
+
private void assertAllReadMethods(byte[] expected, ZipFile zipFile, ZipArchiveEntry entry) {
// simple IOUtil read
try (InputStream stream = zf.getInputStream(entry)) {
[09/28] commons-compress git commit: small fixme
Posted by bo...@apache.org.
small fixme
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/bca3b08b
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/bca3b08b
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/bca3b08b
Branch: refs/heads/master
Commit: bca3b08b2779c1bc95d23455bae441dfaf781e83
Parents: 53ec582
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 07:33:14 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 07:33:14 2018 +0100
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipArchiveInputStream.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/bca3b08b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
index 5ea9c0a..ea2b34f 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStream.java
@@ -314,6 +314,7 @@ public class ZipArchiveInputStream extends ArchiveInputStream {
current.entry.setStreamContiguous(true);
if (current.entry.getCompressedSize() != ArchiveEntry.SIZE_UNKNOWN) {
+ // FIXME this currently leaks bis if the method is not one of the supported ones
InputStream bis = new BoundedInputStream(in, current.entry.getCompressedSize());
if (current.entry.getMethod() == ZipMethod.UNSHRINKING.getCode()) {
current.in = new UnshrinkingInputStream(bis);
[24/28] commons-compress git commit: COMPRESS-380 make second part of
assumption explicit
Posted by bo...@apache.org.
COMPRESS-380 make second part of assumption explicit
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/19ad2871
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/19ad2871
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/19ad2871
Branch: refs/heads/master
Commit: 19ad28711cd7f0b5e2d8a5fab73b978fdb057882
Parents: 3600a5f
Author: Stefan Bodewig <bo...@apache.org>
Authored: Mon Jan 8 09:15:24 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Mon Jan 8 09:15:24 2018 +0100
----------------------------------------------------------------------
.../commons/compress/compressors/deflate64/HuffmanDecoder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/19ad2871/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 8b006c3..f1f1042 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -197,7 +197,7 @@ class HuffmanDecoder implements Closeable {
@Override
int read(byte[] b, int off, int len) throws IOException {
- // as len is an int the min must fit into an int as well
+ // as len is an int and (blockLength - read) is >= 0 the min must fit into an int as well
int max = (int) Math.min(blockLength - read, len);
for (int i = 0; i < max; i++) {
byte next = (byte) (readBits(Byte.SIZE) & 0xFF);
[03/28] commons-compress git commit: COMPRESS-380 change package name
Posted by bo...@apache.org.
COMPRESS-380 change package name
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/2042aa00
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/2042aa00
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/2042aa00
Branch: refs/heads/master
Commit: 2042aa001474179a031a368f9d6ec73c2c686965
Parents: 3e9b0e8
Author: Stefan Bodewig <bo...@apache.org>
Authored: Wed Jan 3 15:20:33 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Wed Jan 3 15:20:33 2018 +0100
----------------------------------------------------------------------
.../commons/compress/archivers/zip/ZipFile.java | 2 +-
.../deflate64/Deflate64InputStream.java | 88 ++++
.../compressors/deflate64/HuffmanDecoder.java | 462 +++++++++++++++++++
.../compressors/deflate64/HuffmanState.java | 25 +
.../compressors/zip/Deflate64InputStream.java | 88 ----
.../compressors/zip/HuffmanDecoder.java | 462 -------------------
.../compress/compressors/zip/HuffmanState.java | 25 -
.../deflate64/Deflate64InputStreamTest.java | 128 +++++
.../deflate64/HuffmanDecoderTest.java | 222 +++++++++
.../zip/Deflate64InputStreamTest.java | 128 -----
.../compressors/zip/HuffmanDecoderTest.java | 222 ---------
11 files changed, 926 insertions(+), 926 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
index b7a41a3..c10fc1a 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipFile.java
@@ -42,7 +42,7 @@ import java.util.zip.InflaterInputStream;
import java.util.zip.ZipException;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-import org.apache.commons.compress.compressors.zip.Deflate64InputStream;
+import org.apache.commons.compress.compressors.deflate64.Deflate64InputStream;
import org.apache.commons.compress.utils.IOUtils;
import static org.apache.commons.compress.archivers.zip.ZipConstants.DWORD;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
new file mode 100644
index 0000000..e45f87f
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStream.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+public class Deflate64InputStream extends InputStream {
+ private HuffmanDecoder decoder;
+ private long uncompressedSize;
+ private long totalRead = 0;
+
+ public Deflate64InputStream(InputStream in, long uncompressedSize) {
+ this(new HuffmanDecoder(in), uncompressedSize);
+ }
+
+ Deflate64InputStream(HuffmanDecoder decoder, long uncompressedSize) {
+ this.uncompressedSize = uncompressedSize;
+ this.decoder = decoder;
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] b = new byte[1];
+ while (true) {
+ int r = read(b);
+ switch (r) {
+ case 1:
+ return b[0] & 0xFF;
+ case -1:
+ return -1;
+ case 0:
+ continue;
+ default:
+ throw new IllegalStateException("Invalid return value from read: " + r);
+ }
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int read = -1;
+ if (decoder != null) {
+ read = decoder.decode(b, off, len);
+ if (read == -1) {
+ close();
+ } else {
+ totalRead += read;
+ }
+ }
+ return read;
+ }
+
+ @Override
+ public int available() throws IOException {
+ long available = 0;
+ if (decoder != null) {
+ available = uncompressedSize - totalRead;
+ if (Long.compare(available, Integer.MAX_VALUE) > 0) {
+ available = Integer.MAX_VALUE;
+ }
+ }
+ return (int) available;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeQuietly(decoder);
+ decoder = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
new file mode 100644
index 0000000..bf1c4ad
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -0,0 +1,462 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import org.apache.commons.compress.utils.BitInputStream;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+
+import static org.apache.commons.compress.compressors.deflate64.HuffmanState.*;
+import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
+
+public class HuffmanDecoder implements Closeable {
+ /**
+ * --------------------------------------------------------------------
+ * idx xtra base idx xtra base idx xtra base
+ * --------------------------------------------------------------------
+ * 257 0 3 267 1 15,16 277 4 67-82
+ * 258 0 4 268 1 17,18 278 4 83-98
+ * 259 0 5 269 2 19-22 279 4 99-114
+ * 260 0 6 270 2 23-26 280 4 115-130
+ * 261 0 7 271 2 27-30 281 5 131-162
+ * 262 0 8 272 2 31-34 282 5 163-194
+ * 263 0 9 273 3 35-42 283 5 195-226
+ * 264 0 10 274 3 43-50 284 5 227-257
+ * 265 1 11,12 275 3 51-58 285 96 3
+ * 266 1 13,14 276 3 59-66
+ * --------------------------------------------------------------------
+ * value = (base of run length) << 5 | (number of extra bits to read)
+ */
+ private static final short[] RUN_LENGTH_TABLE = {
+ 96, 128, 160, 192, 224, 256, 288, 320, 353, 417, 481, 545, 610, 738, 866,
+ 994, 1123, 1379, 1635, 1891, 2148, 2660, 3172, 3684, 4197, 5221, 6245, 7269, 112
+ };
+
+ /**
+ * --------------------------------------------------------------------
+ * idx xtra dist idx xtra dist idx xtra dist
+ * --------------------------------------------------------------------
+ * 0 0 1 10 4 33-48 20 9 1025-1536
+ * 1 0 2 11 4 49-64 21 9 1537-2048
+ * 2 0 3 12 5 65-96 22 10 2049-3072
+ * 3 0 4 13 5 97-128 23 10 3073-4096
+ * 4 1 5,6 14 6 129-192 24 11 4097-6144
+ * 5 1 7,8 15 6 193-256 25 11 6145-8192
+ * 6 2 9-12 16 7 257-384 26 12 8193-12288
+ * 7 2 13-16 17 7 385-512 27 12 12289-16384
+ * 8 3 17-24 18 8 513-768 28 13 16385-24576
+ * 9 3 25-32 19 8 769-1024 29 13 24577-32768
+ * 30 14 32769-49152
+ * 31 14 49153-65536
+ * --------------------------------------------------------------------
+ * value = (base of distance) << 4 | (number of extra bits to read)
+ */
+ private static final int[] DISTANCE_TABLE = {
+ 16, 32, 48, 64, 81, 113, 146, 210, 275, 403, // 0-9
+ 532, 788, 1045, 1557, 2070, 3094, 4119, 6167, 8216, 12312, // 10-19
+ 16409, 24601, 32794, 49178, 65563, 98331, 131100, 196636, 262173, 393245, // 20-29
+ 524318, 786462 // 30-31
+ };
+
+ /**
+ * When using dynamic huffman codes the order in which the values are stored
+ * follows the positioning below
+ */
+ private static final int[] CODE_LENGTHS_ORDER =
+ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
+
+ /**
+ * Huffman Fixed Literal / Distance tables for mode 1
+ */
+ private static final int[] FIXED_LITERALS;
+ private static final int[] FIXED_DISTANCE;
+
+ static {
+ FIXED_LITERALS = new int[288];
+ Arrays.fill(FIXED_LITERALS, 0, 144, 8);
+ Arrays.fill(FIXED_LITERALS, 144, 256, 9);
+ Arrays.fill(FIXED_LITERALS, 256, 280, 7);
+ Arrays.fill(FIXED_LITERALS, 280, 288, 8);
+
+ FIXED_DISTANCE = new int[32];
+ Arrays.fill(FIXED_DISTANCE, 5);
+ }
+
+ private boolean finalBlock = false;
+ private DecoderState state;
+ private BitInputStream reader;
+
+ private final DecodingMemory memory = new DecodingMemory();
+
+ HuffmanDecoder(InputStream in) {
+ this(new BitInputStream(in, ByteOrder.LITTLE_ENDIAN));
+ }
+
+ private HuffmanDecoder(BitInputStream reader) {
+ this.reader = reader;
+ state = new InitialState();
+ }
+
+ @Override
+ public void close() {
+ closeQuietly(reader);
+ reader = null;
+ }
+
+ public int decode(byte[] b) throws IOException {
+ return decode(b, 0, b.length);
+ }
+
+ public int decode(byte[] b, int off, int len) throws IOException {
+ while (!finalBlock || state.hasData()) {
+ switch (state.state()) {
+ case INITIAL:
+ finalBlock = reader.readBits(1) == 1;
+ int mode = (int) reader.readBits(2);
+ switch (mode) {
+ case 0:
+ reader.readBits(Byte.SIZE - 3);
+ long bLen = reader.readBits(16);
+ long bNLen = reader.readBits(16);
+ if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
+ //noinspection DuplicateStringLiteralInspection
+ throw new IllegalStateException("Illegal LEN / NLEN values");
+ }
+ state = new UncompressedState((int) bLen);
+ break;
+ case 1:
+ state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
+ break;
+ case 2:
+ int literals = (int) (reader.readBits(5) + 257);
+ int[] literalTable = new int[literals];
+
+ int distances = (int) (reader.readBits(5) + 1);
+ int[] distanceTable = new int[distances];
+
+ populateDynamicTables(reader, literalTable, distanceTable);
+
+ state = new HuffmanCodes(DYNAMIC_CODES, literalTable, distanceTable);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported compression: " + mode);
+ }
+ break;
+ default:
+ return state.read(b, off, len);
+ }
+ }
+ return -1;
+ }
+
+
+ private static abstract class DecoderState {
+ abstract HuffmanState state();
+
+ abstract int read(byte[] b, int off, int len) throws IOException;
+
+ abstract boolean hasData();
+ }
+
+ private class UncompressedState extends DecoderState {
+ private final int blockLength;
+ private int read;
+
+ private UncompressedState(int blockLength) {
+ this.blockLength = blockLength;
+ }
+
+ @Override
+ HuffmanState state() {
+ return read < blockLength ? STORED : INITIAL;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ int max = Math.min(blockLength - read, len);
+ for (int i = 0; i < max; i++) {
+ byte next = (byte) (reader.readBits(Byte.SIZE) & 0xFF);
+ b[off + i] = memory.add(next);
+ read++;
+ }
+ return max;
+ }
+
+ @Override
+ boolean hasData() {
+ return read < blockLength;
+ }
+ }
+
+ private class InitialState extends DecoderState {
+ @Override
+ HuffmanState state() {
+ return INITIAL;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ throw new IllegalStateException("Cannot read in this state");
+ }
+
+ @Override
+ boolean hasData() {
+ return false;
+ }
+ }
+
+ private class HuffmanCodes extends DecoderState {
+ private boolean endOfBlock = false;
+ private final HuffmanState state;
+ private final BinaryTreeNode lengthTree;
+ private final BinaryTreeNode distanceTree;
+
+ private int runBufferPos = 0;
+ private byte[] runBuffer = new byte[0];
+
+ HuffmanCodes(HuffmanState state, int[] lengths, int[] distance) {
+ this.state = state;
+ lengthTree = buildTree(lengths);
+ distanceTree = buildTree(distance);
+ }
+
+ @Override
+ HuffmanState state() {
+ return endOfBlock ? INITIAL : state;
+ }
+
+ @Override
+ int read(byte[] b, int off, int len) throws IOException {
+ return decodeNext(b, off, len);
+ }
+
+ private int decodeNext(byte[] b, int off, int len) throws IOException {
+ if (endOfBlock) {
+ return -1;
+ }
+ int result = copyFromRunBuffer(b, off, len);
+
+ while (result < len) {
+ int symbol = nextSymbol(reader, lengthTree);
+ if (symbol < 256) {
+ b[off + result++] = memory.add((byte) (symbol & 0xFF));
+ } else if (symbol > 256) {
+ int runMask = RUN_LENGTH_TABLE[symbol - 257];
+ int run = runMask >>> 5;
+ int runXtra = runMask & 0x1F;
+ run += reader.readBits(runXtra);
+
+ int distSym = nextSymbol(reader, distanceTree);
+
+ int distMask = DISTANCE_TABLE[distSym];
+ int dist = distMask >>> 4;
+ int distXtra = distMask & 0xF;
+ dist += reader.readBits(distXtra);
+
+ runBuffer = new byte[run];
+ runBufferPos = 0;
+ memory.recordToBuffer(dist, run, runBuffer);
+
+ result += copyFromRunBuffer(b, off + result, len - result);
+ } else {
+ endOfBlock = true;
+ return result;
+ }
+ }
+
+ return result;
+ }
+
+ private int copyFromRunBuffer(byte[] b, int off, int len) {
+ int bytesInBuffer = runBuffer.length - runBufferPos;
+ int copiedBytes = 0;
+ if (bytesInBuffer > 0) {
+ copiedBytes = Math.min(len, bytesInBuffer);
+ System.arraycopy(runBuffer, runBufferPos, b, off, copiedBytes);
+ runBufferPos += copiedBytes;
+ }
+ return copiedBytes;
+ }
+
+ @Override
+ boolean hasData() {
+ return !endOfBlock;
+ }
+ }
+
+ private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
+ BinaryTreeNode node = tree;
+ while (node != null && node.literal == -1) {
+ long bit = reader.readBits(1);
+ node = bit == 0 ? node.left : node.right;
+ }
+ return node != null ? node.literal : -1;
+ }
+
+ private static void populateDynamicTables(BitInputStream reader, int[] literals, int[] distances) throws IOException {
+ int codeLengths = (int) (reader.readBits(4) + 4);
+
+ int[] codeLengthValues = new int[19];
+ for (int cLen = 0; cLen < codeLengths; cLen++) {
+ codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) reader.readBits(3);
+ }
+
+ BinaryTreeNode codeLengthTree = buildTree(codeLengthValues);
+
+ int[] auxBuffer = new int[literals.length + distances.length];
+
+ int value = -1, length = 0;
+ for (int i = 0; i < auxBuffer.length; ) {
+ if (length > 0) {
+ auxBuffer[i++] = value;
+ length--;
+ } else {
+ int symbol = nextSymbol(reader, codeLengthTree);
+ if (symbol < 16) {
+ value = symbol;
+ auxBuffer[i++] = value;
+ } else if (symbol == 16) {
+ length = (int) (reader.readBits(2) + 3);
+ } else if (symbol == 17) {
+ value = 0;
+ length = (int) (reader.readBits(3) + 3);
+ } else if (symbol == 18) {
+ value = 0;
+ length = (int) (reader.readBits(7) + 11);
+ }
+ }
+ }
+
+ System.arraycopy(auxBuffer, 0, literals, 0, literals.length);
+ System.arraycopy(auxBuffer, literals.length, distances, 0, distances.length);
+ }
+
+ private static class BinaryTreeNode {
+ private final int bits;
+ int literal = -1;
+ BinaryTreeNode left, right;
+
+ private BinaryTreeNode(int bits) {
+ this.bits = bits;
+ }
+
+ void leaf(int symbol) {
+ literal = symbol;
+ left = null;
+ right = null;
+ }
+
+ BinaryTreeNode left() {
+ if (left == null && literal == -1) {
+ left = new BinaryTreeNode(bits + 1);
+ }
+ return left;
+ }
+
+ BinaryTreeNode right() {
+ if (right == null && literal == -1) {
+ right = new BinaryTreeNode(bits + 1);
+ }
+ return right;
+ }
+ }
+
+ private static BinaryTreeNode buildTree(int[] litTable) {
+ int[] literalCodes = getCodes(litTable);
+
+ BinaryTreeNode root = new BinaryTreeNode(0);
+
+ for (int i = 0; i < litTable.length; i++) {
+ int len = litTable[i];
+ if (len != 0) {
+ BinaryTreeNode node = root;
+ int lit = literalCodes[len - 1];
+ for (int p = len - 1; p >= 0; p--) {
+ int bit = lit & (1 << p);
+ node = bit == 0 ? node.left() : node.right();
+ }
+ node.leaf(i);
+ literalCodes[len - 1]++;
+ }
+ }
+ return root;
+ }
+
+ private static int[] getCodes(int[] litTable) {
+ int max = 0;
+ int[] blCount = new int[65];
+
+ for (int aLitTable : litTable) {
+ max = Math.max(max, aLitTable);
+ blCount[aLitTable]++;
+ }
+ blCount = Arrays.copyOf(blCount, max + 1);
+
+ int code = 0;
+ int[] nextCode = new int[max + 1];
+ for (int i = 0; i <= max; i++) {
+ code = (code + blCount[i]) << 1;
+ nextCode[i] = code;
+ }
+
+ return nextCode;
+ }
+
+ private static class DecodingMemory {
+ private final byte[] memory;
+ private final int mask;
+ private int wHead;
+
+ private DecodingMemory() {
+ this(16);
+ }
+
+ private DecodingMemory(int bits) {
+ memory = new byte[1 << bits];
+ Arrays.fill(memory, (byte) -1);
+ mask = memory.length - 1;
+ }
+
+ byte add(byte b) {
+ memory[wHead] = b;
+ wHead = incCounter(wHead);
+ return b;
+ }
+
+ void recordToBuffer(int distance, int length, byte[] buff) {
+ if (distance > memory.length) {
+ throw new IllegalStateException("Illegal distance parameter: " + distance);
+ }
+ int start = (wHead - distance) & mask;
+ if (memory[start] == -1) {
+ throw new IllegalStateException("Attempt to read beyond memory: dist=" + distance);
+ }
+ for (int i = 0, pos = start; i < length; i++, pos = incCounter(pos)) {
+ buff[i] = add(memory[pos]);
+ }
+ }
+
+ private int incCounter(int counter) {
+ return (counter + 1) & mask;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java
new file mode 100644
index 0000000..b34bb7e
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanState.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+enum HuffmanState {
+ INITIAL,
+ STORED,
+ DYNAMIC_CODES,
+ FIXED_CODES
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java b/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
deleted file mode 100644
index 98bf792..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.zip;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
-
-public class Deflate64InputStream extends InputStream {
- private HuffmanDecoder decoder;
- private long uncompressedSize;
- private long totalRead = 0;
-
- public Deflate64InputStream(InputStream in, long uncompressedSize) {
- this(new HuffmanDecoder(in), uncompressedSize);
- }
-
- Deflate64InputStream(HuffmanDecoder decoder, long uncompressedSize) {
- this.uncompressedSize = uncompressedSize;
- this.decoder = decoder;
- }
-
- @Override
- public int read() throws IOException {
- byte[] b = new byte[1];
- while (true) {
- int r = read(b);
- switch (r) {
- case 1:
- return b[0] & 0xFF;
- case -1:
- return -1;
- case 0:
- continue;
- default:
- throw new IllegalStateException("Invalid return value from read: " + r);
- }
- }
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- int read = -1;
- if (decoder != null) {
- read = decoder.decode(b, off, len);
- if (read == -1) {
- close();
- } else {
- totalRead += read;
- }
- }
- return read;
- }
-
- @Override
- public int available() throws IOException {
- long available = 0;
- if (decoder != null) {
- available = uncompressedSize - totalRead;
- if (Long.compare(available, Integer.MAX_VALUE) > 0) {
- available = Integer.MAX_VALUE;
- }
- }
- return (int) available;
- }
-
- @Override
- public void close() throws IOException {
- closeQuietly(decoder);
- decoder = null;
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
deleted file mode 100644
index 0b6b173..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.zip;
-
-import org.apache.commons.compress.utils.BitInputStream;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteOrder;
-import java.util.Arrays;
-
-import static org.apache.commons.compress.compressors.zip.HuffmanState.*;
-import static org.apache.commons.compress.utils.IOUtils.closeQuietly;
-
-public class HuffmanDecoder implements Closeable {
- /**
- * --------------------------------------------------------------------
- * idx xtra base idx xtra base idx xtra base
- * --------------------------------------------------------------------
- * 257 0 3 267 1 15,16 277 4 67-82
- * 258 0 4 268 1 17,18 278 4 83-98
- * 259 0 5 269 2 19-22 279 4 99-114
- * 260 0 6 270 2 23-26 280 4 115-130
- * 261 0 7 271 2 27-30 281 5 131-162
- * 262 0 8 272 2 31-34 282 5 163-194
- * 263 0 9 273 3 35-42 283 5 195-226
- * 264 0 10 274 3 43-50 284 5 227-257
- * 265 1 11,12 275 3 51-58 285 96 3
- * 266 1 13,14 276 3 59-66
- * --------------------------------------------------------------------
- * value = (base of run length) << 5 | (number of extra bits to read)
- */
- private static final short[] RUN_LENGTH_TABLE = {
- 96, 128, 160, 192, 224, 256, 288, 320, 353, 417, 481, 545, 610, 738, 866,
- 994, 1123, 1379, 1635, 1891, 2148, 2660, 3172, 3684, 4197, 5221, 6245, 7269, 112
- };
-
- /**
- * --------------------------------------------------------------------
- * idx xtra dist idx xtra dist idx xtra dist
- * --------------------------------------------------------------------
- * 0 0 1 10 4 33-48 20 9 1025-1536
- * 1 0 2 11 4 49-64 21 9 1537-2048
- * 2 0 3 12 5 65-96 22 10 2049-3072
- * 3 0 4 13 5 97-128 23 10 3073-4096
- * 4 1 5,6 14 6 129-192 24 11 4097-6144
- * 5 1 7,8 15 6 193-256 25 11 6145-8192
- * 6 2 9-12 16 7 257-384 26 12 8193-12288
- * 7 2 13-16 17 7 385-512 27 12 12289-16384
- * 8 3 17-24 18 8 513-768 28 13 16385-24576
- * 9 3 25-32 19 8 769-1024 29 13 24577-32768
- * 30 14 32769-49152
- * 31 14 49153-65536
- * --------------------------------------------------------------------
- * value = (base of distance) << 4 | (number of extra bits to read)
- */
- private static final int[] DISTANCE_TABLE = {
- 16, 32, 48, 64, 81, 113, 146, 210, 275, 403, // 0-9
- 532, 788, 1045, 1557, 2070, 3094, 4119, 6167, 8216, 12312, // 10-19
- 16409, 24601, 32794, 49178, 65563, 98331, 131100, 196636, 262173, 393245, // 20-29
- 524318, 786462 // 30-31
- };
-
- /**
- * When using dynamic huffman codes the order in which the values are stored
- * follows the positioning below
- */
- private static final int[] CODE_LENGTHS_ORDER =
- {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
-
- /**
- * Huffman Fixed Literal / Distance tables for mode 1
- */
- private static final int[] FIXED_LITERALS;
- private static final int[] FIXED_DISTANCE;
-
- static {
- FIXED_LITERALS = new int[288];
- Arrays.fill(FIXED_LITERALS, 0, 144, 8);
- Arrays.fill(FIXED_LITERALS, 144, 256, 9);
- Arrays.fill(FIXED_LITERALS, 256, 280, 7);
- Arrays.fill(FIXED_LITERALS, 280, 288, 8);
-
- FIXED_DISTANCE = new int[32];
- Arrays.fill(FIXED_DISTANCE, 5);
- }
-
- private boolean finalBlock = false;
- private DecoderState state;
- private BitInputStream reader;
-
- private final DecodingMemory memory = new DecodingMemory();
-
- HuffmanDecoder(InputStream in) {
- this(new BitInputStream(in, ByteOrder.LITTLE_ENDIAN));
- }
-
- private HuffmanDecoder(BitInputStream reader) {
- this.reader = reader;
- state = new InitialState();
- }
-
- @Override
- public void close() {
- closeQuietly(reader);
- reader = null;
- }
-
- public int decode(byte[] b) throws IOException {
- return decode(b, 0, b.length);
- }
-
- public int decode(byte[] b, int off, int len) throws IOException {
- while (!finalBlock || state.hasData()) {
- switch (state.state()) {
- case INITIAL:
- finalBlock = reader.readBits(1) == 1;
- int mode = (int) reader.readBits(2);
- switch (mode) {
- case 0:
- reader.readBits(Byte.SIZE - 3);
- long bLen = reader.readBits(16);
- long bNLen = reader.readBits(16);
- if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
- //noinspection DuplicateStringLiteralInspection
- throw new IllegalStateException("Illegal LEN / NLEN values");
- }
- state = new UncompressedState((int) bLen);
- break;
- case 1:
- state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
- break;
- case 2:
- int literals = (int) (reader.readBits(5) + 257);
- int[] literalTable = new int[literals];
-
- int distances = (int) (reader.readBits(5) + 1);
- int[] distanceTable = new int[distances];
-
- populateDynamicTables(reader, literalTable, distanceTable);
-
- state = new HuffmanCodes(DYNAMIC_CODES, literalTable, distanceTable);
- break;
- default:
- throw new IllegalStateException("Unsupported compression: " + mode);
- }
- break;
- default:
- return state.read(b, off, len);
- }
- }
- return -1;
- }
-
-
- private static abstract class DecoderState {
- abstract HuffmanState state();
-
- abstract int read(byte[] b, int off, int len) throws IOException;
-
- abstract boolean hasData();
- }
-
- private class UncompressedState extends DecoderState {
- private final int blockLength;
- private int read;
-
- private UncompressedState(int blockLength) {
- this.blockLength = blockLength;
- }
-
- @Override
- HuffmanState state() {
- return read < blockLength ? STORED : INITIAL;
- }
-
- @Override
- int read(byte[] b, int off, int len) throws IOException {
- int max = Math.min(blockLength - read, len);
- for (int i = 0; i < max; i++) {
- byte next = (byte) (reader.readBits(Byte.SIZE) & 0xFF);
- b[off + i] = memory.add(next);
- read++;
- }
- return max;
- }
-
- @Override
- boolean hasData() {
- return read < blockLength;
- }
- }
-
- private class InitialState extends DecoderState {
- @Override
- HuffmanState state() {
- return INITIAL;
- }
-
- @Override
- int read(byte[] b, int off, int len) throws IOException {
- throw new IllegalStateException("Cannot read in this state");
- }
-
- @Override
- boolean hasData() {
- return false;
- }
- }
-
- private class HuffmanCodes extends DecoderState {
- private boolean endOfBlock = false;
- private final HuffmanState state;
- private final BinaryTreeNode lengthTree;
- private final BinaryTreeNode distanceTree;
-
- private int runBufferPos = 0;
- private byte[] runBuffer = new byte[0];
-
- HuffmanCodes(HuffmanState state, int[] lengths, int[] distance) {
- this.state = state;
- lengthTree = buildTree(lengths);
- distanceTree = buildTree(distance);
- }
-
- @Override
- HuffmanState state() {
- return endOfBlock ? INITIAL : state;
- }
-
- @Override
- int read(byte[] b, int off, int len) throws IOException {
- return decodeNext(b, off, len);
- }
-
- private int decodeNext(byte[] b, int off, int len) throws IOException {
- if (endOfBlock) {
- return -1;
- }
- int result = copyFromRunBuffer(b, off, len);
-
- while (result < len) {
- int symbol = nextSymbol(reader, lengthTree);
- if (symbol < 256) {
- b[off + result++] = memory.add((byte) (symbol & 0xFF));
- } else if (symbol > 256) {
- int runMask = RUN_LENGTH_TABLE[symbol - 257];
- int run = runMask >>> 5;
- int runXtra = runMask & 0x1F;
- run += reader.readBits(runXtra);
-
- int distSym = nextSymbol(reader, distanceTree);
-
- int distMask = DISTANCE_TABLE[distSym];
- int dist = distMask >>> 4;
- int distXtra = distMask & 0xF;
- dist += reader.readBits(distXtra);
-
- runBuffer = new byte[run];
- runBufferPos = 0;
- memory.recordToBuffer(dist, run, runBuffer);
-
- result += copyFromRunBuffer(b, off + result, len - result);
- } else {
- endOfBlock = true;
- return result;
- }
- }
-
- return result;
- }
-
- private int copyFromRunBuffer(byte[] b, int off, int len) {
- int bytesInBuffer = runBuffer.length - runBufferPos;
- int copiedBytes = 0;
- if (bytesInBuffer > 0) {
- copiedBytes = Math.min(len, bytesInBuffer);
- System.arraycopy(runBuffer, runBufferPos, b, off, copiedBytes);
- runBufferPos += copiedBytes;
- }
- return copiedBytes;
- }
-
- @Override
- boolean hasData() {
- return !endOfBlock;
- }
- }
-
- private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
- BinaryTreeNode node = tree;
- while (node != null && node.literal == -1) {
- long bit = reader.readBits(1);
- node = bit == 0 ? node.left : node.right;
- }
- return node != null ? node.literal : -1;
- }
-
- private static void populateDynamicTables(BitInputStream reader, int[] literals, int[] distances) throws IOException {
- int codeLengths = (int) (reader.readBits(4) + 4);
-
- int[] codeLengthValues = new int[19];
- for (int cLen = 0; cLen < codeLengths; cLen++) {
- codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) reader.readBits(3);
- }
-
- BinaryTreeNode codeLengthTree = buildTree(codeLengthValues);
-
- int[] auxBuffer = new int[literals.length + distances.length];
-
- int value = -1, length = 0;
- for (int i = 0; i < auxBuffer.length; ) {
- if (length > 0) {
- auxBuffer[i++] = value;
- length--;
- } else {
- int symbol = nextSymbol(reader, codeLengthTree);
- if (symbol < 16) {
- value = symbol;
- auxBuffer[i++] = value;
- } else if (symbol == 16) {
- length = (int) (reader.readBits(2) + 3);
- } else if (symbol == 17) {
- value = 0;
- length = (int) (reader.readBits(3) + 3);
- } else if (symbol == 18) {
- value = 0;
- length = (int) (reader.readBits(7) + 11);
- }
- }
- }
-
- System.arraycopy(auxBuffer, 0, literals, 0, literals.length);
- System.arraycopy(auxBuffer, literals.length, distances, 0, distances.length);
- }
-
- private static class BinaryTreeNode {
- private final int bits;
- int literal = -1;
- BinaryTreeNode left, right;
-
- private BinaryTreeNode(int bits) {
- this.bits = bits;
- }
-
- void leaf(int symbol) {
- literal = symbol;
- left = null;
- right = null;
- }
-
- BinaryTreeNode left() {
- if (left == null && literal == -1) {
- left = new BinaryTreeNode(bits + 1);
- }
- return left;
- }
-
- BinaryTreeNode right() {
- if (right == null && literal == -1) {
- right = new BinaryTreeNode(bits + 1);
- }
- return right;
- }
- }
-
- private static BinaryTreeNode buildTree(int[] litTable) {
- int[] literalCodes = getCodes(litTable);
-
- BinaryTreeNode root = new BinaryTreeNode(0);
-
- for (int i = 0; i < litTable.length; i++) {
- int len = litTable[i];
- if (len != 0) {
- BinaryTreeNode node = root;
- int lit = literalCodes[len - 1];
- for (int p = len - 1; p >= 0; p--) {
- int bit = lit & (1 << p);
- node = bit == 0 ? node.left() : node.right();
- }
- node.leaf(i);
- literalCodes[len - 1]++;
- }
- }
- return root;
- }
-
- private static int[] getCodes(int[] litTable) {
- int max = 0;
- int[] blCount = new int[65];
-
- for (int aLitTable : litTable) {
- max = Math.max(max, aLitTable);
- blCount[aLitTable]++;
- }
- blCount = Arrays.copyOf(blCount, max + 1);
-
- int code = 0;
- int[] nextCode = new int[max + 1];
- for (int i = 0; i <= max; i++) {
- code = (code + blCount[i]) << 1;
- nextCode[i] = code;
- }
-
- return nextCode;
- }
-
- private static class DecodingMemory {
- private final byte[] memory;
- private final int mask;
- private int wHead;
-
- private DecodingMemory() {
- this(16);
- }
-
- private DecodingMemory(int bits) {
- memory = new byte[1 << bits];
- Arrays.fill(memory, (byte) -1);
- mask = memory.length - 1;
- }
-
- byte add(byte b) {
- memory[wHead] = b;
- wHead = incCounter(wHead);
- return b;
- }
-
- void recordToBuffer(int distance, int length, byte[] buff) {
- if (distance > memory.length) {
- throw new IllegalStateException("Illegal distance parameter: " + distance);
- }
- int start = (wHead - distance) & mask;
- if (memory[start] == -1) {
- throw new IllegalStateException("Attempt to read beyond memory: dist=" + distance);
- }
- for (int i = 0, pos = start; i < length; i++, pos = incCounter(pos)) {
- buff[i] = add(memory[pos]);
- }
- }
-
- private int incCounter(int counter) {
- return (counter + 1) & mask;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
deleted file mode 100644
index 473be6c..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.zip;
-
-enum HuffmanState {
- INITIAL,
- STORED,
- DYNAMIC_CODES,
- FIXED_CODES
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
new file mode 100644
index 0000000..f9d1f3b
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64InputStreamTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.times;
+
+@RunWith(MockitoJUnitRunner.class)
+public class Deflate64InputStreamTest
+{
+ private final HuffmanDecoder nullDecoder = null;
+
+ @Mock
+ private HuffmanDecoder decoder;
+
+ @Test
+ public void readWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
+ assertEquals(-1, input.read());
+ assertEquals(-1, input.read(new byte[1]));
+ assertEquals(-1, input.read(new byte[1], 0, 1));
+ }
+
+ @Test
+ public void properSizeWhenClosed() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
+ assertEquals(0, input.available());
+ }
+
+ @Test
+ public void properSizeWhenInRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE - 1;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(size, input.available());
+ }
+
+ @Test
+ public void properSizeWhenOutOfRange() throws Exception
+ {
+ long size = Integer.MAX_VALUE + 1L;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(Integer.MAX_VALUE, input.available());
+ }
+
+ @Test
+ public void properSizeAfterReading() throws Exception
+ {
+ byte[] buf = new byte[4096];
+ int offset = 1000;
+ int length = 3096;
+
+ Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
+
+ long size = Integer.MAX_VALUE + 2047L;
+ Deflate64InputStream input = new Deflate64InputStream(decoder, size);
+ assertEquals(2048, input.read(buf, offset, length));
+ assertEquals(Integer.MAX_VALUE - 1, input.available());
+ }
+
+ @Test
+ public void closeCallsDecoder() throws Exception
+ {
+
+ Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void closeIsDelegatedJustOnce() throws Exception
+ {
+
+ Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
+
+ input.close();
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void uncompressedBlock() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64InputStream input = new Deflate64InputStream(new ByteArrayInputStream(data), 11);
+ BufferedReader br = new BufferedReader(new InputStreamReader(input)))
+ {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java
new file mode 100644
index 0000000..57249f0
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoderTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class HuffmanDecoderTest {
+ @Test
+ public void decodeUncompressedBlock() throws Exception {
+ byte[] data = {
+ 0b1, // end of block + no compression mode
+ 11, 0, -12, -1, // len & ~len
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(11, len);
+ assertEquals("Hello World", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeUncompressedBlockWithInvalidLenNLenValue() throws Exception {
+ byte[] data = {
+ 0b1, // end of block + no compression mode
+ 11, 0, -12, -2, // len & ~len
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ try {
+ int len = decoder.decode(result);
+ fail("Should have failed but returned " + len + " entries: " + Arrays.toString(Arrays.copyOf(result, len)));
+ } catch (IllegalStateException e) {
+ assertEquals("Illegal LEN / NLEN values", e.getMessage());
+ }
+ }
+
+ @Test
+ public void decodeSimpleFixedHuffmanBlock() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b00000000000000000000000000000001, // d + end of block
+ 0b11111111111111111111111111111100 // end of block (00) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(11, len);
+ assertEquals("Hello World", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeSimpleFixedHuffmanBlockToSmallBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b00000000000000000000000000000001, // d + end of block
+ 0b11111111111111111111111111111100 // end of block (00) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[10];
+ int len;
+ len = decoder.decode(result);
+ assertEquals(10, len);
+ assertEquals("Hello Worl", new String(result, 0, len));
+ len = decoder.decode(result);
+ assertEquals(1, len);
+ assertEquals("d", new String(result, 0, len));
+ }
+
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookup() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[100];
+ int len = decoder.decode(result);
+
+ assertEquals(48, len);
+ assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookupInSmallBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[30];
+ int len;
+
+ len = decoder.decode(result);
+ assertEquals(30, len);
+ assertEquals("Hello World\nHello World\nHello ", new String(result, 0, len));
+
+ len = decoder.decode(result);
+ assertEquals(18, len);
+ assertEquals("World\nHello World\n", new String(result, 0, len));
+ }
+
+ @Test
+ public void decodeFixedHuffmanBlockWithMemoryLookupInExactBuffer() throws Exception {
+ byte[] data = {
+ //|--- binary filling ---|76543210
+ 0b11111111111111111111111111110011, // final block + fixed huffman + H
+ 0b00000000000000000000000001001000, // H + e
+ 0b11111111111111111111111111001101, // e + l
+ 0b11111111111111111111111111001001, // l + l
+ 0b11111111111111111111111111001001, // l + o
+ 0b00000000000000000000000001010111, // o + ' '
+ 0b00000000000000000000000000001000, // ' ' + W
+ 0b11111111111111111111111111001111, // W + o
+ 0b00000000000000000000000000101111, // o + r
+ 0b11111111111111111111111111001010, // r + l
+ 0b00000000000000000000000001001001, // l + d
+ 0b11111111111111111111111111100001, // d + '\n'
+ 0b00000000000000000000000000100010, // '\n' + <len>
+ 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
+ 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
+ 0b11111111111111111111111111111000 // end of block (0000) + garbage
+ };
+
+ HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
+ byte[] result = new byte[48];
+ int len;
+
+ len = decoder.decode(result);
+ assertEquals(48, len);
+ assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
+
+ len = decoder.decode(result);
+ assertEquals(0, len);
+
+ len = decoder.decode(result);
+ assertEquals(-1, len);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
deleted file mode 100644
index a96c319..0000000
--- a/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.zip;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.InputStreamReader;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.times;
-
-@RunWith(MockitoJUnitRunner.class)
-public class Deflate64InputStreamTest
-{
- private final HuffmanDecoder nullDecoder = null;
-
- @Mock
- private HuffmanDecoder decoder;
-
- @Test
- public void readWhenClosed() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
- assertEquals(-1, input.read());
- assertEquals(-1, input.read(new byte[1]));
- assertEquals(-1, input.read(new byte[1], 0, 1));
- }
-
- @Test
- public void properSizeWhenClosed() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(nullDecoder, size);
- assertEquals(0, input.available());
- }
-
- @Test
- public void properSizeWhenInRange() throws Exception
- {
- long size = Integer.MAX_VALUE - 1;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(size, input.available());
- }
-
- @Test
- public void properSizeWhenOutOfRange() throws Exception
- {
- long size = Integer.MAX_VALUE + 1L;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(Integer.MAX_VALUE, input.available());
- }
-
- @Test
- public void properSizeAfterReading() throws Exception
- {
- byte[] buf = new byte[4096];
- int offset = 1000;
- int length = 3096;
-
- Mockito.when(decoder.decode(buf, offset, length)).thenReturn(2048);
-
- long size = Integer.MAX_VALUE + 2047L;
- Deflate64InputStream input = new Deflate64InputStream(decoder, size);
- assertEquals(2048, input.read(buf, offset, length));
- assertEquals(Integer.MAX_VALUE - 1, input.available());
- }
-
- @Test
- public void closeCallsDecoder() throws Exception
- {
-
- Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void closeIsDelegatedJustOnce() throws Exception
- {
-
- Deflate64InputStream input = new Deflate64InputStream(decoder, 10);
-
- input.close();
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void uncompressedBlock() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- try (Deflate64InputStream input = new Deflate64InputStream(new ByteArrayInputStream(data), 11);
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- assertEquals(null, br.readLine());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2042aa00/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
deleted file mode 100644
index b21756d..0000000
--- a/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.zip;
-
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class HuffmanDecoderTest {
- @Test
- public void decodeUncompressedBlock() throws Exception {
- byte[] data = {
- 0b1, // end of block + no compression mode
- 11, 0, -12, -1, // len & ~len
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[100];
- int len = decoder.decode(result);
-
- assertEquals(11, len);
- assertEquals("Hello World", new String(result, 0, len));
- }
-
- @Test
- public void decodeUncompressedBlockWithInvalidLenNLenValue() throws Exception {
- byte[] data = {
- 0b1, // end of block + no compression mode
- 11, 0, -12, -2, // len & ~len
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[100];
- try {
- int len = decoder.decode(result);
- fail("Should have failed but returned " + len + " entries: " + Arrays.toString(Arrays.copyOf(result, len)));
- } catch (IllegalStateException e) {
- assertEquals("Illegal LEN / NLEN values", e.getMessage());
- }
- }
-
- @Test
- public void decodeSimpleFixedHuffmanBlock() throws Exception {
- byte[] data = {
- //|--- binary filling ---|76543210
- 0b11111111111111111111111111110011, // final block + fixed huffman + H
- 0b00000000000000000000000001001000, // H + e
- 0b11111111111111111111111111001101, // e + l
- 0b11111111111111111111111111001001, // l + l
- 0b11111111111111111111111111001001, // l + o
- 0b00000000000000000000000001010111, // o + ' '
- 0b00000000000000000000000000001000, // ' ' + W
- 0b11111111111111111111111111001111, // W + o
- 0b00000000000000000000000000101111, // o + r
- 0b11111111111111111111111111001010, // r + l
- 0b00000000000000000000000001001001, // l + d
- 0b00000000000000000000000000000001, // d + end of block
- 0b11111111111111111111111111111100 // end of block (00) + garbage
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[100];
- int len = decoder.decode(result);
-
- assertEquals(11, len);
- assertEquals("Hello World", new String(result, 0, len));
- }
-
- @Test
- public void decodeSimpleFixedHuffmanBlockToSmallBuffer() throws Exception {
- byte[] data = {
- //|--- binary filling ---|76543210
- 0b11111111111111111111111111110011, // final block + fixed huffman + H
- 0b00000000000000000000000001001000, // H + e
- 0b11111111111111111111111111001101, // e + l
- 0b11111111111111111111111111001001, // l + l
- 0b11111111111111111111111111001001, // l + o
- 0b00000000000000000000000001010111, // o + ' '
- 0b00000000000000000000000000001000, // ' ' + W
- 0b11111111111111111111111111001111, // W + o
- 0b00000000000000000000000000101111, // o + r
- 0b11111111111111111111111111001010, // r + l
- 0b00000000000000000000000001001001, // l + d
- 0b00000000000000000000000000000001, // d + end of block
- 0b11111111111111111111111111111100 // end of block (00) + garbage
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[10];
- int len;
- len = decoder.decode(result);
- assertEquals(10, len);
- assertEquals("Hello Worl", new String(result, 0, len));
- len = decoder.decode(result);
- assertEquals(1, len);
- assertEquals("d", new String(result, 0, len));
- }
-
-
- @Test
- public void decodeFixedHuffmanBlockWithMemoryLookup() throws Exception {
- byte[] data = {
- //|--- binary filling ---|76543210
- 0b11111111111111111111111111110011, // final block + fixed huffman + H
- 0b00000000000000000000000001001000, // H + e
- 0b11111111111111111111111111001101, // e + l
- 0b11111111111111111111111111001001, // l + l
- 0b11111111111111111111111111001001, // l + o
- 0b00000000000000000000000001010111, // o + ' '
- 0b00000000000000000000000000001000, // ' ' + W
- 0b11111111111111111111111111001111, // W + o
- 0b00000000000000000000000000101111, // o + r
- 0b11111111111111111111111111001010, // r + l
- 0b00000000000000000000000001001001, // l + d
- 0b11111111111111111111111111100001, // d + '\n'
- 0b00000000000000000000000000100010, // '\n' + <len>
- 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
- 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
- 0b11111111111111111111111111111000 // end of block (0000) + garbage
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[100];
- int len = decoder.decode(result);
-
- assertEquals(48, len);
- assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
- }
-
- @Test
- public void decodeFixedHuffmanBlockWithMemoryLookupInSmallBuffer() throws Exception {
- byte[] data = {
- //|--- binary filling ---|76543210
- 0b11111111111111111111111111110011, // final block + fixed huffman + H
- 0b00000000000000000000000001001000, // H + e
- 0b11111111111111111111111111001101, // e + l
- 0b11111111111111111111111111001001, // l + l
- 0b11111111111111111111111111001001, // l + o
- 0b00000000000000000000000001010111, // o + ' '
- 0b00000000000000000000000000001000, // ' ' + W
- 0b11111111111111111111111111001111, // W + o
- 0b00000000000000000000000000101111, // o + r
- 0b11111111111111111111111111001010, // r + l
- 0b00000000000000000000000001001001, // l + d
- 0b11111111111111111111111111100001, // d + '\n'
- 0b00000000000000000000000000100010, // '\n' + <len>
- 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
- 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
- 0b11111111111111111111111111111000 // end of block (0000) + garbage
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[30];
- int len;
-
- len = decoder.decode(result);
- assertEquals(30, len);
- assertEquals("Hello World\nHello World\nHello ", new String(result, 0, len));
-
- len = decoder.decode(result);
- assertEquals(18, len);
- assertEquals("World\nHello World\n", new String(result, 0, len));
- }
-
- @Test
- public void decodeFixedHuffmanBlockWithMemoryLookupInExactBuffer() throws Exception {
- byte[] data = {
- //|--- binary filling ---|76543210
- 0b11111111111111111111111111110011, // final block + fixed huffman + H
- 0b00000000000000000000000001001000, // H + e
- 0b11111111111111111111111111001101, // e + l
- 0b11111111111111111111111111001001, // l + l
- 0b11111111111111111111111111001001, // l + o
- 0b00000000000000000000000001010111, // o + ' '
- 0b00000000000000000000000000001000, // ' ' + W
- 0b11111111111111111111111111001111, // W + o
- 0b00000000000000000000000000101111, // o + r
- 0b11111111111111111111111111001010, // r + l
- 0b00000000000000000000000001001001, // l + d
- 0b11111111111111111111111111100001, // d + '\n'
- 0b00000000000000000000000000100010, // '\n' + <len>
- 0b11111111111111111111111110000110, // <len> + offset <001> + dist6
- 0b00000000000000000000000000001101, // dist6 + offset <11> + end of block (000000)
- 0b11111111111111111111111111111000 // end of block (0000) + garbage
- };
-
- HuffmanDecoder decoder = new HuffmanDecoder(new ByteArrayInputStream(data));
- byte[] result = new byte[48];
- int len;
-
- len = decoder.decode(result);
- assertEquals(48, len);
- assertEquals("Hello World\nHello World\nHello World\nHello World\n", new String(result, 0, len));
-
- len = decoder.decode(result);
- assertEquals(0, len);
-
- len = decoder.decode(result);
- assertEquals(-1, len);
- }
-}
[20/28] commons-compress git commit: license header and whitespace
Posted by bo...@apache.org.
license header and whitespace
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/073fe245
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/073fe245
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/073fe245
Branch: refs/heads/master
Commit: 073fe2452f0514f100fa10781f081647d3384cd7
Parents: a123142
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Jan 7 11:00:07 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Jan 7 11:00:07 2018 +0100
----------------------------------------------------------------------
.../compressors/deflate64/Deflate64BugTest.java | 47 +++++++++++++-------
1 file changed, 32 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/073fe245/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
index bd8bb7a..7a51747 100644
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.deflate64;
import static org.apache.commons.compress.AbstractTestCase.getFile;
@@ -10,23 +27,23 @@ import org.apache.commons.compress.archivers.zip.ZipFile;
import org.junit.Test;
public class Deflate64BugTest {
- @Test
- public void readBeyondMemoryException() throws Exception {
- try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380-deflatebug.zip"))) {
- Enumeration<ZipArchiveEntry> entries = zfile.getEntries();
- while (entries.hasMoreElements()) {
- ZipArchiveEntry e = entries.nextElement();
- byte [] buf = new byte [1024 * 8];
- try (InputStream is = zfile.getInputStream(e)) {
- while (true) {
- int read = is.read(buf);
- if (read == -1) {
- break;
+ @Test
+ public void readBeyondMemoryException() throws Exception {
+ try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380-deflatebug.zip"))) {
+ Enumeration<ZipArchiveEntry> entries = zfile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry e = entries.nextElement();
+ byte [] buf = new byte [1024 * 8];
+ try (InputStream is = zfile.getInputStream(e)) {
+ while (true) {
+ int read = is.read(buf);
+ if (read == -1) {
+ break;
+ }
+ }
+ }
}
- }
}
- }
}
- }
}
[26/28] commons-compress git commit: COMPRESS-380 directly read
uncompressed data from the underlying stream
Posted by bo...@apache.org.
COMPRESS-380 directly read uncompressed data from the underlying stream
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/32d507b0
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/32d507b0
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/32d507b0
Branch: refs/heads/master
Commit: 32d507b028b15fd21775f25c83f41059cdc58e75
Parents: 782b156
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Jan 9 12:00:50 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Jan 9 12:00:50 2018 +0100
----------------------------------------------------------------------
.../compressors/deflate64/HuffmanDecoder.java | 30 ++++++++++++++------
.../commons/compress/utils/BitInputStream.java | 9 ++++++
2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/32d507b0/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index 506b35d..75f02fa 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -104,15 +104,13 @@ class HuffmanDecoder implements Closeable {
private boolean finalBlock = false;
private DecoderState state;
private BitInputStream reader;
+ private final InputStream in;
private final DecodingMemory memory = new DecodingMemory();
HuffmanDecoder(InputStream in) {
- this(new BitInputStream(in, ByteOrder.LITTLE_ENDIAN));
- }
-
- private HuffmanDecoder(BitInputStream reader) {
- this.reader = reader;
+ this.reader = new BitInputStream(in, ByteOrder.LITTLE_ENDIAN);
+ this.in = in;
state = new InitialState();
}
@@ -200,9 +198,19 @@ class HuffmanDecoder implements Closeable {
// as len is an int and (blockLength - read) is >= 0 the min must fit into an int as well
int max = (int) Math.min(blockLength - read, len);
for (int i = 0; i < max; i++) {
- byte next = (byte) (readBits(Byte.SIZE) & 0xFF);
- b[off + i] = memory.add(next);
- read++;
+ if (reader.bitsCached() > 0) {
+ byte next = (byte) (readBits(Byte.SIZE) & 0xFF);
+ b[off + i] = memory.add(next);
+ read++;
+ } else {
+ int readNow = in.read(b, off + i, max - i);
+ if (readNow == -1) {
+ throw new EOFException("Truncated Deflate64 Stream");
+ }
+ memory.add(b, off + i, readNow);
+ read += readNow;
+ i += readNow;
+ }
}
return max;
}
@@ -463,6 +471,12 @@ class HuffmanDecoder implements Closeable {
return b;
}
+ void add(byte[] b, int off, int len) {
+ for (int i = off; i < off + len; i++) {
+ add(b[i]);
+ }
+ }
+
void recordToBuffer(int distance, int length, byte[] buff) {
if (distance > memory.length) {
throw new IllegalStateException("Illegal distance parameter: " + distance);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/32d507b0/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
index 7f29ac0..d847eb9 100644
--- a/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
+++ b/src/main/java/org/apache/commons/compress/utils/BitInputStream.java
@@ -101,6 +101,15 @@ public class BitInputStream implements Closeable {
}
/**
+ * Returns the number of bits that can be read from this input
+ * stream without readong from the underlying input stream at all.
+ * @since 1.16
+ */
+ public int bitsCached() {
+ return bitsCachedSize;
+ }
+
+ /**
* Returns an estimate of the number of bits that can be read from
* this input stream without blocking by the next invocation of a
* method for this input stream.
[19/28] commons-compress git commit: An example failing deflate64 zip
file
Posted by bo...@apache.org.
An example failing deflate64 zip file
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/a123142d
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/a123142d
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/a123142d
Branch: refs/heads/master
Commit: a123142d947dbf38fc053c29291b555872c91ff2
Parents: 07ed545
Author: Dawid Weiss <da...@carrotsearch.com>
Authored: Sat Jan 6 17:14:39 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Jan 7 10:58:51 2018 +0100
----------------------------------------------------------------------
.../compressors/deflate64/Deflate64BugTest.java | 32 +++++++++++++++++++
src/test/resources/COMPRESS-380-deflatebug.zip | Bin 0 -> 15290 bytes
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a123142d/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
new file mode 100644
index 0000000..bd8bb7a
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
@@ -0,0 +1,32 @@
+package org.apache.commons.compress.compressors.deflate64;
+
+import static org.apache.commons.compress.AbstractTestCase.getFile;
+
+import java.io.InputStream;
+import java.util.Enumeration;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.junit.Test;
+
+public class Deflate64BugTest {
+ @Test
+ public void readBeyondMemoryException() throws Exception {
+ try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380-deflatebug.zip"))) {
+ Enumeration<ZipArchiveEntry> entries = zfile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry e = entries.nextElement();
+
+ byte [] buf = new byte [1024 * 8];
+ try (InputStream is = zfile.getInputStream(e)) {
+ while (true) {
+ int read = is.read(buf);
+ if (read == -1) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/a123142d/src/test/resources/COMPRESS-380-deflatebug.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-deflatebug.zip b/src/test/resources/COMPRESS-380-deflatebug.zip
new file mode 100644
index 0000000..99f352d
Binary files /dev/null and b/src/test/resources/COMPRESS-380-deflatebug.zip differ
[02/28] commons-compress git commit: COMPRESS-380 add license headers
Posted by bo...@apache.org.
COMPRESS-380 add license headers
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/3e9b0e88
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/3e9b0e88
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/3e9b0e88
Branch: refs/heads/master
Commit: 3e9b0e88ff11a38d05b303f1bd1f13a0afa9380e
Parents: d07f04b
Author: Stefan Bodewig <bo...@apache.org>
Authored: Wed Jan 3 15:15:31 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Wed Jan 3 15:15:31 2018 +0100
----------------------------------------------------------------------
.../compressors/zip/Deflate64InputStream.java | 17 +++++++++++++++++
.../compress/compressors/zip/HuffmanDecoder.java | 17 +++++++++++++++++
.../compress/compressors/zip/HuffmanState.java | 17 +++++++++++++++++
.../zip/Deflate64InputStreamTest.java | 19 ++++++++++++++++++-
.../compressors/zip/HuffmanDecoderTest.java | 19 ++++++++++++++++++-
5 files changed, 87 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3e9b0e88/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java b/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
index aeceb2a..98bf792 100644
--- a/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/Deflate64InputStream.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.zip;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3e9b0e88/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
index de4984b..0b6b173 100644
--- a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanDecoder.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.zip;
import org.apache.commons.compress.utils.BitInputStream;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3e9b0e88/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
index c871955..473be6c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
+++ b/src/main/java/org/apache/commons/compress/compressors/zip/HuffmanState.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.zip;
enum HuffmanState {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3e9b0e88/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
index f5f7943..a96c319 100644
--- a/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/zip/Deflate64InputStreamTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.zip;
import org.junit.Test;
@@ -108,4 +125,4 @@ public class Deflate64InputStreamTest
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/3e9b0e88/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java b/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
index 27d0905..b21756d 100644
--- a/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/zip/HuffmanDecoderTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
package org.apache.commons.compress.compressors.zip;
import org.junit.Test;
@@ -202,4 +219,4 @@ public class HuffmanDecoderTest {
len = decoder.decode(result);
assertEquals(-1, len);
}
-}
\ No newline at end of file
+}
[18/28] commons-compress git commit: COMPRESS-380 -1 is certainly a
legal value for a byte
Posted by bo...@apache.org.
COMPRESS-380 -1 is certainly a legal value for a byte
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/07ed5450
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/07ed5450
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/07ed5450
Branch: refs/heads/master
Commit: 07ed545022f30604d0a4fc4ad5e4866e24e6a172
Parents: 87a3cfa
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Jan 7 10:52:16 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Jan 7 10:52:16 2018 +0100
----------------------------------------------------------------------
.../compress/compressors/deflate64/HuffmanDecoder.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/07ed5450/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index f74c7de..9d6585a 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -445,6 +445,7 @@ class HuffmanDecoder implements Closeable {
private final byte[] memory;
private final int mask;
private int wHead;
+ private boolean wrappedAround;
private DecodingMemory() {
this(16);
@@ -452,7 +453,6 @@ class HuffmanDecoder implements Closeable {
private DecodingMemory(int bits) {
memory = new byte[1 << bits];
- Arrays.fill(memory, (byte) -1);
mask = memory.length - 1;
}
@@ -467,7 +467,7 @@ class HuffmanDecoder implements Closeable {
throw new IllegalStateException("Illegal distance parameter: " + distance);
}
int start = (wHead - distance) & mask;
- if (memory[start] == -1) {
+ if (!wrappedAround && start >= wHead) {
throw new IllegalStateException("Attempt to read beyond memory: dist=" + distance);
}
for (int i = 0, pos = start; i < length; i++, pos = incCounter(pos)) {
@@ -476,7 +476,11 @@ class HuffmanDecoder implements Closeable {
}
private int incCounter(int counter) {
- return (counter + 1) & mask;
+ final int newCounter = (counter + 1) & mask;
+ if (!wrappedAround && newCounter < counter) {
+ wrappedAround = true;
+ }
+ return newCounter;
}
}
[13/28] commons-compress git commit: fix indentation
Posted by bo...@apache.org.
fix indentation
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/334f1de9
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/334f1de9
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/334f1de9
Branch: refs/heads/master
Commit: 334f1de9a3efa31303c2788c209d249d7b33d3aa
Parents: f4b7501
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 11:57:51 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 11:57:51 2018 +0100
----------------------------------------------------------------------
.../Deflate64CompressorInputStreamTest.java | 246 +++++++++----------
1 file changed, 117 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/334f1de9/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
index b7b3cd9..3526224 100644
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64CompressorInputStreamTest.java
@@ -34,134 +34,122 @@ import static org.mockito.Mockito.times;
@RunWith(MockitoJUnitRunner.class)
public class Deflate64CompressorInputStreamTest {
- private final HuffmanDecoder nullDecoder = null;
-
- @Mock
- private HuffmanDecoder decoder;
-
- @Test
- public void readWhenClosed() throws Exception
- {
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
- assertEquals(-1, input.read());
- assertEquals(-1, input.read(new byte[1]));
- assertEquals(-1, input.read(new byte[1], 0, 1));
- }
-
- @Test
- public void properSizeWhenClosed() throws Exception
- {
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
- assertEquals(0, input.available());
- }
-
- @Test
- public void delegatesAvailable() throws Exception
- {
- Mockito.when(decoder.available()).thenReturn(1024);
-
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
- assertEquals(1024, input.available());
- }
-
- @Test
- public void closeCallsDecoder() throws Exception
- {
-
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void closeIsDelegatedJustOnce() throws Exception
- {
-
- Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
-
- input.close();
- input.close();
-
- Mockito.verify(decoder, times(1)).close();
- }
-
- @Test
- public void uncompressedBlock() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- assertEquals(null, br.readLine());
- }
- }
-
- @Test
- public void uncompressedBlockViaFactory() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- try (InputStream input = new CompressorStreamFactory()
- .createCompressorInputStream(CompressorStreamFactory.DEFLATE64, new ByteArrayInputStream(data));
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- assertEquals(null, br.readLine());
- }
- }
-
- @Test
- public void uncompressedBlockAvailable() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
- };
-
- try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data))) {
- assertEquals('H', input.read());
- assertEquals(10, input.available());
- }
- }
-
- @Test
- public void streamIgnoresExtraBytesAfterDeflatedInput() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', 'X'
- };
-
- try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- assertEquals(null, br.readLine());
- }
- }
-
- @Test(expected = java.io.EOFException.class)
- public void throwsEOFExceptionOnTruncatedStreams() throws Exception
- {
- byte[] data = {
- 1, 11, 0, -12, -1,
- 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l',
- };
-
- try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
- BufferedReader br = new BufferedReader(new InputStreamReader(input)))
- {
- assertEquals("Hello World", br.readLine());
- }
- }
+ private final HuffmanDecoder nullDecoder = null;
+
+ @Mock
+ private HuffmanDecoder decoder;
+
+ @Test
+ public void readWhenClosed() throws Exception {
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
+ assertEquals(-1, input.read());
+ assertEquals(-1, input.read(new byte[1]));
+ assertEquals(-1, input.read(new byte[1], 0, 1));
+ }
+
+ @Test
+ public void properSizeWhenClosed() throws Exception {
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(nullDecoder);
+ assertEquals(0, input.available());
+ }
+
+ @Test
+ public void delegatesAvailable() throws Exception {
+ Mockito.when(decoder.available()).thenReturn(1024);
+
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
+ assertEquals(1024, input.available());
+ }
+
+ @Test
+ public void closeCallsDecoder() throws Exception {
+
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void closeIsDelegatedJustOnce() throws Exception {
+
+ Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(decoder);
+
+ input.close();
+ input.close();
+
+ Mockito.verify(decoder, times(1)).close();
+ }
+
+ @Test
+ public void uncompressedBlock() throws Exception {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input))) {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+ @Test
+ public void uncompressedBlockViaFactory() throws Exception {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (InputStream input = new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.DEFLATE64, new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input))) {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+ @Test
+ public void uncompressedBlockAvailable() throws Exception {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data))) {
+ assertEquals('H', input.read());
+ assertEquals(10, input.available());
+ }
+ }
+
+ @Test
+ public void streamIgnoresExtraBytesAfterDeflatedInput() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', 'X'
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input))) {
+ assertEquals("Hello World", br.readLine());
+ assertEquals(null, br.readLine());
+ }
+ }
+
+ @Test(expected = java.io.EOFException.class)
+ public void throwsEOFExceptionOnTruncatedStreams() throws Exception
+ {
+ byte[] data = {
+ 1, 11, 0, -12, -1,
+ 'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l',
+ };
+
+ try (Deflate64CompressorInputStream input = new Deflate64CompressorInputStream(new ByteArrayInputStream(data));
+ BufferedReader br = new BufferedReader(new InputStreamReader(input))) {
+ assertEquals("Hello World", br.readLine());
+ }
+ }
}
[11/28] commons-compress git commit: COMPRESS-380 ensure end if
stream is detected properly
Posted by bo...@apache.org.
COMPRESS-380 ensure end if stream is detected properly
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b97a02ca
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b97a02ca
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b97a02ca
Branch: refs/heads/master
Commit: b97a02ca1c1c220f18971bce3dc41d781da8f5f0
Parents: 0e74bef
Author: Stefan Bodewig <bo...@apache.org>
Authored: Fri Jan 5 10:45:47 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Fri Jan 5 10:45:47 2018 +0100
----------------------------------------------------------------------
.../compressors/deflate64/HuffmanDecoder.java | 45 +++++++++++++-------
1 file changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b97a02ca/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
index ec4a6f0..e96959c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
+++ b/src/main/java/org/apache/commons/compress/compressors/deflate64/HuffmanDecoder.java
@@ -20,6 +20,7 @@ package org.apache.commons.compress.compressors.deflate64;
import org.apache.commons.compress.utils.BitInputStream;
import java.io.Closeable;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
@@ -130,13 +131,13 @@ class HuffmanDecoder implements Closeable {
while (!finalBlock || state.hasData()) {
switch (state.state()) {
case INITIAL:
- finalBlock = reader.readBits(1) == 1;
- int mode = (int) reader.readBits(2);
+ finalBlock = readBits(1) == 1;
+ int mode = (int) readBits(2);
switch (mode) {
case 0:
- reader.readBits(Byte.SIZE - 3);
- long bLen = reader.readBits(16);
- long bNLen = reader.readBits(16);
+ readBits(Byte.SIZE - 3);
+ long bLen = readBits(16);
+ long bNLen = readBits(16);
if (((bLen ^ 0xFFFF) & 0xFFFF) != bNLen) {
//noinspection DuplicateStringLiteralInspection
throw new IllegalStateException("Illegal LEN / NLEN values");
@@ -147,10 +148,10 @@ class HuffmanDecoder implements Closeable {
state = new HuffmanCodes(FIXED_CODES, FIXED_LITERALS, FIXED_DISTANCE);
break;
case 2:
- int literals = (int) (reader.readBits(5) + 257);
+ int literals = (int) (readBits(5) + 257);
int[] literalTable = new int[literals];
- int distances = (int) (reader.readBits(5) + 1);
+ int distances = (int) (readBits(5) + 1);
int[] distanceTable = new int[distances];
populateDynamicTables(reader, literalTable, distanceTable);
@@ -199,7 +200,7 @@ class HuffmanDecoder implements Closeable {
int read(byte[] b, int off, int len) throws IOException {
int max = Math.min(blockLength - read, len);
for (int i = 0; i < max; i++) {
- byte next = (byte) (reader.readBits(Byte.SIZE) & 0xFF);
+ byte next = (byte) (readBits(Byte.SIZE) & 0xFF);
b[off + i] = memory.add(next);
read++;
}
@@ -278,14 +279,14 @@ class HuffmanDecoder implements Closeable {
int runMask = RUN_LENGTH_TABLE[symbol - 257];
int run = runMask >>> 5;
int runXtra = runMask & 0x1F;
- run += reader.readBits(runXtra);
+ run += readBits(runXtra);
int distSym = nextSymbol(reader, distanceTree);
int distMask = DISTANCE_TABLE[distSym];
int dist = distMask >>> 4;
int distXtra = distMask & 0xF;
- dist += reader.readBits(distXtra);
+ dist += readBits(distXtra);
runBuffer = new byte[run];
runBufferPos = 0;
@@ -326,18 +327,18 @@ class HuffmanDecoder implements Closeable {
private static int nextSymbol(BitInputStream reader, BinaryTreeNode tree) throws IOException {
BinaryTreeNode node = tree;
while (node != null && node.literal == -1) {
- long bit = reader.readBits(1);
+ long bit = readBits(reader, 1);
node = bit == 0 ? node.left : node.right;
}
return node != null ? node.literal : -1;
}
private static void populateDynamicTables(BitInputStream reader, int[] literals, int[] distances) throws IOException {
- int codeLengths = (int) (reader.readBits(4) + 4);
+ int codeLengths = (int) (readBits(reader, 4) + 4);
int[] codeLengthValues = new int[19];
for (int cLen = 0; cLen < codeLengths; cLen++) {
- codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) reader.readBits(3);
+ codeLengthValues[CODE_LENGTHS_ORDER[cLen]] = (int) readBits(reader, 3);
}
BinaryTreeNode codeLengthTree = buildTree(codeLengthValues);
@@ -355,13 +356,13 @@ class HuffmanDecoder implements Closeable {
value = symbol;
auxBuffer[i++] = value;
} else if (symbol == 16) {
- length = (int) (reader.readBits(2) + 3);
+ length = (int) (readBits(reader, 2) + 3);
} else if (symbol == 17) {
value = 0;
- length = (int) (reader.readBits(3) + 3);
+ length = (int) (readBits(reader, 3) + 3);
} else if (symbol == 18) {
value = 0;
- length = (int) (reader.readBits(7) + 11);
+ length = (int) (readBits(reader, 7) + 11);
}
}
}
@@ -479,4 +480,16 @@ class HuffmanDecoder implements Closeable {
return (counter + 1) & mask;
}
}
+
+ private long readBits(int numBits) throws IOException {
+ return readBits(reader, numBits);
+ }
+
+ private static long readBits(BitInputStream reader, int numBits) throws IOException {
+ long r = reader.readBits(numBits);
+ if (r == -1) {
+ throw new EOFException();
+ }
+ return r;
+ }
}
[21/28] commons-compress git commit: move a few files around
Posted by bo...@apache.org.
move a few files around
closes #58
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/77a0a69e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/77a0a69e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/77a0a69e
Branch: refs/heads/master
Commit: 77a0a69e3135253b3762672ef6bdf96ebaa8a882
Parents: 073fe24
Author: Stefan Bodewig <bo...@apache.org>
Authored: Sun Jan 7 11:06:12 2018 +0100
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Sun Jan 7 11:06:12 2018 +0100
----------------------------------------------------------------------
.../zip/ZipArchiveInputStreamTest.java | 6 +--
.../compress/archivers/zip/ZipFileTest.java | 4 +-
.../compressors/deflate64/Deflate64BugTest.java | 49 -------------------
.../deflate64/Deflate64BugsTest.java | 49 +++++++++++++++++++
src/test/resources/COMPRESS-380-dd.zip | Bin 1391 -> 0 bytes
src/test/resources/COMPRESS-380-deflatebug.zip | Bin 15290 -> 0 bytes
src/test/resources/COMPRESS-380-input | Bin 3072 -> 0 bytes
src/test/resources/COMPRESS-380.zip | Bin 2257 -> 0 bytes
.../resources/COMPRESS-380/COMPRESS-380-dd.zip | Bin 0 -> 1391 bytes
.../resources/COMPRESS-380/COMPRESS-380-input | Bin 0 -> 3072 bytes
.../COMPRESS-380-readbeyondmemory.zip | Bin 0 -> 15290 bytes
.../resources/COMPRESS-380/COMPRESS-380.zip | Bin 0 -> 2257 bytes
12 files changed, 54 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
index 04269ad..f09b205 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipArchiveInputStreamTest.java
@@ -210,8 +210,8 @@ public class ZipArchiveInputStreamTest {
*/
@Test
public void readDeflate64CompressedStream() throws Exception {
- final File input = getFile("COMPRESS-380-input");
- final File archive = getFile("COMPRESS-380.zip");
+ final File input = getFile("COMPRESS-380/COMPRESS-380-input");
+ final File archive = getFile("COMPRESS-380/COMPRESS-380.zip");
try (FileInputStream in = new FileInputStream(input);
ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) {
byte[] orig = IOUtils.toByteArray(in);
@@ -224,7 +224,7 @@ public class ZipArchiveInputStreamTest {
@Test
public void readDeflate64CompressedStreamWithDataDescriptor() throws Exception {
// this is a copy of bla.jar with META-INF/MANIFEST.MF's method manually changed to ENHANCED_DEFLATED
- final File archive = getFile("COMPRESS-380-dd.zip");
+ final File archive = getFile("COMPRESS-380/COMPRESS-380-dd.zip");
try (ZipArchiveInputStream zin = new ZipArchiveInputStream(new FileInputStream(archive))) {
ZipArchiveEntry e = zin.getNextZipEntry();
assertEquals(-1, e.getSize());
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
index 2903f77..e29f080 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipFileTest.java
@@ -586,8 +586,8 @@ public class ZipFileTest {
*/
@Test
public void readDeflate64CompressedStream() throws Exception {
- final File input = getFile("COMPRESS-380-input");
- final File archive = getFile("COMPRESS-380.zip");
+ final File input = getFile("COMPRESS-380/COMPRESS-380-input");
+ final File archive = getFile("COMPRESS-380/COMPRESS-380.zip");
try (FileInputStream in = new FileInputStream(input);
ZipFile zf = new ZipFile(archive)) {
byte[] orig = IOUtils.toByteArray(in);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
deleted file mode 100644
index 7a51747..0000000
--- a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.commons.compress.compressors.deflate64;
-
-import static org.apache.commons.compress.AbstractTestCase.getFile;
-
-import java.io.InputStream;
-import java.util.Enumeration;
-
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipFile;
-import org.junit.Test;
-
-public class Deflate64BugTest {
-
- @Test
- public void readBeyondMemoryException() throws Exception {
- try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380-deflatebug.zip"))) {
- Enumeration<ZipArchiveEntry> entries = zfile.getEntries();
- while (entries.hasMoreElements()) {
- ZipArchiveEntry e = entries.nextElement();
- byte [] buf = new byte [1024 * 8];
- try (InputStream is = zfile.getInputStream(e)) {
- while (true) {
- int read = is.read(buf);
- if (read == -1) {
- break;
- }
- }
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java
new file mode 100644
index 0000000..d21f052
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/compressors/deflate64/Deflate64BugsTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.commons.compress.compressors.deflate64;
+
+import static org.apache.commons.compress.AbstractTestCase.getFile;
+
+import java.io.InputStream;
+import java.util.Enumeration;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.junit.Test;
+
+public class Deflate64BugsTest {
+
+ @Test
+ public void readBeyondMemoryException() throws Exception {
+ try (ZipFile zfile = new ZipFile(getFile("COMPRESS-380/COMPRESS-380-readbeyondmemory.zip"))) {
+ Enumeration<ZipArchiveEntry> entries = zfile.getEntries();
+ while (entries.hasMoreElements()) {
+ ZipArchiveEntry e = entries.nextElement();
+ byte [] buf = new byte [1024 * 8];
+ try (InputStream is = zfile.getInputStream(e)) {
+ while (true) {
+ int read = is.read(buf);
+ if (read == -1) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380-dd.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-dd.zip b/src/test/resources/COMPRESS-380-dd.zip
deleted file mode 100644
index 9557996..0000000
Binary files a/src/test/resources/COMPRESS-380-dd.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380-deflatebug.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-deflatebug.zip b/src/test/resources/COMPRESS-380-deflatebug.zip
deleted file mode 100644
index 99f352d..0000000
Binary files a/src/test/resources/COMPRESS-380-deflatebug.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380-input
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380-input b/src/test/resources/COMPRESS-380-input
deleted file mode 100644
index daf1f56..0000000
Binary files a/src/test/resources/COMPRESS-380-input and /dev/null differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380.zip b/src/test/resources/COMPRESS-380.zip
deleted file mode 100644
index d9146be..0000000
Binary files a/src/test/resources/COMPRESS-380.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip b/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip
new file mode 100644
index 0000000..9557996
Binary files /dev/null and b/src/test/resources/COMPRESS-380/COMPRESS-380-dd.zip differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380/COMPRESS-380-input
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380/COMPRESS-380-input b/src/test/resources/COMPRESS-380/COMPRESS-380-input
new file mode 100644
index 0000000..daf1f56
Binary files /dev/null and b/src/test/resources/COMPRESS-380/COMPRESS-380-input differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip b/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip
new file mode 100644
index 0000000..99f352d
Binary files /dev/null and b/src/test/resources/COMPRESS-380/COMPRESS-380-readbeyondmemory.zip differ
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/77a0a69e/src/test/resources/COMPRESS-380/COMPRESS-380.zip
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-380/COMPRESS-380.zip b/src/test/resources/COMPRESS-380/COMPRESS-380.zip
new file mode 100644
index 0000000..d9146be
Binary files /dev/null and b/src/test/resources/COMPRESS-380/COMPRESS-380.zip differ