You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/25 18:46:01 UTC
[01/10] commons-compress git commit: COMPRESS-382 first draft of
preventing OOM in LZMA
Repository: commons-compress
Updated Branches:
refs/heads/master 08f0aba10 -> 5def510d0
COMPRESS-382 first draft of preventing OOM in LZMA
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/70594bca
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/70594bca
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/70594bca
Branch: refs/heads/master
Commit: 70594bcaff1d553cbd35312bb63a33bcba0680d3
Parents: 0811386
Author: tballison <ta...@mitre.org>
Authored: Fri Apr 14 15:04:20 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Fri Apr 14 15:04:20 2017 -0400
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 15 ++++++++++--
.../lzma/LZMACompressorInputStream.java | 14 +++++++++--
.../compressors/DetectCompressorTestCase.java | 24 +++++++++++++++----
src/test/resources/COMPRESS-382 | Bin 0 -> 19 bytes
4 files changed, 45 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 55da36a..d569a8a 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -349,6 +349,8 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
*/
private volatile boolean decompressConcatenated = false;
+ private volatile int lzmaMemoryLimitKb = -1;
+
/**
* Create an instance with the decompress Concatenated option set to false.
*/
@@ -431,7 +433,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (LZMAUtils.matches(signature, signatureLength) && LZMAUtils.isLZMACompressionAvailable()) {
- return new LZMACompressorInputStream(in);
+ return new LZMACompressorInputStream(in, lzmaMemoryLimitKb);
}
if (FramedLZ4CompressorInputStream.matches(signature, signatureLength)) {
@@ -666,5 +668,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
this.decompressConcatenated = decompressConcatenated;
}
-
+
+ /**
+ * Set the maximum calculated memory usage for LZMA
+ * in KB.
+ *
+ * @param lzmaMemoryLimitKb
+ */
+ public void setLzmaMemoryLimitKb(int lzmaMemoryLimitKb) {
+ this.lzmaMemoryLimitKb = lzmaMemoryLimitKb;
+ }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 556c78c..ea8a498 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -31,21 +31,31 @@ import org.apache.commons.compress.compressors.CompressorInputStream;
public class LZMACompressorInputStream extends CompressorInputStream {
private final InputStream in;
+ public LZMACompressorInputStream(final InputStream inputStream)
+ throws IOException {
+ in = new LZMAInputStream(inputStream, -1);
+ }
+
/**
* Creates a new input stream that decompresses LZMA-compressed data
* from the specified input stream.
*
* @param inputStream where to read the compressed data
*
+ * @param memoryLimitKb calculated memory use threshold. Throws MemoryLimitException
+ * if calculate memory use is above this threshold
+ *
* @throws IOException if the input is not in the .lzma format,
* the input is corrupt or truncated, the .lzma
* headers specify sizes that are not supported
* by this implementation, or the underlying
* <code>inputStream</code> throws an exception
+ *
+ * @since 1.14
*/
- public LZMACompressorInputStream(final InputStream inputStream)
+ public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitKb)
throws IOException {
- in = new LZMAInputStream(inputStream);
+ in = new LZMAInputStream(inputStream, memoryLimitKb);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 674a2a0..9c446fa 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -19,22 +19,25 @@
package org.apache.commons.compress.compressors;
import static org.apache.commons.compress.AbstractTestCase.getFile;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
-import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorInputStream;
-import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.junit.Test;
+import org.tukaani.xz.MemoryLimitException;
@SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
public final class DetectCompressorTestCase {
@@ -160,6 +163,19 @@ public final class DetectCompressorTestCase {
}
}
+ @Test
+ public void testLZMAMemoryLimit() throws Exception {
+ CompressorStreamFactory compressorStreamFactory = new CompressorStreamFactory();
+ compressorStreamFactory.setLzmaMemoryLimitKb(500);
+ try {
+ InputStream is = compressorStreamFactory.createCompressorInputStream(
+ new BufferedInputStream(new FileInputStream(getFile("COMPRESS-382"))));
+ fail("Should have thrown memory limit exception");
+ } catch (CompressorException e) {
+ assertTrue(e.getCause() instanceof MemoryLimitException);
+ }
+ }
+
private CompressorInputStream getStreamFor(final String resource)
throws CompressorException, IOException {
return factory.createCompressorInputStream(
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/test/resources/COMPRESS-382
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-382 b/src/test/resources/COMPRESS-382
new file mode 100644
index 0000000..be257f2
Binary files /dev/null and b/src/test/resources/COMPRESS-382 differ
[05/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
take 3, create static MemoryLimit and remove new ctors.
Posted by bo...@apache.org.
COMPRESS-382 and COMPRESS-386 -- take 3, create static MemoryLimit
and remove new ctors.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/0e8ff9c4
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/0e8ff9c4
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/0e8ff9c4
Branch: refs/heads/master
Commit: 0e8ff9c44058afbca9c9126a8feebe41cd682626
Parents: b10528a
Author: tballison <ta...@mitre.org>
Authored: Mon Apr 24 21:06:35 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Apr 24 21:06:35 2017 -0400
----------------------------------------------------------------------
.../apache/commons/compress/MemoryLimit.java | 65 ++++++++++++++++++++
.../compressors/CompressorStreamFactory.java | 36 ++---------
.../lzma/LZMACompressorInputStream.java | 19 +++---
.../compressors/xz/XZCompressorInputStream.java | 32 +---------
.../compressors/z/ZCompressorInputStream.java | 10 +--
.../commons/compress/MemoryLimitTest.java | 29 +++++++++
.../compressors/DetectCompressorTestCase.java | 21 ++++++-
7 files changed, 131 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/main/java/org/apache/commons/compress/MemoryLimit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimit.java b/src/main/java/org/apache/commons/compress/MemoryLimit.java
new file mode 100644
index 0000000..e03e8e7
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/MemoryLimit.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress;
+
+/**
+ *
+ * During initialization, some streams compute expected memory use.
+ * They should check this value and throw a MemoryLimitException if the
+ * estimated memory use is greater that {@link MemoryLimit#MEMORY_LIMIT_IN_KB}.
+ * <p/>
+ * During compression/archiving, streams can allocate byte arrays based
+ * on a value read in from the stream. Corrupt files can cause compressors/archivers
+ * to cause {@link OutOfMemoryError}s. Compressors/archivers should check
+ * this maximum threshold before allocating memory and throw a {@link MemoryLimitException}
+ * if the allocation would exceed this limit.
+ * <p/>
+ * To avoid changes in legacy behavior, {@link MemoryLimit#MEMORY_LIMIT_IN_KB}
+ * is set to {@link MemoryLimit#NO_LIMIT}. However, in applications that might
+ * encounter untrusted/corrupt files, we encourage setting the limit to something
+ * reasonable for the application.
+ * <p/>
+ * As of 1.14, this limit should be observed when instantiating CompressorStreams.
+ * Work remains to propagate memory limit checks throughout the codebase.
+ *
+ * @since 1.14
+ */
+public class MemoryLimit {
+
+ public static final int NO_LIMIT = -1;
+ public static volatile int MEMORY_LIMIT_IN_KB = NO_LIMIT;
+
+ /**
+ * Sets {@link MemoryLimit#MEMORY_LIMIT_IN_KB}.
+ * @param memoryLimitInKb limit in kilobytes
+ *
+ * @throws IllegalArgumentException if value is < -1
+ */
+ public static void setMemoryLimitInKb(int memoryLimitInKb) {
+ if (memoryLimitInKb < -1) {
+ throw new IllegalArgumentException("MemoryLimit must be > -2");
+ }
+ //TODO: do we want to set an absolute upper limit?!
+ MEMORY_LIMIT_IN_KB = memoryLimitInKb;
+ }
+
+ public static int getMemoryLimitInKb() {
+ return MEMORY_LIMIT_IN_KB;
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index f3433d9..7daa291 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -349,52 +349,28 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
*/
private volatile boolean decompressConcatenated = false;
- private final int memoryLimitInKb;
/**
* Create an instance with the decompress Concatenated option set to false.
*/
public CompressorStreamFactory() {
this.decompressUntilEOF = null;
- this.memoryLimitInKb = -1;
}
/**
* Create an instance with the provided decompress Concatenated option.
- *
+ *
* @param decompressUntilEOF
* if true, decompress until the end of the input; if false, stop
* after the first stream and leave the input position to point
* to the next byte after the stream. This setting applies to the
* gzip, bzip2 and xz formats only.
- *
- * @param memoryLimitInKb
- * Some streams require allocation of potentially significant
- * byte arrays/tables, and they can offer checks to prevent OOMs
- * on corrupt files. Set the maximum allowed memory allocation in KBs.
- *
- * @since 1.14
+ * @since 1.10
*/
- public CompressorStreamFactory(final boolean decompressUntilEOF, final int memoryLimitInKb) {
+ public CompressorStreamFactory(final boolean decompressUntilEOF) {
this.decompressUntilEOF = Boolean.valueOf(decompressUntilEOF);
// Also copy to existing variable so can continue to use that as the
// current value
this.decompressConcatenated = decompressUntilEOF;
- this.memoryLimitInKb = memoryLimitInKb;
- }
-
-
- /**
- * Create an instance with the provided decompress Concatenated option.
- *
- * @param decompressUntilEOF
- * if true, decompress until the end of the input; if false, stop
- * after the first stream and leave the input position to point
- * to the next byte after the stream. This setting applies to the
- * gzip, bzip2 and xz formats only.
- * @since 1.10
- */
- public CompressorStreamFactory(final boolean decompressUntilEOF) {
- this(decompressUntilEOF, -1);
}
/**
@@ -529,14 +505,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (!XZUtils.isXZCompressionAvailable()) {
throw new CompressorException("XZ compression is not available.");
}
- return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
+ return new XZCompressorInputStream(in, actualDecompressConcatenated);
}
if (LZMA.equalsIgnoreCase(name)) {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
}
- return new LZMACompressorInputStream(in, memoryLimitInKb);
+ return new LZMACompressorInputStream(in);
}
if (PACK200.equalsIgnoreCase(name)) {
@@ -552,7 +528,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (Z.equalsIgnoreCase(name)) {
- return new ZCompressorInputStream(in, memoryLimitInKb);
+ return new ZCompressorInputStream(in);
}
if (DEFLATE.equalsIgnoreCase(name)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 7782be8..6931541 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -21,6 +21,7 @@ package org.apache.commons.compress.compressors.lzma;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.LZMAInputStream;
@@ -33,32 +34,26 @@ import org.apache.commons.compress.compressors.CompressorInputStream;
public class LZMACompressorInputStream extends CompressorInputStream {
private final InputStream in;
- public LZMACompressorInputStream(final InputStream inputStream)
- throws IOException {
- in = new LZMAInputStream(inputStream, -1);
- }
-
/**
* Creates a new input stream that decompresses LZMA-compressed data
* from the specified input stream.
*
* @param inputStream where to read the compressed data
*
- * @param memoryLimitInKb calculated memory use threshold. Throws MemoryLimitException
- * if calculate memory use is above this threshold
- *
* @throws IOException if the input is not in the .lzma format,
* the input is corrupt or truncated, the .lzma
* headers specify sizes that are not supported
- * by this implementation, or the underlying
- * <code>inputStream</code> throws an exception
+ * by this implementation, the underlying
+ * <code>inputStream</code> throws an exception or
+ * if the calculated memory usage
+ * is > {@link MemoryLimit#MEMORY_LIMIT_IN_KB}.
*
* @since 1.14
*/
- public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
+ public LZMACompressorInputStream(final InputStream inputStream)
throws IOException {
try {
- in = new LZMAInputStream(inputStream, memoryLimitInKb);
+ in = new LZMAInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress exception
throw new MemoryLimitException("exceeded calculated memory limit", e);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index b378212..3f977ef 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -21,6 +21,7 @@ package org.apache.commons.compress.compressors.xz;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.XZ;
import org.tukaani.xz.SingleXZInputStream;
@@ -94,38 +95,11 @@ public class XZCompressorInputStream extends CompressorInputStream {
public XZCompressorInputStream(final InputStream inputStream,
final boolean decompressConcatenated)
throws IOException {
- this(inputStream, decompressConcatenated, -1);
- }
- /**
- * Creates a new input stream that decompresses XZ-compressed data
- * from the specified input stream.
- *
- * @param inputStream where to read the compressed data
- * @param decompressConcatenated
- * if true, decompress until the end of the
- * input; if false, stop after the first .xz
- * stream and leave the input position to point
- * to the next byte after the .xz stream
- * @param memoryLimitInKb memory limit used when reading blocks. If
- * the estimated memory limit is exceeded on {@link #read()},
- * a {@link MemoryLimitException} is thrown.
- *
- * @throws IOException if the input is not in the .xz format,
- * the input is corrupt or truncated, the .xz
- * headers specify options that are not supported
- * by this implementation,
- * or the underlying <code>inputStream</code> throws an exception
- *
- * @since 1.14
- */
- public XZCompressorInputStream(InputStream inputStream,
- boolean decompressConcatenated, int memoryLimitInKb)
- throws IOException {
if (decompressConcatenated) {
- in = new XZInputStream(inputStream, memoryLimitInKb);
+ in = new XZInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
} else {
- in = new SingleXZInputStream(inputStream, memoryLimitInKb);
+ in = new SingleXZInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 64387e3..ca61cd3 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -38,8 +39,7 @@ public class ZCompressorInputStream extends LZWInputStream {
private final int maxCodeSize;
private long totalCodesRead = 0;
- public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
- throws IOException {
+ public ZCompressorInputStream(final InputStream inputStream) throws IOException {
super(inputStream, ByteOrder.LITTLE_ENDIAN);
final int firstByte = (int) in.readBits(8);
final int secondByte = (int) in.readBits(8);
@@ -52,13 +52,9 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- initializeTables(maxCodeSize, memoryLimitInKb);
+ initializeTables(maxCodeSize, MemoryLimit.getMemoryLimitInKb());
clearEntries();
}
-
- public ZCompressorInputStream(final InputStream inputStream) throws IOException {
- this(inputStream, -1);
- }
private void clearEntries() {
setTableSize((1 << 8) + (blockMode ? 1 : 0));
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/test/java/org/apache/commons/compress/MemoryLimitTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/MemoryLimitTest.java b/src/test/java/org/apache/commons/compress/MemoryLimitTest.java
new file mode 100644
index 0000000..fdbaa86
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/MemoryLimitTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.commons.compress;
+
+import org.junit.Test;
+
+public class MemoryLimitTest {
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangeCheck() throws Exception {
+ MemoryLimit.setMemoryLimitInKb(-2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/0e8ff9c4/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index b70d3c7..6c8543f 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -31,6 +31,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.MockEvilInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -38,6 +39,8 @@ import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStr
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.Test;
@SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
@@ -71,6 +74,18 @@ public final class DetectCompressorTestCase {
}
}
+ @Before
+ public void setUp() {
+ //make sure to reset this before each test
+ MemoryLimit.setMemoryLimitInKb(MemoryLimit.NO_LIMIT);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ //make sure this is really, truly reset after all the tests
+ MemoryLimit.setMemoryLimitInKb(MemoryLimit.NO_LIMIT);
+ }
+
private final TestData[] tests = {
new TestData("multiple.bz2", new char[]{'a','b'}, factoryTrue, true),
new TestData("multiple.bz2", new char[]{'a','b'}, factorySetTrue, true),
@@ -190,7 +205,7 @@ public final class DetectCompressorTestCase {
//This test is here instead of the xz unit test to make sure
//that the parameter is properly passed via the CompressorStreamFactory
try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) {
- int c = compressorIs.read();
+ compressorIs.read();
}
}
@@ -202,8 +217,8 @@ public final class DetectCompressorTestCase {
}
private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception {
- CompressorStreamFactory fac = new CompressorStreamFactory(true,
- memoryLimitInKb);
+ MemoryLimit.setMemoryLimitInKb(memoryLimitInKb);
+ CompressorStreamFactory fac = new CompressorStreamFactory(true);
InputStream is = new BufferedInputStream(
new FileInputStream(getFile(fileName)));
try {
[04/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
break out unit tests; add memory limit for xz.
Posted by bo...@apache.org.
COMPRESS-382 and COMPRESS-386 -- break out unit tests;
add memory limit for xz.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b10528a6
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b10528a6
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b10528a6
Branch: refs/heads/master
Commit: b10528a62e51b2c5fdc0c4b7884cc93f03f8ce96
Parents: 7d73baf
Author: tballison <ta...@mitre.org>
Authored: Mon Apr 24 10:10:30 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Apr 24 10:10:30 2017 -0400
----------------------------------------------------------------------
.../commons/compress/MemoryLimitException.java | 40 +++++++++++++
.../CompressorMemoryLimitException.java | 36 ------------
.../compressors/CompressorStreamFactory.java | 19 ++----
.../lzma/LZMACompressorInputStream.java | 9 ++-
.../compressors/lzw/LZWInputStream.java | 8 +--
.../compressors/xz/XZCompressorInputStream.java | 62 +++++++++++++++++---
.../compressors/z/ZCompressorInputStream.java | 15 +----
.../compressors/DetectCompressorTestCase.java | 54 +++++++++++++----
8 files changed, 154 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
new file mode 100644
index 0000000..8922ed2
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress;
+
+import java.io.IOException;
+
+/**
+ * If a stream checks for estimated memory allocation, and the estimate
+ * goes above the memory limit, this is thrown. This can also be thrown
+ * if a stream tries to allocate a byte array that is larger than
+ * the allowable limit.
+ *
+ * @since 1.14
+ */
+public class MemoryLimitException extends IOException {
+
+ public MemoryLimitException(String message) {
+ super(message);
+ }
+
+ public MemoryLimitException(String message, Exception e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java b/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
deleted file mode 100644
index 4c87d1e..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.commons.compress.compressors;
-
-/**
- * If a stream checks for estimated memory allocation, and the estimate
- * goes above the memory limit, this is thrown.
- *
- * @since 1.14
- */
-public class CompressorMemoryLimitException extends CompressorException {
-
- public CompressorMemoryLimitException(String message) {
- super(message);
- }
-
- public CompressorMemoryLimitException(String message, Exception e) {
- super(message, e);
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index a0a816f..f3433d9 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -57,7 +57,6 @@ import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.compress.utils.ServiceLoaderIterator;
import org.apache.commons.compress.utils.Sets;
-import org.tukaani.xz.MemoryLimitException;
/**
* <p>
@@ -498,7 +497,9 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
* the input stream
* @return compressor input stream
* @throws CompressorException
- * if the compressor name is not known or not available
+ * if the compressor name is not known or not available,
+ * or if there's an IOException or MemoryLimitException thrown
+ * during initialization
* @throws IllegalArgumentException
* if the name or input stream is null
*/
@@ -528,18 +529,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (!XZUtils.isXZCompressionAvailable()) {
throw new CompressorException("XZ compression is not available.");
}
- return new XZCompressorInputStream(in, actualDecompressConcatenated);
+ return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
}
if (LZMA.equalsIgnoreCase(name)) {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
}
- try {
- return new LZMACompressorInputStream(in, memoryLimitInKb);
- } catch (MemoryLimitException e) {
- throw new CompressorMemoryLimitException("exceeded calculated memory limit", e);
- }
+ return new LZMACompressorInputStream(in, memoryLimitInKb);
}
if (PACK200.equalsIgnoreCase(name)) {
@@ -555,11 +552,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (Z.equalsIgnoreCase(name)) {
- try {
- return new ZCompressorInputStream(in, memoryLimitInKb);
- } catch (ZCompressorInputStream.IOExceptionWrappingMemoryLimitException e) {
- throw new CompressorMemoryLimitException(e.getMessage());
- }
+ return new ZCompressorInputStream(in, memoryLimitInKb);
}
if (DEFLATE.equalsIgnoreCase(name)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 0520e62..7782be8 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.lzma;
import java.io.IOException;
import java.io.InputStream;
+
+import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.LZMAInputStream;
import org.apache.commons.compress.compressors.CompressorInputStream;
@@ -55,7 +57,12 @@ public class LZMACompressorInputStream extends CompressorInputStream {
*/
public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
throws IOException {
- in = new LZMAInputStream(inputStream, memoryLimitInKb);
+ try {
+ in = new LZMAInputStream(inputStream, memoryLimitInKb);
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress exception
+ throw new MemoryLimitException("exceeded calculated memory limit", e);
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index b61d9a1..350b4b0 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.compressors.CompressorInputStream;
-import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.utils.BitInputStream;
/**
@@ -116,13 +116,13 @@ public abstract class LZWInputStream extends CompressorInputStream {
* Initializes the arrays based on the maximum code size.
* @param maxCodeSize maximum code size
* @param memoryLimitInKb maximum allowed table size in Kb
- * @throws CompressorMemoryLimitException if maxTableSize is > memoryLimitInKb
+ * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
*/
protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
- throws CompressorMemoryLimitException {
+ throws MemoryLimitException {
final int maxTableSize = 1 << maxCodeSize;
if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new CompressorMemoryLimitException("Tried to allocate "+maxTableSize +
+ throw new MemoryLimitException("Tried to allocate "+maxTableSize +
" but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
}
initializeTables(maxCodeSize);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index 27d70d2..b378212 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.xz;
import java.io.IOException;
import java.io.InputStream;
+
+import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.XZ;
import org.tukaani.xz.SingleXZInputStream;
import org.tukaani.xz.XZInputStream;
@@ -92,30 +94,72 @@ public class XZCompressorInputStream extends CompressorInputStream {
public XZCompressorInputStream(final InputStream inputStream,
final boolean decompressConcatenated)
throws IOException {
+ this(inputStream, decompressConcatenated, -1);
+ }
+
+ /**
+ * Creates a new input stream that decompresses XZ-compressed data
+ * from the specified input stream.
+ *
+ * @param inputStream where to read the compressed data
+ * @param decompressConcatenated
+ * if true, decompress until the end of the
+ * input; if false, stop after the first .xz
+ * stream and leave the input position to point
+ * to the next byte after the .xz stream
+ * @param memoryLimitInKb memory limit used when reading blocks. If
+ * the estimated memory limit is exceeded on {@link #read()},
+ * a {@link MemoryLimitException} is thrown.
+ *
+ * @throws IOException if the input is not in the .xz format,
+ * the input is corrupt or truncated, the .xz
+ * headers specify options that are not supported
+ * by this implementation,
+ * or the underlying <code>inputStream</code> throws an exception
+ *
+ * @since 1.14
+ */
+ public XZCompressorInputStream(InputStream inputStream,
+ boolean decompressConcatenated, int memoryLimitInKb)
+ throws IOException {
if (decompressConcatenated) {
- in = new XZInputStream(inputStream);
+ in = new XZInputStream(inputStream, memoryLimitInKb);
} else {
- in = new SingleXZInputStream(inputStream);
+ in = new SingleXZInputStream(inputStream, memoryLimitInKb);
}
}
@Override
public int read() throws IOException {
- final int ret = in.read();
- count(ret == -1 ? -1 : 1);
- return ret;
+ try {
+ final int ret = in.read();
+ count(ret == -1 ? -1 : 1);
+ return ret;
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ throw new MemoryLimitException("Exceeded memory limit", e);
+ }
}
@Override
public int read(final byte[] buf, final int off, final int len) throws IOException {
- final int ret = in.read(buf, off, len);
- count(ret);
- return ret;
+ try {
+ final int ret = in.read(buf, off, len);
+ count(ret);
+ return ret;
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress MemoryLimtException
+ throw new MemoryLimitException("Exceeded memory limit", e);
+ }
}
@Override
public long skip(final long n) throws IOException {
- return in.skip(n);
+ try {
+ return in.skip(n);
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress MemoryLimtException
+ throw new MemoryLimitException("Excedded memory limit", e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 22d7c0c..64387e3 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
-import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -53,11 +52,7 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- try {
- initializeTables(maxCodeSize, memoryLimitInKb);
- } catch (CompressorMemoryLimitException e) {
- throw new IOExceptionWrappingMemoryLimitException(e.getMessage());
- }
+ initializeTables(maxCodeSize, memoryLimitInKb);
clearEntries();
}
@@ -173,12 +168,4 @@ public class ZCompressorInputStream extends LZWInputStream {
return length > 3 && signature[0] == MAGIC_1 && signature[1] == (byte) MAGIC_2;
}
- /**
- * Wrapper that subclasses IOException to wrap a MemoryLimitException
- */
- public static class IOExceptionWrappingMemoryLimitException extends IOException {
- public IOExceptionWrappingMemoryLimitException(String message) {
- super(message);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 7cec5df..b70d3c7 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -31,6 +31,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.MockEvilInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
@@ -170,21 +171,50 @@ public final class DetectCompressorTestCase {
return name;
}
- @Test
- public void testMemoryLimit() throws Exception {
- testMemoryLimit("COMPRESS-382");
- testMemoryLimit("COMPRESS-386");
+ @Test(expected = MemoryLimitException.class)
+ public void testLZMAMemoryLimit() throws Exception {
+ getStreamFor("COMPRESS-382", 100);
}
- private void testMemoryLimit(String fileName) throws IOException, CompressorException {
- CompressorStreamFactory fac = new CompressorStreamFactory(true,
- 100);
- try (InputStream is = new BufferedInputStream(
- new FileInputStream(getFile(fileName)))) {
- InputStream compressorInputStream = fac.createCompressorInputStream(is);
- fail("Should have thrown CompressorMemoryLimitException");
- } catch (CompressorMemoryLimitException e) {
+ @Test(expected = MemoryLimitException.class)
+ public void testZMemoryLimit() throws Exception {
+ getStreamFor("COMPRESS-386", 100);
+ }
+
+ @Test(expected = MemoryLimitException.class)
+ public void testXZMemoryLimitOnRead() throws Exception {
+ //Even though the file is very small, the memory limit
+ //has to be quite large (8296 KiB) because of the dictionary size
+
+ //This is triggered on read(); not during initialization.
+ //This test is here instead of the xz unit test to make sure
+ //that the parameter is properly passed via the CompressorStreamFactory
+ try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) {
+ int c = compressorIs.read();
+ }
+ }
+ @Test(expected = MemoryLimitException.class)
+ public void testXZMemoryLimitOnSkip() throws Exception {
+ try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) {
+ compressorIs.skip(10);
+ }
+ }
+
+ private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception {
+ CompressorStreamFactory fac = new CompressorStreamFactory(true,
+ memoryLimitInKb);
+ InputStream is = new BufferedInputStream(
+ new FileInputStream(getFile(fileName)));
+ try {
+ return fac.createCompressorInputStream(is);
+ } catch (CompressorException e) {
+ if (e.getCause() != null && e.getCause() instanceof Exception) {
+ //unwrap cause to reveal MemoryLimiteException
+ throw (Exception)e.getCause();
+ } else {
+ throw e;
+ }
}
}
[02/10] commons-compress git commit: Merge remote-tracking branch
'apache/master' into COMPRESS-382
Posted by bo...@apache.org.
Merge remote-tracking branch 'apache/master' into COMPRESS-382
# Conflicts:
# src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
# src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/3ecbdd79
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/3ecbdd79
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/3ecbdd79
Branch: refs/heads/master
Commit: 3ecbdd79ee964d325b7ce8a1ca982935cc7d92f7
Parents: 70594bc 13a0390
Author: tballison <ta...@mitre.org>
Authored: Fri Apr 21 14:22:39 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Fri Apr 21 14:22:39 2017 -0400
----------------------------------------------------------------------
src/changes/changes.xml | 7 ++
.../archivers/ArchiveStreamFactory.java | 111 ++++++++++-------
.../compressors/CompressorStreamFactory.java | 123 ++++++++++---------
.../pack200/Pack200CompressorOutputStream.java | 10 +-
.../commons/compress/ArchiveReadTest.java | 11 +-
.../commons/compress/MockEvilInputStream.java | 41 +++++++
.../archivers/ArchiveStreamFactoryTest.java | 46 +++++++
.../compress/archivers/LongPathTest.java | 15 ++-
.../compress/archivers/LongSymLinkTest.java | 13 +-
.../compressors/DetectCompressorTestCase.java | 62 +++++++---
10 files changed, 306 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
[03/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
add memoryLimit to CompressorStreamFactory and ZCompressorInputStream and
LZMACompressorInputStream
Posted by bo...@apache.org.
COMPRESS-382 and COMPRESS-386 -- add memoryLimit to CompressorStreamFactory
and ZCompressorInputStream and LZMACompressorInputStream
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/7d73baf1
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/7d73baf1
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/7d73baf1
Branch: refs/heads/master
Commit: 7d73baf10e435fcaa4927495afc185fb473c416b
Parents: 3ecbdd7
Author: tballison <ta...@mitre.org>
Authored: Fri Apr 21 19:21:42 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Fri Apr 21 19:21:42 2017 -0400
----------------------------------------------------------------------
.../CompressorMemoryLimitException.java | 36 ++++++++++++++++
.../compressors/CompressorStreamFactory.java | 43 +++++++++++++++++---
.../lzma/LZMACompressorInputStream.java | 6 +--
.../compressors/lzw/LZWInputStream.java | 17 ++++++++
.../compressors/z/ZCompressorInputStream.java | 24 +++++++++--
.../compressors/DetectCompressorTestCase.java | 24 +++++++++++
src/test/resources/COMPRESS-386 | 1 +
7 files changed, 140 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java b/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
new file mode 100644
index 0000000..4c87d1e
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors;
+
+/**
+ * If a stream checks for estimated memory allocation, and the estimate
+ * goes above the memory limit, this is thrown.
+ *
+ * @since 1.14
+ */
+public class CompressorMemoryLimitException extends CompressorException {
+
+ public CompressorMemoryLimitException(String message) {
+ super(message);
+ }
+
+ public CompressorMemoryLimitException(String message, Exception e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index a29179c..a0a816f 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -57,6 +57,7 @@ import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.compress.utils.ServiceLoaderIterator;
import org.apache.commons.compress.utils.Sets;
+import org.tukaani.xz.MemoryLimitException;
/**
* <p>
@@ -349,28 +350,52 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
*/
private volatile boolean decompressConcatenated = false;
+ private final int memoryLimitInKb;
/**
* Create an instance with the decompress Concatenated option set to false.
*/
public CompressorStreamFactory() {
this.decompressUntilEOF = null;
+ this.memoryLimitInKb = -1;
}
/**
* Create an instance with the provided decompress Concatenated option.
- *
+ *
* @param decompressUntilEOF
* if true, decompress until the end of the input; if false, stop
* after the first stream and leave the input position to point
* to the next byte after the stream. This setting applies to the
* gzip, bzip2 and xz formats only.
- * @since 1.10
+ *
+ * @param memoryLimitInKb
+ * Some streams require allocation of potentially significant
+ * byte arrays/tables, and they can offer checks to prevent OOMs
+ * on corrupt files. Set the maximum allowed memory allocation in KBs.
+ *
+ * @since 1.14
*/
- public CompressorStreamFactory(final boolean decompressUntilEOF) {
+ public CompressorStreamFactory(final boolean decompressUntilEOF, final int memoryLimitInKb) {
this.decompressUntilEOF = Boolean.valueOf(decompressUntilEOF);
// Also copy to existing variable so can continue to use that as the
// current value
this.decompressConcatenated = decompressUntilEOF;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+
+ /**
+ * Create an instance with the provided decompress Concatenated option.
+ *
+ * @param decompressUntilEOF
+ * if true, decompress until the end of the input; if false, stop
+ * after the first stream and leave the input position to point
+ * to the next byte after the stream. This setting applies to the
+ * gzip, bzip2 and xz formats only.
+ * @since 1.10
+ */
+ public CompressorStreamFactory(final boolean decompressUntilEOF) {
+ this(decompressUntilEOF, -1);
}
/**
@@ -510,7 +535,11 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
}
- return new LZMACompressorInputStream(in);
+ try {
+ return new LZMACompressorInputStream(in, memoryLimitInKb);
+ } catch (MemoryLimitException e) {
+ throw new CompressorMemoryLimitException("exceeded calculated memory limit", e);
+ }
}
if (PACK200.equalsIgnoreCase(name)) {
@@ -526,7 +555,11 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (Z.equalsIgnoreCase(name)) {
- return new ZCompressorInputStream(in);
+ try {
+ return new ZCompressorInputStream(in, memoryLimitInKb);
+ } catch (ZCompressorInputStream.IOExceptionWrappingMemoryLimitException e) {
+ throw new CompressorMemoryLimitException(e.getMessage());
+ }
}
if (DEFLATE.equalsIgnoreCase(name)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index ea8a498..0520e62 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -42,7 +42,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
*
* @param inputStream where to read the compressed data
*
- * @param memoryLimitKb calculated memory use threshold. Throws MemoryLimitException
+ * @param memoryLimitInKb calculated memory use threshold. Throws MemoryLimitException
* if calculate memory use is above this threshold
*
* @throws IOException if the input is not in the .lzma format,
@@ -53,9 +53,9 @@ public class LZMACompressorInputStream extends CompressorInputStream {
*
* @since 1.14
*/
- public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitKb)
+ public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
throws IOException {
- in = new LZMAInputStream(inputStream, memoryLimitKb);
+ in = new LZMAInputStream(inputStream, memoryLimitInKb);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 5967f66..b61d9a1 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.nio.ByteOrder;
import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.utils.BitInputStream;
/**
@@ -114,6 +115,22 @@ public abstract class LZWInputStream extends CompressorInputStream {
/**
* Initializes the arrays based on the maximum code size.
* @param maxCodeSize maximum code size
+ * @param memoryLimitInKb maximum allowed table size in Kb
+ * @throws CompressorMemoryLimitException if maxTableSize is > memoryLimitInKb
+ */
+ protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
+ throws CompressorMemoryLimitException {
+ final int maxTableSize = 1 << maxCodeSize;
+ if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
+ throw new CompressorMemoryLimitException("Tried to allocate "+maxTableSize +
+ " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
+ }
+ initializeTables(maxCodeSize);
+ }
+
+ /**
+ * Initializes the arrays based on the maximum code size.
+ * @param maxCodeSize maximum code size
*/
protected void initializeTables(final int maxCodeSize) {
final int maxTableSize = 1 << maxCodeSize;
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 799e58d..22d7c0c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -37,8 +38,9 @@ public class ZCompressorInputStream extends LZWInputStream {
private final boolean blockMode;
private final int maxCodeSize;
private long totalCodesRead = 0;
-
- public ZCompressorInputStream(final InputStream inputStream) throws IOException {
+
+ public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
+ throws IOException {
super(inputStream, ByteOrder.LITTLE_ENDIAN);
final int firstByte = (int) in.readBits(8);
final int secondByte = (int) in.readBits(8);
@@ -51,9 +53,17 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- initializeTables(maxCodeSize);
+ try {
+ initializeTables(maxCodeSize, memoryLimitInKb);
+ } catch (CompressorMemoryLimitException e) {
+ throw new IOExceptionWrappingMemoryLimitException(e.getMessage());
+ }
clearEntries();
}
+
+ public ZCompressorInputStream(final InputStream inputStream) throws IOException {
+ this(inputStream, -1);
+ }
private void clearEntries() {
setTableSize((1 << 8) + (blockMode ? 1 : 0));
@@ -163,4 +173,12 @@ public class ZCompressorInputStream extends LZWInputStream {
return length > 3 && signature[0] == MAGIC_1 && signature[1] == (byte) MAGIC_2;
}
+ /**
+ * Wrapper that subclasses IOException to wrap a MemoryLimitException
+ */
+ public static class IOExceptionWrappingMemoryLimitException extends IOException {
+ public IOExceptionWrappingMemoryLimitException(String message) {
+ super(message);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 67abcd3..7cec5df 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -133,6 +133,10 @@ public final class DetectCompressorTestCase {
assertEquals(CompressorStreamFactory.SNAPPY_FRAMED, detect("bla.tar.sz"));
assertEquals(CompressorStreamFactory.Z, detect("bla.tar.Z"));
+ //make sure we don't oom on detect
+ assertEquals(CompressorStreamFactory.Z, detect("COMPRESS-386"));
+ assertEquals(CompressorStreamFactory.LZMA, detect("COMPRESS-382"));
+
try {
CompressorStreamFactory.detect(new BufferedInputStream(new ByteArrayInputStream(new byte[0])));
fail("shouldn't be able to detect empty stream");
@@ -167,6 +171,26 @@ public final class DetectCompressorTestCase {
}
@Test
+ public void testMemoryLimit() throws Exception {
+ testMemoryLimit("COMPRESS-382");
+ testMemoryLimit("COMPRESS-386");
+ }
+
+ private void testMemoryLimit(String fileName) throws IOException, CompressorException {
+ CompressorStreamFactory fac = new CompressorStreamFactory(true,
+ 100);
+ try (InputStream is = new BufferedInputStream(
+ new FileInputStream(getFile(fileName)))) {
+ InputStream compressorInputStream = fac.createCompressorInputStream(is);
+ fail("Should have thrown CompressorMemoryLimitException");
+ } catch (CompressorMemoryLimitException e) {
+
+ }
+
+ }
+
+
+ @Test
public void testOverride() {
CompressorStreamFactory fac = new CompressorStreamFactory();
assertFalse(fac.getDecompressConcatenated());
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/7d73baf1/src/test/resources/COMPRESS-386
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-386 b/src/test/resources/COMPRESS-386
new file mode 100644
index 0000000..36d7f52
--- /dev/null
+++ b/src/test/resources/COMPRESS-386
@@ -0,0 +1 @@
+ \ufffd B
\ No newline at end of file
[09/10] commons-compress git commit: merge hard reset -- move from
other change list...ugh
Posted by bo...@apache.org.
merge hard reset -- move from other change list...ugh
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/2247ff96
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/2247ff96
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/2247ff96
Branch: refs/heads/master
Commit: 2247ff9601374845012558ecf5963ebb7e1066c5
Parents: 00b5e39
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 25 09:01:28 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 25 09:01:28 2017 -0400
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 36 ++++++++++++++++----
.../compressors/DetectCompressorTestCase.java | 19 ++---------
2 files changed, 32 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2247ff96/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 7daa291..f3433d9 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -349,28 +349,52 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
*/
private volatile boolean decompressConcatenated = false;
+ private final int memoryLimitInKb;
/**
* Create an instance with the decompress Concatenated option set to false.
*/
public CompressorStreamFactory() {
this.decompressUntilEOF = null;
+ this.memoryLimitInKb = -1;
}
/**
* Create an instance with the provided decompress Concatenated option.
- *
+ *
* @param decompressUntilEOF
* if true, decompress until the end of the input; if false, stop
* after the first stream and leave the input position to point
* to the next byte after the stream. This setting applies to the
* gzip, bzip2 and xz formats only.
- * @since 1.10
+ *
+ * @param memoryLimitInKb
+ * Some streams require allocation of potentially significant
+ * byte arrays/tables, and they can offer checks to prevent OOMs
+ * on corrupt files. Set the maximum allowed memory allocation in KBs.
+ *
+ * @since 1.14
*/
- public CompressorStreamFactory(final boolean decompressUntilEOF) {
+ public CompressorStreamFactory(final boolean decompressUntilEOF, final int memoryLimitInKb) {
this.decompressUntilEOF = Boolean.valueOf(decompressUntilEOF);
// Also copy to existing variable so can continue to use that as the
// current value
this.decompressConcatenated = decompressUntilEOF;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+
+ /**
+ * Create an instance with the provided decompress Concatenated option.
+ *
+ * @param decompressUntilEOF
+ * if true, decompress until the end of the input; if false, stop
+ * after the first stream and leave the input position to point
+ * to the next byte after the stream. This setting applies to the
+ * gzip, bzip2 and xz formats only.
+ * @since 1.10
+ */
+ public CompressorStreamFactory(final boolean decompressUntilEOF) {
+ this(decompressUntilEOF, -1);
}
/**
@@ -505,14 +529,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (!XZUtils.isXZCompressionAvailable()) {
throw new CompressorException("XZ compression is not available.");
}
- return new XZCompressorInputStream(in, actualDecompressConcatenated);
+ return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
}
if (LZMA.equalsIgnoreCase(name)) {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
}
- return new LZMACompressorInputStream(in);
+ return new LZMACompressorInputStream(in, memoryLimitInKb);
}
if (PACK200.equalsIgnoreCase(name)) {
@@ -528,7 +552,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (Z.equalsIgnoreCase(name)) {
- return new ZCompressorInputStream(in);
+ return new ZCompressorInputStream(in, memoryLimitInKb);
}
if (DEFLATE.equalsIgnoreCase(name)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/2247ff96/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 2be690a..6fde36d 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -31,7 +31,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.MockEvilInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -39,8 +38,6 @@ import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStr
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
-import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.Test;
@SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
@@ -74,18 +71,6 @@ public final class DetectCompressorTestCase {
}
}
- @Before
- public void setUp() {
- //make sure to reset this before each test
- MemoryLimit.setMemoryLimitInKb(MemoryLimit.NO_LIMIT);
- }
-
- @AfterClass
- public static void tearDown() {
- //make sure this is really, truly reset after all the tests
- MemoryLimit.setMemoryLimitInKb(MemoryLimit.NO_LIMIT);
- }
-
private final TestData[] tests = {
new TestData("multiple.bz2", new char[]{'a','b'}, factoryTrue, true),
new TestData("multiple.bz2", new char[]{'a','b'}, factorySetTrue, true),
@@ -217,8 +202,8 @@ public final class DetectCompressorTestCase {
}
private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception {
- MemoryLimit.setMemoryLimitInKb(memoryLimitInKb);
- CompressorStreamFactory fac = new CompressorStreamFactory(true);
+ CompressorStreamFactory fac = new CompressorStreamFactory(true,
+ memoryLimitInKb);
InputStream is = new BufferedInputStream(
new FileInputStream(getFile(fileName)));
try {
[07/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
hard reset to expunge static parameter proposal. Updated MemoryLimitException
to include cause when it exists. Updated LZWInputStream to calculate
estimated memory on maxTableSize*6 t
Posted by bo...@apache.org.
COMPRESS-382 and COMPRESS-386 -- hard reset to expunge static parameter
proposal. Updated MemoryLimitException to include cause when it exists.
Updated LZWInputStream to calculate estimated memory on maxTableSize*6
to account for int[] and byte[], byte[].
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4e442bb4
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4e442bb4
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4e442bb4
Branch: refs/heads/master
Commit: 4e442bb44280243f6e0e72d845fa8275d665ca84
Parents: b10528a
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 25 08:44:21 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 25 08:44:21 2017 -0400
----------------------------------------------------------------------
.../commons/compress/MemoryLimitException.java | 30 +++++++++++++++++---
.../lzma/LZMACompressorInputStream.java | 2 +-
.../compressors/lzw/LZWInputStream.java | 20 +++++++++----
.../compressors/xz/XZCompressorInputStream.java | 8 +++---
.../compressors/z/ZCompressorInputStream.java | 2 +-
.../compressors/DetectCompressorTestCase.java | 2 +-
6 files changed, 47 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..122edb1 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -30,11 +30,33 @@ import java.io.IOException;
*/
public class MemoryLimitException extends IOException {
- public MemoryLimitException(String message) {
- super(message);
+ //long instead of int to account for overflow for corrupt files
+ private final long memoryNeededInKb;
+ private final int memoryLimitInKb;
+
+ public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb) {
+ super(buildMessage(memoryNeededInKb, memoryLimitInKb));
+ this.memoryNeededInKb = memoryNeededInKb;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+ public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb, Exception e) {
+ super(buildMessage(memoryNeededInKb, memoryLimitInKb), e);
+ this.memoryNeededInKb = memoryNeededInKb;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+ public long getMemoryNeededInKb() {
+ return memoryNeededInKb;
+ }
+
+ public int getMemoryLimitInKb() {
+ return memoryLimitInKb;
}
- public MemoryLimitException(String message, Exception e) {
- super(message, e);
+ private static String buildMessage(long memoryNeededInKb, int memoryLimitInKb) {
+ return "" + memoryNeededInKb + " kb of memory would be needed; limit was "
+ + memoryLimitInKb + " kb. " +
+ "If the file is not corrupt, consider increasing the memory limit.";
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 7782be8..5723b5c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -61,7 +61,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
in = new LZMAInputStream(inputStream, memoryLimitInKb);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress exception
- throw new MemoryLimitException("exceeded calculated memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..2a6127d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -114,16 +114,24 @@ public abstract class LZWInputStream extends CompressorInputStream {
/**
* Initializes the arrays based on the maximum code size.
+ * First checks that the estimated memory usage is below memoryLimitInKb
+ *
* @param maxCodeSize maximum code size
- * @param memoryLimitInKb maximum allowed table size in Kb
- * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+ * @param memoryLimitInKb maximum allowed estimated memory usage in Kb
+ * @throws MemoryLimitException if estimated memory usage is greater than memoryLimitInKb
*/
protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
throws MemoryLimitException {
- final int maxTableSize = 1 << maxCodeSize;
- if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new MemoryLimitException("Tried to allocate "+maxTableSize +
- " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
+
+ if (memoryLimitInKb > -1) {
+ final int maxTableSize = 1 << maxCodeSize;
+ //account for potential overflow
+ long memoryUsageInBytes = (long) maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack))
+ long memoryUsageInKb = memoryUsageInBytes >> 10;
+
+ if (memoryUsageInKb > (long)memoryLimitInKb) {
+ throw new MemoryLimitException(memoryUsageInKb, memoryLimitInKb);
+ }
}
initializeTables(maxCodeSize);
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index b378212..fc0cbf5 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -120,7 +120,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
* @since 1.14
*/
public XZCompressorInputStream(InputStream inputStream,
- boolean decompressConcatenated, int memoryLimitInKb)
+ boolean decompressConcatenated, final int memoryLimitInKb)
throws IOException {
if (decompressConcatenated) {
in = new XZInputStream(inputStream, memoryLimitInKb);
@@ -136,7 +136,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
count(ret == -1 ? -1 : 1);
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
@@ -148,7 +148,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
@@ -158,7 +158,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return in.skip(n);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Excedded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 64387e3..994e102 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -38,7 +38,7 @@ public class ZCompressorInputStream extends LZWInputStream {
private final int maxCodeSize;
private long totalCodesRead = 0;
- public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
+ public ZCompressorInputStream(final InputStream inputStream, final int memoryLimitInKb)
throws IOException {
super(inputStream, ByteOrder.LITTLE_ENDIAN);
final int firstByte = (int) in.readBits(8);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index b70d3c7..844b168 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -210,7 +210,7 @@ public final class DetectCompressorTestCase {
return fac.createCompressorInputStream(is);
} catch (CompressorException e) {
if (e.getCause() != null && e.getCause() instanceof Exception) {
- //unwrap cause to reveal MemoryLimiteException
+ //unwrap cause to reveal MemoryLimitException
throw (Exception)e.getCause();
} else {
throw e;
[08/10] commons-compress git commit: merge hard reset
Posted by bo...@apache.org.
merge hard reset
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/00b5e391
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/00b5e391
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/00b5e391
Branch: refs/heads/master
Commit: 00b5e3911a774af003b7d7291c147fc50bc21c32
Parents: 4e442bb 26eab98
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 25 08:53:31 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 25 08:53:31 2017 -0400
----------------------------------------------------------------------
.../compressors/CompressorStreamFactory.java | 36 ++++----------------
.../compressors/DetectCompressorTestCase.java | 21 ++++++++++--
2 files changed, 24 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/00b5e391/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
[06/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
take 4, clean up and allow for overflow via longs.
Posted by bo...@apache.org.
COMPRESS-382 and COMPRESS-386 -- take 4, clean up and allow
for overflow via longs.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/26eab98e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/26eab98e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/26eab98e
Branch: refs/heads/master
Commit: 26eab98e3a0d403587cc915eaa8f7d4b7b894cef
Parents: 0e8ff9c
Author: tballison <ta...@mitre.org>
Authored: Mon Apr 24 22:09:59 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Apr 24 22:09:59 2017 -0400
----------------------------------------------------------------------
.../apache/commons/compress/MemoryLimit.java | 15 +++++++++++
.../commons/compress/MemoryLimitException.java | 18 +++++++++-----
.../lzma/LZMACompressorInputStream.java | 2 +-
.../compressors/lzw/LZWInputStream.java | 26 ++++++++------------
.../compressors/xz/XZCompressorInputStream.java | 6 ++---
.../compressors/z/ZCompressorInputStream.java | 3 +--
6 files changed, 42 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimit.java b/src/main/java/org/apache/commons/compress/MemoryLimit.java
index e03e8e7..b43eb86 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimit.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimit.java
@@ -62,4 +62,19 @@ public class MemoryLimit {
public static int getMemoryLimitInKb() {
return MEMORY_LIMIT_IN_KB;
}
+
+ public static void checkLimitInKb(long memoryNeeded) throws MemoryLimitException {
+ if (memoryNeeded < 0) {
+ throw new IllegalArgumentException("MemoryLimit must be > -1");
+ }
+
+ if (memoryNeeded >> 10 > Integer.MAX_VALUE) {
+ throw new MemoryLimitException(memoryNeeded,
+ (MEMORY_LIMIT_IN_KB < 0) ? Integer.MAX_VALUE : MEMORY_LIMIT_IN_KB);
+ }
+
+ if (MEMORY_LIMIT_IN_KB > -1 && memoryNeeded > MEMORY_LIMIT_IN_KB) {
+ throw new MemoryLimitException(memoryNeeded, MEMORY_LIMIT_IN_KB);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..a243a17 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -26,15 +26,21 @@ import java.io.IOException;
* if a stream tries to allocate a byte array that is larger than
* the allowable limit.
*
+ * <p/>
+ * Set the global memory limit via {@link MemoryLimit#setMemoryLimitInKb(int)}.
+ *
* @since 1.14
*/
public class MemoryLimitException extends IOException {
- public MemoryLimitException(String message) {
- super(message);
- }
-
- public MemoryLimitException(String message, Exception e) {
- super(message, e);
+ /**
+ *
+ * @param memoryNeeded estimated memory needed
+ * @param memoryLimit memory limit applied
+ */
+ public MemoryLimitException(long memoryNeeded, int memoryLimit) {
+ super("" + memoryNeeded + " KiB of memory would be needed; limit was "
+ + memoryLimit + " KiB. If the file is not corrupt, consider " +
+ "increasing MemoryLimit.MEMORY_LIMIT_IN_KB.");
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 6931541..35d7e12 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -56,7 +56,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
in = new LZMAInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress exception
- throw new MemoryLimitException("exceeded calculated memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..ed2c476 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.utils.BitInputStream;
@@ -115,25 +116,18 @@ public abstract class LZWInputStream extends CompressorInputStream {
/**
* Initializes the arrays based on the maximum code size.
* @param maxCodeSize maximum code size
- * @param memoryLimitInKb maximum allowed table size in Kb
- * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+ *
+ * @throws MemoryLimitException
+ * if the calculated memory usage, based on the maxTableSize,
+ * is > {@link MemoryLimit#MEMORY_LIMIT_IN_KB}
*/
- protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
- throws MemoryLimitException {
+ protected void initializeTables(final int maxCodeSize) throws MemoryLimitException {
final int maxTableSize = 1 << maxCodeSize;
- if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new MemoryLimitException("Tried to allocate "+maxTableSize +
- " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
- }
- initializeTables(maxCodeSize);
- }
- /**
- * Initializes the arrays based on the maximum code size.
- * @param maxCodeSize maximum code size
- */
- protected void initializeTables(final int maxCodeSize) {
- final int maxTableSize = 1 << maxCodeSize;
+ //account for potential overflow
+ long memoryUsageInBytes = (long)maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack))
+ MemoryLimit.checkLimitInKb(memoryUsageInBytes >> 10);
+
prefixes = new int[maxTableSize];
characters = new byte[maxTableSize];
outputStack = new byte[maxTableSize];
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index 3f977ef..e80cbcf 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -110,7 +110,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
count(ret == -1 ? -1 : 1);
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
@@ -122,7 +122,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
@@ -132,7 +132,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return in.skip(n);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Excedded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index ca61cd3..28b69a2 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
-import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -52,7 +51,7 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- initializeTables(maxCodeSize, MemoryLimit.getMemoryLimitInKb());
+ initializeTables(maxCodeSize);
clearEntries();
}
[10/10] commons-compress git commit: Merge branch 'COMPRESS-382'
Posted by bo...@apache.org.
Merge branch 'COMPRESS-382'
closes #20
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/5def510d
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/5def510d
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/5def510d
Branch: refs/heads/master
Commit: 5def510d00f73b25db8ce44303df49050f99dfbc
Parents: 08f0aba 2247ff9
Author: Stefan Bodewig <bo...@apache.org>
Authored: Tue Apr 25 20:41:32 2017 +0200
Committer: Stefan Bodewig <bo...@apache.org>
Committed: Tue Apr 25 20:45:38 2017 +0200
----------------------------------------------------------------------
src/changes/changes.xml | 15 +++++
.../commons/compress/MemoryLimitException.java | 62 +++++++++++++++++++
.../compressors/CompressorStreamFactory.java | 40 +++++++++---
.../lzma/LZMACompressorInputStream.java | 21 ++++++-
.../compressors/lzw/LZWInputStream.java | 25 ++++++++
.../compressors/xz/XZCompressorInputStream.java | 62 ++++++++++++++++---
.../compressors/z/ZCompressorInputStream.java | 11 +++-
.../compressors/DetectCompressorTestCase.java | 54 ++++++++++++++++
src/test/resources/COMPRESS-382 | Bin 0 -> 19 bytes
src/test/resources/COMPRESS-386 | 1 +
10 files changed, 270 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/5def510d/src/changes/changes.xml
----------------------------------------------------------------------
diff --cc src/changes/changes.xml
index ce0105f,8b1bedc..3b24da7
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@@ -79,11 -79,6 +79,26 @@@ The <action> type attribute can be add,
<action issue="COMPRESS-387" type="fix" date="2017-04-18">
Make unit tests work on Windows paths with spaces in their names.
</action>
+ <action issue="COMPRESS-388" type="update" date="2017-04-25"
+ due-to="Zbynek Vyskovsky">
+ Improved performance for concurrent reads from ZipFile when
+ reading from a file.
+ </action>
++ <action issue="COMPRESS-382" type="add" date="2017-04-25"
++ due-to="Tim Allison">
++ Added a way to limit amount of memory ZCompressorStream may
++ use.
++ </action>
++ <action issue="COMPRESS-386" type="add" date="2017-04-25"
++ due-to="Tim Allison">
++ Added a way to limit amount of memory ZCompressorStream may
++ use.
++ </action>
++ <action issue="COMPRESS-382" type="add" date="2017-04-25"
++ due-to="Tim Allison">
++ Added a way to limit amount of memory LZMACompressorStream and
++ XZCompressorInputStream may use.
++ </action>
</release>
<release version="1.13" date="2016-12-29"
description="Release 1.13 - API compatible to 1.12 but requires Java 7 at runtime.">