You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/25 18:46:04 UTC
[04/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
break out unit tests; add memory limit for xz.
COMPRESS-382 and COMPRESS-386 -- break out unit tests;
add memory limit for xz.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b10528a6
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b10528a6
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b10528a6
Branch: refs/heads/master
Commit: b10528a62e51b2c5fdc0c4b7884cc93f03f8ce96
Parents: 7d73baf
Author: tballison <ta...@mitre.org>
Authored: Mon Apr 24 10:10:30 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Apr 24 10:10:30 2017 -0400
----------------------------------------------------------------------
.../commons/compress/MemoryLimitException.java | 40 +++++++++++++
.../CompressorMemoryLimitException.java | 36 ------------
.../compressors/CompressorStreamFactory.java | 19 ++----
.../lzma/LZMACompressorInputStream.java | 9 ++-
.../compressors/lzw/LZWInputStream.java | 8 +--
.../compressors/xz/XZCompressorInputStream.java | 62 +++++++++++++++++---
.../compressors/z/ZCompressorInputStream.java | 15 +----
.../compressors/DetectCompressorTestCase.java | 54 +++++++++++++----
8 files changed, 154 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
new file mode 100644
index 0000000..8922ed2
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress;
+
+import java.io.IOException;
+
+/**
+ * If a stream checks for estimated memory allocation, and the estimate
+ * goes above the memory limit, this is thrown. This can also be thrown
+ * if a stream tries to allocate a byte array that is larger than
+ * the allowable limit.
+ *
+ * @since 1.14
+ */
+public class MemoryLimitException extends IOException {
+
+ public MemoryLimitException(String message) {
+ super(message);
+ }
+
+ public MemoryLimitException(String message, Exception e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java b/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
deleted file mode 100644
index 4c87d1e..0000000
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.commons.compress.compressors;
-
-/**
- * If a stream checks for estimated memory allocation, and the estimate
- * goes above the memory limit, this is thrown.
- *
- * @since 1.14
- */
-public class CompressorMemoryLimitException extends CompressorException {
-
- public CompressorMemoryLimitException(String message) {
- super(message);
- }
-
- public CompressorMemoryLimitException(String message, Exception e) {
- super(message, e);
- }
-}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index a0a816f..f3433d9 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -57,7 +57,6 @@ import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.compress.utils.Lists;
import org.apache.commons.compress.utils.ServiceLoaderIterator;
import org.apache.commons.compress.utils.Sets;
-import org.tukaani.xz.MemoryLimitException;
/**
* <p>
@@ -498,7 +497,9 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
* the input stream
* @return compressor input stream
* @throws CompressorException
- * if the compressor name is not known or not available
+ * if the compressor name is not known or not available,
+ * or if there's an IOException or MemoryLimitException thrown
+ * during initialization
* @throws IllegalArgumentException
* if the name or input stream is null
*/
@@ -528,18 +529,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
if (!XZUtils.isXZCompressionAvailable()) {
throw new CompressorException("XZ compression is not available.");
}
- return new XZCompressorInputStream(in, actualDecompressConcatenated);
+ return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb);
}
if (LZMA.equalsIgnoreCase(name)) {
if (!LZMAUtils.isLZMACompressionAvailable()) {
throw new CompressorException("LZMA compression is not available");
}
- try {
- return new LZMACompressorInputStream(in, memoryLimitInKb);
- } catch (MemoryLimitException e) {
- throw new CompressorMemoryLimitException("exceeded calculated memory limit", e);
- }
+ return new LZMACompressorInputStream(in, memoryLimitInKb);
}
if (PACK200.equalsIgnoreCase(name)) {
@@ -555,11 +552,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider {
}
if (Z.equalsIgnoreCase(name)) {
- try {
- return new ZCompressorInputStream(in, memoryLimitInKb);
- } catch (ZCompressorInputStream.IOExceptionWrappingMemoryLimitException e) {
- throw new CompressorMemoryLimitException(e.getMessage());
- }
+ return new ZCompressorInputStream(in, memoryLimitInKb);
}
if (DEFLATE.equalsIgnoreCase(name)) {
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 0520e62..7782be8 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.lzma;
import java.io.IOException;
import java.io.InputStream;
+
+import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.LZMAInputStream;
import org.apache.commons.compress.compressors.CompressorInputStream;
@@ -55,7 +57,12 @@ public class LZMACompressorInputStream extends CompressorInputStream {
*/
public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
throws IOException {
- in = new LZMAInputStream(inputStream, memoryLimitInKb);
+ try {
+ in = new LZMAInputStream(inputStream, memoryLimitInKb);
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress exception
+ throw new MemoryLimitException("exceeded calculated memory limit", e);
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index b61d9a1..350b4b0 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.compressors.CompressorInputStream;
-import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.utils.BitInputStream;
/**
@@ -116,13 +116,13 @@ public abstract class LZWInputStream extends CompressorInputStream {
* Initializes the arrays based on the maximum code size.
* @param maxCodeSize maximum code size
* @param memoryLimitInKb maximum allowed table size in Kb
- * @throws CompressorMemoryLimitException if maxTableSize is > memoryLimitInKb
+ * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
*/
protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
- throws CompressorMemoryLimitException {
+ throws MemoryLimitException {
final int maxTableSize = 1 << maxCodeSize;
if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new CompressorMemoryLimitException("Tried to allocate "+maxTableSize +
+ throw new MemoryLimitException("Tried to allocate "+maxTableSize +
" but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
}
initializeTables(maxCodeSize);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index 27d70d2..b378212 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.xz;
import java.io.IOException;
import java.io.InputStream;
+
+import org.apache.commons.compress.MemoryLimitException;
import org.tukaani.xz.XZ;
import org.tukaani.xz.SingleXZInputStream;
import org.tukaani.xz.XZInputStream;
@@ -92,30 +94,72 @@ public class XZCompressorInputStream extends CompressorInputStream {
public XZCompressorInputStream(final InputStream inputStream,
final boolean decompressConcatenated)
throws IOException {
+ this(inputStream, decompressConcatenated, -1);
+ }
+
+ /**
+ * Creates a new input stream that decompresses XZ-compressed data
+ * from the specified input stream.
+ *
+ * @param inputStream where to read the compressed data
+ * @param decompressConcatenated
+ * if true, decompress until the end of the
+ * input; if false, stop after the first .xz
+ * stream and leave the input position to point
+ * to the next byte after the .xz stream
+ * @param memoryLimitInKb memory limit used when reading blocks. If
+ * the estimated memory limit is exceeded on {@link #read()},
+ * a {@link MemoryLimitException} is thrown.
+ *
+ * @throws IOException if the input is not in the .xz format,
+ * the input is corrupt or truncated, the .xz
+ * headers specify options that are not supported
+ * by this implementation,
+ * or the underlying <code>inputStream</code> throws an exception
+ *
+ * @since 1.14
+ */
+ public XZCompressorInputStream(InputStream inputStream,
+ boolean decompressConcatenated, int memoryLimitInKb)
+ throws IOException {
if (decompressConcatenated) {
- in = new XZInputStream(inputStream);
+ in = new XZInputStream(inputStream, memoryLimitInKb);
} else {
- in = new SingleXZInputStream(inputStream);
+ in = new SingleXZInputStream(inputStream, memoryLimitInKb);
}
}
@Override
public int read() throws IOException {
- final int ret = in.read();
- count(ret == -1 ? -1 : 1);
- return ret;
+ try {
+ final int ret = in.read();
+ count(ret == -1 ? -1 : 1);
+ return ret;
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ throw new MemoryLimitException("Exceeded memory limit", e);
+ }
}
@Override
public int read(final byte[] buf, final int off, final int len) throws IOException {
- final int ret = in.read(buf, off, len);
- count(ret);
- return ret;
+ try {
+ final int ret = in.read(buf, off, len);
+ count(ret);
+ return ret;
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress MemoryLimtException
+ throw new MemoryLimitException("Exceeded memory limit", e);
+ }
}
@Override
public long skip(final long n) throws IOException {
- return in.skip(n);
+ try {
+ return in.skip(n);
+ } catch (org.tukaani.xz.MemoryLimitException e) {
+ //convert to commons-compress MemoryLimtException
+ throw new MemoryLimitException("Excedded memory limit", e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 22d7c0c..64387e3 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
-import org.apache.commons.compress.compressors.CompressorMemoryLimitException;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -53,11 +52,7 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- try {
- initializeTables(maxCodeSize, memoryLimitInKb);
- } catch (CompressorMemoryLimitException e) {
- throw new IOExceptionWrappingMemoryLimitException(e.getMessage());
- }
+ initializeTables(maxCodeSize, memoryLimitInKb);
clearEntries();
}
@@ -173,12 +168,4 @@ public class ZCompressorInputStream extends LZWInputStream {
return length > 3 && signature[0] == MAGIC_1 && signature[1] == (byte) MAGIC_2;
}
- /**
- * Wrapper that subclasses IOException to wrap a MemoryLimitException
- */
- public static class IOExceptionWrappingMemoryLimitException extends IOException {
- public IOExceptionWrappingMemoryLimitException(String message) {
- super(message);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 7cec5df..b70d3c7 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -31,6 +31,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.MockEvilInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
@@ -170,21 +171,50 @@ public final class DetectCompressorTestCase {
return name;
}
- @Test
- public void testMemoryLimit() throws Exception {
- testMemoryLimit("COMPRESS-382");
- testMemoryLimit("COMPRESS-386");
+ @Test(expected = MemoryLimitException.class)
+ public void testLZMAMemoryLimit() throws Exception {
+ getStreamFor("COMPRESS-382", 100);
}
- private void testMemoryLimit(String fileName) throws IOException, CompressorException {
- CompressorStreamFactory fac = new CompressorStreamFactory(true,
- 100);
- try (InputStream is = new BufferedInputStream(
- new FileInputStream(getFile(fileName)))) {
- InputStream compressorInputStream = fac.createCompressorInputStream(is);
- fail("Should have thrown CompressorMemoryLimitException");
- } catch (CompressorMemoryLimitException e) {
+ @Test(expected = MemoryLimitException.class)
+ public void testZMemoryLimit() throws Exception {
+ getStreamFor("COMPRESS-386", 100);
+ }
+
+ @Test(expected = MemoryLimitException.class)
+ public void testXZMemoryLimitOnRead() throws Exception {
+ //Even though the file is very small, the memory limit
+ //has to be quite large (8296 KiB) because of the dictionary size
+
+ //This is triggered on read(); not during initialization.
+ //This test is here instead of the xz unit test to make sure
+ //that the parameter is properly passed via the CompressorStreamFactory
+ try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) {
+ int c = compressorIs.read();
+ }
+ }
+ @Test(expected = MemoryLimitException.class)
+ public void testXZMemoryLimitOnSkip() throws Exception {
+ try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) {
+ compressorIs.skip(10);
+ }
+ }
+
+ private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception {
+ CompressorStreamFactory fac = new CompressorStreamFactory(true,
+ memoryLimitInKb);
+ InputStream is = new BufferedInputStream(
+ new FileInputStream(getFile(fileName)));
+ try {
+ return fac.createCompressorInputStream(is);
+ } catch (CompressorException e) {
+ if (e.getCause() != null && e.getCause() instanceof Exception) {
+ //unwrap cause to reveal MemoryLimiteException
+ throw (Exception)e.getCause();
+ } else {
+ throw e;
+ }
}
}