You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/25 18:46:07 UTC
[07/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
hard reset to expunge static parameter proposal. Updated MemoryLimitException
to include cause when it exists. Updated LZWInputStream to calculate
estimated memory on maxTableSize*6 t
COMPRESS-382 and COMPRESS-386 -- hard reset to expunge static parameter
proposal. Updated MemoryLimitException to include cause when it exists.
Updated LZWInputStream to calculate estimated memory on maxTableSize*6
to account for int[] and byte[], byte[].
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4e442bb4
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4e442bb4
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4e442bb4
Branch: refs/heads/master
Commit: 4e442bb44280243f6e0e72d845fa8275d665ca84
Parents: b10528a
Author: tballison <ta...@mitre.org>
Authored: Tue Apr 25 08:44:21 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Tue Apr 25 08:44:21 2017 -0400
----------------------------------------------------------------------
.../commons/compress/MemoryLimitException.java | 30 +++++++++++++++++---
.../lzma/LZMACompressorInputStream.java | 2 +-
.../compressors/lzw/LZWInputStream.java | 20 +++++++++----
.../compressors/xz/XZCompressorInputStream.java | 8 +++---
.../compressors/z/ZCompressorInputStream.java | 2 +-
.../compressors/DetectCompressorTestCase.java | 2 +-
6 files changed, 47 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..122edb1 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -30,11 +30,33 @@ import java.io.IOException;
*/
public class MemoryLimitException extends IOException {
- public MemoryLimitException(String message) {
- super(message);
+ //long instead of int to account for overflow for corrupt files
+ private final long memoryNeededInKb;
+ private final int memoryLimitInKb;
+
+ public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb) {
+ super(buildMessage(memoryNeededInKb, memoryLimitInKb));
+ this.memoryNeededInKb = memoryNeededInKb;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+ public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb, Exception e) {
+ super(buildMessage(memoryNeededInKb, memoryLimitInKb), e);
+ this.memoryNeededInKb = memoryNeededInKb;
+ this.memoryLimitInKb = memoryLimitInKb;
+ }
+
+ public long getMemoryNeededInKb() {
+ return memoryNeededInKb;
+ }
+
+ public int getMemoryLimitInKb() {
+ return memoryLimitInKb;
}
- public MemoryLimitException(String message, Exception e) {
- super(message, e);
+ private static String buildMessage(long memoryNeededInKb, int memoryLimitInKb) {
+ return "" + memoryNeededInKb + " kb of memory would be needed; limit was "
+ + memoryLimitInKb + " kb. " +
+ "If the file is not corrupt, consider increasing the memory limit.";
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 7782be8..5723b5c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -61,7 +61,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
in = new LZMAInputStream(inputStream, memoryLimitInKb);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress exception
- throw new MemoryLimitException("exceeded calculated memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..2a6127d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -114,16 +114,24 @@ public abstract class LZWInputStream extends CompressorInputStream {
/**
* Initializes the arrays based on the maximum code size.
+ * First checks that the estimated memory usage is below memoryLimitInKb
+ *
* @param maxCodeSize maximum code size
- * @param memoryLimitInKb maximum allowed table size in Kb
- * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+ * @param memoryLimitInKb maximum allowed estimated memory usage in Kb
+ * @throws MemoryLimitException if estimated memory usage is greater than memoryLimitInKb
*/
protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
throws MemoryLimitException {
- final int maxTableSize = 1 << maxCodeSize;
- if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new MemoryLimitException("Tried to allocate "+maxTableSize +
- " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
+
+ if (memoryLimitInKb > -1) {
+ final int maxTableSize = 1 << maxCodeSize;
+ //account for potential overflow
+ long memoryUsageInBytes = (long) maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack))
+ long memoryUsageInKb = memoryUsageInBytes >> 10;
+
+ if (memoryUsageInKb > (long)memoryLimitInKb) {
+ throw new MemoryLimitException(memoryUsageInKb, memoryLimitInKb);
+ }
}
initializeTables(maxCodeSize);
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index b378212..fc0cbf5 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -120,7 +120,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
* @since 1.14
*/
public XZCompressorInputStream(InputStream inputStream,
- boolean decompressConcatenated, int memoryLimitInKb)
+ boolean decompressConcatenated, final int memoryLimitInKb)
throws IOException {
if (decompressConcatenated) {
in = new XZInputStream(inputStream, memoryLimitInKb);
@@ -136,7 +136,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
count(ret == -1 ? -1 : 1);
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
@@ -148,7 +148,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
@@ -158,7 +158,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return in.skip(n);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Excedded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 64387e3..994e102 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -38,7 +38,7 @@ public class ZCompressorInputStream extends LZWInputStream {
private final int maxCodeSize;
private long totalCodesRead = 0;
- public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
+ public ZCompressorInputStream(final InputStream inputStream, final int memoryLimitInKb)
throws IOException {
super(inputStream, ByteOrder.LITTLE_ENDIAN);
final int firstByte = (int) in.readBits(8);
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index b70d3c7..844b168 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -210,7 +210,7 @@ public final class DetectCompressorTestCase {
return fac.createCompressorInputStream(is);
} catch (CompressorException e) {
if (e.getCause() != null && e.getCause() instanceof Exception) {
- //unwrap cause to reveal MemoryLimiteException
+ //unwrap cause to reveal MemoryLimitException
throw (Exception)e.getCause();
} else {
throw e;